454 lines
33 KiB
Go
454 lines
33 KiB
Go
package service
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/platform/domain"
|
|
)
|
|
|
|
func TestCoreServiceRunJobLifecycle(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
Capabilities: []string{"process.start"},
|
|
Capacity: domain.RunCapacity{MaxJobs: 4},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
if !claim.Accepted || !claim.HasJob || claim.Job.JobID != "job-1" || claim.Job.State != domain.JobStateAccepted {
|
|
t.Fatalf("expected claimed job, got %+v", claim)
|
|
}
|
|
|
|
ack, err := svc.AckRunJob(domain.RunJobAck{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: claim.Job.LeaseToken,
|
|
Attempt: claim.Job.Attempt,
|
|
Message: "starting",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("ack job: %v", err)
|
|
}
|
|
if ack.Job.State != domain.JobStateRunning || ack.Job.Progress.Message != "starting" {
|
|
t.Fatalf("expected running ack job, got %+v", ack)
|
|
}
|
|
|
|
progress, err := svc.UpdateRunJobProgress(domain.RunJobProgress{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: claim.Job.LeaseToken,
|
|
Attempt: claim.Job.Attempt,
|
|
Progress: domain.RunJobProgressReport{Percent: 50, Message: "half"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("progress job: %v", err)
|
|
}
|
|
if progress.Job.Progress.Percent != 50 || progress.Job.Progress.Message != "half" {
|
|
t.Fatalf("expected progress update, got %+v", progress)
|
|
}
|
|
|
|
result, err := svc.CompleteRunJob(domain.RunJobResult{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: claim.Job.LeaseToken,
|
|
Attempt: claim.Job.Attempt,
|
|
State: domain.JobStateSucceeded,
|
|
Progress: domain.RunJobProgressReport{Percent: 100, Message: "done"},
|
|
ResultRef: "artifact://jobs/job-1/result",
|
|
Message: "done",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("complete job: %v", err)
|
|
}
|
|
if result.Job.State != domain.JobStateSucceeded || result.Job.ResultRef != "artifact://jobs/job-1/result" {
|
|
t.Fatalf("expected succeeded result, got %+v", result)
|
|
}
|
|
|
|
stored, err := svc.GetJob("job-1")
|
|
if err != nil {
|
|
t.Fatalf("get completed job: %v", err)
|
|
}
|
|
if stored.State != domain.JobStateSucceeded || stored.Progress.Percent != 100 {
|
|
t.Fatalf("expected stored terminal job, got %+v", stored)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobClaimNoJob(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
Capabilities: []string{"process.start"},
|
|
Capacity: domain.RunCapacity{MaxJobs: 4},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("claim no job: %v", err)
|
|
}
|
|
if !claim.Accepted || claim.HasJob || claim.Job != nil || claim.NextPollSeconds <= 0 {
|
|
t.Fatalf("expected empty claim response, got %+v", claim)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobRejectsInvalidSessionAndLease(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
|
|
_, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: "stale", Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err == nil || !strings.Contains(err.Error(), "sessionToken") {
|
|
t.Fatalf("expected invalid session rejection, got %v", err)
|
|
}
|
|
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
_, err = svc.UpdateRunJobProgress(domain.RunJobProgress{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: "bad-lease",
|
|
Attempt: claim.Job.Attempt,
|
|
Progress: domain.RunJobProgressReport{Percent: 10},
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "leaseToken") {
|
|
t.Fatalf("expected invalid lease rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobRejectsInvalidProgress(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
|
|
_, err = svc.UpdateRunJobProgress(domain.RunJobProgress{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: claim.Job.LeaseToken,
|
|
Attempt: claim.Job.Attempt,
|
|
Progress: domain.RunJobProgressReport{Percent: 101},
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "progress.percent") {
|
|
t.Fatalf("expected invalid progress rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobCancelPoll(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
if _, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}); err != nil {
|
|
t.Fatalf("ack job: %v", err)
|
|
}
|
|
|
|
cancel, err := svc.RequestRunJobCancel(domain.RunJobCancelRequest{JobID: "job-1", Reason: "operator requested"})
|
|
if err != nil {
|
|
t.Fatalf("request cancel: %v", err)
|
|
}
|
|
if !cancel.Accepted || cancel.Reason != "operator requested" {
|
|
t.Fatalf("unexpected cancel request: %+v", cancel)
|
|
}
|
|
|
|
poll, err := svc.PollRunJobCancel(domain.RunJobCancelPoll{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: "job-1", LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
|
if err != nil {
|
|
t.Fatalf("poll cancel: %v", err)
|
|
}
|
|
if !poll.HasCancel || poll.JobID != "job-1" || poll.Reason != "operator requested" {
|
|
t.Fatalf("expected cancel poll result, got %+v", poll)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobTerminalResultIsIdempotent(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
|
|
request := domain.RunJobResult{
|
|
RunEndpointID: "run-local",
|
|
SessionToken: sessionToken,
|
|
JobID: claim.Job.JobID,
|
|
LeaseToken: claim.Job.LeaseToken,
|
|
Attempt: claim.Job.Attempt,
|
|
State: domain.JobStateSucceeded,
|
|
Progress: domain.RunJobProgressReport{Percent: 100, Message: "done"},
|
|
ResultRef: "artifact://jobs/job-1/result",
|
|
Message: "done",
|
|
}
|
|
first, err := svc.CompleteRunJob(request)
|
|
if err != nil {
|
|
t.Fatalf("complete first: %v", err)
|
|
}
|
|
second, err := svc.CompleteRunJob(request)
|
|
if err != nil {
|
|
t.Fatalf("complete duplicate: %v", err)
|
|
}
|
|
if second.Job.State != first.Job.State || second.Job.ResultRef != first.Job.ResultRef {
|
|
t.Fatalf("expected duplicate result to be idempotent, got %+v %+v", first, second)
|
|
}
|
|
|
|
request.State = domain.JobStateFailed
|
|
request.Message = "failed"
|
|
_, err = svc.CompleteRunJob(request)
|
|
if err == nil || !strings.Contains(err.Error(), "conflicts") {
|
|
t.Fatalf("expected conflicting result rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobSQLiteTemplateEnvelopeIsFencedToLease(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
request := domain.SCUMSQLiteTemplateRequest{RequestID: "request-query", JobID: "job-query", Binding: scumTemplateTestBinding(), Capability: domain.SCUMDataCapabilityPlayerRead, TargetKey: "scum-database", TemplateKey: "players.active.v1", AdapterVersion: "adapter-1", RequiredSchemaFingerprint: scumTemplateTestHash(), AssetDigest: scumTemplateTestHash(), ParameterDigest: scumTemplateTestHash(), Parameters: map[string]any{"limit": 100.0}, Bounds: domain.DefaultSCUMSQLiteTemplateBounds(), RequestedAt: time.Now()}
|
|
createSCUMTemplateServerFixture(t, svc, request.Binding)
|
|
job, err := svc.CreateJob(domain.Job{ID: request.JobID, ServerInstanceID: request.Binding.ServerInstanceID, RunEndpointID: request.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunDBSQLiteQuery, TargetKey: request.TargetKey, InputRef: "input://sqlite-template/request-query", IdempotencyKey: "idem-query", ExecutionInput: domain.JobExecutionInput{SQLiteTemplate: &request}})
|
|
if err != nil {
|
|
t.Fatalf("create sqlite query job: %v", err)
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunDBSQLiteQuery}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID || claim.Job.ExecutionInput.SQLiteTemplate == nil {
|
|
t.Fatalf("claim sqlite query job: claim=%+v err=%v", claim, err)
|
|
}
|
|
result := scumTemplateTestResult(request)
|
|
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "query complete"}, Message: "query complete", ExecutionResult: domain.JobExecutionResult{Kind: scumSQLiteTemplateExecutionKind, SQLiteTemplate: &result, AuditSummary: "redacted sqlite template query"}}); err != nil {
|
|
t.Fatalf("complete matching sqlite query: %v", err)
|
|
}
|
|
|
|
badRequest := request
|
|
badRequest.RequestID = "request-query-bad"
|
|
badRequest.JobID = "job-query-bad"
|
|
if _, err := svc.CreateJob(domain.Job{ID: badRequest.JobID, ServerInstanceID: badRequest.Binding.ServerInstanceID, RunEndpointID: badRequest.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunDBSQLiteQuery, TargetKey: badRequest.TargetKey, InputRef: "input://sqlite-template/request-query-bad", IdempotencyKey: "idem-query-bad", ExecutionInput: domain.JobExecutionInput{SQLiteTemplate: &badRequest}}); err != nil {
|
|
t.Fatalf("create bad sqlite query job: %v", err)
|
|
}
|
|
badClaim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunDBSQLiteQuery}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim bad sqlite query job: %v", err)
|
|
}
|
|
badResult := scumTemplateTestResult(badRequest)
|
|
badResult.AssetDigest = "sha256:" + strings.Repeat("b", 64)
|
|
_, err = svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, JobID: badClaim.Job.JobID, LeaseToken: badClaim.Job.LeaseToken, Attempt: badClaim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "query complete"}, Message: "query complete", ExecutionResult: domain.JobExecutionResult{Kind: scumSQLiteTemplateExecutionKind, SQLiteTemplate: &badResult, AuditSummary: "redacted sqlite template query"}})
|
|
if err == nil || !strings.Contains(err.Error(), "digest") {
|
|
t.Fatalf("expected digest-fenced query result rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobTypedRCONTemplateEnvelopeIsFencedToLease(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
request := scumTypedRCONTemplateTestRequest()
|
|
createSCUMTemplateServerFixture(t, svc, request.Binding)
|
|
job, err := svc.CreateJob(domain.Job{ID: request.JobID, ServerInstanceID: request.Binding.ServerInstanceID, RunEndpointID: request.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunProtectedRCON, TargetKey: request.TargetKey, InputRef: "input://rcon-template/request-rcon", IdempotencyKey: request.IdempotencyKey, RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1}, ExecutionInput: domain.JobExecutionInput{RemoteAdapterKind: "protected-rcon", SourceRCON: scumTypedRCONSourcePlan(), RCONTemplate: &request}})
|
|
if err != nil {
|
|
t.Fatalf("create typed RCON job: %v", err)
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunProtectedRCON}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID || claim.Job.ExecutionInput.RCONTemplate == nil || claim.Job.ExecutionInput.RCONTemplate.Payload["absoluteValue"].(float64) != 100.0 {
|
|
t.Fatalf("claim typed RCON job: claim=%+v err=%v", claim, err)
|
|
}
|
|
result := scumTypedRCONTemplateTestResult(request)
|
|
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "rcon complete"}, Message: "rcon complete", ExecutionResult: domain.JobExecutionResult{Kind: scumRCONTemplateExecutionKind, RCONTemplate: &result, AuditSummary: "redacted typed RCON template command"}}); err != nil {
|
|
t.Fatalf("complete matching typed RCON job: %v", err)
|
|
}
|
|
|
|
badRequest := request
|
|
badRequest.RequestID = "request-rcon-bad"
|
|
badRequest.JobID = "job-rcon-bad"
|
|
badRequest.IdempotencyKey = "idem-rcon-bad"
|
|
if _, err := svc.CreateJob(domain.Job{ID: badRequest.JobID, ServerInstanceID: badRequest.Binding.ServerInstanceID, RunEndpointID: badRequest.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunProtectedRCON, TargetKey: badRequest.TargetKey, InputRef: "input://rcon-template/request-rcon-bad", IdempotencyKey: badRequest.IdempotencyKey, RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1}, ExecutionInput: domain.JobExecutionInput{RemoteAdapterKind: "protected-rcon", SourceRCON: scumTypedRCONSourcePlan(), RCONTemplate: &badRequest}}); err != nil {
|
|
t.Fatalf("create bad typed RCON job: %v", err)
|
|
}
|
|
badClaim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunProtectedRCON}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim bad typed RCON job: %v", err)
|
|
}
|
|
badResult := scumTypedRCONTemplateTestResult(badRequest)
|
|
badResult.PayloadDigest = "sha256:" + strings.Repeat("b", 64)
|
|
_, err = svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, JobID: badClaim.Job.JobID, LeaseToken: badClaim.Job.LeaseToken, Attempt: badClaim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "rcon complete"}, Message: "rcon complete", ExecutionResult: domain.JobExecutionResult{Kind: scumRCONTemplateExecutionKind, RCONTemplate: &badResult, AuditSummary: "redacted typed RCON template command"}})
|
|
if err == nil || !strings.Contains(err.Error(), "digest") {
|
|
t.Fatalf("expected digest-fenced typed RCON result rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobGuardedMutationEnvelopeIsFencedToLease(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
request := scumGuardedMutationTestRequest()
|
|
createSCUMTemplateServerFixture(t, svc, request.Binding)
|
|
job, err := svc.CreateJob(domain.Job{ID: request.JobID, ServerInstanceID: request.Binding.ServerInstanceID, RunEndpointID: request.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunProtectedSQL, TargetKey: request.TargetKey, InputRef: "input://guarded-mutation/request-mutation", IdempotencyKey: request.IdempotencyKey, RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1}, ExecutionInput: domain.JobExecutionInput{RemoteAdapterKind: "protected-sql", RemoteAdapterKey: request.TargetKey, GuardedMutation: &request}})
|
|
if err != nil {
|
|
t.Fatalf("create guarded mutation job: %v", err)
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunProtectedSQL}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID || claim.Job.ExecutionInput.GuardedMutation == nil || claim.Job.ExecutionInput.GuardedMutation.Payload["attributeKey"].(string) != "Strength" {
|
|
t.Fatalf("claim guarded mutation job: claim=%+v err=%v", claim, err)
|
|
}
|
|
result := scumGuardedMutationTestResult(request)
|
|
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: request.Binding.RunEndpointID, SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "mutation complete"}, Message: "mutation complete", ExecutionResult: domain.JobExecutionResult{Kind: scumGuardedMutationExecutionKind, GuardedMutation: &result, AuditSummary: "redacted guarded mutation"}}); err != nil {
|
|
t.Fatalf("complete matching guarded mutation job: %v", err)
|
|
}
|
|
|
|
badRequest := request
|
|
badRequest.RequestID = "request-mutation-bad"
|
|
badRequest.JobID = "job-mutation-bad"
|
|
badRequest.IdempotencyKey = "idem-mutation-bad"
|
|
if _, err := svc.CreateJob(domain.Job{ID: badRequest.JobID, ServerInstanceID: badRequest.Binding.ServerInstanceID, RunEndpointID: badRequest.Binding.RunEndpointID, Capability: domain.JobCapabilityRemoteRunProtectedSQL, TargetKey: badRequest.TargetKey, InputRef: "input://guarded-mutation/request-mutation-bad", IdempotencyKey: badRequest.IdempotencyKey, RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1}, ExecutionInput: domain.JobExecutionInput{RemoteAdapterKind: "protected-sql", RemoteAdapterKey: badRequest.TargetKey, GuardedMutation: &badRequest}}); err != nil {
|
|
t.Fatalf("create bad guarded mutation job: %v", err)
|
|
}
|
|
badClaim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunProtectedSQL}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim bad guarded mutation job: %v", err)
|
|
}
|
|
badResult := scumGuardedMutationTestResult(badRequest)
|
|
badResult.PatchDigest = "sha256:" + strings.Repeat("b", 64)
|
|
_, err = svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: badRequest.Binding.RunEndpointID, SessionToken: sessionToken, JobID: badClaim.Job.JobID, LeaseToken: badClaim.Job.LeaseToken, Attempt: badClaim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "mutation complete"}, Message: "mutation complete", ExecutionResult: domain.JobExecutionResult{Kind: scumGuardedMutationExecutionKind, GuardedMutation: &badResult, AuditSummary: "redacted guarded mutation"}})
|
|
if err == nil || !strings.Contains(err.Error(), "digest") {
|
|
t.Fatalf("expected digest-fenced guarded mutation result rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobParsedLogBatchEnvelopeIsFencedToLease(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
binding := scumTemplateTestBinding()
|
|
createSCUMTemplateServerFixture(t, svc, binding)
|
|
source := &domain.RuntimeLogSource{Key: "scum-login-events", Kind: "file.tail", TargetKey: "logs/login", StreamKey: "scum.login", CursorKind: "fingerprint", RetentionDays: 90}
|
|
job, err := svc.CreateJob(domain.Job{ID: "job-log", ServerInstanceID: binding.ServerInstanceID, RunEndpointID: binding.RunEndpointID, Capability: domain.JobCapabilityLogsBackfill, TargetKey: "logs/scum-login-events", InputRef: "artifact://logs/checkpoint/1", IdempotencyKey: "idem-log", ExecutionInput: domain.JobExecutionInput{PluginID: binding.PluginID, TargetVersion: binding.PluginVersion, LogSource: source, Inputs: map[string]string{"parserKey": "scum-login-log-login-parser", "parserVersion": "scum-login-log-v1", "parserDigest": scumTemplateTestHash(), "adapterVersion": binding.AdapterVersion}}})
|
|
if err != nil {
|
|
t.Fatalf("create parsed log backfill job: %v", err)
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityLogsBackfill}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID || claim.Job.ExecutionInput.LogSource == nil {
|
|
t.Fatalf("claim parsed log job: claim=%+v err=%v", claim, err)
|
|
}
|
|
result := scumParsedLogBatchTestResult(binding, job.ID, *source)
|
|
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: binding.RunEndpointID, SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "log parse complete"}, Message: "log parse complete", ExecutionResult: domain.JobExecutionResult{Kind: scumParsedLogBatchExecutionKind, ParsedLogBatch: &result, AuditSummary: "redacted parsed log batch"}}); err != nil {
|
|
t.Fatalf("complete matching parsed log job: %v", err)
|
|
}
|
|
|
|
badJob, err := svc.CreateJob(domain.Job{ID: "job-log-bad", ServerInstanceID: binding.ServerInstanceID, RunEndpointID: binding.RunEndpointID, Capability: domain.JobCapabilityLogsBackfill, TargetKey: "logs/scum-login-events", InputRef: "artifact://logs/checkpoint/2", IdempotencyKey: "idem-log-bad", ExecutionInput: domain.JobExecutionInput{PluginID: binding.PluginID, TargetVersion: binding.PluginVersion, LogSource: source, Inputs: map[string]string{"parserKey": "scum-login-log-login-parser", "parserVersion": "scum-login-log-v1", "parserDigest": scumTemplateTestHash(), "adapterVersion": binding.AdapterVersion}}})
|
|
if err != nil {
|
|
t.Fatalf("create bad parsed log job: %v", err)
|
|
}
|
|
badClaim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: binding.RunEndpointID, SessionToken: sessionToken, Capabilities: []string{domain.JobCapabilityLogsBackfill}, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil || !badClaim.HasJob || badClaim.Job.JobID != badJob.ID {
|
|
t.Fatalf("claim bad parsed log job: claim=%+v err=%v", badClaim, err)
|
|
}
|
|
badResult := scumParsedLogBatchTestResult(binding, badJob.ID, *source)
|
|
badResult.ParserDigest = "sha256:" + strings.Repeat("b", 64)
|
|
_, err = svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: binding.RunEndpointID, SessionToken: sessionToken, JobID: badClaim.Job.JobID, LeaseToken: badClaim.Job.LeaseToken, Attempt: badClaim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "log parse complete"}, Message: "log parse complete", ExecutionResult: domain.JobExecutionResult{Kind: scumParsedLogBatchExecutionKind, ParsedLogBatch: &badResult, AuditSummary: "redacted parsed log batch"}})
|
|
if err == nil || !strings.Contains(err.Error(), "parser identity") {
|
|
t.Fatalf("expected parser-fenced parsed log result rejection, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceRunJobReconcile(t *testing.T) {
|
|
svc, sessionToken := newRegisteredRunJobService(t)
|
|
createQueuedRunJob(t, svc, "job-1", "idem-1")
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capacity: domain.RunCapacity{MaxJobs: 4}})
|
|
if err != nil {
|
|
t.Fatalf("claim job: %v", err)
|
|
}
|
|
if _, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}); err != nil {
|
|
t.Fatalf("ack job: %v", err)
|
|
}
|
|
|
|
reconcile, err := svc.ReconcileRunJobs(domain.RunJobReconcile{RunEndpointID: "run-local", SessionToken: sessionToken, ActiveJobs: []domain.RunJobReconcileEntry{
|
|
{JobID: "job-1", LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt},
|
|
{JobID: "local-only", LeaseToken: "local-lease", Attempt: 1},
|
|
}})
|
|
if err != nil {
|
|
t.Fatalf("reconcile jobs: %v", err)
|
|
}
|
|
if len(reconcile.ConfirmedJobs) != 1 || reconcile.ConfirmedJobs[0].JobID != "job-1" {
|
|
t.Fatalf("expected platform active job, got %+v", reconcile)
|
|
}
|
|
if len(reconcile.DiscardJobIDs) != 1 || reconcile.DiscardJobIDs[0] != "local-only" {
|
|
t.Fatalf("expected unknown local job, got %+v", reconcile.DiscardJobIDs)
|
|
}
|
|
}
|
|
|
|
func newRegisteredRunJobService(t *testing.T) (*CoreService, string) {
|
|
t.Helper()
|
|
svc := newTestCoreService()
|
|
helloRequest := validRunControlHello()
|
|
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, "process.start", domain.JobCapabilityRemoteRunDBSQLiteQuery, domain.JobCapabilityRemoteRunProtectedRCON, domain.JobCapabilityRemoteRunProtectedSQL, domain.JobCapabilityLogsBackfill)
|
|
helloRequest.CapabilityReport.Fingerprint = "cap-jobs"
|
|
hello, err := svc.RegisterRunHello(helloRequest)
|
|
if err != nil {
|
|
t.Fatalf("register run hello: %v", err)
|
|
}
|
|
return svc, hello.SessionToken
|
|
}
|
|
|
|
func scumTemplateTestHash() string { return "sha256:" + strings.Repeat("a", 64) }
|
|
|
|
func scumTemplateTestBinding() domain.SCUMBindingIdentity {
|
|
return domain.SCUMBindingIdentity{ServerInstanceID: "server-scum", RunBindingID: "binding-scum", RunEndpointID: "run-local", PluginID: "game.scum", PluginVersion: "0.1.6", AdapterVersion: "adapter-1", GameVersion: "scum-1", DatabaseIdentity: "scum-database"}
|
|
}
|
|
|
|
func scumTypedRCONTemplateTestRequest() domain.SCUMTypedRCONTemplateRequest {
|
|
return domain.SCUMTypedRCONTemplateRequest{RequestID: "request-rcon", JobID: "job-rcon", Binding: scumTemplateTestBinding(), Capability: domain.SCUMDataCapabilityEconomyCommand, TransportKey: "scum-rcon", TargetKey: "scum-rcon", TemplateKey: "economy.fame.set.v1", AdapterVersion: "adapter-1", RequiredSchemaFingerprint: scumTemplateTestHash(), AssetDigest: scumTemplateTestHash(), PayloadDigest: scumTemplateTestHash(), ConfirmationDigest: scumTemplateTestHash(), TargetIdentityDigest: scumTemplateTestHash(), IdempotencyKey: "idem-rcon", Payload: map[string]any{"externalPlayerId": "player-redacted", "absoluteValue": 100.0}, ReviewReason: "operator reviewed absolute fame update", Bounds: domain.DefaultSCUMTypedRCONTemplateBounds(), RequestedAt: time.Now()}
|
|
}
|
|
|
|
func scumGuardedMutationTestRequest() domain.SCUMGuardedMutationRequest {
|
|
return domain.SCUMGuardedMutationRequest{RequestID: "request-mutation", JobID: "job-mutation", Binding: scumTemplateTestBinding(), Capability: domain.SCUMDataCapabilityProfileXMLWrite, TargetKey: "scum-mutation-db", TemplateKey: "profile.attributes.patch.v1", AdapterVersion: "adapter-1", RequiredSchemaFingerprint: scumTemplateTestHash(), AssetDigest: scumTemplateTestHash(), TargetIdentityDigest: scumTemplateTestHash(), ExpectedRowDigest: scumTemplateTestHash(), ExpectedValueDigest: scumTemplateTestHash(), ExpectedXMLDigest: scumTemplateTestHash(), PatchDigest: scumTemplateTestHash(), BackupEvidenceDigest: scumTemplateTestHash(), OfflineEvidenceDigest: scumTemplateTestHash(), DangerConfirmationDigest: scumTemplateTestHash(), ReadbackExpectationDigest: scumTemplateTestHash(), IdempotencyKey: "idem-mutation", Payload: map[string]any{"attributeKey": "Strength", "absoluteValue": 8.5}, ReviewReason: "operator confirmed offline profile attribute patch", Bounds: domain.DefaultSCUMGuardedMutationBounds(), RequestedAt: time.Now()}
|
|
}
|
|
|
|
func scumTypedRCONSourcePlan() *domain.RuntimeSourceRCONPlan {
|
|
return &domain.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/scum-rcon/release.json", Port: 27015}
|
|
}
|
|
|
|
func createSCUMTemplateServerFixture(t *testing.T, svc *CoreService, binding domain.SCUMBindingIdentity) {
|
|
t.Helper()
|
|
if _, err := svc.CreateGamePlugin(domain.GamePlugin{ID: binding.PluginID, Name: "SCUM", Version: binding.PluginVersion, ServerType: "scum", ManifestRef: "artifact://manifests/game.scum/0.1.6", CreateFormSchemaRef: "artifact://schemas/game.scum/create-form/0.1.6", RequiredRunCapabilities: []string{domain.JobCapabilityRemoteRunDBSQLiteQuery, domain.JobCapabilityRemoteRunProtectedRCON, domain.JobCapabilityRemoteRunProtectedSQL, domain.JobCapabilityLogsBackfill}, DeclaredPermissions: []string{"server.remote.access", "server.game-client.command", "server.game-client.maintenance", "server.logs.read"}, Permissions: domain.PluginPermissions{RemoteAccess: true}}); err != nil {
|
|
t.Fatalf("create SCUM plugin fixture: %v", err)
|
|
}
|
|
if _, err := svc.CreateServerInstance(domain.ServerInstance{ID: binding.ServerInstanceID, PluginID: binding.PluginID, RunEndpointID: binding.RunEndpointID, Name: "SCUM"}); err != nil {
|
|
t.Fatalf("create SCUM server fixture: %v", err)
|
|
}
|
|
}
|
|
|
|
func scumTemplateTestResult(request domain.SCUMSQLiteTemplateRequest) domain.SCUMSQLiteTemplateResult {
|
|
return domain.SCUMSQLiteTemplateResult{RequestID: request.RequestID, JobID: request.JobID, Binding: request.Binding, Status: domain.SCUMTerminalResultSucceeded, Capability: request.Capability, TargetKey: request.TargetKey, TemplateKey: request.TemplateKey, AdapterVersion: request.AdapterVersion, SchemaFingerprint: request.RequiredSchemaFingerprint, AssetDigest: request.AssetDigest, ParameterDigest: request.ParameterDigest, SourceFingerprint: scumTemplateTestHash(), ObservedAt: time.Now(), ResultDigest: scumTemplateTestHash(), RowCount: 1, Rows: []map[string]any{{"externalPlayerId": "player-redacted", "displayName": "Known Player"}}, Limits: request.Bounds, SafeError: domain.SCUMSafeError{Code: domain.SCUMSafeErrorNone}}
|
|
}
|
|
|
|
func scumTypedRCONTemplateTestResult(request domain.SCUMTypedRCONTemplateRequest) domain.SCUMTypedRCONTemplateResult {
|
|
return domain.SCUMTypedRCONTemplateResult{RequestID: request.RequestID, JobID: request.JobID, Binding: request.Binding, Status: domain.SCUMTerminalResultSucceeded, Capability: request.Capability, TransportKey: request.TransportKey, TargetKey: request.TargetKey, TemplateKey: request.TemplateKey, AdapterVersion: request.AdapterVersion, SchemaFingerprint: request.RequiredSchemaFingerprint, AssetDigest: request.AssetDigest, PayloadDigest: request.PayloadDigest, ConfirmationDigest: request.ConfirmationDigest, TargetIdentityDigest: request.TargetIdentityDigest, ObservedAt: time.Now(), ResultDigest: scumTemplateTestHash(), ResponseDigest: scumTemplateTestHash(), ConfirmationStatus: domain.SCUMRCONConfirmationConfirmed, ConfirmationDigestID: scumTemplateTestHash(), SafeSummary: "confirmed by declared readback", Limits: request.Bounds, SafeError: domain.SCUMSafeError{Code: domain.SCUMSafeErrorNone}}
|
|
}
|
|
|
|
func scumGuardedMutationTestResult(request domain.SCUMGuardedMutationRequest) domain.SCUMGuardedMutationResult {
|
|
return domain.SCUMGuardedMutationResult{RequestID: request.RequestID, JobID: request.JobID, Binding: request.Binding, Status: domain.SCUMTerminalResultSucceeded, Capability: request.Capability, TargetKey: request.TargetKey, TemplateKey: request.TemplateKey, AdapterVersion: request.AdapterVersion, SchemaFingerprint: request.RequiredSchemaFingerprint, AssetDigest: request.AssetDigest, SourceFingerprint: scumTemplateTestHash(), TargetIdentityDigest: request.TargetIdentityDigest, ExpectedRowDigest: request.ExpectedRowDigest, ExpectedValueDigest: request.ExpectedValueDigest, ExpectedXMLDigest: request.ExpectedXMLDigest, PatchDigest: request.PatchDigest, BackupEvidenceDigest: request.BackupEvidenceDigest, OfflineEvidenceDigest: request.OfflineEvidenceDigest, DangerConfirmationDigest: request.DangerConfirmationDigest, ReadbackExpectationDigest: request.ReadbackExpectationDigest, ObservedAt: time.Now(), ResultDigest: scumTemplateTestHash(), BeforeDigest: scumTemplateTestHash(), AfterDigest: scumTemplateTestHash(), ReadbackDigest: scumTemplateTestHash(), AffectedRows: 1, ReadbackStatus: domain.SCUMMutationReadbackConfirmed, SafeSummary: "confirmed by declared readback", Limits: request.Bounds, SafeError: domain.SCUMSafeError{Code: domain.SCUMSafeErrorNone}}
|
|
}
|
|
|
|
func scumParsedLogBatchTestResult(binding domain.SCUMBindingIdentity, jobID string, source domain.RuntimeLogSource) domain.SCUMParsedLogBatchResult {
|
|
cursor := domain.SCUMParsedLogCursor{SourceIdentityDigest: scumTemplateTestHash(), StreamGeneration: scumTemplateTestHash(), Sequence: 7}
|
|
return domain.SCUMParsedLogBatchResult{RequestID: "request-log", JobID: jobID, Binding: binding, Status: domain.SCUMTerminalResultSucceeded, SourceKey: source.Key, StreamKey: source.StreamKey, ParserKey: "scum-login-log-login-parser", ParserVersion: "scum-login-log-v1", AdapterVersion: binding.AdapterVersion, AssetDigest: scumTemplateTestHash(), ParserDigest: scumTemplateTestHash(), ObservedAt: time.Now(), ResultDigest: scumTemplateTestHash(), FirstCursor: cursor, LastCursor: cursor, TailState: domain.SCUMLogTailRotated, Replay: true, EventCount: 1, Events: []domain.SCUMParsedLogEvent{{EventType: "scum.login", OccurredAt: time.Now(), Cursor: cursor, LogicalEventDigest: scumTemplateTestHash(), EventDigest: scumTemplateTestHash(), PayloadDigest: scumTemplateTestHash(), Payload: map[string]any{"externalPlayerId": "player-redacted", "displayName": "Known Player", "profileLocalId": "profile-redacted"}}}, SafeSummary: "one sanitized login event parsed from declared source", Limits: domain.DefaultSCUMParsedLogBatchBounds(), SafeError: domain.SCUMSafeError{Code: domain.SCUMSafeErrorNone}}
|
|
}
|
|
|
|
func createQueuedRunJob(t *testing.T, svc *CoreService, id string, idempotencyKey string) domain.Job {
|
|
t.Helper()
|
|
job, err := svc.CreateJob(domain.Job{
|
|
ID: id,
|
|
RunEndpointID: "run-local",
|
|
Capability: "process.start",
|
|
IdempotencyKey: idempotencyKey,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create queued job: %v", err)
|
|
}
|
|
return job
|
|
}
|