Remove pre-1.0 audit and protected request scaffolding
This commit is contained in:
@@ -91,16 +91,6 @@ func (svc *CoreService) InvokeAIForSession(sessionID string, request domain.AIIn
|
||||
}
|
||||
result, err := svc.aiProviderClient.Invoke(provider, request)
|
||||
if err != nil {
|
||||
auditID, auditErr := svc.recordAuditEventWithID(user.ID, "ai.provider.invoke.failed", "ai-provider", provider.ID, domain.AuditResultFailed, "AI provider invocation failed safely")
|
||||
if auditErr != nil {
|
||||
return domain.AIInvocationResponse{}, auditErr
|
||||
}
|
||||
svc.productionMu.Lock()
|
||||
_, alertErr := svc.upsertAlert(domain.AlertRecord{SourceKind: "ai-provider", SourceID: provider.ID, RuleKey: "ai.provider.failed", Severity: domain.AlertSeverityWarning, Title: "AI provider invocation failed", Message: "AI provider invocation failed safely", Retryable: false, LastAuditEventID: auditID})
|
||||
svc.productionMu.Unlock()
|
||||
if alertErr != nil {
|
||||
return domain.AIInvocationResponse{}, alertErr
|
||||
}
|
||||
return domain.CopyAIInvocationResponse(domain.AIInvocationResponse{
|
||||
RequestID: request.RequestID,
|
||||
Purpose: request.Purpose,
|
||||
|
||||
@@ -58,9 +58,6 @@ func (svc *CoreService) OpenArtifactDownloadForSession(sessionID string, request
|
||||
if err := validator.ValidateArtifactDownloadReference(reference); err != nil {
|
||||
return domain.ArtifactDownloadReference{}, err
|
||||
}
|
||||
if err := svc.auditArtifactDownload(sessionID, artifact); err != nil {
|
||||
return domain.ArtifactDownloadReference{}, err
|
||||
}
|
||||
return domain.CopyArtifactDownloadReference(reference), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -28,13 +28,12 @@ func (svc *CoreService) DeployClientManagerForSession(sessionID string, request
|
||||
if err := validator.ValidateClientManagerDeployRequest(request); err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
user, instance, plugin, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerDeploy, "client-manager.deploy.denied")
|
||||
_, instance, plugin, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerDeploy, "client-manager.deploy.denied")
|
||||
if err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
distribution, err := svc.authorizedClientManagerDistribution(instance, profile, request.DistributionID)
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.deploy.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager deployment denied: distribution ownership, target, revision, or key fence is invalid")
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
installation, err := svc.ensureClientManagerInstallationFromDistribution(instance, plugin, distribution)
|
||||
@@ -93,9 +92,6 @@ func (svc *CoreService) DeployClientManagerForSession(sessionID string, request
|
||||
if job.ID != installation.CurrentJobID {
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager deploy job fence is invalid")
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "client-manager.deploy", "client-manager-installation", installation.ID, domain.AuditResultQueued, "queued typed client-manager deployment for current artifact and key generation"); err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
@@ -107,12 +103,11 @@ func (svc *CoreService) ControlClientManagerForSession(sessionID string, request
|
||||
if request.Operation == domain.ClientManagerOperationRollback {
|
||||
capability = domain.JobCapabilityClientManagerRollback
|
||||
}
|
||||
user, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, capability, "client-manager."+string(request.Operation)+".denied")
|
||||
_, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, capability, "client-manager."+string(request.Operation)+".denied")
|
||||
if err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
if !containsString(profile.Lifecycle.Actions, string(request.Operation)) {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager."+string(request.Operation)+".denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager control denied: action is not declared")
|
||||
return domain.ClientManagerLifecycleView{}, ErrForbidden
|
||||
}
|
||||
installation, err := svc.getClientManagerInstallation(instance.ID, profile.Key)
|
||||
@@ -175,7 +170,6 @@ func (svc *CoreService) ControlClientManagerForSession(sessionID string, request
|
||||
if job.ID != installation.CurrentJobID {
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager control job fence is invalid")
|
||||
}
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager."+string(request.Operation), "client-manager-installation", installation.ID, domain.AuditResultQueued, "queued typed client-manager "+string(request.Operation)+" operation")
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
@@ -183,7 +177,7 @@ func (svc *CoreService) UpdateClientManagerForSession(sessionID string, request
|
||||
if err := validator.ValidateClientManagerUpdateRequest(request); err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
user, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerUpdate, "client-manager.update.denied")
|
||||
_, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerUpdate, "client-manager.update.denied")
|
||||
if err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
@@ -199,11 +193,9 @@ func (svc *CoreService) UpdateClientManagerForSession(sessionID string, request
|
||||
}
|
||||
distribution, err := svc.authorizedClientManagerDistribution(instance, profile, request.DistributionID)
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.update.denied", "client-manager-installation", installation.ID, domain.AuditResultDenied, "client-manager update denied: artifact scope or generation is invalid")
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
if distribution.ArtifactID == installation.ActiveArtifactID || !clientManagerVersionAllowed(profile, installation.ActiveVersion, clientManagerDistributionVersion(distribution, profile)) {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.update.denied", "client-manager-installation", installation.ID, domain.AuditResultDenied, "client-manager update denied: artifact is not compatible with the active deployment")
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager update artifact is incompatible")
|
||||
}
|
||||
if existing, err := svc.store.Jobs().GetByIdempotency(endpoint.ID, request.IdempotencyKey); err == nil {
|
||||
@@ -246,7 +238,6 @@ func (svc *CoreService) UpdateClientManagerForSession(sessionID string, request
|
||||
if job.ID != installation.CurrentJobID {
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager update job fence is invalid")
|
||||
}
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.update", "client-manager-installation", installation.ID, domain.AuditResultQueued, "queued approved staged client-manager update with rollback retention")
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
@@ -254,7 +245,7 @@ func (svc *CoreService) UninstallClientManagerForSession(sessionID string, reque
|
||||
if err := validator.ValidateClientManagerUninstallRequest(request); err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
user, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerUninstall, "client-manager.uninstall.denied")
|
||||
_, instance, _, profile, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, domain.JobCapabilityClientManagerUninstall, "client-manager.uninstall.denied")
|
||||
if err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
@@ -304,13 +295,11 @@ func (svc *CoreService) UninstallClientManagerForSession(sessionID string, reque
|
||||
if job.ID != installation.CurrentJobID {
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager uninstall job fence is invalid")
|
||||
}
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.uninstall", "client-manager-installation", installation.ID, domain.AuditResultQueued, "queued safe controlled-workspace uninstall")
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
func (svc *CoreService) RevokeClientManagerSessionForSession(sessionID string, request domain.ClientManagerRevokeSessionRequest) (domain.ClientManagerLifecycleView, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
if _, err := svc.GetCurrentUser(sessionID); err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID)
|
||||
@@ -334,7 +323,6 @@ func (svc *CoreService) RevokeClientManagerSessionForSession(sessionID string, r
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
}
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.revoke", "client-manager-installation", installation.ID, domain.AuditResultSuccess, "revoked Client Manager component session without exposing token material")
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
@@ -360,7 +348,7 @@ func (svc *CoreService) RetryClientManagerLifecycleForSession(sessionID string,
|
||||
case domain.ClientManagerOperationUninstall:
|
||||
capability = domain.JobCapabilityClientManagerUninstall
|
||||
}
|
||||
user, _, _, _, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, capability, "client-manager.retry.denied")
|
||||
_, _, _, _, endpoint, err := svc.authorizeClientManagerLifecycle(sessionID, request.ServerInstanceID, request.ProfileKey, capability, "client-manager.retry.denied")
|
||||
if err != nil {
|
||||
return domain.ClientManagerLifecycleView{}, err
|
||||
}
|
||||
@@ -402,7 +390,6 @@ func (svc *CoreService) RetryClientManagerLifecycleForSession(sessionID string,
|
||||
if job.ID != installation.CurrentJobID {
|
||||
return domain.ClientManagerLifecycleView{}, validationError("client-manager retry job fence is invalid")
|
||||
}
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.retry", "client-manager-installation", installation.ID, domain.AuditResultQueued, "queued bounded retry using the existing deployment generation fence")
|
||||
return svc.clientManagerLifecycleView(installation)
|
||||
}
|
||||
|
||||
@@ -472,7 +459,6 @@ func (svc *CoreService) authorizeClientManagerLifecycle(sessionID, serverInstanc
|
||||
}
|
||||
profile, err := findRuntimeClientManagerProfile(plugin, profileKey)
|
||||
if err != nil || profile.Deployment.Mode != "run-supervised" || !containsString(profile.Deployment.RequiredRunCapabilities, capability) {
|
||||
_ = svc.recordAuditEvent(user.ID, deniedAction, "server-instance", instance.ID, domain.AuditResultDenied, "client-manager lifecycle denied: profile or capability is not declared")
|
||||
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, ErrForbidden
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
@@ -480,7 +466,6 @@ func (svc *CoreService) authorizeClientManagerLifecycle(sessionID, serverInstanc
|
||||
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
||||
}
|
||||
if err := svc.validateRunnableEndpoint(endpoint, capability); err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, deniedAction, "server-instance", instance.ID, domain.AuditResultDenied, "client-manager lifecycle denied: assigned Run endpoint is offline or unsupported")
|
||||
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
||||
}
|
||||
return user, instance, plugin, profile, endpoint, nil
|
||||
@@ -974,7 +959,7 @@ func (svc *CoreService) projectClientManagerLifecycleResult(job domain.Job, stam
|
||||
installation.LastHeartbeatSequence = 0
|
||||
case domain.ClientManagerOperationUninstall:
|
||||
installation.Status = domain.ClientManagerLifecycleUninstalled
|
||||
installation.Phase = "controlled workspace removed"
|
||||
installation.Phase = "managed workspace removed"
|
||||
installation.Health = domain.ClientManagerHealthOffline
|
||||
installation.HealthReason = "uninstalled"
|
||||
installation.ActiveArtifactID = ""
|
||||
@@ -994,11 +979,7 @@ func (svc *CoreService) projectClientManagerLifecycleResult(job domain.Job, stam
|
||||
if err := svc.store.ClientManagerInstallations().Update(installation); err != nil {
|
||||
return err
|
||||
}
|
||||
result := domain.AuditResultSuccess
|
||||
if !success {
|
||||
result = domain.AuditResultFailed
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+installation.RunEndpointID, "client-manager."+string(installation.LastOperation), "client-manager-installation", installation.ID, result, "client-manager lifecycle job reached a bounded terminal result")
|
||||
return nil
|
||||
}
|
||||
return repo.ErrNotFound
|
||||
}
|
||||
@@ -1134,7 +1115,6 @@ func (svc *CoreService) RegisterClientManager(request domain.ClientManagerRegist
|
||||
}
|
||||
stamp := svc.now()
|
||||
if request.Timestamp.Before(stamp.Add(-clientManagerRegistrationWindow)) || request.Timestamp.After(stamp.Add(clientManagerRegistrationWindow)) {
|
||||
_ = svc.recordAuditEvent("client-manager:"+request.InstallationID, "client-manager.register.denied", "client-manager-installation", request.InstallationID, domain.AuditResultDenied, "client-manager registration denied: timestamp expired")
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
}
|
||||
installation, err := svc.store.ClientManagerInstallations().Get(request.InstallationID)
|
||||
@@ -1142,7 +1122,6 @@ func (svc *CoreService) RegisterClientManager(request domain.ClientManagerRegist
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
}
|
||||
if installation.ServerInstanceID != request.ServerInstanceID || installation.ProfileKey != request.ProfileKey || installation.ActiveArtifactID != request.ArtifactID || installation.ActiveVersion != request.Version || installation.ActiveRevision != request.SourceRevision || installation.TargetOS != request.TargetOS || installation.TargetArch != request.TargetArch || installation.KeyGeneration != request.KeyGeneration || installation.DeploymentGeneration != request.DeploymentGeneration || installation.RequiresRedeploy || installation.Status == domain.ClientManagerLifecycleUninstalled {
|
||||
_ = svc.recordAuditEvent("client-manager:"+request.InstallationID, "client-manager.register.denied", "client-manager-installation", request.InstallationID, domain.AuditResultDenied, "client-manager registration denied: identity or deployment fence is stale")
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(installation.ServerInstanceID)
|
||||
@@ -1155,7 +1134,6 @@ func (svc *CoreService) RegisterClientManager(request domain.ClientManagerRegist
|
||||
}
|
||||
profile, err := findRuntimeClientManagerProfile(plugin, installation.ProfileKey)
|
||||
if err != nil || !clientManagerCapabilitiesMatch(profile.Health.RequiredCapabilities, request.Capabilities) {
|
||||
_ = svc.recordAuditEvent("client-manager:"+request.InstallationID, "client-manager.register.denied", "client-manager-installation", request.InstallationID, domain.AuditResultDenied, "client-manager registration denied: capabilities do not match declaration")
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
}
|
||||
key, err := svc.activeComponentKey(installation.ServerInstanceID, domain.DistributionComponentClientManager, installation.ProfileKey)
|
||||
@@ -1168,12 +1146,10 @@ func (svc *CoreService) RegisterClientManager(request domain.ClientManagerRegist
|
||||
}
|
||||
expected := clientManagerRegistrationSignature(plainKey, request)
|
||||
if subtle.ConstantTimeCompare([]byte(expected), []byte(request.Signature)) != 1 {
|
||||
_ = svc.recordAuditEvent("client-manager:"+request.InstallationID, "client-manager.register.denied", "client-manager-installation", request.InstallationID, domain.AuditResultDenied, "client-manager registration denied: signature mismatch")
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
}
|
||||
nonceID := clientManagerNonceID(request.InstallationID, request.Nonce)
|
||||
if _, err := svc.store.ClientManagerNonces().Get(nonceID); err == nil {
|
||||
_ = svc.recordAuditEvent("client-manager:"+request.InstallationID, "client-manager.register.denied", "client-manager-installation", request.InstallationID, domain.AuditResultDenied, "client-manager registration denied: nonce replay")
|
||||
return domain.ClientManagerRegisterResult{}, ErrUnauthorized
|
||||
} else if !errors.Is(err, repo.ErrNotFound) {
|
||||
return domain.ClientManagerRegisterResult{}, err
|
||||
@@ -1209,7 +1185,6 @@ func (svc *CoreService) RegisterClientManager(request domain.ClientManagerRegist
|
||||
if err := svc.store.ClientManagerInstallations().Update(installation); err != nil {
|
||||
return domain.ClientManagerRegisterResult{}, err
|
||||
}
|
||||
_ = svc.recordAuditEvent("client-manager:"+installation.ID, "client-manager.register", "client-manager-installation", installation.ID, domain.AuditResultSuccess, "Client Manager registered with an isolated expiring component session")
|
||||
return domain.ClientManagerRegisterResult{Accepted: true, InstallationID: installation.ID, SessionToken: token, ExpiresAt: session.ExpiresAt, HeartbeatEvery: profile.Health.IntervalSeconds, ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -265,7 +265,7 @@ func claimClientManagerJob(t *testing.T, svc *CoreService, sessionToken, capabil
|
||||
|
||||
func completeClientManagerJob(t *testing.T, svc *CoreService, sessionToken string, claim domain.RunJobClaimResult, state domain.JobState, kind, processState string) {
|
||||
t.Helper()
|
||||
_, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: state, Progress: domain.RunJobProgressReport{Percent: 100, Message: "client-manager lifecycle terminal"}, Message: "client-manager lifecycle terminal", ExecutionResult: domain.JobExecutionResult{Kind: kind, ProcessState: processState, AuditSummary: "bounded lifecycle result"}})
|
||||
_, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: state, Progress: domain.RunJobProgressReport{Percent: 100, Message: "client-manager lifecycle terminal"}, Message: "client-manager lifecycle terminal", ExecutionResult: domain.JobExecutionResult{Kind: kind, ProcessState: processState, Summary: "bounded lifecycle result"}})
|
||||
if err != nil {
|
||||
t.Fatalf("complete lifecycle job: %v", err)
|
||||
}
|
||||
|
||||
@@ -441,7 +441,7 @@ func TestCoreServiceRunLifecycleReportProjectsGeneratedRunFacts(t *testing.T) {
|
||||
}
|
||||
registered := registerGeneratedRunForStatusTest(t, svc, instance, plugin.ID)
|
||||
|
||||
reported, err := svc.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, ServerInstanceID: instance.ID, Capability: domain.LifecycleCapabilityStart, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "autonomous start complete"}, Message: "autonomous start complete", ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running", AuditSummary: "private supervised process identity"}})
|
||||
reported, err := svc.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, ServerInstanceID: instance.ID, Capability: domain.LifecycleCapabilityStart, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "autonomous start complete"}, Message: "autonomous start complete", ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running", Summary: "private supervised process identity"}})
|
||||
if err != nil || !reported.Accepted || reported.ProjectedState != domain.ServerInstanceStateRunning {
|
||||
t.Fatalf("expected accepted lifecycle report projected running, result=%+v err=%v", reported, err)
|
||||
}
|
||||
|
||||
@@ -461,7 +461,7 @@ func (svc *CoreService) projectDependencyAndRunUpdateResult(job domain.Job, stam
|
||||
if err := svc.store.DependencyStatuses().Update(status); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "dependency.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), status.Message)
|
||||
return nil
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return nil
|
||||
@@ -494,7 +494,7 @@ func (svc *CoreService) projectDependencyAndRunUpdateResult(job domain.Job, stam
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "run.update.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), update.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectDependencyAndRunUpdateProgress(job domain.Job, stamp time.Time) error {
|
||||
@@ -589,26 +589,9 @@ func (svc *CoreService) ReportRunUpdateHealth(report domain.RunUpdateHealthRepor
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if report.Outcome == "rolled-back" {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
if err := svc.recordAuditEvent("run", "run.update.health", "server-instance", update.ServerInstanceID, auditResult, update.Message); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func auditResultForJob(job domain.Job) domain.AuditResult {
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
return domain.AuditResultSuccess
|
||||
}
|
||||
if job.State == domain.JobStateCancelled {
|
||||
return domain.AuditResultDenied
|
||||
}
|
||||
return domain.AuditResultFailed
|
||||
}
|
||||
|
||||
func sameRunUpdateTarget(existing, expected domain.RunUpdateJob) bool {
|
||||
return existing.ServerInstanceID == expected.ServerInstanceID && existing.RunEndpointID == expected.RunEndpointID && existing.ArtifactID == expected.ArtifactID && existing.Checksum == expected.Checksum && existing.TargetOS == expected.TargetOS && existing.TargetArch == expected.TargetArch && existing.TargetRelease == expected.TargetRelease && existing.JobID == expected.JobID && existing.IdempotencyKey == expected.IdempotencyKey
|
||||
}
|
||||
|
||||
@@ -35,18 +35,6 @@ func TestDependencyCatalogRequiresCurrentReviewedDigest(t *testing.T) {
|
||||
t.Fatalf("stale digest created a job: %+v", job)
|
||||
}
|
||||
}
|
||||
audits, err := svc.ListAuditEvents(domain.AuditEventFilter{ResourceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list audits: %v", err)
|
||||
}
|
||||
foundDenied := false
|
||||
for _, audit := range audits {
|
||||
foundDenied = foundDenied || audit.Action == "dependency.install.denied"
|
||||
}
|
||||
if !foundDenied {
|
||||
t.Fatalf("expected stale digest audit, got %+v", audits)
|
||||
}
|
||||
|
||||
request.PlanDigest = catalog.Plans[0].Digest
|
||||
request.IdempotencyKey = "dependency-current-digest"
|
||||
job, err := svc.QueueDependencyJobForSession(session, request)
|
||||
@@ -136,7 +124,7 @@ func TestDependencyInputFencingCancellationAndTerminalProjection(t *testing.T) {
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.DependencyExecutionEvidence{ProbeKey: input.Probe.Key, PlanDigest: input.PlanDigest, State: string(domain.DependencyStatePresent), Evidence: "OpenJDK 21"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "dependency probe completed"}, ResultRef: "artifact://jobs/dependency-check/result", Message: "dependency probe completed", ExecutionResult: domain.JobExecutionResult{Kind: "dependency.check", Checksum: input.PlanDigest, AuditSummary: "dependency probe completed", Content: string(evidence)}}); err != nil {
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "dependency probe completed"}, ResultRef: "artifact://jobs/dependency-check/result", Message: "dependency probe completed", ExecutionResult: domain.JobExecutionResult{Kind: "dependency.check", Checksum: input.PlanDigest, Summary: "dependency probe completed", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete dependency result: %v", err)
|
||||
}
|
||||
projected, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
@@ -248,7 +236,7 @@ func TestRunUpdateTargetFencingChunksHealthAndRollbackProjection(t *testing.T) {
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.RunUpdateExecutionEvidence{TargetRelease: update.TargetRelease, Phase: "staged"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: "artifact://jobs/run-update/staged", Message: "Run update verified and staged", ExecutionResult: domain.JobExecutionResult{Kind: "run.update.staged", Checksum: update.Checksum, SizeBytes: int64(len(payload)), AuditSummary: "verified update staged", Content: string(evidence)}}); err != nil {
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: "artifact://jobs/run-update/staged", Message: "Run update verified and staged", ExecutionResult: domain.JobExecutionResult{Kind: "run.update.staged", Checksum: update.Checksum, SizeBytes: int64(len(payload)), Summary: "verified update staged", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete staged Run update: %v", err)
|
||||
}
|
||||
updates, err := svc.ListRunUpdateJobsForSession(session, instance.ID)
|
||||
|
||||
@@ -38,7 +38,6 @@ func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, requ
|
||||
return domain.RunDistribution{}, err
|
||||
}
|
||||
if err := validatePluginTarget(plugin, request.TargetOS); err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.generate.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run generation denied: unsupported target")
|
||||
return domain.RunDistribution{}, err
|
||||
}
|
||||
if deploymentNeedsCompleteRuntimeBinding(plugin, instance.Deployment) {
|
||||
@@ -47,7 +46,6 @@ func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, requ
|
||||
}
|
||||
}
|
||||
if ready, reason := svc.distributionBuilderReadiness(); !ready {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.generate.denied", "server-instance", instance.ID, domain.AuditResultDenied, reason)
|
||||
return domain.RunDistribution{}, validationError(reason)
|
||||
}
|
||||
|
||||
@@ -114,9 +112,6 @@ func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, requ
|
||||
return domain.RunDistribution{}, validationError("distribution build idempotency key conflicts with another job")
|
||||
}
|
||||
svc.enqueueDistributionBuild(job)
|
||||
if err := svc.recordAuditEvent(user.ID, "run.generate", "server-instance", instance.ID, domain.AuditResultQueued, "queued run binary build job in the platform builder with redacted runtime key ref"); err != nil {
|
||||
return domain.RunDistribution{}, err
|
||||
}
|
||||
return domain.CopyRunDistribution(distribution), nil
|
||||
}
|
||||
|
||||
@@ -144,26 +139,22 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if err := validatePluginTarget(plugin, request.TargetOS); err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: unsupported target")
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
profile, err := findRuntimeClientManagerProfile(plugin, request.ProfileKey)
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: profile is not declared")
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if strings.TrimSpace(request.SourceRevision) == "" {
|
||||
request.SourceRevision = clientManagerProfileRevision(profile)
|
||||
}
|
||||
if !clientManagerProfileSupportsTarget(profile, request.TargetOS, request.TargetArch) || request.RepositoryURL != profile.RepositoryURL || !clientManagerProfileAllowsRevision(profile, request.SourceRevision) {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: repository, revision, or target is not declared")
|
||||
return domain.ClientManagerDistribution{}, validationError("client-manager build must match the declared profile repository, revision, and target")
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "client-manager.build.denied"); err != nil {
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if ready, reason := svc.distributionBuilderReadiness(); !ready {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, reason)
|
||||
return domain.ClientManagerDistribution{}, validationError(reason)
|
||||
}
|
||||
|
||||
@@ -271,9 +262,6 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
return domain.ClientManagerDistribution{}, validationError("distribution build idempotency key conflicts with another job")
|
||||
}
|
||||
svc.enqueueDistributionBuild(job)
|
||||
if err := svc.recordAuditEvent(user.ID, "client-manager.build", "server-instance", instance.ID, domain.AuditResultQueued, "queued client-manager source build in the platform builder with redacted runtime key ref"); err != nil {
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
return domain.CopyClientManagerDistribution(distribution), nil
|
||||
}
|
||||
|
||||
@@ -376,9 +364,6 @@ func (svc *CoreService) ResetComponentKeyForSession(sessionID string, request do
|
||||
return domain.EncryptedComponentKey{}, err
|
||||
}
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "runtime-key.reset", "server-instance", instance.ID, domain.AuditResultSuccess, "reset "+string(request.ComponentKind)+" key; previous packages revoked"); err != nil {
|
||||
return domain.EncryptedComponentKey{}, err
|
||||
}
|
||||
return domain.CopyEncryptedComponentKey(newKey), nil
|
||||
}
|
||||
|
||||
@@ -404,7 +389,6 @@ func (svc *CoreService) AuthenticateComponent(request domain.ComponentAuthentica
|
||||
}
|
||||
if key.Generation != request.Generation {
|
||||
result.Reason = "key generation is no longer current"
|
||||
_ = svc.recordAuditEvent("runtime", "runtime-key.auth", "server-instance", request.ServerInstanceID, domain.AuditResultDenied, "component authentication denied: stale generation")
|
||||
return domain.CopyComponentAuthenticationResult(result), nil
|
||||
}
|
||||
plainKey, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
@@ -413,7 +397,6 @@ func (svc *CoreService) AuthenticateComponent(request domain.ComponentAuthentica
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(plainKey), []byte(request.Key)) != 1 {
|
||||
result.Reason = "key is not current"
|
||||
_ = svc.recordAuditEvent("runtime", "runtime-key.auth", "server-instance", request.ServerInstanceID, domain.AuditResultDenied, "component authentication denied: key mismatch")
|
||||
return domain.CopyComponentAuthenticationResult(result), nil
|
||||
}
|
||||
result.Allowed = true
|
||||
@@ -523,14 +506,12 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
if artifact.State != domain.ArtifactStateAvailable {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: artifact is unavailable")
|
||||
return domain.RunUpdateJob{}, validationError("artifact must be available")
|
||||
}
|
||||
if request.Checksum == "" {
|
||||
request.Checksum = artifact.Checksum
|
||||
}
|
||||
if request.Checksum != artifact.Checksum {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: checksum mismatch")
|
||||
return domain.RunUpdateJob{}, validationError("checksum must match artifact")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
@@ -546,7 +527,6 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
}
|
||||
distribution, err := findRunDistributionForArtifact(distributions, artifact.ID)
|
||||
if err != nil || distribution.RunEndpointID != endpoint.ID || distribution.TargetOS != endpoint.Platform || distribution.TargetArch != endpoint.Architecture || distribution.Checksum != artifact.Checksum || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != distribution.BuildJobID {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: artifact is not an approved target-matched Run distribution")
|
||||
return domain.RunUpdateJob{}, validationError("artifact must be an approved target-matched Run distribution")
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
@@ -560,7 +540,6 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
Progress: domain.JobProgress{Percent: 0, Message: "run self-update queued"},
|
||||
})
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: endpoint unsupported or offline")
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
stamp := svc.now()
|
||||
@@ -598,9 +577,6 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
}
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "run.update", "server-instance", instance.ID, domain.AuditResultQueued, "queued run self-update job with artifact checksum"); err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
return domain.CopyRunUpdateJob(updateJob), nil
|
||||
}
|
||||
|
||||
@@ -625,7 +601,6 @@ func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request d
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if !pluginDeclares(plugin, "server.dependencies.manage") {
|
||||
_ = svc.recordAuditEvent(user.ID, "dependency.install.denied", "server-instance", instance.ID, domain.AuditResultDenied, "dependency operation denied: plugin permission is not declared")
|
||||
return domain.Job{}, forbiddenError("plugin does not declare required permission: server.dependencies.manage")
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "dependency.install.denied"); err != nil {
|
||||
@@ -656,20 +631,17 @@ func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request d
|
||||
}
|
||||
expectedDigest := dependencyPlanDigest(resolution, probe, plan)
|
||||
if request.Install && request.PlanDigest != expectedDigest {
|
||||
_ = svc.recordAuditEvent(user.ID, "dependency.install.denied", "server-instance", instance.ID, domain.AuditResultDenied, "dependency install denied: reviewed plan digest is stale or missing")
|
||||
return domain.Job{}, validationError("planDigest must match the current reviewed install plan")
|
||||
}
|
||||
request.PlanDigest = expectedDigest
|
||||
capability := domain.JobCapabilityDependenciesCheck
|
||||
targetKey := "dependencies/" + request.ProbeKey
|
||||
message := "dependency check queued"
|
||||
auditAction := "dependency.check"
|
||||
state := domain.DependencyStateUnknown
|
||||
if request.Install {
|
||||
capability = domain.JobCapabilityDependenciesInstall
|
||||
targetKey = "dependencies/install/" + request.InstallPlanKey
|
||||
message = "dependency install queued"
|
||||
auditAction = "dependency.install"
|
||||
state = domain.DependencyStateInstalling
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
@@ -682,15 +654,11 @@ func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request d
|
||||
Progress: domain.JobProgress{Percent: 0, Message: message},
|
||||
})
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, auditAction+".denied", "server-instance", instance.ID, domain.AuditResultDenied, "dependency operation denied: endpoint unsupported or offline")
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.upsertDependencyStatus(instance, request, job.ID, probe.Required, state, "queued through platform job"); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, auditAction, "server-instance", instance.ID, domain.AuditResultQueued, message); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
return domain.CopyJob(job), nil
|
||||
}
|
||||
|
||||
@@ -715,12 +683,10 @@ func (svc *CoreService) QueueLogBackfillForSession(sessionID string, request dom
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if !pluginSupports(plugin, domain.JobCapabilityLogsBackfill) {
|
||||
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: plugin capability is not declared")
|
||||
return domain.Job{}, ErrForbidden
|
||||
}
|
||||
source, err := declaredFileLogSource(plugin, request.SourceKey)
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: log source is not declared")
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "logs.backfill.denied"); err != nil {
|
||||
@@ -738,10 +704,6 @@ func (svc *CoreService) QueueLogBackfillForSession(sessionID string, request dom
|
||||
ExecutionInput: domain.JobExecutionInput{LogSource: &source},
|
||||
})
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: endpoint unsupported or offline")
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "logs.backfill", "server-instance", instance.ID, domain.AuditResultQueued, "queued historical log backfill without log bodies in job result"); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
return domain.CopyJob(job), nil
|
||||
@@ -763,11 +725,9 @@ func declaredFileLogSource(plugin domain.GamePlugin, sourceKey string) (domain.R
|
||||
|
||||
func (svc *CoreService) validateDistributionPluginPermission(actorID string, plugin domain.GamePlugin, serverInstanceID string, permission string, deniedAction string) error {
|
||||
if plugin.Status != domain.GamePluginStatusInstalled {
|
||||
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "distribution operation denied: plugin is not installed")
|
||||
return forbiddenError("plugin is not installed")
|
||||
}
|
||||
if !containsString(plugin.DeclaredPermissions, permission) {
|
||||
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "distribution operation denied: plugin permission is not declared")
|
||||
return forbiddenError("plugin does not declare required permission: " + permission)
|
||||
}
|
||||
return nil
|
||||
@@ -1033,63 +993,6 @@ func (svc *CoreService) upsertDependencyStatus(instance domain.ServerInstance, r
|
||||
return svc.store.DependencyStatuses().Create(status)
|
||||
}
|
||||
|
||||
func (svc *CoreService) recordAuditEvent(actorID string, action string, resourceKind string, resourceID string, result domain.AuditResult, summary string) error {
|
||||
_, err := svc.recordAuditEventWithID(actorID, action, resourceKind, resourceID, result, summary)
|
||||
return err
|
||||
}
|
||||
|
||||
func (svc *CoreService) recordAuditEventWithID(actorID string, action string, resourceKind string, resourceID string, result domain.AuditResult, summary string) (string, error) {
|
||||
svc.auditMu.Lock()
|
||||
svc.auditSeq++
|
||||
seq := svc.auditSeq
|
||||
svc.auditMu.Unlock()
|
||||
|
||||
stamp := svc.now()
|
||||
event := domain.AuditEvent{
|
||||
ID: fmt.Sprintf("audit-%s-%d-%d", strings.ReplaceAll(action, ".", "-"), stamp.UnixNano(), seq),
|
||||
ActorID: actorID,
|
||||
Action: action,
|
||||
ResourceKind: resourceKind,
|
||||
ResourceID: resourceID,
|
||||
Result: result,
|
||||
Summary: safeBridgeReason(summary),
|
||||
CreatedAt: stamp,
|
||||
}
|
||||
if err := validator.ValidateAuditEvent(event); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := svc.store.AuditEvents().Create(event); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return event.ID, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) auditArtifactDownload(sessionID string, artifact domain.Artifact) error {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runDistributions, err := svc.store.RunDistributions().List(domain.RunDistributionFilter{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, distribution := range runDistributions {
|
||||
if distribution.ArtifactID == artifact.ID {
|
||||
return svc.recordAuditEvent(user.ID, "run.download", "server-instance", distribution.ServerInstanceID, domain.AuditResultSuccess, "downloaded run package artifact with redacted runtime key ref")
|
||||
}
|
||||
}
|
||||
clientDistributions, err := svc.store.ClientManagerDistributions().List(domain.ClientManagerDistributionFilter{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, distribution := range clientDistributions {
|
||||
if distribution.ArtifactID == artifact.ID {
|
||||
return svc.recordAuditEvent(user.ID, "client-manager.download", "server-instance", distribution.ServerInstanceID, domain.AuditResultSuccess, "downloaded client-manager package artifact with redacted runtime key ref")
|
||||
}
|
||||
}
|
||||
return svc.recordAuditEvent(user.ID, "artifact.download", string(artifact.OwnerKind), artifact.OwnerID, domain.AuditResultSuccess, "downloaded platform artifact")
|
||||
}
|
||||
|
||||
func validatePluginTarget(plugin domain.GamePlugin, targetOS string) error {
|
||||
if len(plugin.SupportedOS) == 0 || containsString(plugin.SupportedOS, targetOS) {
|
||||
return nil
|
||||
@@ -1251,7 +1154,6 @@ func (svc *CoreService) requireCompleteRuntimeBindings(actorID string, serverIns
|
||||
if complete {
|
||||
return nil
|
||||
}
|
||||
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "operation denied: "+reason)
|
||||
return validationError(reason)
|
||||
}
|
||||
|
||||
|
||||
@@ -496,7 +496,7 @@ func TestCoreServiceResetRunKeyRevokesOldPackagesAndRequiresRegeneration(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceBuildsClientManagerWithDistinctKeyAndAuditsSensitiveOperations(t *testing.T) {
|
||||
func TestCoreServiceBuildsClientManagerWithDistinctKeyAndRedactsSensitiveOperations(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
runDistribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -578,24 +578,6 @@ func TestCoreServiceBuildsClientManagerWithDistinctKeyAndAuditsSensitiveOperatio
|
||||
t.Fatalf("expected old client-manager key to be denied after reset, got %+v", auth)
|
||||
}
|
||||
|
||||
audits, err := svc.ListAuditEvents(domain.AuditEventFilter{ResourceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list audits: %v", err)
|
||||
}
|
||||
actions := map[string]bool{}
|
||||
for _, audit := range audits {
|
||||
actions[audit.Action] = true
|
||||
for _, forbidden := range []string{runConfig.AuthKey, clientConfig.AuthKey, "password=", "unix://", "/Users/"} {
|
||||
if strings.Contains(audit.Summary, forbidden) {
|
||||
t.Fatalf("audit leaked forbidden fragment %q in %+v", forbidden, audit)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, action := range []string{"run.generate", "run.download", "client-manager.build", "client-manager.build.denied", "runtime-key.reset"} {
|
||||
if !actions[action] {
|
||||
t.Fatalf("expected audit action %q in %+v", action, audits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.ServerInstance) {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
@@ -252,30 +251,6 @@ func gameClientBridgeQueryTemplateKeys(declarations []domain.GameClientBridgeQue
|
||||
return values
|
||||
}
|
||||
|
||||
func validateProtectedGameClientBridgePayload(declaration *domain.GameClientBridgeProtectedRequestDeclaration, payload map[string]any) error {
|
||||
if declaration == nil {
|
||||
return nil
|
||||
}
|
||||
if len(payload) != 1 {
|
||||
return validationError("protected bridge request must contain only its declared text field")
|
||||
}
|
||||
value, exists := payload[declaration.TextField]
|
||||
if !exists {
|
||||
return validationError("protected bridge request text field is required")
|
||||
}
|
||||
text, ok := value.(string)
|
||||
if !ok || len([]byte(text)) == 0 || len([]byte(text)) > declaration.MaxTextBytes {
|
||||
return validationError("protected bridge request text is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func protectedGameClientBridgeAuditSummary(declaration *domain.GameClientBridgeProtectedRequestDeclaration, payload map[string]any) string {
|
||||
text, _ := payload[declaration.TextField].(string)
|
||||
digest := sha256.Sum256([]byte(text))
|
||||
return fmt.Sprintf("queued protected %s request transport=%s target=%s text=redacted sha256=%x", declaration.Kind, declaration.TransportKey, declaration.TargetKey, digest[:8])
|
||||
}
|
||||
|
||||
func (svc *CoreService) queueGameClientBridgeCommand(requesterID string, request domain.GameClientBridgeQueueRequest) (domain.GameClientBridgeCommand, error) {
|
||||
request.Payload = domain.CopyGameClientBridgePayload(request.Payload)
|
||||
if err := validator.ValidateGameClientBridgeQueueRequest(request); err != nil {
|
||||
@@ -299,13 +274,6 @@ func (svc *CoreService) queueGameClientBridgeCommand(requesterID string, request
|
||||
if request.ExpiresAt.After(stamp.Add(time.Duration(declaration.TimeoutSeconds) * time.Second)) {
|
||||
return domain.GameClientBridgeCommand{}, validationError("bridge command expiry exceeds declared timeout")
|
||||
}
|
||||
if err := validateProtectedGameClientBridgePayload(declaration.ProtectedRequest, request.Payload); err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
if declaration.ProtectedRequest != nil && declaration.TimeoutSeconds > protectedRequestMaxTimeoutSeconds {
|
||||
return domain.GameClientBridgeCommand{}, validationError("protected bridge request timeout exceeds Run policy")
|
||||
}
|
||||
|
||||
existing, err := svc.store.GameClientBridgeCommands().GetByIdempotency(request.ServerInstanceID, requesterID, request.CommandType, request.IdempotencyKey)
|
||||
if err == nil {
|
||||
return domain.CopyGameClientBridgeCommand(existing), nil
|
||||
@@ -343,32 +311,9 @@ func (svc *CoreService) queueGameClientBridgeCommand(requesterID string, request
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
if declaration.ProtectedRequest != nil {
|
||||
command.Payload = redactedProtectedRequestPayload(declaration.ProtectedRequest)
|
||||
if approvalState == domain.GameClientBridgeApprovalApproved {
|
||||
command.RunJobID = jobIDFromParts("job-protected-request", command.ServerInstanceID, command.ID)
|
||||
}
|
||||
}
|
||||
summary := "queued declared game client bridge command"
|
||||
if declaration.ProtectedRequest != nil {
|
||||
summary = protectedGameClientBridgeAuditSummary(declaration.ProtectedRequest, request.Payload)
|
||||
}
|
||||
auditID, err := svc.recordAuditEventWithID(requesterID, "game-client-bridge.command.queue", "game-client-bridge-command", command.ID, domain.AuditResultQueued, summary)
|
||||
if err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
command.AuditReferences = []string{auditID}
|
||||
if err := svc.store.GameClientBridgeCommands().Create(command); err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
if declaration.ProtectedRequest != nil && command.RunJobID != "" {
|
||||
if err := svc.dispatchProtectedRequest(command, declaration, request.Payload); err != nil {
|
||||
if deleteErr := svc.store.GameClientBridgeCommands().Delete(command.ID); deleteErr != nil {
|
||||
return domain.GameClientBridgeCommand{}, deleteErr
|
||||
}
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
}
|
||||
return domain.CopyGameClientBridgeCommand(command), nil
|
||||
}
|
||||
|
||||
@@ -413,11 +358,6 @@ func (svc *CoreService) claimGameClientBridgeCommands(component gameClientBridge
|
||||
command.State = domain.GameClientBridgeCommandClaimed
|
||||
command.Claim = domain.GameClientBridgeClaim{SessionID: component.Session.ID, InstallationID: component.Installation.ID, DeploymentGeneration: component.Session.DeploymentGeneration, FencingToken: fencingToken, LeaseExpiresAt: gameClientBridgeClaimLeaseExpiry(stamp, command.ExpiresAt), ClaimedAt: stamp}
|
||||
command.UpdatedAt = stamp
|
||||
auditID, auditErr := svc.recordAuditEventWithID("component:"+component.Session.ProfileKey, "game-client-bridge.command.claim", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "companion claimed bridge command")
|
||||
if auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
if err := svc.store.GameClientBridgeCommands().Update(command); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -440,11 +380,6 @@ func (svc *CoreService) ackGameClientBridgeCommand(component gameClientBridgeCom
|
||||
command.Claim.AcknowledgedAt = stamp
|
||||
command.Claim.LeaseExpiresAt = gameClientBridgeClaimLeaseExpiry(stamp, command.ExpiresAt)
|
||||
command.UpdatedAt = stamp
|
||||
auditID, err := svc.recordAuditEventWithID("component:"+component.Session.ProfileKey, "game-client-bridge.command.ack", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "companion acknowledged bridge command")
|
||||
if err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
if err := svc.store.GameClientBridgeCommands().Update(command); err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
@@ -486,17 +421,9 @@ func (svc *CoreService) completeGameClientBridgeCommand(component gameClientBrid
|
||||
command.Result = domain.GameClientBridgeResult{Status: request.Status, Summary: request.Summary, Payload: domain.CopyGameClientBridgePayload(request.Payload), CompletedBy: component.Session.ID, CompletedAt: stamp}
|
||||
command.CompletedAt = stamp
|
||||
command.UpdatedAt = stamp
|
||||
auditID, err := svc.recordAuditEventWithID("component:"+component.Session.ProfileKey, "game-client-bridge.command.result", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "companion recorded terminal bridge command result")
|
||||
if err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
if err := svc.store.GameClientBridgeCommands().Update(command); err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
if command.RunJobID != "" {
|
||||
svc.protectedRequests.Delete(command.RunJobID)
|
||||
}
|
||||
return domain.CopyGameClientBridgeCommand(command), nil
|
||||
}
|
||||
|
||||
@@ -539,17 +466,9 @@ func (svc *CoreService) CancelGameClientBridgeCommandForSession(sessionID string
|
||||
command.Result = domain.GameClientBridgeResult{Status: domain.GameClientBridgeResultCancelled, Summary: "cancelled by operator", CompletedAt: stamp}
|
||||
command.CompletedAt = stamp
|
||||
command.UpdatedAt = stamp
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "game-client-bridge.command.cancel", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "operator cancelled bridge command")
|
||||
if err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
if err := svc.store.GameClientBridgeCommands().Update(command); err != nil {
|
||||
return domain.GameClientBridgeCommand{}, err
|
||||
}
|
||||
if command.RunJobID != "" {
|
||||
svc.protectedRequests.Delete(command.RunJobID)
|
||||
}
|
||||
return domain.CopyGameClientBridgeCommand(command), nil
|
||||
}
|
||||
|
||||
@@ -682,11 +601,6 @@ func (svc *CoreService) sweepGameClientBridgeCommandsLocked(stamp time.Time) err
|
||||
command.State = domain.GameClientBridgeCommandPending
|
||||
command.Claim = domain.GameClientBridgeClaim{FencingToken: fencingToken}
|
||||
command.UpdatedAt = stamp
|
||||
auditID, auditErr := svc.recordAuditEventWithID("platform", "game-client-bridge.command.lease-expire", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "expired bridge claim returned to pending")
|
||||
if auditErr != nil {
|
||||
return auditErr
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
if err := svc.store.GameClientBridgeCommands().Update(command); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -722,11 +636,6 @@ func (svc *CoreService) expireGameClientBridgeCommandLocked(command domain.GameC
|
||||
command.State = domain.GameClientBridgeCommandExpired
|
||||
command.CompletedAt = stamp
|
||||
command.UpdatedAt = stamp
|
||||
auditID, err := svc.recordAuditEventWithID("platform", "game-client-bridge.command.expire", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "bridge command expired before completion")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
return svc.store.GameClientBridgeCommands().Update(command)
|
||||
}
|
||||
|
||||
|
||||
@@ -111,11 +111,6 @@ func (svc *CoreService) UploadGameClientBridgeSnapshot(request domain.GameClient
|
||||
CreatedAt: stamp,
|
||||
ExpiresAt: stamp.Add(time.Duration(request.Retention.KeepForSeconds) * time.Second),
|
||||
}
|
||||
auditID, err := svc.recordAuditEventWithID("component:"+component.Session.ProfileKey, "game-client-bridge.snapshot.ingest", "game-client-bridge-snapshot", snapshot.ID, domain.AuditResultSuccess, "companion uploaded typed bridge snapshot")
|
||||
if err != nil {
|
||||
return domain.GameClientBridgeSnapshot{}, err
|
||||
}
|
||||
snapshot.AuditReferences = []string{auditID}
|
||||
if err := svc.store.GameClientBridgeSnapshots().Create(snapshot); err != nil {
|
||||
return domain.GameClientBridgeSnapshot{}, err
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,7 +12,7 @@ func newGameClientBridgeService(t *testing.T) (*CoreService, *time.Time) {
|
||||
t.Helper()
|
||||
now := time.Date(2026, 7, 20, 10, 0, 0, 0, time.UTC)
|
||||
store := repo.NewMemoryStore()
|
||||
plugin := domain.GamePlugin{ID: "game.scum", RuntimeProfiles: domain.GamePluginRuntimeProfiles{ClientManagers: []domain.RuntimeClientManagerProfile{{Key: "scum-client", Health: domain.RuntimeClientManagerHealth{RequiredCapabilities: []string{gameClientBridgeCapability}}}}}, GameClientBridge: domain.GameClientBridgeManifest{Commands: []domain.GameClientBridgeCommandDeclaration{{Type: "announcement.send", ApprovalLevel: domain.GameClientBridgeApprovalLevelOperator, TimeoutSeconds: 600, MaxPayloadBytes: 4096}}, Snapshots: []domain.GameClientBridgeSnapshotDeclaration{{Type: "players", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 3600, MaxRecords: 100}}, {Type: "health", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 60}}, {Type: "companion.health", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 3600, MaxRecords: 100}}}, Retention: domain.GameClientBridgeRetention{KeepForSeconds: 86400, MaxRecords: 1000}}}
|
||||
plugin := domain.GamePlugin{ID: "game.scum", RuntimeProfiles: domain.GamePluginRuntimeProfiles{ClientManagers: []domain.RuntimeClientManagerProfile{{Key: "scum-client", Health: domain.RuntimeClientManagerHealth{RequiredCapabilities: []string{gameClientBridgeCapability}}}}}, GameClientBridge: domain.GameClientBridgeManifest{Commands: []domain.GameClientBridgeCommandDeclaration{{Type: "diagnostic.ping", ApprovalLevel: domain.GameClientBridgeApprovalLevelNone, TimeoutSeconds: 600, MaxPayloadBytes: 4096}}, Snapshots: []domain.GameClientBridgeSnapshotDeclaration{{Type: "players", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 3600, MaxRecords: 100}}, {Type: "health", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 60}}, {Type: "companion.health", SchemaVersion: "1", Retention: domain.GameClientBridgeRetention{KeepForSeconds: 3600, MaxRecords: 100}}}, Retention: domain.GameClientBridgeRetention{KeepForSeconds: 86400, MaxRecords: 1000}}}
|
||||
if err := store.GamePlugins().Create(plugin); err != nil {
|
||||
t.Fatalf("seed bridge plugin: %v", err)
|
||||
}
|
||||
@@ -23,7 +21,7 @@ func newGameClientBridgeService(t *testing.T) (*CoreService, *time.Time) {
|
||||
}
|
||||
|
||||
func bridgeQueueRequest(now time.Time, key string) domain.GameClientBridgeQueueRequest {
|
||||
return domain.GameClientBridgeQueueRequest{ServerInstanceID: "server-1", PluginID: "game.scum", ProfileKey: "scum-client", CommandType: "announcement.send", Payload: map[string]any{"message": "hello"}, IdempotencyKey: key, Priority: 10, ExpiresAt: now.Add(5 * time.Minute)}
|
||||
return domain.GameClientBridgeQueueRequest{ServerInstanceID: "server-1", PluginID: "game.scum", ProfileKey: "scum-client", CommandType: "diagnostic.ping", Payload: map[string]any{"message": "hello"}, IdempotencyKey: key, Priority: 10, ExpiresAt: now.Add(5 * time.Minute)}
|
||||
}
|
||||
|
||||
func bridgeComponent() gameClientBridgeComponentSession {
|
||||
@@ -45,8 +43,8 @@ func TestGameClientBridgeCommandLifecycleAndIdempotency(t *testing.T) {
|
||||
t.Fatalf("idempotency reuse: command=%#v err=%v", duplicate, err)
|
||||
}
|
||||
commands, _ := svc.store.GameClientBridgeCommands().List(domain.GameClientBridgeCommandFilter{})
|
||||
if len(commands) != 1 || len(command.AuditReferences) != 1 {
|
||||
t.Fatalf("expected one durable audited command: %#v", commands)
|
||||
if len(commands) != 1 {
|
||||
t.Fatalf("expected one durable command: %#v", commands)
|
||||
}
|
||||
|
||||
component := bridgeComponent()
|
||||
@@ -71,12 +69,8 @@ func TestGameClientBridgeCommandLifecycleAndIdempotency(t *testing.T) {
|
||||
if err != nil || completed.State != domain.GameClientBridgeCommandSucceeded || completed.Result.Status != domain.GameClientBridgeResultSucceeded || completed.CompletedAt.IsZero() {
|
||||
t.Fatalf("complete bridge command: %#v err=%v", completed, err)
|
||||
}
|
||||
if len(completed.AuditReferences) < 3 {
|
||||
t.Fatalf("expected queue, claim, and result audit references: %#v", completed.AuditReferences)
|
||||
}
|
||||
auditReferenceCount := len(completed.AuditReferences)
|
||||
replayed, err := svc.completeGameClientBridgeCommand(component, resultRequest)
|
||||
if err != nil || replayed.ID != completed.ID || replayed.State != completed.State || !replayed.CompletedAt.Equal(completed.CompletedAt) || len(replayed.AuditReferences) != auditReferenceCount {
|
||||
if err != nil || replayed.ID != completed.ID || replayed.State != completed.State || !replayed.CompletedAt.Equal(completed.CompletedAt) {
|
||||
t.Fatalf("exact terminal result retry was not idempotent: replayed=%#v err=%v", replayed, err)
|
||||
}
|
||||
if _, err := svc.completeGameClientBridgeCommand(component, domain.GameClientBridgeResultRequest{SessionToken: "session-token", CommandID: command.ID, FencingToken: 1, Status: domain.GameClientBridgeResultFailed, Summary: "conflict"}); err == nil {
|
||||
@@ -84,111 +78,6 @@ func TestGameClientBridgeCommandLifecycleAndIdempotency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedGameClientBridgeRequestIsScopedAndRedacted(t *testing.T) {
|
||||
svc, clock := newGameClientBridgeService(t)
|
||||
plugin, err := svc.store.GamePlugins().Get("game.scum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plugin.RuntimeProfiles.TransportProfiles = []domain.RuntimeTransportProfile{{Key: "database", Kind: "sqlite", TargetKey: "database", Capabilities: []string{domain.JobCapabilityRemoteRunProtectedSQL}}}
|
||||
plugin.GameClientBridge.Commands = append(plugin.GameClientBridge.Commands, domain.GameClientBridgeCommandDeclaration{Type: "database.request", ApprovalLevel: domain.GameClientBridgeApprovalLevelPlatformAdmin, TimeoutSeconds: 60, MaxPayloadBytes: 4096, ProtectedRequest: &domain.GameClientBridgeProtectedRequestDeclaration{Kind: "sql", TransportKey: "database", TargetKey: "database", TextField: "requestText", MaxTextBytes: 1024}})
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := "UPDATE players SET rank = 2 WHERE id = 7"
|
||||
request := domain.GameClientBridgeQueueRequest{ServerInstanceID: "server-1", PluginID: "game.scum", ProfileKey: "scum-client", CommandType: "database.request", Payload: map[string]any{"requestText": text}, IdempotencyKey: "protected-1", ExpiresAt: clock.Add(time.Minute)}
|
||||
command, err := svc.queueGameClientBridgeCommand("user-1", request)
|
||||
if err != nil {
|
||||
t.Fatalf("queue protected request: %v", err)
|
||||
}
|
||||
if command.ApprovalState != domain.GameClientBridgeApprovalPending {
|
||||
t.Fatalf("protected request bypassed approval: %#v", command)
|
||||
}
|
||||
if _, err := svc.queueGameClientBridgeCommand("user-1", domain.GameClientBridgeQueueRequest{ServerInstanceID: "server-1", PluginID: "game.scum", ProfileKey: "scum-client", CommandType: "database.request", Payload: map[string]any{"requestText": text, "unexpected": true}, IdempotencyKey: "protected-extra", ExpiresAt: clock.Add(time.Minute)}); err == nil {
|
||||
t.Fatal("protected request accepted undeclared payload field")
|
||||
}
|
||||
events, err := svc.store.AuditEvents().List(domain.AuditEventFilter{ResourceID: command.ID})
|
||||
if err != nil || len(events) != 1 {
|
||||
t.Fatalf("protected request audit: events=%#v err=%v", events, err)
|
||||
}
|
||||
if strings.Contains(events[0].Summary, text) || !strings.Contains(events[0].Summary, "text=redacted") {
|
||||
t.Fatalf("audit leaked protected request: %#v", events[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedGameClientBridgeRequestDispatchesOneTimeRunInput(t *testing.T) {
|
||||
svc, clock := newGameClientBridgeService(t)
|
||||
plugin, err := svc.store.GamePlugins().Get("game.scum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plugin.RequiredRunCapabilities = []string{domain.JobCapabilityRemoteRunProtectedSQL}
|
||||
plugin.RuntimeProfiles.TransportProfiles = []domain.RuntimeTransportProfile{{Key: "scum-database", Kind: "sqlite", TargetKey: "scum-database", Capabilities: []string{domain.JobCapabilityRemoteRunProtectedSQL}}}
|
||||
plugin.GameClientBridge.Commands = append(plugin.GameClientBridge.Commands, domain.GameClientBridgeCommandDeclaration{Type: "database.request", ApprovalLevel: domain.GameClientBridgeApprovalLevelPlatformAdmin, TimeoutSeconds: 120, MaxPayloadBytes: 4096, ProtectedRequest: &domain.GameClientBridgeProtectedRequestDeclaration{Kind: "sql", TransportKey: "scum-database", TargetKey: "scum-database", TextField: "requestText", MaxTextBytes: 1024}})
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.store.Users().Create(domain.User{ID: "platform-admin", Email: "admin@example.test", Roles: []string{"platform-admin"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.store.ServerInstances().Create(domain.ServerInstance{ID: "server-1", PluginID: plugin.ID, RunEndpointID: "run-local", Name: "Protected Bridge", State: domain.ServerInstanceStateRunning}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hello := validRunControlHello()
|
||||
hello.CapabilityReport.Capabilities = []string{domain.JobCapabilityRemoteRunProtectedSQL}
|
||||
hello.CapabilityReport.Fingerprint = "protected-request-capabilities"
|
||||
run, err := svc.RegisterRunHello(hello)
|
||||
if err != nil {
|
||||
t.Fatalf("register Run: %v", err)
|
||||
}
|
||||
|
||||
text := "SELECT player_id, position FROM players WHERE player_id = 7"
|
||||
command, err := svc.queueGameClientBridgeCommand("platform-admin", domain.GameClientBridgeQueueRequest{ServerInstanceID: "server-1", PluginID: plugin.ID, ProfileKey: "scum-client", CommandType: "database.request", Payload: map[string]any{"requestText": text}, IdempotencyKey: "protected-run-1", ExpiresAt: clock.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("queue protected request: %v", err)
|
||||
}
|
||||
if command.RunJobID == "" || command.Payload["requestText"] != "redacted" || command.ApprovalState != domain.GameClientBridgeApprovalApproved {
|
||||
t.Fatalf("protected command was not redacted and dispatched: %#v", command)
|
||||
}
|
||||
commandJSON, _ := json.Marshal(command)
|
||||
if strings.Contains(string(commandJSON), text) {
|
||||
t.Fatalf("protected bridge command persisted request text: %s", commandJSON)
|
||||
}
|
||||
if claimed, err := svc.claimGameClientBridgeCommands(bridgeComponent(), 10); err != nil || len(claimed) != 0 {
|
||||
t.Fatalf("protected request must not be exposed to the Companion: commands=%#v err=%v", claimed, err)
|
||||
}
|
||||
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: run.SessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunProtectedSQL}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job == nil || claim.Job.JobID != command.RunJobID || claim.Job.FencingToken == 0 {
|
||||
t.Fatalf("claim protected Run job: claim=%#v err=%v", claim, err)
|
||||
}
|
||||
assignmentJSON, _ := json.Marshal(claim.Job)
|
||||
if strings.Contains(string(assignmentJSON), text) {
|
||||
t.Fatalf("Run assignment exposed protected request text: %s", assignmentJSON)
|
||||
}
|
||||
ack, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: run.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Message: "accepted"})
|
||||
if err != nil || !ack.Accepted {
|
||||
t.Fatalf("ack protected Run job: ack=%#v err=%v", ack, err)
|
||||
}
|
||||
if _, err := svc.GetProtectedRequestExecutionInput(domain.ProtectedRequestExecutionInputRequest{RunEndpointID: "run-local", SessionToken: run.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, FencingToken: claim.Job.FencingToken + 1}); err == nil {
|
||||
t.Fatal("expected fencing mismatch rejection")
|
||||
}
|
||||
input, err := svc.GetProtectedRequestExecutionInput(domain.ProtectedRequestExecutionInputRequest{RunEndpointID: "run-local", SessionToken: run.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, FencingToken: claim.Job.FencingToken})
|
||||
if err != nil || input.RequestText != text || input.Kind != "sql" || input.TransportKey != "scum-database" || !input.Authorized {
|
||||
t.Fatalf("read protected Run input: input=%#v err=%v", input, err)
|
||||
}
|
||||
if _, err := svc.GetProtectedRequestExecutionInput(domain.ProtectedRequestExecutionInputRequest{RunEndpointID: "run-local", SessionToken: run.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, FencingToken: claim.Job.FencingToken}); err == nil {
|
||||
t.Fatal("expected one-time protected input rejection")
|
||||
}
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: run.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateFailed, Progress: domain.RunJobProgressReport{Percent: 100, Message: "unknown request"}, ErrorCode: "protected_request_unknown", ExecutionResult: domain.JobExecutionResult{Kind: "protected.sql.unknown", AuditSummary: "protected request outcome is unknown"}}); err != nil {
|
||||
t.Fatalf("complete protected Run job: %v", err)
|
||||
}
|
||||
completed, err := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if err != nil || completed.State != domain.GameClientBridgeCommandUnknown || completed.Result.Status != domain.GameClientBridgeResultUnknown || strings.Contains(completed.Result.Summary, text) {
|
||||
t.Fatalf("project protected Run result: command=%#v err=%v", completed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGameClientBridgeIdempotencyScopeIsAppliedByService(t *testing.T) {
|
||||
svc, clock := newGameClientBridgeService(t)
|
||||
request := bridgeQueueRequest(*clock, "scope-key")
|
||||
@@ -266,7 +155,7 @@ func TestGameClientBridgePendingCommandExpiresBeforeFirstClaim(t *testing.T) {
|
||||
t.Fatalf("expired pending command was claimable: %#v err=%v", claimed, err)
|
||||
}
|
||||
expired, err := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() || expired.Claim.FencingToken != 0 || len(expired.AuditReferences) != 2 {
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() || expired.Claim.FencingToken != 0 {
|
||||
t.Fatalf("first claim did not persist pending command expiry: %#v err=%v", expired, err)
|
||||
}
|
||||
}
|
||||
@@ -293,7 +182,7 @@ func TestGameClientBridgeExpiredLeaseRejectsMutationsBeforeReclaim(t *testing.T)
|
||||
}
|
||||
for _, command := range claimed {
|
||||
protected, getErr := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if getErr != nil || protected.State != domain.GameClientBridgeCommandClaimed || !protected.Claim.AcknowledgedAt.IsZero() || protected.Result.Status != "" || !protected.CompletedAt.IsZero() || protected.Claim.FencingToken != command.Claim.FencingToken || len(protected.AuditReferences) != 2 {
|
||||
if getErr != nil || protected.State != domain.GameClientBridgeCommandClaimed || !protected.Claim.AcknowledgedAt.IsZero() || protected.Result.Status != "" || !protected.CompletedAt.IsZero() || protected.Claim.FencingToken != command.Claim.FencingToken {
|
||||
t.Fatalf("expired lease mutation changed protected command: %#v err=%v", protected, getErr)
|
||||
}
|
||||
}
|
||||
@@ -341,8 +230,8 @@ func TestGameClientBridgeClaimMutationsExpireAtCommandDeadline(t *testing.T) {
|
||||
}
|
||||
for _, command := range commands {
|
||||
expired, err := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() || len(expired.AuditReferences) < 3 {
|
||||
t.Fatalf("deadline mutation did not persist audited expiry: %#v err=%v", expired, err)
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() {
|
||||
t.Fatalf("deadline mutation did not persist expiry: %#v err=%v", expired, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -360,11 +249,11 @@ func TestGameClientBridgeFailedResultIsPersisted(t *testing.T) {
|
||||
}
|
||||
request := domain.GameClientBridgeResultRequest{SessionToken: "session-token", CommandID: command.ID, FencingToken: claimed[0].Claim.FencingToken, Status: domain.GameClientBridgeResultFailed, Summary: "game window unavailable", Payload: map[string]any{"retryable": true}}
|
||||
failed, err := svc.completeGameClientBridgeCommand(component, request)
|
||||
if err != nil || failed.State != domain.GameClientBridgeCommandFailed || failed.Result.Status != domain.GameClientBridgeResultFailed || failed.Result.Summary != request.Summary || failed.Result.CompletedBy != component.Session.ID || failed.CompletedAt.IsZero() || len(failed.AuditReferences) != 3 {
|
||||
if err != nil || failed.State != domain.GameClientBridgeCommandFailed || failed.Result.Status != domain.GameClientBridgeResultFailed || failed.Result.Summary != request.Summary || failed.Result.CompletedBy != component.Session.ID || failed.CompletedAt.IsZero() {
|
||||
t.Fatalf("record failed result: %#v err=%v", failed, err)
|
||||
}
|
||||
persisted, err := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if err != nil || persisted.State != domain.GameClientBridgeCommandFailed || persisted.Result.Status != domain.GameClientBridgeResultFailed || persisted.Result.Payload["retryable"] != true || !persisted.CompletedAt.Equal(failed.CompletedAt) || len(persisted.AuditReferences) != len(failed.AuditReferences) {
|
||||
if err != nil || persisted.State != domain.GameClientBridgeCommandFailed || persisted.Result.Status != domain.GameClientBridgeResultFailed || persisted.Result.Payload["retryable"] != true || !persisted.CompletedAt.Equal(failed.CompletedAt) {
|
||||
t.Fatalf("failed result was not persisted: %#v err=%v", persisted, err)
|
||||
}
|
||||
}
|
||||
@@ -393,8 +282,8 @@ func TestGameClientBridgeOperatorCancellationExpiresAtCommandDeadline(t *testing
|
||||
t.Fatal("expected cancellation at command deadline to be rejected")
|
||||
}
|
||||
expired, err := svc.store.GameClientBridgeCommands().Get(command.ID)
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() || expired.Cancellation.RequestedBy != "" || len(expired.AuditReferences) != 2 {
|
||||
t.Fatalf("deadline cancellation did not preserve audited expiry: %#v err=%v", expired, err)
|
||||
if err != nil || expired.State != domain.GameClientBridgeCommandExpired || expired.CompletedAt.IsZero() || expired.Cancellation.RequestedBy != "" {
|
||||
t.Fatalf("deadline cancellation did not preserve expiry: %#v err=%v", expired, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,9 +313,8 @@ func TestGameClientBridgeOperatorCancellationRejectsLateSuccess(t *testing.T) {
|
||||
if err != nil || cancelled.State != domain.GameClientBridgeCommandCancelled || cancelled.Cancellation.RequestedBy != user.ID {
|
||||
t.Fatalf("cancel bridge command: %#v err=%v", cancelled, err)
|
||||
}
|
||||
auditReferenceCount := len(cancelled.AuditReferences)
|
||||
repeated, err := svc.CancelGameClientBridgeCommandForSession(auth.SessionID, domain.GameClientBridgeCancelRequest{CommandID: command.ID, Reason: "operator requested"})
|
||||
if err != nil || repeated.State != domain.GameClientBridgeCommandCancelled || !repeated.Cancellation.CancelledAt.Equal(cancelled.Cancellation.CancelledAt) || len(repeated.AuditReferences) != auditReferenceCount {
|
||||
if err != nil || repeated.State != domain.GameClientBridgeCommandCancelled || !repeated.Cancellation.CancelledAt.Equal(cancelled.Cancellation.CancelledAt) {
|
||||
t.Fatalf("repeated cancellation was not idempotent: %#v err=%v", repeated, err)
|
||||
}
|
||||
remaining, err := svc.claimGameClientBridgeCommands(component, 1)
|
||||
|
||||
@@ -269,9 +269,6 @@ func (svc *CoreService) CompleteRunJob(result domain.RunJobResult) (domain.RunJo
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectProtectedRequestJobResult(job, result, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectLifecycleJobResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
@@ -345,7 +342,7 @@ func validateExecutionResultForJob(job domain.Job, result domain.RunJobResult) e
|
||||
return validationError("client-manager deploy result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerControl:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.controlled" {
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.control" {
|
||||
return validationError("client-manager control result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerUpdate:
|
||||
@@ -630,10 +627,6 @@ func firstEligibleSupportedJob(jobs []domain.Job, capabilities []string, stamp t
|
||||
}
|
||||
|
||||
func assignmentFromJob(job domain.Job, leaseToken string) domain.RunJobAssignment {
|
||||
fencingToken := uint64(0)
|
||||
if isProtectedRequestCapability(job.Capability) {
|
||||
fencingToken = uint64(job.Attempt)
|
||||
}
|
||||
return domain.RunJobAssignment{
|
||||
JobID: job.ID,
|
||||
ServerInstanceID: job.ServerInstanceID,
|
||||
@@ -648,7 +641,7 @@ func assignmentFromJob(job domain.Job, leaseToken string) domain.RunJobAssignmen
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: job.ExecutionInput.WorkspaceScope, Content: job.ExecutionInput.Content, ExpectedVersion: job.ExecutionInput.ExpectedVersion, ExpectedChecksum: job.ExecutionInput.ExpectedChecksum, MaxReadBytes: job.ExecutionInput.MaxReadBytes, RemoteAdapterKey: job.ExecutionInput.RemoteAdapterKey, RemoteAdapterKind: job.ExecutionInput.RemoteAdapterKind, TimeoutSeconds: job.ExecutionInput.TimeoutSeconds, PluginID: job.ExecutionInput.PluginID, LifecycleOperation: job.ExecutionInput.LifecycleOperation, TargetVersion: job.ExecutionInput.TargetVersion, Inputs: domain.CopyStringMap(job.ExecutionInput.Inputs), LogSource: domain.CopyRuntimeLogSourcePtr(job.ExecutionInput.LogSource), LogSources: domain.CopyRuntimeLogSources(job.ExecutionInput.LogSources), DLLExtensions: append([]domain.RuntimeDLLExtensionPlan(nil), job.ExecutionInput.DLLExtensions...), SourceRCON: domain.CopyRuntimeSourceRCONPlan(job.ExecutionInput.SourceRCON), Deployment: deploymentPlanForDispatchValue(job.ExecutionInput.Deployment), ServerDeploymentPlan: domain.CopyServerDeploymentPlan(job.ExecutionInput.ServerDeploymentPlan)},
|
||||
LeaseToken: leaseToken,
|
||||
Attempt: job.Attempt,
|
||||
FencingToken: fencingToken,
|
||||
FencingToken: 0,
|
||||
MaxAttempts: job.RetryPolicy.MaxAttempts,
|
||||
AckDeadlineAt: job.AckDeadlineAt,
|
||||
LeaseExpiresAt: job.LeaseExpiresAt,
|
||||
|
||||
@@ -132,13 +132,6 @@ func (svc *CoreService) CreateBackupForSession(sessionID string, record domain.B
|
||||
if err := svc.store.Backups().Create(record); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "backup.create", "server-instance", instance.ID, domain.AuditResultQueued, "created bounded backup record with artifact checksum"); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.pruneBackups(instance.ID); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
@@ -189,9 +182,6 @@ func (svc *CoreService) RecoverIncompleteBackups() error {
|
||||
if err := svc.store.Backups().Update(record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.recordAuditEvent("platform-recovery", "backup.recover", "server-instance", record.ServerInstanceID, domain.AuditResultFailed, "marked interrupted backup recoverable without exposing storage details"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -207,7 +197,7 @@ func (svc *CoreService) pruneMetricSamples(serverInstanceID string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return svc.recordAuditEvent("platform-retention", "metrics.retention", "server-instance", serverInstanceID, domain.AuditResultSuccess, "pruned oldest metric samples to bounded retention")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) pruneBackups(serverInstanceID string) error {
|
||||
@@ -222,7 +212,6 @@ func (svc *CoreService) pruneBackups(serverInstanceID string) error {
|
||||
total += record.SizeBytes
|
||||
}
|
||||
}
|
||||
pruned := false
|
||||
for len(items) > maxBackupsPerServer || total > maxBackupBytesPerServer {
|
||||
record := items[0]
|
||||
items = items[1:]
|
||||
@@ -235,10 +224,6 @@ func (svc *CoreService) pruneBackups(serverInstanceID string) error {
|
||||
if err := svc.store.Backups().Update(record); err != nil {
|
||||
return err
|
||||
}
|
||||
pruned = true
|
||||
}
|
||||
if pruned {
|
||||
return svc.recordAuditEvent("platform-retention", "backup.retention", "server-instance", serverInstanceID, domain.AuditResultSuccess, "expired oldest backup records to bounded retention")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -184,17 +184,6 @@ func (svc *CoreService) applyPluginLogProjection(instance domain.ServerInstance,
|
||||
if insideWindow && !sameObservation {
|
||||
return nil
|
||||
}
|
||||
announcementAlreadyQueued := false
|
||||
announcementIdempotencyKey := ""
|
||||
if projection.Presence != nil {
|
||||
announcementIdempotencyKey = fmt.Sprintf("log-projection:%s:%s:%d", projection.Key, key, observedAt.Unix()/int64(projection.Presence.ActiveWindowSeconds))
|
||||
_, commandErr := svc.store.GameClientBridgeCommands().GetByIdempotency(instance.ID, "system:log-projection", projection.Presence.Announcement.CommandType, announcementIdempotencyKey)
|
||||
if commandErr == nil {
|
||||
announcementAlreadyQueued = true
|
||||
} else if !errors.Is(commandErr, repo.ErrNotFound) {
|
||||
return commandErr
|
||||
}
|
||||
}
|
||||
if !isNew {
|
||||
value = mergePluginDataValues(existing.Value, value)
|
||||
}
|
||||
@@ -211,39 +200,9 @@ func (svc *CoreService) applyPluginLogProjection(instance domain.ServerInstance,
|
||||
return applyErr
|
||||
}
|
||||
}
|
||||
if projection.Presence != nil && !announcementAlreadyQueued {
|
||||
announcement := projection.Presence.Announcement
|
||||
template := announcement.ReturningTextTemplate
|
||||
if isNew || sameObservation {
|
||||
template = announcement.NewTextTemplate
|
||||
}
|
||||
requestText := renderLogProjectionTemplate(template, captures)
|
||||
expiresAt := svc.now().Add(gameClientBridgeCommandTimeout(plugin, announcement.CommandType))
|
||||
if _, err := svc.queueGameClientBridgeCommand("system:log-projection", domain.GameClientBridgeQueueRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
PluginID: plugin.ID,
|
||||
ProfileKey: announcement.ProfileKey,
|
||||
CommandType: announcement.CommandType,
|
||||
Payload: map[string]any{announcement.TextField: requestText},
|
||||
IdempotencyKey: announcementIdempotencyKey,
|
||||
Priority: 100,
|
||||
ExpiresAt: expiresAt,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gameClientBridgeCommandTimeout(plugin domain.GamePlugin, commandType string) time.Duration {
|
||||
for _, declaration := range plugin.GameClientBridge.Commands {
|
||||
if declaration.Type == commandType && declaration.TimeoutSeconds > 0 {
|
||||
return time.Duration(declaration.TimeoutSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return time.Minute
|
||||
}
|
||||
|
||||
func pluginLogProjectionValue(target domain.GameClientBridgeLogProjectionTargetDeclaration, captures map[string]string, observedAt time.Time) map[string]any {
|
||||
value := make(map[string]any, len(target.CaptureMappings)+len(target.FixedValues)+1)
|
||||
for destination, capture := range target.CaptureMappings {
|
||||
|
||||
@@ -1,25 +1,16 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func TestDurableStdoutProjectionCreatesUsersSuppressesRapidDuplicatesAndAnnouncesReturns(t *testing.T) {
|
||||
func TestDurableStdoutProjectionCreatesUsersAndSuppressesRapidDuplicates(t *testing.T) {
|
||||
svc := newTestCoreService()
|
||||
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
|
||||
capability := domain.JobCapabilityRemoteRunProtectedRCON
|
||||
plugin.RequiredRunCapabilities = append(plugin.RequiredRunCapabilities, capability)
|
||||
plugin.RuntimeProfiles.TransportProfiles = append(plugin.RuntimeProfiles.TransportProfiles, domain.RuntimeTransportProfile{Key: "scum-management", Kind: "rcon", TargetKey: "scum-management", Capabilities: []string{capability}})
|
||||
plugin.RuntimeProfiles.ClientManagers = append(plugin.RuntimeProfiles.ClientManagers, domain.RuntimeClientManagerProfile{Key: "scum-client", Health: domain.RuntimeClientManagerHealth{RequiredCapabilities: []string{"game-client.bridge"}}})
|
||||
plugin.GameClientBridge.Commands = []domain.GameClientBridgeCommandDeclaration{{
|
||||
Type: "presence.announce", Title: "Presence announcement", Permission: "server.game-client.command", ApprovalLevel: domain.GameClientBridgeApprovalLevelOperator,
|
||||
PayloadSchemaRef: "schemas/presence-announcement.json", TimeoutSeconds: 60, MaxPayloadBytes: 4096,
|
||||
ProtectedRequest: &domain.GameClientBridgeProtectedRequestDeclaration{Kind: "rcon", TransportKey: "scum-management", TargetKey: "scum-management", TextField: "requestText", MaxTextBytes: 1024},
|
||||
}}
|
||||
plugin.GameClientBridge.LogProjections = []domain.GameClientBridgeLogProjectionDeclaration{{
|
||||
Key: "player.login", StreamKeys: []string{"stdout"}, CorrelationFields: []string{"playerSlot"}, MaxInterveningLines: 4,
|
||||
Steps: []domain.GameClientBridgeLogProjectionStepDeclaration{
|
||||
@@ -37,25 +28,16 @@ func TestDurableStdoutProjectionCreatesUsersSuppressesRapidDuplicatesAndAnnounce
|
||||
Collection: "scum_activity_events", UpsertKeys: []string{"steamId", "observedAt"},
|
||||
CaptureMappings: map[string]string{"steamId": "steamId", "displayName": "displayName"}, FixedValues: map[string]string{"eventType": "login"}, ObservedAtField: "observedAt",
|
||||
},
|
||||
Announcement: domain.GameClientBridgeLogProjectionAnnouncementDeclaration{
|
||||
ProfileKey: "scum-client", CommandType: "presence.announce", TextField: "requestText",
|
||||
NewTextTemplate: "#announce Welcome {{displayName}}", ReturningTextTemplate: "#announce Welcome back {{displayName}}",
|
||||
},
|
||||
},
|
||||
}}
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("update plugin projection: %v", err)
|
||||
}
|
||||
endpoint.Capabilities = append(endpoint.Capabilities, capability)
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("update Run capability: %v", err)
|
||||
}
|
||||
instance, err := svc.CreateServerInstance(domain.ServerInstance{ID: "server-log-projection", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "SCUM projection", State: domain.ServerInstanceStateRunning})
|
||||
if err != nil {
|
||||
t.Fatalf("create server: %v", err)
|
||||
}
|
||||
helloRequest := validRunControlHello()
|
||||
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, capability)
|
||||
helloRequest.CapabilityReport.Fingerprint = "cap-log-projection"
|
||||
hello, err := svc.RegisterRunHello(helloRequest)
|
||||
if err != nil {
|
||||
@@ -74,29 +56,19 @@ func TestDurableStdoutProjectionCreatesUsersSuppressesRapidDuplicatesAndAnnounce
|
||||
ingestProjectionLines(t, svc, hello.SessionToken, endpoint.ID, instance.ID, stream.ID, 3, base.Add(2*time.Second), []string{
|
||||
`LogBattlEye: Display: Player 0 SteamID (assumed): 76561199510658111`,
|
||||
})
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 1, 1)
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 1, 0)
|
||||
|
||||
ingestProjectionLines(t, svc, hello.SessionToken, endpoint.ID, instance.ID, stream.ID, 4, base.Add(5*time.Minute), []string{
|
||||
`LogBattlEye: Display: Player "love_fitting" reported as player 0`,
|
||||
`LogBattlEye: Display: Player 0 SteamID (assumed): 76561199510658111`,
|
||||
})
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 1, 1)
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 1, 0)
|
||||
|
||||
ingestProjectionLines(t, svc, hello.SessionToken, endpoint.ID, instance.ID, stream.ID, 6, base.Add(11*time.Minute), []string{
|
||||
`LogBattlEye: Display: Player "love_fitting" reported as player 0`,
|
||||
`LogBattlEye: Display: Player 0 SteamID (assumed): 76561199510658111`,
|
||||
})
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 2, 2)
|
||||
|
||||
svc.protectedRequests.mu.Lock()
|
||||
texts := make([]string, 0, len(svc.protectedRequests.payloads))
|
||||
for _, payload := range svc.protectedRequests.payloads {
|
||||
texts = append(texts, payload.requestText)
|
||||
}
|
||||
svc.protectedRequests.mu.Unlock()
|
||||
if len(texts) != 2 || !containsText(texts, "#announce Welcome love_fitting") || !containsText(texts, "#announce Welcome back love_fitting") {
|
||||
t.Fatalf("unexpected plugin-declared announcement requests: %v", texts)
|
||||
}
|
||||
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 2, 0)
|
||||
}
|
||||
|
||||
func ingestProjectionLines(t *testing.T, svc *CoreService, sessionToken, endpointID, serverID, streamID string, firstSeq uint64, observedAt time.Time, lines []string) {
|
||||
@@ -124,15 +96,6 @@ func assertPresenceProjectionCounts(t *testing.T, svc *CoreService, pluginID, se
|
||||
}
|
||||
queued, err := svc.store.GameClientBridgeCommands().List(domain.GameClientBridgeCommandFilter{ServerInstanceID: serverID, PluginID: pluginID})
|
||||
if err != nil || len(queued) != commands {
|
||||
t.Fatalf("presence announcements=%+v err=%v", queued, err)
|
||||
t.Fatalf("presence bridge commands=%+v err=%v", queued, err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsText(values []string, expected string) bool {
|
||||
for _, value := range values {
|
||||
if strings.Contains(value, expected) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -14,297 +14,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
capacityHeartbeatStaleAfter = 2 * time.Minute
|
||||
capacityRetryAfterSeconds = 30
|
||||
capacityLogBacklogLimit = 256
|
||||
capacityArtifactBacklogLimit = 128
|
||||
aiConfigDiffTTL = 30 * time.Minute
|
||||
aiConfigDiffTTL = 30 * time.Minute
|
||||
)
|
||||
|
||||
func (svc *CoreService) GetProductionCapacityForSession(sessionID string) (domain.ProductionCapacitySummary, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.ProductionCapacitySummary{}, err
|
||||
}
|
||||
endpoints, err := svc.store.RunEndpoints().List(domain.RunEndpointFilter{})
|
||||
if err != nil {
|
||||
return domain.ProductionCapacitySummary{}, err
|
||||
}
|
||||
visibleEndpointIDs, err := svc.visibleEndpointIDs(user)
|
||||
if err != nil {
|
||||
return domain.ProductionCapacitySummary{}, err
|
||||
}
|
||||
alerts, err := svc.store.Alerts().List(domain.AlertFilter{})
|
||||
if err != nil {
|
||||
return domain.ProductionCapacitySummary{}, err
|
||||
}
|
||||
|
||||
summary := domain.ProductionCapacitySummary{GeneratedAt: svc.now()}
|
||||
for _, endpoint := range endpoints {
|
||||
if !isPlatformAdmin(user) {
|
||||
if _, visible := visibleEndpointIDs[endpoint.ID]; !visible {
|
||||
continue
|
||||
}
|
||||
}
|
||||
running, queued, err := svc.capacityJobCounts(endpoint.ID)
|
||||
if err != nil {
|
||||
return domain.ProductionCapacitySummary{}, err
|
||||
}
|
||||
projection := svc.capacityProjection(endpoint, running, queued)
|
||||
for _, alert := range alerts {
|
||||
if alert.SourceKind == "run-endpoint" && alert.SourceID == endpoint.ID && alert.RuleKey == "capacity.pressure" && alert.State != domain.AlertStateResolved {
|
||||
projection.LastAdmissionDecision = domain.CapacityAdmissionDeferred
|
||||
projection.LastAdmissionReason = alert.Message
|
||||
projection.LastAdmissionCheckedAt = alert.LastSeenAt
|
||||
}
|
||||
}
|
||||
summary.Endpoints = append(summary.Endpoints, projection)
|
||||
summary.TotalMaxJobs += projection.MaxJobs
|
||||
summary.TotalRunningJobs += projection.RunningJobs
|
||||
summary.TotalQueuedJobs += projection.QueuedJobs
|
||||
}
|
||||
for _, alert := range alerts {
|
||||
if alert.State != domain.AlertStateResolved && svc.canAccessAlert(user, alert) {
|
||||
summary.ActiveAlerts++
|
||||
}
|
||||
}
|
||||
sort.Slice(summary.Endpoints, func(i, j int) bool { return summary.Endpoints[i].RunEndpointID < summary.Endpoints[j].RunEndpointID })
|
||||
return domain.CopyProductionCapacitySummary(summary), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CheckCapacityAdmissionForSession(sessionID string, request domain.CapacityAdmissionRequest) (domain.CapacityAdmissionDecision, error) {
|
||||
if err := validator.ValidateCapacityAdmissionRequest(request); err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
request, err = svc.authorizeCapacityRequest(user, request)
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
svc.productionMu.Lock()
|
||||
defer svc.productionMu.Unlock()
|
||||
return svc.checkCapacityAdmission(user.ID, request)
|
||||
}
|
||||
|
||||
func (svc *CoreService) checkCapacityAdmission(actorID string, request domain.CapacityAdmissionRequest) (domain.CapacityAdmissionDecision, error) {
|
||||
endpoint, err := svc.store.RunEndpoints().Get(request.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
running, queued, err := svc.capacityJobCounts(endpoint.ID)
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
projection := svc.capacityProjection(endpoint, running, queued)
|
||||
decision := domain.CapacityAdmissionDecision{
|
||||
Accepted: true, State: domain.CapacityAdmissionAccepted, Reason: "capacity available",
|
||||
ServerInstanceID: request.ServerInstanceID, RunEndpointID: endpoint.ID, Capability: request.Capability,
|
||||
TargetKey: request.TargetKey, MaxJobs: projection.MaxJobs, RunningJobs: projection.RunningJobs,
|
||||
QueuedJobs: projection.QueuedJobs, CheckedAt: svc.now(),
|
||||
}
|
||||
pressure := append([]domain.CapacityPressureCode(nil), projection.PressureCodes...)
|
||||
if len(validator.MissingCapabilities(endpoint.Capabilities, []string{request.Capability})) > 0 {
|
||||
pressure = appendCapacityPressure(pressure, domain.CapacityPressureCapabilityGap)
|
||||
}
|
||||
decision.PressureCodes = pressure
|
||||
|
||||
hardDenied := containsCapacityPressure(pressure, domain.CapacityPressureEndpointOffline) || containsCapacityPressure(pressure, domain.CapacityPressureCapabilityGap)
|
||||
if hardDenied {
|
||||
decision.Accepted = false
|
||||
decision.State = domain.CapacityAdmissionDenied
|
||||
decision.Reason = "endpoint is unavailable or missing the required capability"
|
||||
} else if len(pressure) > 0 {
|
||||
decision.Accepted = false
|
||||
decision.State = domain.CapacityAdmissionDeferred
|
||||
decision.Reason = "endpoint capacity is temporarily under pressure"
|
||||
decision.RetryAfterSeconds = capacityRetryAfterSeconds
|
||||
}
|
||||
|
||||
auditResult := domain.AuditResultSuccess
|
||||
auditAction := "capacity.admission.accepted"
|
||||
if !decision.Accepted {
|
||||
auditResult = domain.AuditResultDenied
|
||||
auditAction = "capacity.admission.denied"
|
||||
}
|
||||
auditID, err := svc.recordAuditEventWithID(actorID, auditAction, "run-endpoint", endpoint.ID, auditResult, decision.Reason)
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
decision.AuditEventID = auditID
|
||||
if !decision.Accepted {
|
||||
severity := domain.AlertSeverityWarning
|
||||
if hardDenied {
|
||||
severity = domain.AlertSeverityCritical
|
||||
}
|
||||
alert, err := svc.upsertAlert(domain.AlertRecord{
|
||||
SourceKind: "run-endpoint", SourceID: endpoint.ID, RuleKey: "capacity.pressure", Severity: severity,
|
||||
Title: "Run endpoint capacity admission blocked", Message: decision.Reason, Retryable: true,
|
||||
RetryAfterSeconds: decision.RetryAfterSeconds, LastAuditEventID: auditID,
|
||||
})
|
||||
if err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
decision.AlertID = alert.ID
|
||||
} else if err := svc.resolveAlertForSource("run-endpoint", endpoint.ID, "capacity.pressure", actorID, "capacity returned to an admissible state", auditID); err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
if err := validator.ValidateCapacityAdmissionDecision(decision); err != nil {
|
||||
return domain.CapacityAdmissionDecision{}, err
|
||||
}
|
||||
return domain.CopyCapacityAdmissionDecision(decision), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListAlertsForSession(sessionID string, filter domain.AlertFilter) ([]domain.AlertRecord, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
alerts, err := svc.store.Alerts().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := make([]domain.AlertRecord, 0, len(alerts))
|
||||
for _, alert := range alerts {
|
||||
if svc.canAccessAlert(user, alert) {
|
||||
visible = append(visible, alert)
|
||||
}
|
||||
}
|
||||
sort.Slice(visible, func(i, j int) bool { return visible[i].UpdatedAt.After(visible[j].UpdatedAt) })
|
||||
return domain.CopyAlertRecords(visible), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) AcknowledgeAlertForSession(sessionID string, request domain.AlertAcknowledgeRequest) (domain.AlertRecord, error) {
|
||||
if err := validator.ValidateAlertAcknowledgeRequest(request); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
svc.productionMu.Lock()
|
||||
defer svc.productionMu.Unlock()
|
||||
alert, err := svc.store.Alerts().Get(request.AlertID)
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if !svc.canAccessAlert(user, alert) {
|
||||
return domain.AlertRecord{}, ErrForbidden
|
||||
}
|
||||
if alert.State == domain.AlertStateResolved {
|
||||
return domain.AlertRecord{}, validationError("resolved alerts cannot be acknowledged")
|
||||
}
|
||||
stamp := svc.now()
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "alert.acknowledge", "alert", alert.ID, domain.AuditResultSuccess, defaultAlertNote(request.Note, "alert acknowledged"))
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
alert.State = domain.AlertStateAcknowledged
|
||||
alert.AcknowledgedBy = user.ID
|
||||
alert.AcknowledgedAt = stamp
|
||||
alert.LastAuditEventID = auditID
|
||||
alert.UpdatedAt = stamp
|
||||
if err := validator.ValidateAlertRecord(alert); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if err := svc.store.Alerts().Update(alert); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
return domain.CopyAlertRecord(alert), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ResolveAlertForSession(sessionID string, request domain.AlertResolveRequest) (domain.AlertRecord, error) {
|
||||
if err := validator.ValidateAlertResolveRequest(request); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
svc.productionMu.Lock()
|
||||
defer svc.productionMu.Unlock()
|
||||
alert, err := svc.store.Alerts().Get(request.AlertID)
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if !svc.canAccessAlert(user, alert) {
|
||||
return domain.AlertRecord{}, ErrForbidden
|
||||
}
|
||||
if alert.State == domain.AlertStateResolved {
|
||||
return domain.CopyAlertRecord(alert), nil
|
||||
}
|
||||
stamp := svc.now()
|
||||
note := defaultAlertNote(request.Note, "alert resolved after operator review")
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "alert.resolve", "alert", alert.ID, domain.AuditResultSuccess, note)
|
||||
if err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
alert.State = domain.AlertStateResolved
|
||||
alert.ResolvedBy = user.ID
|
||||
alert.ResolvedAt = stamp
|
||||
alert.ResolutionNote = note
|
||||
alert.LastAuditEventID = auditID
|
||||
alert.UpdatedAt = stamp
|
||||
if err := validator.ValidateAlertRecord(alert); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if err := svc.store.Alerts().Update(alert); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
return domain.CopyAlertRecord(alert), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RetryAlertForSession(sessionID string, request domain.AlertRetryRequest) (domain.AlertRetryResult, error) {
|
||||
if err := validator.ValidateAlertRetryRequest(request); err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
alert, err := svc.store.Alerts().Get(request.AlertID)
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
if !svc.canAccessAlert(user, alert) {
|
||||
return domain.AlertRetryResult{}, ErrForbidden
|
||||
}
|
||||
if !alert.Retryable {
|
||||
return domain.AlertRetryResult{}, validationError("alert source is not retryable")
|
||||
}
|
||||
switch alert.SourceKind {
|
||||
case "run-endpoint":
|
||||
endpoint, err := svc.store.RunEndpoints().Get(alert.SourceID)
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
capability := firstCapacityCapability(endpoint.Capabilities)
|
||||
decision, err := svc.CheckCapacityAdmissionForSession(sessionID, domain.CapacityAdmissionRequest{RunEndpointID: endpoint.ID, Capability: capability, IdempotencyKey: request.IdempotencyKey})
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
updated, err := svc.store.Alerts().Get(alert.ID)
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
return domain.CopyAlertRetryResult(domain.AlertRetryResult{Alert: updated, Decision: decision, Status: string(decision.State)}), nil
|
||||
case "plugin-lifecycle":
|
||||
installation, err := svc.store.PluginLifecycles().Get(alert.SourceID)
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
result, err := svc.RunPluginLifecycleForSession(sessionID, domain.PluginLifecycleRequest{PluginID: installation.PluginID, ServerInstanceID: installation.ServerInstanceID, Operation: installation.LastOperation, TargetVersion: installation.TargetVersion, IdempotencyKey: request.IdempotencyKey, Confirmed: true})
|
||||
if err != nil {
|
||||
return domain.AlertRetryResult{}, err
|
||||
}
|
||||
return domain.CopyAlertRetryResult(domain.AlertRetryResult{Alert: alert, Decision: result.Decision, Status: result.Status}), nil
|
||||
default:
|
||||
return domain.AlertRetryResult{}, validationError("alert source does not support scoped retry")
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListPluginLifecyclesForSession(sessionID string, filter domain.PluginLifecycleFilter) ([]domain.PluginLifecycleInstallation, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
@@ -383,13 +95,6 @@ func (svc *CoreService) RunPluginLifecycleForSession(sessionID string, request d
|
||||
if !errors.Is(jobErr, repo.ErrNotFound) {
|
||||
return domain.PluginLifecycleResult{}, jobErr
|
||||
}
|
||||
decision, err := svc.checkCapacityAdmission(user.ID, domain.CapacityAdmissionRequest{ServerInstanceID: instance.ID, RunEndpointID: endpoint.ID, Capability: capability, TargetKey: targetKey, IdempotencyKey: request.IdempotencyKey})
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
if !decision.Accepted {
|
||||
return domain.CopyPluginLifecycleResult(domain.PluginLifecycleResult{Installation: installation, Decision: decision, Status: string(decision.State)}), nil
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: jobIDFromParts("job-plugin-lifecycle", installation.ID, request.IdempotencyKey), ServerInstanceID: instance.ID,
|
||||
RunEndpointID: endpoint.ID, Capability: capability, TargetKey: targetKey, IdempotencyKey: request.IdempotencyKey,
|
||||
@@ -409,11 +114,6 @@ func (svc *CoreService) RunPluginLifecycleForSession(sessionID string, request d
|
||||
installation.IdempotencyKey = request.IdempotencyKey
|
||||
installation.FailureReason = ""
|
||||
installation.UpdatedAt = stamp
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "plugin.lifecycle."+string(request.Operation), "plugin-lifecycle", installation.ID, domain.AuditResultQueued, "plugin lifecycle operation admitted and queued")
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
installation.AuditEventID = auditID
|
||||
if err := validator.ValidatePluginLifecycleInstallation(installation); err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
@@ -425,7 +125,7 @@ func (svc *CoreService) RunPluginLifecycleForSession(sessionID string, request d
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
return domain.CopyPluginLifecycleResult(domain.PluginLifecycleResult{Installation: installation, Job: job, Decision: decision, Status: "queued"}), nil
|
||||
return domain.CopyPluginLifecycleResult(domain.PluginLifecycleResult{Installation: installation, Job: job, Status: "queued"}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListAIConfigDiffsForSession(sessionID string, filter domain.AIConfigDiffFilter) ([]domain.AIConfigDiffPreview, error) {
|
||||
@@ -503,9 +203,6 @@ func (svc *CoreService) ApproveAIConfigDiffForSession(sessionID string, request
|
||||
if err := svc.store.AIConfigDiffs().Update(preview); err != nil {
|
||||
return domain.AIConfigDiffApprovalResult{}, err
|
||||
}
|
||||
if _, err := svc.recordAuditEventWithID(user.ID, "ai.config-diff.approve", "ai-config-diff", preview.ID, domain.AuditResultQueued, "approved reviewed AI config diff and queued one config write job"); err != nil {
|
||||
return domain.AIConfigDiffApprovalResult{}, err
|
||||
}
|
||||
return domain.CopyAIConfigDiffApprovalResult(domain.AIConfigDiffApprovalResult{Preview: preview, Dispatch: dispatch}), nil
|
||||
}
|
||||
|
||||
@@ -525,17 +222,9 @@ func (svc *CoreService) projectProductionOpsJobResult(job domain.Job, stamp time
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
applyPluginLifecycleSuccess(&installation)
|
||||
installation.FailureReason = ""
|
||||
if err := svc.resolveAlertForSource("plugin-lifecycle", installation.ID, "plugin.lifecycle.failed", "run:"+job.RunEndpointID, "plugin lifecycle job completed", installation.AuditEventID); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
installation.CurrentState = domain.PluginLifecycleStateFailed
|
||||
installation.FailureReason = "plugin lifecycle job did not complete successfully"
|
||||
alert, err := svc.upsertAlert(domain.AlertRecord{SourceKind: "plugin-lifecycle", SourceID: installation.ID, RuleKey: "plugin.lifecycle.failed", Severity: domain.AlertSeverityWarning, Title: "Plugin lifecycle operation failed", Message: installation.FailureReason, Retryable: true, LastJobID: job.ID, LastAuditEventID: installation.AuditEventID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
installation.AlertID = alert.ID
|
||||
}
|
||||
installation.UpdatedAt = stamp
|
||||
return svc.store.PluginLifecycles().Update(installation)
|
||||
@@ -598,193 +287,6 @@ func (svc *CoreService) getServerConfigForUser(userID, serverInstanceID string)
|
||||
return config, validator.ValidateServerConfig(config)
|
||||
}
|
||||
|
||||
func (svc *CoreService) authorizeCapacityRequest(user domain.User, request domain.CapacityAdmissionRequest) (domain.CapacityAdmissionRequest, error) {
|
||||
if request.ServerInstanceID != "" {
|
||||
instance, err := svc.store.ServerInstances().Get(request.ServerInstanceID)
|
||||
if err != nil {
|
||||
return request, err
|
||||
}
|
||||
if !canAccessServer(user, instance) {
|
||||
return request, ErrForbidden
|
||||
}
|
||||
if request.RunEndpointID != "" && request.RunEndpointID != instance.RunEndpointID {
|
||||
return request, validationError("runEndpointId must match server instance")
|
||||
}
|
||||
request.RunEndpointID = instance.RunEndpointID
|
||||
return request, nil
|
||||
}
|
||||
if request.RunEndpointID == "" {
|
||||
return request, validationError("serverInstanceId or runEndpointId is required")
|
||||
}
|
||||
if isPlatformAdmin(user) {
|
||||
return request, nil
|
||||
}
|
||||
visible, err := svc.visibleEndpointIDs(user)
|
||||
if err != nil {
|
||||
return request, err
|
||||
}
|
||||
if _, ok := visible[request.RunEndpointID]; !ok {
|
||||
return request, ErrForbidden
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) visibleEndpointIDs(user domain.User) (map[string]struct{}, error) {
|
||||
instances, err := svc.store.ServerInstances().List(domain.ServerInstanceFilter{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := map[string]struct{}{}
|
||||
for _, instance := range instances {
|
||||
if isPlatformAdmin(user) || canAccessServer(user, instance) {
|
||||
ids[instance.RunEndpointID] = struct{}{}
|
||||
}
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) capacityJobCounts(endpointID string) (int, int, error) {
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: endpointID})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
running, queued := 0, 0
|
||||
for _, job := range jobs {
|
||||
switch job.State {
|
||||
case domain.JobStateAccepted, domain.JobStateRunning:
|
||||
running++
|
||||
case domain.JobStateQueued, domain.JobStateRetrying:
|
||||
queued++
|
||||
}
|
||||
}
|
||||
return running, queued, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) capacityProjection(endpoint domain.RunEndpoint, durableRunning, durableQueued int) domain.EndpointCapacityProjection {
|
||||
running := maxInt(endpoint.Capacity.RunningJobs, durableRunning)
|
||||
queued := maxInt(endpoint.Capacity.QueuedJobs, durableQueued)
|
||||
projection := domain.EndpointCapacityProjection{RunEndpointID: endpoint.ID, DisplayName: endpoint.DisplayName, Status: endpoint.Status, Capabilities: endpoint.Capabilities, MaxJobs: endpoint.Capacity.MaxJobs, RunningJobs: running, QueuedJobs: queued, LogBacklogBatches: endpoint.Capacity.LogBacklogBatches, ArtifactBacklogChunks: endpoint.Capacity.ArtifactBacklogChunks, Summary: safeBridgeReason(endpoint.Capacity.Summary), LastHeartbeatAt: endpoint.LastHeartbeatAt}
|
||||
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
||||
projection.PressureCodes = appendCapacityPressure(projection.PressureCodes, domain.CapacityPressureEndpointOffline)
|
||||
}
|
||||
if endpoint.LastHeartbeatAt.IsZero() || svc.now().Sub(endpoint.LastHeartbeatAt) > capacityHeartbeatStaleAfter {
|
||||
projection.PressureCodes = appendCapacityPressure(projection.PressureCodes, domain.CapacityPressureEndpointStale)
|
||||
}
|
||||
if projection.MaxJobs <= 0 || running >= projection.MaxJobs {
|
||||
projection.PressureCodes = appendCapacityPressure(projection.PressureCodes, domain.CapacityPressureJobLimit)
|
||||
}
|
||||
queueLimit := maxInt(4, projection.MaxJobs*2)
|
||||
if queued >= queueLimit {
|
||||
projection.PressureCodes = appendCapacityPressure(projection.PressureCodes, domain.CapacityPressureQueueLimit)
|
||||
}
|
||||
if projection.LogBacklogBatches >= capacityLogBacklogLimit || projection.ArtifactBacklogChunks >= capacityArtifactBacklogLimit || len(endpoint.Capacity.PressureCodes) > 0 {
|
||||
projection.PressureCodes = appendCapacityPressure(projection.PressureCodes, domain.CapacityPressureBacklog)
|
||||
}
|
||||
return projection
|
||||
}
|
||||
|
||||
func (svc *CoreService) upsertAlert(candidate domain.AlertRecord) (domain.AlertRecord, error) {
|
||||
stamp := svc.now()
|
||||
candidate.ID = alertIDForSource(candidate.SourceKind, candidate.SourceID, candidate.RuleKey)
|
||||
existing, err := svc.store.Alerts().Get(candidate.ID)
|
||||
if err == nil {
|
||||
existing.Severity = candidate.Severity
|
||||
existing.State = domain.AlertStateActive
|
||||
existing.Title = candidate.Title
|
||||
existing.Message = safeBridgeReason(candidate.Message)
|
||||
existing.OccurrenceCount++
|
||||
existing.Retryable = candidate.Retryable
|
||||
existing.RetryAfterSeconds = candidate.RetryAfterSeconds
|
||||
existing.LastJobID = candidate.LastJobID
|
||||
existing.LastAuditEventID = candidate.LastAuditEventID
|
||||
existing.LastSeenAt = stamp
|
||||
existing.ResolvedBy = ""
|
||||
existing.ResolvedAt = time.Time{}
|
||||
existing.ResolutionNote = ""
|
||||
existing.UpdatedAt = stamp
|
||||
if err := validator.ValidateAlertRecord(existing); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if err := svc.store.Alerts().Update(existing); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
if !errors.Is(err, repo.ErrNotFound) {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
candidate.State = domain.AlertStateActive
|
||||
candidate.Message = safeBridgeReason(candidate.Message)
|
||||
candidate.OccurrenceCount = 1
|
||||
candidate.LastSeenAt = stamp
|
||||
candidate.CreatedAt = stamp
|
||||
candidate.UpdatedAt = stamp
|
||||
if err := validator.ValidateAlertRecord(candidate); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
if err := svc.store.Alerts().Create(candidate); err != nil {
|
||||
return domain.AlertRecord{}, err
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveAlertForSource(sourceKind, sourceID, ruleKey, actorID, note, auditID string) error {
|
||||
alert, err := svc.store.Alerts().Get(alertIDForSource(sourceKind, sourceID, ruleKey))
|
||||
if errors.Is(err, repo.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if alert.State == domain.AlertStateResolved {
|
||||
return nil
|
||||
}
|
||||
stamp := svc.now()
|
||||
alert.State = domain.AlertStateResolved
|
||||
alert.ResolvedBy = actorID
|
||||
alert.ResolvedAt = stamp
|
||||
alert.ResolutionNote = note
|
||||
alert.LastAuditEventID = auditID
|
||||
alert.UpdatedAt = stamp
|
||||
return svc.store.Alerts().Update(alert)
|
||||
}
|
||||
|
||||
func (svc *CoreService) canAccessAlert(user domain.User, alert domain.AlertRecord) bool {
|
||||
if isPlatformAdmin(user) {
|
||||
return true
|
||||
}
|
||||
switch alert.SourceKind {
|
||||
case "server-instance":
|
||||
instance, err := svc.store.ServerInstances().Get(alert.SourceID)
|
||||
return err == nil && canAccessServer(user, instance)
|
||||
case "run-endpoint":
|
||||
instances, err := svc.store.ServerInstances().List(domain.ServerInstanceFilter{RunEndpointID: alert.SourceID})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, instance := range instances {
|
||||
if canAccessServer(user, instance) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case "plugin-lifecycle":
|
||||
installation, err := svc.store.PluginLifecycles().Get(alert.SourceID)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(installation.ServerInstanceID)
|
||||
return err == nil && canAccessServer(user, instance)
|
||||
case "ai-config-diff":
|
||||
preview, err := svc.store.AIConfigDiffs().Get(alert.SourceID)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(preview.ServerInstanceID)
|
||||
return err == nil && canAccessServer(user, instance)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (svc *CoreService) pluginLifecycleDenied(actorID string, instance domain.ServerInstance, plugin domain.GamePlugin, request domain.PluginLifecycleRequest, reason string) (domain.PluginLifecycleResult, error) {
|
||||
svc.productionMu.Lock()
|
||||
defer svc.productionMu.Unlock()
|
||||
@@ -798,21 +300,11 @@ func (svc *CoreService) pluginLifecycleDenied(actorID string, instance domain.Se
|
||||
|
||||
func (svc *CoreService) pluginLifecycleDeniedLocked(actorID string, installation domain.PluginLifecycleInstallation, request domain.PluginLifecycleRequest, reason string) (domain.PluginLifecycleResult, error) {
|
||||
stamp := svc.now()
|
||||
auditID, err := svc.recordAuditEventWithID(actorID, "plugin.lifecycle.denied", "plugin-lifecycle", installation.ID, domain.AuditResultDenied, reason)
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
installation.CurrentState = domain.PluginLifecycleStateFailed
|
||||
installation.LastOperation = request.Operation
|
||||
installation.TargetVersion = request.TargetVersion
|
||||
installation.FailureReason = safeBridgeReason(reason)
|
||||
installation.AuditEventID = auditID
|
||||
installation.UpdatedAt = stamp
|
||||
alert, err := svc.upsertAlert(domain.AlertRecord{SourceKind: "plugin-lifecycle", SourceID: installation.ID, RuleKey: "plugin.lifecycle.compatibility", Severity: domain.AlertSeverityWarning, Title: "Plugin lifecycle compatibility check failed", Message: installation.FailureReason, Retryable: true, LastAuditEventID: auditID})
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
installation.AlertID = alert.ID
|
||||
if err := validator.ValidatePluginLifecycleInstallation(installation); err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
@@ -824,7 +316,7 @@ func (svc *CoreService) pluginLifecycleDeniedLocked(actorID string, installation
|
||||
if err != nil {
|
||||
return domain.PluginLifecycleResult{}, err
|
||||
}
|
||||
return domain.CopyPluginLifecycleResult(domain.PluginLifecycleResult{Installation: installation, Alert: &alert, Status: "denied"}), nil
|
||||
return domain.CopyPluginLifecycleResult(domain.PluginLifecycleResult{Installation: installation, Status: "denied"}), nil
|
||||
}
|
||||
|
||||
func pluginLifecycleDispatchMetadata(plugin domain.GamePlugin, operation domain.PluginLifecycleOperation) (string, string, error) {
|
||||
@@ -937,11 +429,6 @@ func applyPluginLifecycleSuccess(installation *domain.PluginLifecycleInstallatio
|
||||
}
|
||||
}
|
||||
|
||||
func alertIDForSource(sourceKind, sourceID, ruleKey string) string {
|
||||
sum := sha256.Sum256([]byte(sourceKind + "\x00" + sourceID + "\x00" + ruleKey))
|
||||
return "alert-" + hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
func pluginLifecycleInstallationID(pluginID, serverInstanceID string) string {
|
||||
sum := sha256.Sum256([]byte(pluginID + "\x00" + serverInstanceID))
|
||||
return "plugin-lifecycle-" + hex.EncodeToString(sum[:12])
|
||||
@@ -951,42 +438,3 @@ func aiConfigDiffID(requestID, serverInstanceID string) string {
|
||||
sum := sha256.Sum256([]byte(requestID + "\x00" + serverInstanceID))
|
||||
return "ai-config-diff-" + hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
func appendCapacityPressure(codes []domain.CapacityPressureCode, code domain.CapacityPressureCode) []domain.CapacityPressureCode {
|
||||
if !containsCapacityPressure(codes, code) {
|
||||
return append(codes, code)
|
||||
}
|
||||
return codes
|
||||
}
|
||||
|
||||
func containsCapacityPressure(codes []domain.CapacityPressureCode, target domain.CapacityPressureCode) bool {
|
||||
for _, code := range codes {
|
||||
if code == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstCapacityCapability(capabilities []string) string {
|
||||
for _, capability := range capabilities {
|
||||
if strings.TrimSpace(capability) != "" {
|
||||
return capability
|
||||
}
|
||||
}
|
||||
return "control.heartbeat"
|
||||
}
|
||||
|
||||
func defaultAlertNote(note, fallback string) string {
|
||||
if strings.TrimSpace(note) == "" {
|
||||
return fallback
|
||||
}
|
||||
return safeBridgeReason(note)
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
@@ -9,38 +9,6 @@ import (
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
func TestProductionCapacityCreatesDurableAlertAndSupportsClosure(t *testing.T) {
|
||||
svc, session, instance := newProductionOpsFixture(t)
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
t.Fatalf("get endpoint: %v", err)
|
||||
}
|
||||
endpoint.Capacity.RunningJobs = endpoint.Capacity.MaxJobs
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("update endpoint pressure: %v", err)
|
||||
}
|
||||
|
||||
decision, err := svc.CheckCapacityAdmissionForSession(session, domain.CapacityAdmissionRequest{ServerInstanceID: instance.ID, Capability: domain.LifecycleCapabilityInstall, IdempotencyKey: "capacity-pressure"})
|
||||
if err != nil {
|
||||
t.Fatalf("check capacity: %v", err)
|
||||
}
|
||||
if decision.Accepted || decision.State != domain.CapacityAdmissionDeferred || decision.AlertID == "" || decision.AuditEventID == "" {
|
||||
t.Fatalf("expected durable deferred decision, got %+v", decision)
|
||||
}
|
||||
alerts, err := svc.ListAlertsForSession(session, domain.AlertFilter{State: domain.AlertStateActive})
|
||||
if err != nil || len(alerts) != 1 || alerts[0].OccurrenceCount != 1 {
|
||||
t.Fatalf("expected one active alert, got %+v err=%v", alerts, err)
|
||||
}
|
||||
acknowledged, err := svc.AcknowledgeAlertForSession(session, domain.AlertAcknowledgeRequest{AlertID: decision.AlertID, Note: "operator reviewing queue pressure"})
|
||||
if err != nil || acknowledged.State != domain.AlertStateAcknowledged || acknowledged.AcknowledgedBy == "" {
|
||||
t.Fatalf("acknowledge alert: %+v err=%v", acknowledged, err)
|
||||
}
|
||||
resolved, err := svc.ResolveAlertForSession(session, domain.AlertResolveRequest{AlertID: decision.AlertID, Note: "capacity policy reviewed"})
|
||||
if err != nil || resolved.State != domain.AlertStateResolved || resolved.ResolvedBy == "" {
|
||||
t.Fatalf("resolve alert: %+v err=%v", resolved, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginLifecycleDispatchIsIdempotentAndRejectsInputDrift(t *testing.T) {
|
||||
svc, session, instance := newProductionOpsFixture(t)
|
||||
request := domain.PluginLifecycleRequest{PluginID: instance.PluginID, ServerInstanceID: instance.ID, Operation: domain.PluginLifecycleOperationInstall, TargetVersion: "1.0.0", IdempotencyKey: "plugin-install-v1"}
|
||||
@@ -66,7 +34,7 @@ func TestPluginLifecycleDispatchIsIdempotentAndRejectsInputDrift(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginLifecycleBridgeDispatchesOnlyPlatformGovernedJob(t *testing.T) {
|
||||
func TestPluginLifecycleBridgeDispatchesBoundedJob(t *testing.T) {
|
||||
svc, session, instance := newProductionOpsFixture(t)
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
@@ -90,13 +58,12 @@ func TestPluginLifecycleBridgeDispatchesOnlyPlatformGovernedJob(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("execute lifecycle bridge: %v", err)
|
||||
}
|
||||
if response.Status != "queued" || response.Result["jobId"] == "" || response.Result["installationId"] == "" || response.Result["admissionState"] != string(domain.CapacityAdmissionAccepted) {
|
||||
t.Fatalf("expected Platform-governed lifecycle job, got %+v", response)
|
||||
if response.Status != "queued" || response.Result["jobId"] == "" || response.Result["installationId"] == "" {
|
||||
t.Fatalf("expected plugin lifecycle job, got %+v", response)
|
||||
}
|
||||
serialized := strings.ToLower(strings.Join([]string{
|
||||
response.Result["jobId"], response.Result["installationId"], response.Result["currentState"],
|
||||
response.Result["desiredState"], response.Result["alertId"], response.Result["auditEventId"],
|
||||
response.Result["admissionState"], response.Result["admissionReason"],
|
||||
response.Result["desiredState"],
|
||||
}, " "))
|
||||
for _, forbidden := range []string{"password", "apikey", "token", "secret://", "baseurl", "hostpath", "socket", "pid", "dsn", "rcon", "runendpoint"} {
|
||||
if strings.Contains(serialized, forbidden) {
|
||||
|
||||
@@ -1,254 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const protectedRequestMaxTimeoutSeconds = 120
|
||||
|
||||
type protectedRequestPayload struct {
|
||||
commandID string
|
||||
kind string
|
||||
transportKey string
|
||||
targetKey string
|
||||
requestText string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
// protectedRequestBroker keeps opaque request text out of durable jobs and
|
||||
// bridge records. It releases a payload exactly once to a current Run lease.
|
||||
type protectedRequestBroker struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
payloads map[string]protectedRequestPayload
|
||||
}
|
||||
|
||||
func newProtectedRequestBroker(now func() time.Time) *protectedRequestBroker {
|
||||
return &protectedRequestBroker{now: now, payloads: map[string]protectedRequestPayload{}}
|
||||
}
|
||||
|
||||
func (broker *protectedRequestBroker) Put(jobID string, payload protectedRequestPayload) error {
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
broker.pruneLocked()
|
||||
if _, exists := broker.payloads[jobID]; exists {
|
||||
return validationError("protected request idempotency key is already pending")
|
||||
}
|
||||
broker.payloads[jobID] = payload
|
||||
return nil
|
||||
}
|
||||
|
||||
func (broker *protectedRequestBroker) Consume(jobID string) (protectedRequestPayload, error) {
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
broker.pruneLocked()
|
||||
payload, exists := broker.payloads[jobID]
|
||||
if !exists {
|
||||
return protectedRequestPayload{}, validationError("protected request input is unavailable")
|
||||
}
|
||||
delete(broker.payloads, jobID)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (broker *protectedRequestBroker) Delete(jobID string) {
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
delete(broker.payloads, jobID)
|
||||
}
|
||||
|
||||
func (broker *protectedRequestBroker) pruneLocked() {
|
||||
stamp := broker.now()
|
||||
for jobID, payload := range broker.payloads {
|
||||
if !stamp.Before(payload.expiresAt) {
|
||||
delete(broker.payloads, jobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func protectedRequestCapability(kind string) (string, string, error) {
|
||||
switch kind {
|
||||
case "sql":
|
||||
return domain.JobCapabilityRemoteRunProtectedSQL, "protected-sql", nil
|
||||
case "rcon":
|
||||
return domain.JobCapabilityRemoteRunProtectedRCON, "protected-rcon", nil
|
||||
case "program":
|
||||
return domain.JobCapabilityRemoteRunProgram, "protected-program", nil
|
||||
default:
|
||||
return "", "", validationError("protected request kind is unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func redactedProtectedRequestPayload(declaration *domain.GameClientBridgeProtectedRequestDeclaration) map[string]any {
|
||||
return map[string]any{declaration.TextField: "redacted"}
|
||||
}
|
||||
|
||||
func (svc *CoreService) dispatchProtectedRequest(command domain.GameClientBridgeCommand, declaration domain.GameClientBridgeCommandDeclaration, payload map[string]any) error {
|
||||
if declaration.ProtectedRequest == nil || declaration.TimeoutSeconds < 1 || declaration.TimeoutSeconds > protectedRequestMaxTimeoutSeconds {
|
||||
return validationError("protected request timeout is out of bounds")
|
||||
}
|
||||
requestText, _ := payload[declaration.ProtectedRequest.TextField].(string)
|
||||
capability, adapterKind, err := protectedRequestCapability(declaration.ProtectedRequest.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jobID := command.RunJobID
|
||||
if jobID == "" {
|
||||
return validationError("protected request job binding is missing")
|
||||
}
|
||||
executionInput := domain.JobExecutionInput{
|
||||
WorkspaceScope: command.ProfileKey,
|
||||
RemoteAdapterKey: declaration.ProtectedRequest.TransportKey,
|
||||
RemoteAdapterKind: adapterKind,
|
||||
TimeoutSeconds: declaration.TimeoutSeconds,
|
||||
PluginID: command.PluginID,
|
||||
}
|
||||
if declaration.ProtectedRequest.Kind == "rcon" {
|
||||
if resolution, resolveErr := svc.resolveProtectedSourceRCONDispatch(command.ServerInstanceID, declaration.ProtectedRequest); resolveErr == nil {
|
||||
executionInput.WorkspaceScope = resolution.binding.ProfileKey
|
||||
executionInput.SourceRCON = resolution.plan
|
||||
}
|
||||
}
|
||||
if err := svc.protectedRequests.Put(jobID, protectedRequestPayload{commandID: command.ID, kind: declaration.ProtectedRequest.Kind, transportKey: declaration.ProtectedRequest.TransportKey, targetKey: declaration.ProtectedRequest.TargetKey, requestText: requestText, expiresAt: command.ExpiresAt}); err != nil {
|
||||
return err
|
||||
}
|
||||
job := domain.Job{
|
||||
ID: jobID,
|
||||
ServerInstanceID: command.ServerInstanceID,
|
||||
RunEndpointID: mustProtectedRequestRunEndpoint(svc, command.ServerInstanceID),
|
||||
Capability: capability,
|
||||
TargetKey: declaration.ProtectedRequest.TargetKey,
|
||||
InputRef: "input://protected-request/" + command.ID,
|
||||
IdempotencyKey: "protected-request:" + command.ID,
|
||||
Progress: domain.JobProgress{Percent: 0, Message: "protected request queued"},
|
||||
RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1},
|
||||
ExecutionInput: executionInput,
|
||||
}
|
||||
if job.RunEndpointID == "" {
|
||||
svc.protectedRequests.Delete(jobID)
|
||||
return validationError("protected request server binding is unavailable")
|
||||
}
|
||||
created, err := svc.CreateJob(job)
|
||||
if err != nil {
|
||||
svc.protectedRequests.Delete(jobID)
|
||||
return err
|
||||
}
|
||||
if created.ID != jobID || created.ServerInstanceID != job.ServerInstanceID || created.Capability != capability || created.TargetKey != job.TargetKey || created.ExecutionInput.RemoteAdapterKey != job.ExecutionInput.RemoteAdapterKey || created.ExecutionInput.RemoteAdapterKind != adapterKind {
|
||||
svc.protectedRequests.Delete(jobID)
|
||||
return validationError("protected request idempotency key is already bound")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mustProtectedRequestRunEndpoint(svc *CoreService, serverInstanceID string) string {
|
||||
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return instance.RunEndpointID
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetProtectedRequestExecutionInput(request domain.ProtectedRequestExecutionInputRequest) (domain.ProtectedRequestExecutionInput, error) {
|
||||
if err := validator.ValidateProtectedRequestExecutionInputRequest(request); err != nil {
|
||||
return domain.ProtectedRequestExecutionInput{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.ProtectedRequestExecutionInput{}, err
|
||||
}
|
||||
if request.FencingToken != uint64(job.Attempt) || !isProtectedRequestCapability(job.Capability) || job.RetryPolicy.MaxAttempts != 1 || !strings.HasPrefix(job.InputRef, "input://protected-request/") {
|
||||
return domain.ProtectedRequestExecutionInput{}, validationError("job is not a fenced protected request")
|
||||
}
|
||||
commands, err := svc.store.GameClientBridgeCommands().List(domain.GameClientBridgeCommandFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return domain.ProtectedRequestExecutionInput{}, err
|
||||
}
|
||||
var command domain.GameClientBridgeCommand
|
||||
for _, candidate := range commands {
|
||||
if candidate.RunJobID == job.ID {
|
||||
command = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if command.ID == "" || command.ApprovalState != domain.GameClientBridgeApprovalApproved || command.State != domain.GameClientBridgeCommandPending || !command.ExpiresAt.After(svc.now()) {
|
||||
return domain.ProtectedRequestExecutionInput{}, validationError("protected request is not currently authorized")
|
||||
}
|
||||
payload, err := svc.protectedRequests.Consume(job.ID)
|
||||
if err != nil {
|
||||
return domain.ProtectedRequestExecutionInput{}, err
|
||||
}
|
||||
capability, adapterKind, capabilityErr := protectedRequestCapability(payload.kind)
|
||||
if capabilityErr != nil || capability != job.Capability || payload.targetKey != job.TargetKey || payload.transportKey != job.ExecutionInput.RemoteAdapterKey || adapterKind != job.ExecutionInput.RemoteAdapterKind {
|
||||
return domain.ProtectedRequestExecutionInput{}, validationError("protected request logical binding is invalid")
|
||||
}
|
||||
return domain.CopyProtectedRequestExecutionInput(domain.ProtectedRequestExecutionInput{JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, FencingToken: request.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: payload.expiresAt, Kind: payload.kind, TransportKey: payload.transportKey, TargetKey: payload.targetKey, RequestText: payload.requestText}), nil
|
||||
}
|
||||
|
||||
func isProtectedRequestCapability(capability string) bool {
|
||||
_, _, err := protectedRequestCapabilityForCapability(capability)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func protectedRequestCapabilityForCapability(capability string) (string, string, error) {
|
||||
switch capability {
|
||||
case domain.JobCapabilityRemoteRunProtectedSQL:
|
||||
return "sql", "protected-sql", nil
|
||||
case domain.JobCapabilityRemoteRunProtectedRCON:
|
||||
return "rcon", "protected-rcon", nil
|
||||
case domain.JobCapabilityRemoteRunProgram:
|
||||
return "program", "protected-program", nil
|
||||
default:
|
||||
return "", "", validationError("job is not a protected request")
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectProtectedRequestJobResult(job domain.Job, result domain.RunJobResult, stamp time.Time) error {
|
||||
if !isProtectedRequestCapability(job.Capability) {
|
||||
return nil
|
||||
}
|
||||
svc.protectedRequests.Delete(job.ID)
|
||||
svc.bridgeMu.Lock()
|
||||
defer svc.bridgeMu.Unlock()
|
||||
commands, err := svc.store.GameClientBridgeCommands().List(domain.GameClientBridgeCommandFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, command := range commands {
|
||||
if command.RunJobID != job.ID || isTerminalGameClientBridgeCommandState(command.State) {
|
||||
continue
|
||||
}
|
||||
switch result.State {
|
||||
case domain.JobStateSucceeded:
|
||||
command.State = domain.GameClientBridgeCommandSucceeded
|
||||
command.Result.Status = domain.GameClientBridgeResultSucceeded
|
||||
case domain.JobStateCancelled:
|
||||
command.State = domain.GameClientBridgeCommandCancelled
|
||||
command.Result.Status = domain.GameClientBridgeResultCancelled
|
||||
case domain.JobStateFailed:
|
||||
command.State = domain.GameClientBridgeCommandFailed
|
||||
command.Result.Status = domain.GameClientBridgeResultFailed
|
||||
if result.ErrorCode == "protected_request_unknown" || strings.HasSuffix(result.ExecutionResult.Kind, ".unknown") {
|
||||
command.State = domain.GameClientBridgeCommandUnknown
|
||||
command.Result.Status = domain.GameClientBridgeResultUnknown
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
command.Result.Summary = "protected request completed by Run"
|
||||
command.Result.CompletedBy = "run"
|
||||
command.Result.CompletedAt = stamp
|
||||
command.CompletedAt = stamp
|
||||
command.UpdatedAt = stamp
|
||||
auditID, auditErr := svc.recordAuditEventWithID("run", "game-client-bridge.command.result", "game-client-bridge-command", command.ID, domain.AuditResultSuccess, "Run recorded protected bridge command result")
|
||||
if auditErr != nil {
|
||||
return auditErr
|
||||
}
|
||||
command.AuditReferences = append(command.AuditReferences, auditID)
|
||||
return svc.store.GameClientBridgeCommands().Update(command)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -66,8 +66,6 @@ func (svc *CoreService) RequestRemoteAdapterForSession(sessionID string, request
|
||||
selected = domain.RemoteAdapterDeclaration{Key: "legacy-" + string(remoteAdapterKindForCapability(request.Capability)), Kind: remoteAdapterKindForCapability(request.Capability), TargetKeys: []string{request.TargetKey}, Capabilities: []string{request.Capability}, TimeoutSeconds: 30, MaxAttempts: 3}
|
||||
}
|
||||
if selected.Key == "" {
|
||||
user, _ := svc.GetCurrentUser(sessionID)
|
||||
_ = svc.recordAuditEvent(user.ID, "remote-adapter.authorize", "server-instance", instance.ID, domain.AuditResultDenied, "remote adapter declaration, target, or capability was not approved")
|
||||
return domain.RemoteAdapterResult{}, ErrForbidden
|
||||
}
|
||||
}
|
||||
@@ -102,15 +100,7 @@ func (svc *CoreService) RequestRemoteAdapterForSession(sessionID string, request
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "remote-adapter.authorize", "server-instance", instance.ID, domain.AuditResultQueued, "authorized declared remote adapter target with bounded timeout and retry")
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
return domain.RemoteAdapterResult{RequestID: created.ID, ServerInstanceID: instance.ID, DeclarationKey: selected.Key, TargetKey: request.TargetKey, Kind: selected.Kind, Status: string(created.State), Retryable: attempts > 1, Message: "scoped remote adapter queued", ResultRef: "job://" + created.ID, AuditEventID: auditID}, nil
|
||||
return domain.RemoteAdapterResult{RequestID: created.ID, ServerInstanceID: instance.ID, DeclarationKey: selected.Key, TargetKey: request.TargetKey, Kind: selected.Kind, Status: string(created.State), Retryable: attempts > 1, Message: "scoped remote adapter queued", ResultRef: "job://" + created.ID}, nil
|
||||
}
|
||||
|
||||
func intersectRemoteCapabilities(profile []string, declared []string, endpoint []string) []string {
|
||||
|
||||
@@ -107,12 +107,6 @@ type Core interface {
|
||||
DeleteServerInstanceForSession(string, string, domain.ServerDeletionRequest) (domain.ServerInstance, error)
|
||||
GetPlatformResourceUsage() (domain.PlatformResourceUsage, error)
|
||||
ListServerMetricsForSession(string) ([]domain.ServerMetrics, error)
|
||||
GetProductionCapacityForSession(string) (domain.ProductionCapacitySummary, error)
|
||||
CheckCapacityAdmissionForSession(string, domain.CapacityAdmissionRequest) (domain.CapacityAdmissionDecision, error)
|
||||
ListAlertsForSession(string, domain.AlertFilter) ([]domain.AlertRecord, error)
|
||||
AcknowledgeAlertForSession(string, domain.AlertAcknowledgeRequest) (domain.AlertRecord, error)
|
||||
ResolveAlertForSession(string, domain.AlertResolveRequest) (domain.AlertRecord, error)
|
||||
RetryAlertForSession(string, domain.AlertRetryRequest) (domain.AlertRetryResult, error)
|
||||
ListPluginLifecyclesForSession(string, domain.PluginLifecycleFilter) ([]domain.PluginLifecycleInstallation, error)
|
||||
RunPluginLifecycleForSession(string, domain.PluginLifecycleRequest) (domain.PluginLifecycleResult, error)
|
||||
ListAIConfigDiffsForSession(string, domain.AIConfigDiffFilter) ([]domain.AIConfigDiffPreview, error)
|
||||
@@ -144,7 +138,6 @@ type Core interface {
|
||||
GetDependencyExecutionInput(domain.DependencyExecutionInputRequest) (domain.DependencyExecutionInput, error)
|
||||
DispatchSourceRCONCommandForSession(string, domain.SourceRCONCommandRequest) (domain.SourceRCONCommandDispatch, error)
|
||||
GetSourceRCONExecutionInput(domain.SourceRCONExecutionInputRequest) (domain.SourceRCONExecutionInput, error)
|
||||
GetProtectedRequestExecutionInput(domain.ProtectedRequestExecutionInputRequest) (domain.ProtectedRequestExecutionInput, error)
|
||||
GetRunUpdateInput(domain.RunUpdateInputRequest) (domain.RunUpdateInput, error)
|
||||
ReadRunUpdateChunk(domain.RunUpdateChunkRequest) (domain.RunUpdateChunk, error)
|
||||
ReportRunUpdateHealth(domain.RunUpdateHealthReport) (domain.RunUpdateHealthResult, error)
|
||||
@@ -215,9 +208,6 @@ type Core interface {
|
||||
IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error)
|
||||
GetRunLogStreamProgress(domain.RunLogStreamProgress) (domain.RunLogStreamProgressResult, error)
|
||||
QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
|
||||
CreateAuditEvent(domain.AuditEvent) (domain.AuditEvent, error)
|
||||
GetAuditEvent(string) (domain.AuditEvent, error)
|
||||
ListAuditEvents(domain.AuditEventFilter) ([]domain.AuditEvent, error)
|
||||
SeedPlatformAdmin(string, string) error
|
||||
}
|
||||
|
||||
@@ -244,11 +234,8 @@ type CoreService struct {
|
||||
artifactTransfers map[string]domain.ArtifactTransferSession
|
||||
artifactPayloads map[string][]byte
|
||||
artifactTransferSeq uint64
|
||||
auditMu sync.Mutex
|
||||
auditSeq uint64
|
||||
productionMu sync.Mutex
|
||||
sourceRCONCommands *sourceRCONCommandBroker
|
||||
protectedRequests *protectedRequestBroker
|
||||
aiProviderClient AIProviderClient
|
||||
secretEnvelope SecretEnvelope
|
||||
networkFingerprintKey []byte
|
||||
@@ -288,7 +275,6 @@ func newCoreServiceWithLogStore(store repo.Store, logStore LogBodyStore, now fun
|
||||
artifactTransfers: map[string]domain.ArtifactTransferSession{},
|
||||
artifactPayloads: map[string][]byte{},
|
||||
sourceRCONCommands: newSourceRCONCommandBroker(now),
|
||||
protectedRequests: newProtectedRequestBroker(now),
|
||||
aiProviderClient: MockAIProviderClient{},
|
||||
secretEnvelope: newSecretEnvelope(developmentSecretEnvelopeKey),
|
||||
networkFingerprintKey: []byte(developmentSecretEnvelopeKey),
|
||||
@@ -703,23 +689,6 @@ func (svc *CoreService) TestAIProvider(id string) (domain.AIProviderTestResult,
|
||||
result.Success = false
|
||||
result.Message = "provider invocation failed safely"
|
||||
result.Violations = []string{"provider invocation failed safely"}
|
||||
auditID, auditErr := svc.recordAuditEventWithID("platform", "ai.provider.test.failed", "ai-provider", provider.ID, domain.AuditResultFailed, result.Message)
|
||||
if auditErr != nil {
|
||||
return domain.AIProviderTestResult{}, auditErr
|
||||
}
|
||||
svc.productionMu.Lock()
|
||||
_, alertErr := svc.upsertAlert(domain.AlertRecord{SourceKind: "ai-provider", SourceID: provider.ID, RuleKey: "ai.provider.failed", Severity: domain.AlertSeverityWarning, Title: "AI provider health check failed", Message: result.Message, Retryable: false, LastAuditEventID: auditID})
|
||||
svc.productionMu.Unlock()
|
||||
if alertErr != nil {
|
||||
return domain.AIProviderTestResult{}, alertErr
|
||||
}
|
||||
} else {
|
||||
svc.productionMu.Lock()
|
||||
resolveErr := svc.resolveAlertForSource("ai-provider", provider.ID, "ai.provider.failed", "platform", "AI provider health check passed", "")
|
||||
svc.productionMu.Unlock()
|
||||
if resolveErr != nil {
|
||||
return domain.AIProviderTestResult{}, resolveErr
|
||||
}
|
||||
}
|
||||
return domain.CopyAIProviderTestResult(result), nil
|
||||
}
|
||||
@@ -1020,7 +989,7 @@ func (svc *CoreService) executeBridgePluginLifecycle(sessionID string, base doma
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = result.Status
|
||||
base.Result = map[string]string{"installationId": result.Installation.ID, "currentState": string(result.Installation.CurrentState), "desiredState": string(result.Installation.DesiredState), "jobId": result.Job.ID, "alertId": result.Installation.AlertID, "auditEventId": result.Installation.AuditEventID, "admissionState": string(result.Decision.State), "admissionReason": result.Decision.Reason}
|
||||
base.Result = map[string]string{"installationId": result.Installation.ID, "currentState": string(result.Installation.CurrentState), "desiredState": string(result.Installation.DesiredState), "jobId": result.Job.ID}
|
||||
return base
|
||||
}
|
||||
|
||||
@@ -2668,27 +2637,6 @@ func (svc *CoreService) ListLogStreams(filter domain.LogStreamFilter) ([]domain.
|
||||
return svc.store.LogStreams().List(filter)
|
||||
}
|
||||
|
||||
func (svc *CoreService) CreateAuditEvent(event domain.AuditEvent) (domain.AuditEvent, error) {
|
||||
if event.CreatedAt.IsZero() {
|
||||
event.CreatedAt = svc.now()
|
||||
}
|
||||
if err := validator.ValidateAuditEvent(event); err != nil {
|
||||
return domain.AuditEvent{}, err
|
||||
}
|
||||
if err := svc.store.AuditEvents().Create(event); err != nil {
|
||||
return domain.AuditEvent{}, err
|
||||
}
|
||||
return domain.CopyAuditEvent(event), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetAuditEvent(id string) (domain.AuditEvent, error) {
|
||||
return svc.store.AuditEvents().Get(id)
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListAuditEvents(filter domain.AuditEventFilter) ([]domain.AuditEvent, error) {
|
||||
return svc.store.AuditEvents().List(filter)
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateRunnableEndpoint(endpoint domain.RunEndpoint, capability string) error {
|
||||
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
||||
return validationError("run endpoint must be online or degraded")
|
||||
|
||||
@@ -163,28 +163,6 @@ func TestCoreServiceCreateListGetWorkflows(t *testing.T) {
|
||||
t.Fatalf("list log streams: len=%d err=%v", len(streams), err)
|
||||
}
|
||||
|
||||
audit, err := svc.CreateAuditEvent(domain.AuditEvent{
|
||||
ID: "audit-1",
|
||||
ActorID: user.ID,
|
||||
Action: "server.create",
|
||||
ResourceKind: "server-instance",
|
||||
ResourceID: instance.ID,
|
||||
Result: domain.AuditResultSuccess,
|
||||
Summary: "created server instance",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create audit event: %v", err)
|
||||
}
|
||||
if !audit.CreatedAt.Equal(fixedTime) {
|
||||
t.Fatalf("expected audit timestamp default, got %+v", audit)
|
||||
}
|
||||
if _, err := svc.GetAuditEvent(audit.ID); err != nil {
|
||||
t.Fatalf("get audit event: %v", err)
|
||||
}
|
||||
auditEvents, err := svc.ListAuditEvents(domain.AuditEventFilter{ResourceID: instance.ID})
|
||||
if err != nil || len(auditEvents) != 1 {
|
||||
t.Fatalf("list audit events: len=%d err=%v", len(auditEvents), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceCreateRemoteProgramJobCreatesManagementLogStreams(t *testing.T) {
|
||||
@@ -1070,7 +1048,7 @@ func TestConfigWriteTerminalResultAppliesDurableTypedProjection(t *testing.T) {
|
||||
t.Fatalf("claim typed config job: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
checksum := validator.BytesChecksum([]byte(proposed))
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "config write completed"}, Message: "config write completed", ExecutionResult: domain.JobExecutionResult{Kind: "file.write", Version: dispatch.Job.ExecutionInput.ExpectedVersion + 1, Checksum: checksum, SizeBytes: int64(len(proposed)), AuditSummary: "atomic compare-and-swap file write"}}); err != nil {
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "config write completed"}, Message: "config write completed", ExecutionResult: domain.JobExecutionResult{Kind: "file.write", Version: dispatch.Job.ExecutionInput.ExpectedVersion + 1, Checksum: checksum, SizeBytes: int64(len(proposed)), Summary: "atomic compare-and-swap file write"}}); err != nil {
|
||||
t.Fatalf("complete typed config job: %v", err)
|
||||
}
|
||||
updated, err := svc.GetServerConfigForSession(ownerSession, instance.ID)
|
||||
|
||||
@@ -202,7 +202,7 @@ func TestCoreServiceGuidedPluginLifecycleSuccessDoesNotRequireExecutionReceipt(t
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt,
|
||||
State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "done"}, Message: "done", ResultRef: "artifact://jobs/guided-success/lifecycle-result",
|
||||
ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running", AuditSummary: "bounded process state"},
|
||||
ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running", Summary: "bounded process state"},
|
||||
}); err != nil {
|
||||
t.Fatalf("guided plugin lifecycle success without receipt should be terminal: %v", err)
|
||||
}
|
||||
|
||||
@@ -52,13 +52,6 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
|
||||
}
|
||||
svc.publishLogProcessState(instance)
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if report.State == domain.JobStateFailed || report.State == domain.JobStateCancelled {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
if err := svc.recordAuditEvent("run:"+report.RunEndpointID, "lifecycle.report", "server-instance", instance.ID, auditResult, lifecycleReportSummary(report, nextState, projected)); err != nil {
|
||||
return domain.RunLifecycleReportResult{}, err
|
||||
}
|
||||
return domain.CopyRunLifecycleReportResult(domain.RunLifecycleReportResult{Accepted: true, RunEndpointID: report.RunEndpointID, ServerInstanceID: report.ServerInstanceID, ProjectedState: nextState, ServerTime: stamp}), nil
|
||||
}
|
||||
|
||||
@@ -72,40 +65,17 @@ func lifecycleObservationIsStale(instance domain.ServerInstance, report domain.R
|
||||
return !report.ObservedAt.IsZero() && !instance.LifecycleObservedAt.IsZero() && report.ObservedAt.Before(instance.LifecycleObservedAt)
|
||||
}
|
||||
|
||||
func lifecycleReportSummary(report domain.RunLifecycleReport, projectedState domain.ServerInstanceState, projected bool) string {
|
||||
for _, candidate := range []string{report.ExecutionResult.AuditSummary, report.Progress.Message, report.Message, report.ErrorCode} {
|
||||
if strings.TrimSpace(candidate) != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if projected {
|
||||
return "run reported " + report.Capability + " " + string(report.State) + "; projected server state " + string(projectedState)
|
||||
}
|
||||
return "run reported " + report.Capability + " " + string(report.State)
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectRemoteAdapterJobResult(job domain.Job, stamp time.Time) error {
|
||||
if !strings.HasPrefix(job.Capability, "remote.") || job.ServerInstanceID == "" || !isTerminalJobState(job.State) {
|
||||
return nil
|
||||
}
|
||||
result := domain.AuditResultSuccess
|
||||
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
|
||||
result = domain.AuditResultFailed
|
||||
}
|
||||
summary := "remote adapter " + job.Capability + " completed with bounded result reference"
|
||||
if job.State == domain.JobStateFailed {
|
||||
summary = "remote adapter " + job.Capability + " failed or timed out; retry/fencing remained platform-owned"
|
||||
}
|
||||
if job.State == domain.JobStateCancelled {
|
||||
summary = "remote adapter " + job.Capability + " was cancelled before terminal projection"
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "remote-adapter.result", "server-instance", job.ServerInstanceID, result, summary)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityConfigWrite {
|
||||
if job.State != domain.JobStateSucceeded {
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "config.write.result", "server-instance", job.ServerInstanceID, domain.AuditResultFailed, job.ExecutionResult.AuditSummary)
|
||||
return nil
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(job.ServerInstanceID)
|
||||
if err != nil {
|
||||
@@ -123,7 +93,7 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "config.write.result", "server-instance", instance.ID, domain.AuditResultSuccess, job.ExecutionResult.AuditSummary)
|
||||
return nil
|
||||
}
|
||||
nextState, ok := lifecycleProjectedState(job.Capability, job.State, job.ExecutionResult)
|
||||
if !ok || job.ServerInstanceID == "" {
|
||||
@@ -142,11 +112,7 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
|
||||
return err
|
||||
}
|
||||
svc.publishLogProcessState(instance)
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "lifecycle.result", "server-instance", instance.ID, auditResult, job.Progress.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectServerDeploymentProgress(job domain.Job, stamp time.Time) error {
|
||||
|
||||
@@ -515,9 +515,9 @@ func claimAndCompleteLifecycleJobForServer(t *testing.T, svc *CoreService, sessi
|
||||
if state == domain.JobStateSucceeded {
|
||||
switch capability {
|
||||
case domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStatus:
|
||||
executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "running", AuditSummary: "bounded process state"}
|
||||
executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "running", Summary: "bounded process state"}
|
||||
case domain.LifecycleCapabilityStop:
|
||||
executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop", AuditSummary: "bounded process state"}
|
||||
executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop", Summary: "bounded process state"}
|
||||
}
|
||||
}
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{
|
||||
|
||||
@@ -235,62 +235,6 @@ func sourceRCONTransportForCapability(profiles domain.GamePluginRuntimeProfiles,
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveProtectedSourceRCONDispatch(serverInstanceID string, request *domain.GameClientBridgeProtectedRequestDeclaration) (sourceRCONDispatchResolution, error) {
|
||||
if request == nil || request.Kind != "rcon" {
|
||||
return sourceRCONDispatchResolution{}, validationError("protected request is not RCON")
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
capability := domain.JobCapabilityRemoteRunProtectedRCON
|
||||
if plugin.Status != domain.GamePluginStatusInstalled || plugin.Version != instance.PluginVersion || !plugin.Permissions.RemoteAccess || !plugin.RemoteAccess.RCON || !containsString(plugin.RequiredRunCapabilities, capability) || !containsString(plugin.RemoteAccess.RunCapabilities, capability) {
|
||||
return sourceRCONDispatchResolution{}, forbiddenError("plugin does not declare protected SCUM RCON access")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
if err := svc.validateRunnableEndpoint(endpoint, capability); err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
if !strings.EqualFold(endpoint.Platform, "windows") || !strings.EqualFold(endpoint.Architecture, "amd64") {
|
||||
return sourceRCONDispatchResolution{}, validationError("unsupported_extension_platform: SCUM Source RCON requires windows/amd64")
|
||||
}
|
||||
binding, err := svc.runtimeBindingForServer(instance.ID)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
binding, err = normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
if binding.Status != domain.RuntimeBindingStatusComplete || binding.PluginVersion != plugin.Version {
|
||||
return sourceRCONDispatchResolution{}, validationError("runtime binding is incomplete or stale")
|
||||
}
|
||||
profile, exists := runtimeLifecycleProfileForKey(plugin.RuntimeProfiles, binding.ProfileKey)
|
||||
if !exists || !containsString(profile.Capabilities, capability) || !runtimePlatformsContain(profile.Platforms, "windows") {
|
||||
return sourceRCONDispatchResolution{}, validationError("selected runtime profile does not support protected SCUM RCON")
|
||||
}
|
||||
transport, err := sourceRCONTransportForCapability(plugin.RuntimeProfiles, profile, request.TransportKey, capability)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
if transport.TargetKey != request.TargetKey {
|
||||
return sourceRCONDispatchResolution{}, validationError("protected RCON transport target is invalid")
|
||||
}
|
||||
extension, err := sourceRCONExtension(plugin.RuntimeProfiles, profile, endpoint)
|
||||
if err != nil {
|
||||
return sourceRCONDispatchResolution{}, err
|
||||
}
|
||||
plan := &domain.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: extension.Key, ModKey: extension.ModKey, ConfigRef: "ue4ss/Mods/" + extension.ModKey + "/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/" + extension.TargetKey + "/release.json", Port: extension.RCONPort}
|
||||
return sourceRCONDispatchResolution{plugin: plugin, binding: binding, transport: transport, plan: plan}, nil
|
||||
}
|
||||
|
||||
func sourceRCONExtension(profiles domain.GamePluginRuntimeProfiles, profile domain.RuntimeLifecycleProfile, endpoint domain.RunEndpoint) (domain.RuntimeDLLExtensionProfile, error) {
|
||||
byKey := make(map[string]domain.RuntimeDLLExtensionProfile, len(profiles.DLLExtensions))
|
||||
for _, extension := range profiles.DLLExtensions {
|
||||
|
||||
@@ -80,68 +80,6 @@ func TestSourceRCONDispatchUsesOneTimeRedactedInput(t *testing.T) {
|
||||
if strings.Contains(string(storedJSON), input.Command) || strings.Contains(string(storedJSON), request.Message) {
|
||||
t.Fatalf("consumed command was persisted: %s", storedJSON)
|
||||
}
|
||||
if events, err := svc.store.AuditEvents().List(domain.AuditEventFilter{ResourceID: instance.ID}); err != nil || len(events) != 0 {
|
||||
t.Fatalf("RCON command must not add an audit event, events=%+v err=%v", events, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedRCONBridgeDispatchCarriesSourceRCONPlan(t *testing.T) {
|
||||
svc, _, _, instance := newSourceRCONFixture(t)
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
protectedCapability := domain.JobCapabilityRemoteRunProtectedRCON
|
||||
plugin.RequiredRunCapabilities = append(plugin.RequiredRunCapabilities, protectedCapability)
|
||||
plugin.RemoteAccess.RunCapabilities = append(plugin.RemoteAccess.RunCapabilities, protectedCapability)
|
||||
plugin.RuntimeProfiles.ClientManagers = []domain.RuntimeClientManagerProfile{{Key: "scum-client-manager", Health: domain.RuntimeClientManagerHealth{RequiredCapabilities: []string{gameClientBridgeCapability}}}}
|
||||
plugin.RuntimeProfiles.LifecycleProfiles[0].Capabilities = append(plugin.RuntimeProfiles.LifecycleProfiles[0].Capabilities, protectedCapability)
|
||||
plugin.RuntimeProfiles.LifecycleProfiles[0].TransportKeys = append(plugin.RuntimeProfiles.LifecycleProfiles[0].TransportKeys, "scum-management")
|
||||
plugin.RuntimeProfiles.TransportProfiles = append(plugin.RuntimeProfiles.TransportProfiles, domain.RuntimeTransportProfile{Key: "scum-management", Kind: "rcon", TargetKey: "scum-management", Capabilities: []string{protectedCapability}})
|
||||
plugin.GameClientBridge.Commands = append(plugin.GameClientBridge.Commands, domain.GameClientBridgeCommandDeclaration{Type: "management.rcon.request", ApprovalLevel: domain.GameClientBridgeApprovalLevelOperator, TimeoutSeconds: 120, MaxPayloadBytes: 8192, ProtectedRequest: &domain.GameClientBridgeProtectedRequestDeclaration{Kind: "rcon", TransportKey: "scum-management", TargetKey: "scum-management", TextField: "requestText", MaxTextBytes: 8192}})
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"rcon": "runtime-rcon", "scum-management": "runtime-rcon"}}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("refresh protected RCON binding: %v", err)
|
||||
}
|
||||
if err := svc.store.RuntimeBindings().Update(binding); err != nil {
|
||||
t.Fatalf("store protected RCON binding: %v", err)
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint.Capabilities = append(endpoint.Capabilities, protectedCapability)
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := svc.resolveProtectedSourceRCONDispatch(instance.ID, plugin.GameClientBridge.Commands[len(plugin.GameClientBridge.Commands)-1].ProtectedRequest); err != nil {
|
||||
t.Fatalf("resolve protected Source RCON plan: %v", err)
|
||||
}
|
||||
|
||||
command, err := svc.queueGameClientBridgeCommand("user-rcon-owner", domain.GameClientBridgeQueueRequest{ServerInstanceID: instance.ID, PluginID: plugin.ID, ProfileKey: "scum-client-manager", CommandType: "management.rcon.request", Payload: map[string]any{"requestText": "#ListPlayers"}, IdempotencyKey: "protected-rcon-1", ExpiresAt: fixedTime.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("queue protected RCON: %v", err)
|
||||
}
|
||||
job, err := svc.store.Jobs().Get(command.RunJobID)
|
||||
if err != nil {
|
||||
t.Fatalf("get protected RCON job: %v", err)
|
||||
}
|
||||
if job.Capability != protectedCapability || job.InputRef == "" || !strings.HasPrefix(job.InputRef, "input://protected-request/") || job.ExecutionInput.SourceRCON == nil {
|
||||
t.Fatalf("expected protected RCON job with frozen Source RCON plan, got %+v", job)
|
||||
}
|
||||
if job.ExecutionInput.WorkspaceScope != "local" || job.ExecutionInput.RemoteAdapterKey != "scum-management" || job.ExecutionInput.RemoteAdapterKind != "protected-rcon" || job.ExecutionInput.SourceRCON.Port != 27015 {
|
||||
t.Fatalf("protected RCON plan did not preserve logical runtime binding: %+v", job.ExecutionInput)
|
||||
}
|
||||
serialized, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(serialized), "#ListPlayers") || strings.Contains(string(serialized), "password=") {
|
||||
t.Fatalf("protected RCON job leaked transient input: %s", serialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceRCONDispatchRejectsUnsafeOrIncompatibleState(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user