feat: 完整游戏运维功能
This commit is contained in:
npc0-hue
@@ -0,0 +1,290 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func TestDependencyCatalogRequiresCurrentReviewedDigest(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "dependency-other-owner", DisplayName: "Other Owner", Email: "dependency-other@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
if _, err := svc.GetDependencyCatalogForSession(otherSession, instance.ID); !errors.Is(err, ErrForbidden) {
|
||||
t.Fatalf("expected cross-owner dependency catalog denial, got %v", err)
|
||||
}
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get dependency catalog: %v", err)
|
||||
}
|
||||
if catalog.TargetOS != "linux" || catalog.TargetArch != "amd64" || len(catalog.Probes) != 1 || len(catalog.Plans) != 1 || !strings.HasPrefix(catalog.Plans[0].Digest, "sha256:") {
|
||||
t.Fatalf("unexpected dependency catalog: %+v", catalog)
|
||||
}
|
||||
request := domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, Install: true, InstallPlanKey: catalog.Plans[0].Key, PlanDigest: "sha256:" + strings.Repeat("f", 64), IdempotencyKey: "dependency-stale-digest"}
|
||||
if _, err := svc.QueueDependencyJobForSession(session, request); err == nil || !strings.Contains(err.Error(), "planDigest") {
|
||||
t.Fatalf("expected stale digest rejection, got %v", err)
|
||||
}
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list jobs: %v", err)
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if job.IdempotencyKey == request.IdempotencyKey {
|
||||
t.Fatalf("stale digest created a job: %+v", job)
|
||||
}
|
||||
}
|
||||
audits, err := svc.ListAuditEvents(domain.AuditEventFilter{ResourceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list audits: %v", err)
|
||||
}
|
||||
foundDenied := false
|
||||
for _, audit := range audits {
|
||||
foundDenied = foundDenied || audit.Action == "dependency.install.denied"
|
||||
}
|
||||
if !foundDenied {
|
||||
t.Fatalf("expected stale digest audit, got %+v", audits)
|
||||
}
|
||||
|
||||
request.PlanDigest = catalog.Plans[0].Digest
|
||||
request.IdempotencyKey = "dependency-current-digest"
|
||||
job, err := svc.QueueDependencyJobForSession(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("queue reviewed dependency plan: %v", err)
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityDependenciesInstall || job.TargetKey != "dependencies/install/"+catalog.Plans[0].Key {
|
||||
t.Fatalf("unexpected dependency install job: %+v", job)
|
||||
}
|
||||
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get plugin: %v", err)
|
||||
}
|
||||
plugin.RuntimeProfiles.InstallPlans[0].Steps[0].PackageName = "openjdk-22-jre"
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("mutate plugin declaration: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesInstall}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob {
|
||||
t.Fatalf("claim dependency install: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
_, err = svc.GetDependencyExecutionInput(domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
||||
if err == nil || !strings.Contains(err.Error(), "changed after dispatch") {
|
||||
t.Fatalf("expected changed declaration rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDependencyInputFencingCancellationAndTerminalProjection(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("catalog: %v", err)
|
||||
}
|
||||
job, err := svc.QueueDependencyJobForSession(session, domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, IdempotencyKey: "dependency-check-fencing"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue dependency check: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesCheck}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID {
|
||||
t.Fatalf("claim dependency check: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
base := domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}
|
||||
input, err := svc.GetDependencyExecutionInput(base)
|
||||
if err != nil || input.PlanDigest == "" || input.Bindings["java"] == "" {
|
||||
t.Fatalf("get fenced dependency input: input=%+v err=%v", input, err)
|
||||
}
|
||||
wrongSession := base
|
||||
wrongSession.SessionToken = "stale-session"
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongSession); err == nil {
|
||||
t.Fatal("expected wrong session rejection")
|
||||
}
|
||||
wrongAttempt := base
|
||||
wrongAttempt.Attempt++
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongAttempt); err == nil {
|
||||
t.Fatal("expected wrong attempt rejection")
|
||||
}
|
||||
wrongLease := base
|
||||
wrongLease.LeaseToken = "stale-lease"
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongLease); err == nil {
|
||||
t.Fatal("expected wrong lease rejection")
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.DependencyExecutionEvidence{ProbeKey: input.Probe.Key, PlanDigest: input.PlanDigest, State: string(domain.DependencyStatePresent), Evidence: "OpenJDK 21"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "dependency probe completed"}, ResultRef: "artifact://jobs/dependency-check/result", Message: "dependency probe completed", ExecutionResult: domain.JobExecutionResult{Kind: "dependency.check", Checksum: input.PlanDigest, AuditSummary: "dependency probe completed", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete dependency result: %v", err)
|
||||
}
|
||||
projected, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil || projected.Probes[0].State != domain.DependencyStatePresent || projected.Probes[0].Evidence != "OpenJDK 21" {
|
||||
t.Fatalf("unexpected dependency projection: catalog=%+v err=%v", projected, err)
|
||||
}
|
||||
|
||||
cancelJob, err := svc.QueueDependencyJobForSession(session, domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, IdempotencyKey: "dependency-check-cancel"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue cancellable dependency check: %v", err)
|
||||
}
|
||||
claim, err = svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesCheck}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || claim.Job.JobID != cancelJob.ID {
|
||||
t.Fatalf("claim cancellable dependency check: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
if _, err := svc.RequestRunJobCancelForSession(session, domain.RunJobCancelRequest{JobID: cancelJob.ID, Reason: "operator cancelled"}); err != nil {
|
||||
t.Fatalf("request cancel: %v", err)
|
||||
}
|
||||
if _, err := svc.GetDependencyExecutionInput(domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}); err == nil || !strings.Contains(err.Error(), "cancelled") {
|
||||
t.Fatalf("expected cancelled input rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginBridgeDependencyInstallUsesReviewedPlanDigest(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get plugin: %v", err)
|
||||
}
|
||||
plugin.Pages = append(plugin.Pages, domain.GamePluginPage{Key: "runtime", Title: "Runtime", Path: "/runtime", Permissions: []string{"server.dependencies.manage"}, BridgeActions: []string{string(domain.PluginBridgeActionDependenciesRequest)}})
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("add dependency bridge page: %v", err)
|
||||
}
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get dependency catalog: %v", err)
|
||||
}
|
||||
request := domain.PluginBridgeExecuteRequest{RequestID: "bridge-dependency-install", PluginID: plugin.ID, RouteKey: "runtime", ServerInstanceID: instance.ID, Action: domain.PluginBridgeActionDependenciesRequest, Payload: map[string]string{"operation": "install", "probeKey": catalog.Probes[0].Key, "planKey": catalog.Plans[0].Key, "idempotencyKey": "bridge-dependency-install"}}
|
||||
denied, err := svc.ExecutePluginBridgeAction(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("execute bridge without digest: %v", err)
|
||||
}
|
||||
if denied.Status == "queued" || denied.Error == nil {
|
||||
t.Fatalf("bridge install without reviewed digest must be denied: %+v", denied)
|
||||
}
|
||||
request.Payload["planDigest"] = catalog.Plans[0].Digest
|
||||
request.Payload["idempotencyKey"] = "bridge-dependency-install-approved"
|
||||
approved, err := svc.ExecutePluginBridgeAction(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("execute reviewed bridge install: %v", err)
|
||||
}
|
||||
if approved.Status != "queued" || approved.Result["capability"] != domain.JobCapabilityDependenciesInstall {
|
||||
t.Fatalf("expected reviewed bridge dependency job, got %+v", approved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUpdateTargetFencingChunksHealthAndRollbackProjection(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{ServerInstanceID: instance.ID, TargetOS: "linux", TargetArch: "amd64", IdempotencyKey: "run-update-build"})
|
||||
if err != nil {
|
||||
t.Fatalf("generate update distribution: %v", err)
|
||||
}
|
||||
payload := []byte("compiled target-matched run archive")
|
||||
distribution = completeDistributionBuild(t, svc, distribution, payload)
|
||||
otherInstance, err := svc.CreateServerInstanceForSession(session, domain.ServerInstance{ID: "server-update-other", PluginID: instance.PluginID, RunEndpointID: instance.RunEndpointID, Name: "Other Update Server", State: domain.ServerInstanceStateReady})
|
||||
if err != nil {
|
||||
t.Fatalf("create other update server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, otherInstance, "local")
|
||||
if _, err := svc.PushRunUpdateForSession(session, domain.RunUpdateRequest{ServerInstanceID: otherInstance.ID, ArtifactID: distribution.ArtifactID, Checksum: distribution.Checksum, IdempotencyKey: "run-update-cross-server"}); err == nil {
|
||||
t.Fatal("expected cross-server update artifact rejection")
|
||||
}
|
||||
|
||||
endpoint, _ := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
endpoint.Architecture = "arm64"
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("change endpoint target: %v", err)
|
||||
}
|
||||
request := domain.RunUpdateRequest{ServerInstanceID: instance.ID, ArtifactID: distribution.ArtifactID, Checksum: distribution.Checksum, IdempotencyKey: "run-update-target-check"}
|
||||
if _, err := svc.PushRunUpdateForSession(session, request); err == nil || !strings.Contains(err.Error(), "target-matched") {
|
||||
t.Fatalf("expected cross-target update rejection, got %v", err)
|
||||
}
|
||||
endpoint.Architecture = "amd64"
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("restore endpoint target: %v", err)
|
||||
}
|
||||
request.IdempotencyKey = "run-update-fenced"
|
||||
update, err := svc.PushRunUpdateForSession(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("push target-matched update: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityRunSelfUpdate}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job.JobID != update.JobID {
|
||||
t.Fatalf("claim Run update: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
inputRequest := domain.RunUpdateInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}
|
||||
input, err := svc.GetRunUpdateInput(inputRequest)
|
||||
if err != nil || input.TargetRelease != update.TargetRelease || input.Checksum != distribution.Checksum {
|
||||
t.Fatalf("get Run update input: input=%+v err=%v", input, err)
|
||||
}
|
||||
chunk, err := svc.ReadRunUpdateChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Offset: 0, Length: 8})
|
||||
if err != nil || string(chunk.Payload) != string(payload[:8]) || chunk.Offset != 0 || chunk.TotalBytes != int64(len(payload)) {
|
||||
t.Fatalf("read bounded update chunk: chunk=%+v err=%v", chunk, err)
|
||||
}
|
||||
if _, err := svc.ReadRunUpdateChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt + 1, Offset: 0, Length: 8}); err == nil {
|
||||
t.Fatal("expected stale update chunk attempt rejection")
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.RunUpdateExecutionEvidence{TargetRelease: update.TargetRelease, Phase: "staged"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: "artifact://jobs/run-update/staged", Message: "Run update verified and staged", ExecutionResult: domain.JobExecutionResult{Kind: "run.update.staged", Checksum: update.Checksum, SizeBytes: int64(len(payload)), AuditSummary: "verified update staged", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete staged Run update: %v", err)
|
||||
}
|
||||
updates, err := svc.ListRunUpdateJobsForSession(session, instance.ID)
|
||||
if err != nil || len(updates) != 1 || updates[0].Phase != domain.RunUpdatePhaseRestartRequested {
|
||||
t.Fatalf("expected restart-requested projection, updates=%+v err=%v", updates, err)
|
||||
}
|
||||
|
||||
newHello := dependencyUpdateHello(instance)
|
||||
newHello.Version = update.TargetRelease
|
||||
newRegistration, err := svc.RegisterRunHello(newHello)
|
||||
if err != nil {
|
||||
t.Fatalf("register updated Run: %v", err)
|
||||
}
|
||||
health := domain.RunUpdateHealthReport{RunEndpointID: instance.RunEndpointID, SessionToken: newRegistration.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Outcome: "succeeded", Version: update.TargetRelease}
|
||||
if _, err := svc.ReportRunUpdateHealth(domain.RunUpdateHealthReport{RunEndpointID: health.RunEndpointID, SessionToken: health.SessionToken, JobID: health.JobID, LeaseToken: "stale-lease", Attempt: health.Attempt, Outcome: health.Outcome, Version: health.Version}); err == nil {
|
||||
t.Fatal("expected stale health lease rejection")
|
||||
}
|
||||
result, err := svc.ReportRunUpdateHealth(health)
|
||||
if err != nil || !result.Accepted || result.Phase != domain.RunUpdatePhaseSucceeded {
|
||||
t.Fatalf("report updated Run health: result=%+v err=%v", result, err)
|
||||
}
|
||||
|
||||
rollbackHello := dependencyUpdateHello(instance)
|
||||
rollbackHello.Version = update.PreviousVersion
|
||||
rollbackRegistration, err := svc.RegisterRunHello(rollbackHello)
|
||||
if err != nil {
|
||||
t.Fatalf("register rolled-back Run: %v", err)
|
||||
}
|
||||
health.SessionToken = rollbackRegistration.SessionToken
|
||||
health.Outcome = "rolled-back"
|
||||
health.Version = update.PreviousVersion
|
||||
result, err = svc.ReportRunUpdateHealth(health)
|
||||
if err != nil || result.Phase != domain.RunUpdatePhaseRolledBack {
|
||||
t.Fatalf("report rollback: result=%+v err=%v", result, err)
|
||||
}
|
||||
updates, _ = svc.ListRunUpdateJobsForSession(session, instance.ID)
|
||||
if !updates[0].Rollback || updates[0].Status != domain.DistributionJobStatusFailed || updates[0].Phase != domain.RunUpdatePhaseRolledBack {
|
||||
t.Fatalf("unexpected rollback projection: %+v", updates[0])
|
||||
}
|
||||
}
|
||||
|
||||
func registerDependencyUpdateRun(t *testing.T, svc *CoreService, instance domain.ServerInstance) string {
|
||||
t.Helper()
|
||||
result, err := svc.RegisterRunHello(dependencyUpdateHello(instance))
|
||||
if err != nil || !result.Accepted {
|
||||
t.Fatalf("register dependency/update Run: result=%+v err=%v", result, err)
|
||||
}
|
||||
return result.SessionToken
|
||||
}
|
||||
|
||||
func dependencyUpdateHello(instance domain.ServerInstance) domain.RunControlHello {
|
||||
return domain.RunControlHello{
|
||||
RegistrationToken: "registration-token",
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
DisplayName: "Dependency Update Run",
|
||||
Version: "0.1.0",
|
||||
Status: domain.RunEndpointStatusOnline,
|
||||
Platform: "linux",
|
||||
Architecture: "amd64",
|
||||
CapabilityReport: domain.RunCapabilityReport{Capabilities: []string{domain.JobCapabilityDependenciesCheck, domain.JobCapabilityDependenciesInstall, domain.JobCapabilityRunSelfUpdate}, Fingerprint: "dependency-update-v1"},
|
||||
Capacity: domain.RunCapacity{MaxJobs: 2},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user