package service import ( "strings" "testing" "time" "browser.local/platform/domain" ) func TestClientManagerLifecycleBuildDeployRegisterHealthUpdateRollbackAndUninstall(t *testing.T) { svc, ownerSession, instance := newDistributionTestFixture(t) baseTime := svc.now() svc.now = func() time.Time { return baseTime } distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-build-v1") view, err := svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if err != nil || view.Installation.Status != domain.ClientManagerLifecycleAvailable { t.Fatalf("expected available build projection, view=%+v err=%v", view, err) } view, err = svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"}) if err != nil || view.Installation.Status != domain.ClientManagerLifecycleDeploying || view.Job.State != domain.JobStateQueued { t.Fatalf("queue deployment: view=%+v err=%v", view, err) } if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"}); err != nil { t.Fatalf("idempotent deployment: %v", err) } runSession := registerClientManagerRun(t, svc) claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy) input, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}) if err != nil || input.ArtifactID != distribution.ArtifactID || input.KeyGeneration != distribution.KeyGeneration || input.DeploymentGeneration != view.Installation.DeploymentGeneration || strings.Contains(strings.Join(input.Arguments, " "), "/Users/") { t.Fatalf("get fenced deployment input: input=%+v err=%v", input, err) } chunk, err := svc.ReadClientManagerLifecycleChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Offset: 0, Length: 7}) if err != nil || len(chunk.Payload) == 0 || chunk.ArtifactID != distribution.ArtifactID { t.Fatalf("read deployment chunk: chunk=%+v err=%v", chunk, err) } if _, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt + 1}); err == nil { t.Fatal("expected stale attempt to be rejected") } completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.Status != domain.ClientManagerLifecycleRegistering || view.Installation.ActiveArtifactID != distribution.ArtifactID { t.Fatalf("expected deployed registration state, got %+v", view.Installation) } componentKey := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager") registerRequest := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, baseTime) registerRequest.Signature = clientManagerRegistrationSignature(componentKey, registerRequest) registration, err := svc.RegisterClientManager(registerRequest) if err != nil || !registration.Accepted || registration.SessionToken == "" { t.Fatalf("register client manager: result=%+v err=%v", registration, err) } if _, err := svc.RegisterClientManager(registerRequest); err == nil { t.Fatal("expected registration nonce replay rejection") } if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: runSession, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil { t.Fatal("Run control session must not authenticate as a Client Manager session") } heartbeat, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, HealthReason: "ready", Capabilities: registerRequest.Capabilities, SentAt: baseTime}) if err != nil || heartbeat.Status != domain.ClientManagerLifecycleOnline { t.Fatalf("accept heartbeat: result=%+v err=%v", heartbeat, err) } if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil { t.Fatal("expected replayed heartbeat sequence rejection") } baseTime = baseTime.Add(50 * time.Second) if err := svc.ReconcileClientManagerLifecycle(); err != nil { t.Fatalf("reconcile degraded health: %v", err) } view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.Status != domain.ClientManagerLifecycleDegraded { t.Fatalf("expected degraded heartbeat timeout, got %+v", view.Installation) } baseTime = baseTime.Add(80 * time.Second) if err := svc.ReconcileClientManagerLifecycle(); err != nil { t.Fatalf("reconcile offline health: %v", err) } view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.Status != domain.ClientManagerLifecycleOffline { t.Fatalf("expected offline heartbeat timeout, got %+v", view.Installation) } updatedDistribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.1.0", "lifecycle-build-v2") view, err = svc.UpdateClientManagerForSession(ownerSession, domain.ClientManagerUpdateRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: updatedDistribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Approved: true, IdempotencyKey: "lifecycle-update-v2"}) if err != nil || view.Installation.Status != domain.ClientManagerLifecycleUpdating { t.Fatalf("queue staged update: view=%+v err=%v", view, err) } runSession = registerClientManagerRun(t, svc) claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate) completeClientManagerJob(t, svc, runSession, claim, domain.JobStateFailed, "client-manager.rollback.restored", "running") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.ActiveArtifactID != distribution.ArtifactID || !strings.Contains(view.Installation.Phase, "previous deployment restored") || !view.Installation.Retryable { t.Fatalf("expected failed update to retain previous active slot, got %+v", view.Installation) } view, err = svc.RetryClientManagerLifecycleForSession(ownerSession, domain.ClientManagerRetryRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-update-v2-retry"}) if err != nil { t.Fatalf("retry staged update: %v", err) } runSession = registerClientManagerRun(t, svc) claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate) completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.updated", "running") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.ActiveArtifactID != updatedDistribution.ArtifactID || view.Installation.PreviousArtifactID != distribution.ArtifactID || view.Installation.Status != domain.ClientManagerLifecycleRegistering { t.Fatalf("expected successful update slot commit, got %+v", view.Installation) } view, err = svc.ControlClientManagerForSession(ownerSession, domain.ClientManagerControlRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", Operation: domain.ClientManagerOperationRollback, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-rollback-v1"}) if err != nil { t.Fatalf("queue explicit rollback: %v", err) } runSession = registerClientManagerRun(t, svc) claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerRollback) completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.rolled-back", "running") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.ActiveArtifactID != distribution.ArtifactID || view.Installation.PreviousArtifactID != updatedDistribution.ArtifactID { t.Fatalf("expected rollback slot swap, got %+v", view.Installation) } view, err = svc.UninstallClientManagerForSession(ownerSession, domain.ClientManagerUninstallRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Confirmed: true, IdempotencyKey: "lifecycle-uninstall"}) if err != nil { t.Fatalf("queue uninstall: %v", err) } runSession = registerClientManagerRun(t, svc) claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUninstall) completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.uninstalled", "stopped") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if view.Installation.Status != domain.ClientManagerLifecycleUninstalled || view.Installation.ActiveArtifactID != "" { t.Fatalf("expected durable uninstalled history, got %+v", view.Installation) } if _, err := svc.store.ClientManagerDistributions().Get(distribution.ID); err != nil { t.Fatalf("uninstall must retain distribution history: %v", err) } } func TestClientManagerLifecycleRejectsCrossScopeStaleAndRevokedIdentity(t *testing.T) { svc, ownerSession, instance := newDistributionTestFixture(t) now := svc.now() svc.now = func() time.Time { return now } distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-scope-build") view, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-scope-deploy"}) if err != nil { t.Fatalf("queue deploy: %v", err) } otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "other-owner", DisplayName: "Other", Email: "other-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"}) if _, err := svc.GetClientManagerLifecycleForSession(otherSession, instance.ID, "scum-client-manager"); err == nil { t.Fatal("expected cross-owner lifecycle read denial") } if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration + 1, IdempotencyKey: "lifecycle-stale-deploy"}); err == nil { t.Fatal("expected stale deployment generation denial") } runSession := registerClientManagerRun(t, svc) claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy) completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running") view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") plain := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager") request := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now) request.ArtifactID = "cross-server-artifact" request.Signature = clientManagerRegistrationSignature(plain, request) if _, err := svc.RegisterClientManager(request); err == nil { t.Fatal("expected cross-artifact registration rejection") } request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now) request.KeyGeneration++ request.Nonce = "nonce-stale-key-generation" request.Signature = clientManagerRegistrationSignature(plain, request) if _, err := svc.RegisterClientManager(request); err == nil { t.Fatal("expected stale key generation registration rejection") } request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now) request.Nonce = "nonce-valid-component-identity" request.Signature = clientManagerRegistrationSignature(plain, request) registration, err := svc.RegisterClientManager(request) if err != nil { t.Fatalf("register valid component: %v", err) } now = now.Add(16 * time.Minute) if err := svc.ReconcileClientManagerLifecycle(); err != nil { t.Fatalf("expire component session: %v", err) } if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil { t.Fatal("expected expired component session rejection") } request.Timestamp = now request.Nonce = "nonce-replacement-after-expiry" request.Signature = clientManagerRegistrationSignature(plain, request) registration, err = svc.RegisterClientManager(request) if err != nil { t.Fatalf("register replacement component session: %v", err) } if _, err := svc.ResetComponentKeyForSession(ownerSession, domain.ComponentKeyResetRequest{ServerInstanceID: instance.ID, ComponentKind: domain.DistributionComponentClientManager, ComponentKey: "scum-client-manager"}); err != nil { t.Fatalf("reset component key: %v", err) } if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil { t.Fatal("expected reset to revoke component session") } view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager") if !view.Installation.RequiresRedeploy || view.Installation.Status != domain.ClientManagerLifecycleFailed { t.Fatalf("expected key reset recovery projection, got %+v", view.Installation) } for _, forbidden := range []string{plain, registration.SessionToken, "secret://", "/Users/", "tcp://"} { payload := strings.Join([]string{view.Installation.Phase, view.Installation.HealthReason}, " ") if strings.Contains(payload, forbidden) { t.Fatalf("safe lifecycle view leaked %q: %s", forbidden, payload) } } } func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution { t.Helper() endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID) if err != nil { t.Fatalf("get lifecycle Run endpoint: %v", err) } endpoint.LastHeartbeatAt = svc.now() if err := svc.store.RunEndpoints().Update(endpoint); err != nil { t.Fatalf("refresh lifecycle Run heartbeat: %v", err) } plugin, err := svc.store.GamePlugins().Get(instance.PluginID) if err != nil { t.Fatalf("get lifecycle plugin: %v", err) } for i := range plugin.RuntimeProfiles.ClientManagers { if plugin.RuntimeProfiles.ClientManagers[i].Key == "scum-client-manager" { plugin.RuntimeProfiles.ClientManagers[i].Version = version } } if err := svc.store.GamePlugins().Update(plugin); err != nil { t.Fatalf("update lifecycle version: %v", err) } payload := []byte("client-manager-package-" + version) svc.ConfigureDistributionBuilder(staticDistributionBuilder{payload: payload}) distribution, err := svc.GenerateClientManagerDistributionForSession(session, domain.ClientManagerBuildRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", TargetOS: "linux", TargetArch: "amd64", RepositoryURL: "https://github.com/F88888/scum_client.git", SourceRevision: "main", IdempotencyKey: idempotency}) if err != nil { t.Fatalf("generate lifecycle distribution: %v", err) } return completeClientDistributionBuild(t, svc, distribution, payload) } func registerClientManagerRun(t *testing.T, svc *CoreService) string { t.Helper() hello := validRunControlHello() hello.Platform = "linux" hello.Architecture = "amd64" hello.CapabilityReport.Capabilities = append(hello.CapabilityReport.Capabilities, domain.JobCapabilityClientManagerDeploy, domain.JobCapabilityClientManagerControl, domain.JobCapabilityClientManagerUpdate, domain.JobCapabilityClientManagerRollback, domain.JobCapabilityClientManagerUninstall) result, err := svc.RegisterRunHello(hello) if err != nil { t.Fatalf("register lifecycle Run: %v", err) } return result.SessionToken } func claimClientManagerJob(t *testing.T, svc *CoreService, sessionToken, capability string) domain.RunJobClaimResult { t.Helper() claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capabilities: []string{capability}, Capacity: domain.RunCapacity{MaxJobs: 1}}) if err != nil || !claim.HasJob || claim.Job.Capability != capability { t.Fatalf("claim %s job: claim=%+v err=%v", capability, claim, err) } if _, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Message: "typed lifecycle work started"}); err != nil { t.Fatalf("ack lifecycle job: %v", err) } return claim } func completeClientManagerJob(t *testing.T, svc *CoreService, sessionToken string, claim domain.RunJobClaimResult, state domain.JobState, kind, processState string) { t.Helper() _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: state, Progress: domain.RunJobProgressReport{Percent: 100, Message: "client-manager lifecycle terminal"}, Message: "client-manager lifecycle terminal", ExecutionResult: domain.JobExecutionResult{Kind: kind, ProcessState: processState, AuditSummary: "bounded lifecycle result"}}) if err != nil { t.Fatalf("complete lifecycle job: %v", err) } } func currentClientManagerPlainKey(t *testing.T, svc *CoreService, serverID, profileKey string) string { t.Helper() key, err := svc.activeComponentKey(serverID, domain.DistributionComponentClientManager, profileKey) if err != nil { t.Fatalf("get active component key: %v", err) } plain, err := svc.decryptRuntimeKey(key.EncryptedKey) if err != nil { t.Fatalf("decrypt component key: %v", err) } return plain } func lifecycleRegisterRequest(installation domain.ClientManagerInstallation, capabilities []string, stamp time.Time) domain.ClientManagerRegisterRequest { return domain.ClientManagerRegisterRequest{InstallationID: installation.ID, ServerInstanceID: installation.ServerInstanceID, ProfileKey: installation.ProfileKey, ArtifactID: installation.ActiveArtifactID, Version: installation.ActiveVersion, SourceRevision: installation.ActiveRevision, TargetOS: installation.TargetOS, TargetArch: installation.TargetArch, KeyGeneration: installation.KeyGeneration, DeploymentGeneration: installation.DeploymentGeneration, Capabilities: capabilities, Timestamp: stamp, Nonce: "nonce-client-manager-registration"} }