290 lines
20 KiB
Go
290 lines
20 KiB
Go
package service
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/platform/domain"
|
|
)
|
|
|
|
func TestClientManagerLifecycleBuildDeployRegisterHealthUpdateRollbackAndUninstall(t *testing.T) {
|
|
svc, ownerSession, instance := newDistributionTestFixture(t)
|
|
baseTime := svc.now()
|
|
svc.now = func() time.Time { return baseTime }
|
|
distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-build-v1")
|
|
view, err := svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleAvailable {
|
|
t.Fatalf("expected available build projection, view=%+v err=%v", view, err)
|
|
}
|
|
|
|
view, err = svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"})
|
|
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleDeploying || view.Job.State != domain.JobStateQueued {
|
|
t.Fatalf("queue deployment: view=%+v err=%v", view, err)
|
|
}
|
|
if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"}); err != nil {
|
|
t.Fatalf("idempotent deployment: %v", err)
|
|
}
|
|
runSession := registerClientManagerRun(t, svc)
|
|
claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy)
|
|
input, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
|
if err != nil || input.ArtifactID != distribution.ArtifactID || input.KeyGeneration != distribution.KeyGeneration || input.DeploymentGeneration != view.Installation.DeploymentGeneration || strings.Contains(strings.Join(input.Arguments, " "), "/Users/") {
|
|
t.Fatalf("get fenced deployment input: input=%+v err=%v", input, err)
|
|
}
|
|
chunk, err := svc.ReadClientManagerLifecycleChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Offset: 0, Length: 7})
|
|
if err != nil || len(chunk.Payload) == 0 || chunk.ArtifactID != distribution.ArtifactID {
|
|
t.Fatalf("read deployment chunk: chunk=%+v err=%v", chunk, err)
|
|
}
|
|
if _, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt + 1}); err == nil {
|
|
t.Fatal("expected stale attempt to be rejected")
|
|
}
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.Status != domain.ClientManagerLifecycleRegistering || view.Installation.ActiveArtifactID != distribution.ArtifactID {
|
|
t.Fatalf("expected deployed registration state, got %+v", view.Installation)
|
|
}
|
|
|
|
componentKey := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager")
|
|
registerRequest := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, baseTime)
|
|
registerRequest.Signature = clientManagerRegistrationSignature(componentKey, registerRequest)
|
|
registration, err := svc.RegisterClientManager(registerRequest)
|
|
if err != nil || !registration.Accepted || registration.SessionToken == "" {
|
|
t.Fatalf("register client manager: result=%+v err=%v", registration, err)
|
|
}
|
|
if _, err := svc.RegisterClientManager(registerRequest); err == nil {
|
|
t.Fatal("expected registration nonce replay rejection")
|
|
}
|
|
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: runSession, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil {
|
|
t.Fatal("Run control session must not authenticate as a Client Manager session")
|
|
}
|
|
heartbeat, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, HealthReason: "ready", Capabilities: registerRequest.Capabilities, SentAt: baseTime})
|
|
if err != nil || heartbeat.Status != domain.ClientManagerLifecycleOnline {
|
|
t.Fatalf("accept heartbeat: result=%+v err=%v", heartbeat, err)
|
|
}
|
|
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil {
|
|
t.Fatal("expected replayed heartbeat sequence rejection")
|
|
}
|
|
|
|
baseTime = baseTime.Add(50 * time.Second)
|
|
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
|
t.Fatalf("reconcile degraded health: %v", err)
|
|
}
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.Status != domain.ClientManagerLifecycleDegraded {
|
|
t.Fatalf("expected degraded heartbeat timeout, got %+v", view.Installation)
|
|
}
|
|
baseTime = baseTime.Add(80 * time.Second)
|
|
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
|
t.Fatalf("reconcile offline health: %v", err)
|
|
}
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.Status != domain.ClientManagerLifecycleOffline {
|
|
t.Fatalf("expected offline heartbeat timeout, got %+v", view.Installation)
|
|
}
|
|
|
|
updatedDistribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.1.0", "lifecycle-build-v2")
|
|
view, err = svc.UpdateClientManagerForSession(ownerSession, domain.ClientManagerUpdateRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: updatedDistribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Approved: true, IdempotencyKey: "lifecycle-update-v2"})
|
|
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleUpdating {
|
|
t.Fatalf("queue staged update: view=%+v err=%v", view, err)
|
|
}
|
|
runSession = registerClientManagerRun(t, svc)
|
|
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate)
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateFailed, "client-manager.rollback.restored", "running")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.ActiveArtifactID != distribution.ArtifactID || !strings.Contains(view.Installation.Phase, "previous deployment restored") || !view.Installation.Retryable {
|
|
t.Fatalf("expected failed update to retain previous active slot, got %+v", view.Installation)
|
|
}
|
|
view, err = svc.RetryClientManagerLifecycleForSession(ownerSession, domain.ClientManagerRetryRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-update-v2-retry"})
|
|
if err != nil {
|
|
t.Fatalf("retry staged update: %v", err)
|
|
}
|
|
runSession = registerClientManagerRun(t, svc)
|
|
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate)
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.updated", "running")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.ActiveArtifactID != updatedDistribution.ArtifactID || view.Installation.PreviousArtifactID != distribution.ArtifactID || view.Installation.Status != domain.ClientManagerLifecycleRegistering {
|
|
t.Fatalf("expected successful update slot commit, got %+v", view.Installation)
|
|
}
|
|
|
|
view, err = svc.ControlClientManagerForSession(ownerSession, domain.ClientManagerControlRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", Operation: domain.ClientManagerOperationRollback, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-rollback-v1"})
|
|
if err != nil {
|
|
t.Fatalf("queue explicit rollback: %v", err)
|
|
}
|
|
runSession = registerClientManagerRun(t, svc)
|
|
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerRollback)
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.rolled-back", "running")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.ActiveArtifactID != distribution.ArtifactID || view.Installation.PreviousArtifactID != updatedDistribution.ArtifactID {
|
|
t.Fatalf("expected rollback slot swap, got %+v", view.Installation)
|
|
}
|
|
|
|
view, err = svc.UninstallClientManagerForSession(ownerSession, domain.ClientManagerUninstallRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Confirmed: true, IdempotencyKey: "lifecycle-uninstall"})
|
|
if err != nil {
|
|
t.Fatalf("queue uninstall: %v", err)
|
|
}
|
|
runSession = registerClientManagerRun(t, svc)
|
|
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUninstall)
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.uninstalled", "stopped")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if view.Installation.Status != domain.ClientManagerLifecycleUninstalled || view.Installation.ActiveArtifactID != "" {
|
|
t.Fatalf("expected durable uninstalled history, got %+v", view.Installation)
|
|
}
|
|
if _, err := svc.store.ClientManagerDistributions().Get(distribution.ID); err != nil {
|
|
t.Fatalf("uninstall must retain distribution history: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestClientManagerLifecycleRejectsCrossScopeStaleAndRevokedIdentity(t *testing.T) {
|
|
svc, ownerSession, instance := newDistributionTestFixture(t)
|
|
now := svc.now()
|
|
svc.now = func() time.Time { return now }
|
|
distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-scope-build")
|
|
view, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-scope-deploy"})
|
|
if err != nil {
|
|
t.Fatalf("queue deploy: %v", err)
|
|
}
|
|
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "other-owner", DisplayName: "Other", Email: "other-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
|
if _, err := svc.GetClientManagerLifecycleForSession(otherSession, instance.ID, "scum-client-manager"); err == nil {
|
|
t.Fatal("expected cross-owner lifecycle read denial")
|
|
}
|
|
if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration + 1, IdempotencyKey: "lifecycle-stale-deploy"}); err == nil {
|
|
t.Fatal("expected stale deployment generation denial")
|
|
}
|
|
runSession := registerClientManagerRun(t, svc)
|
|
claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy)
|
|
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running")
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
plain := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager")
|
|
request := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
|
request.ArtifactID = "cross-server-artifact"
|
|
request.Signature = clientManagerRegistrationSignature(plain, request)
|
|
if _, err := svc.RegisterClientManager(request); err == nil {
|
|
t.Fatal("expected cross-artifact registration rejection")
|
|
}
|
|
request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
|
request.KeyGeneration++
|
|
request.Nonce = "nonce-stale-key-generation"
|
|
request.Signature = clientManagerRegistrationSignature(plain, request)
|
|
if _, err := svc.RegisterClientManager(request); err == nil {
|
|
t.Fatal("expected stale key generation registration rejection")
|
|
}
|
|
request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
|
request.Nonce = "nonce-valid-component-identity"
|
|
request.Signature = clientManagerRegistrationSignature(plain, request)
|
|
registration, err := svc.RegisterClientManager(request)
|
|
if err != nil {
|
|
t.Fatalf("register valid component: %v", err)
|
|
}
|
|
now = now.Add(16 * time.Minute)
|
|
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
|
t.Fatalf("expire component session: %v", err)
|
|
}
|
|
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil {
|
|
t.Fatal("expected expired component session rejection")
|
|
}
|
|
request.Timestamp = now
|
|
request.Nonce = "nonce-replacement-after-expiry"
|
|
request.Signature = clientManagerRegistrationSignature(plain, request)
|
|
registration, err = svc.RegisterClientManager(request)
|
|
if err != nil {
|
|
t.Fatalf("register replacement component session: %v", err)
|
|
}
|
|
if _, err := svc.ResetComponentKeyForSession(ownerSession, domain.ComponentKeyResetRequest{ServerInstanceID: instance.ID, ComponentKind: domain.DistributionComponentClientManager, ComponentKey: "scum-client-manager"}); err != nil {
|
|
t.Fatalf("reset component key: %v", err)
|
|
}
|
|
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil {
|
|
t.Fatal("expected reset to revoke component session")
|
|
}
|
|
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
|
if !view.Installation.RequiresRedeploy || view.Installation.Status != domain.ClientManagerLifecycleFailed {
|
|
t.Fatalf("expected key reset recovery projection, got %+v", view.Installation)
|
|
}
|
|
for _, forbidden := range []string{plain, registration.SessionToken, "secret://", "/Users/", "tcp://"} {
|
|
payload := strings.Join([]string{view.Installation.Phase, view.Installation.HealthReason}, " ")
|
|
if strings.Contains(payload, forbidden) {
|
|
t.Fatalf("safe lifecycle view leaked %q: %s", forbidden, payload)
|
|
}
|
|
}
|
|
}
|
|
|
|
func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution {
|
|
t.Helper()
|
|
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
|
if err != nil {
|
|
t.Fatalf("get lifecycle Run endpoint: %v", err)
|
|
}
|
|
endpoint.LastHeartbeatAt = svc.now()
|
|
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
|
t.Fatalf("refresh lifecycle Run heartbeat: %v", err)
|
|
}
|
|
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
|
if err != nil {
|
|
t.Fatalf("get lifecycle plugin: %v", err)
|
|
}
|
|
for i := range plugin.RuntimeProfiles.ClientManagers {
|
|
if plugin.RuntimeProfiles.ClientManagers[i].Key == "scum-client-manager" {
|
|
plugin.RuntimeProfiles.ClientManagers[i].Version = version
|
|
}
|
|
}
|
|
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
|
t.Fatalf("update lifecycle version: %v", err)
|
|
}
|
|
payload := []byte("client-manager-package-" + version)
|
|
svc.ConfigureDistributionBuilder(staticDistributionBuilder{payload: payload})
|
|
distribution, err := svc.GenerateClientManagerDistributionForSession(session, domain.ClientManagerBuildRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", TargetOS: "linux", TargetArch: "amd64", RepositoryURL: "https://github.com/F88888/scum_client.git", SourceRevision: "main", IdempotencyKey: idempotency})
|
|
if err != nil {
|
|
t.Fatalf("generate lifecycle distribution: %v", err)
|
|
}
|
|
return completeClientDistributionBuild(t, svc, distribution, payload)
|
|
}
|
|
|
|
func registerClientManagerRun(t *testing.T, svc *CoreService) string {
|
|
t.Helper()
|
|
hello := validRunControlHello()
|
|
hello.Platform = "linux"
|
|
hello.Architecture = "amd64"
|
|
hello.CapabilityReport.Capabilities = append(hello.CapabilityReport.Capabilities, domain.JobCapabilityClientManagerDeploy, domain.JobCapabilityClientManagerControl, domain.JobCapabilityClientManagerUpdate, domain.JobCapabilityClientManagerRollback, domain.JobCapabilityClientManagerUninstall)
|
|
result, err := svc.RegisterRunHello(hello)
|
|
if err != nil {
|
|
t.Fatalf("register lifecycle Run: %v", err)
|
|
}
|
|
return result.SessionToken
|
|
}
|
|
|
|
func claimClientManagerJob(t *testing.T, svc *CoreService, sessionToken, capability string) domain.RunJobClaimResult {
|
|
t.Helper()
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capabilities: []string{capability}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
|
if err != nil || !claim.HasJob || claim.Job.Capability != capability {
|
|
t.Fatalf("claim %s job: claim=%+v err=%v", capability, claim, err)
|
|
}
|
|
if _, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Message: "typed lifecycle work started"}); err != nil {
|
|
t.Fatalf("ack lifecycle job: %v", err)
|
|
}
|
|
return claim
|
|
}
|
|
|
|
func completeClientManagerJob(t *testing.T, svc *CoreService, sessionToken string, claim domain.RunJobClaimResult, state domain.JobState, kind, processState string) {
|
|
t.Helper()
|
|
_, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: state, Progress: domain.RunJobProgressReport{Percent: 100, Message: "client-manager lifecycle terminal"}, Message: "client-manager lifecycle terminal", ExecutionResult: domain.JobExecutionResult{Kind: kind, ProcessState: processState, Summary: "bounded lifecycle result"}})
|
|
if err != nil {
|
|
t.Fatalf("complete lifecycle job: %v", err)
|
|
}
|
|
}
|
|
|
|
func currentClientManagerPlainKey(t *testing.T, svc *CoreService, serverID, profileKey string) string {
|
|
t.Helper()
|
|
key, err := svc.activeComponentKey(serverID, domain.DistributionComponentClientManager, profileKey)
|
|
if err != nil {
|
|
t.Fatalf("get active component key: %v", err)
|
|
}
|
|
plain, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
|
if err != nil {
|
|
t.Fatalf("decrypt component key: %v", err)
|
|
}
|
|
return plain
|
|
}
|
|
|
|
func lifecycleRegisterRequest(installation domain.ClientManagerInstallation, capabilities []string, stamp time.Time) domain.ClientManagerRegisterRequest {
|
|
return domain.ClientManagerRegisterRequest{InstallationID: installation.ID, ServerInstanceID: installation.ServerInstanceID, ProfileKey: installation.ProfileKey, ArtifactID: installation.ActiveArtifactID, Version: installation.ActiveVersion, SourceRevision: installation.ActiveRevision, TargetOS: installation.TargetOS, TargetArch: installation.TargetArch, KeyGeneration: installation.KeyGeneration, DeploymentGeneration: installation.DeploymentGeneration, Capabilities: capabilities, Timestamp: stamp, Nonce: "nonce-client-manager-registration"}
|
|
}
|