Make Run runtime state authoritative

2026-08-06 10:07:01 +08:00
parent d8d07914da
commit 0875a1b17e
10 changed files with 324 additions and 12 deletions
@@ -113,6 +113,9 @@ func (svc *CoreService) RegisterRunHello(hello domain.RunControlHello) (domain.R
 	if err := svc.queueManagedGuidedDeploymentAfterRegistration(hello); err != nil {
 		return domain.RunControlHelloResult{}, err
 	}
+	if err := svc.queueRuntimeStateReconciliationAfterRegistration(hello, generation); err != nil {
+		return domain.RunControlHelloResult{}, err
+	}
 	featureFlags := []string{"control.hello", "control.heartbeat", "signed-envelope.v1.optional"}
 	if session.RequireSignedRequests {
 		featureFlags[2] = "signed-envelope.v1.required"
@@ -128,6 +131,73 @@ func (svc *CoreService) RegisterRunHello(hello domain.RunControlHello) (domain.R
 	}), nil
 }

+// queueRuntimeStateReconciliationAfterRegistration lets a generated Run correct
+// stale observed state after reconnecting. Platform still owns command dispatch,
+// but the Run-owned process supervisor is authoritative for whether the local
+// managed process exists.
+func (svc *CoreService) queueRuntimeStateReconciliationAfterRegistration(hello domain.RunControlHello, generation int) error {
+	if hello.ComponentKind != domain.DistributionComponentRun || strings.TrimSpace(hello.ServerInstanceID) == "" {
+		return nil
+	}
+	instance, err := svc.store.ServerInstances().Get(hello.ServerInstanceID)
+	if err != nil {
+		return err
+	}
+	if instance.RunEndpointID != hello.RunEndpointID || instance.RunEndpointID != dedicatedRunEndpointID(instance.ID) {
+		return nil
+	}
+	if instance.State != domain.ServerInstanceStateRunning && instance.State != domain.ServerInstanceStateFailed {
+		return nil
+	}
+	if !containsString(hello.CapabilityReport.Capabilities, domain.LifecycleCapabilityStatus) {
+		return nil
+	}
+	active, err := svc.hasActiveServerLifecycleJob(instance.ID)
+	if err != nil || active {
+		return err
+	}
+	plugin, endpoint, err := svc.lifecycleDependencies(instance.PluginID, instance.RunEndpointID)
+	if err != nil {
+		return err
+	}
+	if err := validateLifecycleActionRef(plugin, domain.ServerLifecycleActionStatus); err != nil {
+		return nil
+	}
+	if err := validateServerInstanceLifecycleDependencies(instance, plugin, endpoint, domain.ServerLifecycleActionStatus); err != nil {
+		return nil
+	}
+	if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityStatus); err != nil {
+		return nil
+	}
+	_, err = svc.dispatchLifecycleJob(instance, domain.ServerLifecycleActionStatus, fmt.Sprintf("runtime-state-reconcile:%s:g%d", instance.ID, generation))
+	return err
+}
+
+func (svc *CoreService) hasActiveServerLifecycleJob(serverInstanceID string) (bool, error) {
+	jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: serverInstanceID})
+	if err != nil {
+		return false, err
+	}
+	for _, job := range jobs {
+		if !isLifecycleJobCapability(job.Capability) {
+			continue
+		}
+		if job.State == domain.JobStateQueued || job.State == domain.JobStateAccepted || job.State == domain.JobStateRunning || job.State == domain.JobStateRetrying {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func isLifecycleJobCapability(capability string) bool {
+	switch capability {
+	case domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus:
+		return true
+	default:
+		return false
+	}
+}
+
 // queueManagedGuidedDeploymentAfterRegistration advances only a newly-created,
 // dedicated guided server. Selecting guided-install is the owner's prior
 // authorization for the plugin-declared bootstrap action; reconnects remain
@@ -443,6 +443,93 @@ func TestCoreServiceGeneratedSCUMRunRegistrationQueuesGuidedStart(t *testing.T)
 	}
 }

+func TestCoreServiceGeneratedRunRegistrationReconcilesStaleRunningState(t *testing.T) {
+	svc := newTestCoreService()
+	plugin := createGeneratedRunStatusPlugin(t, svc)
+	instance := domain.ServerInstance{ID: "managed-stale-running", PluginID: plugin.ID, PluginVersion: plugin.Version, RunEndpointID: dedicatedRunEndpointID("managed-stale-running"), Name: "Managed Stale Running", State: domain.ServerInstanceStateRunning, ConfigVersion: 1, Deployment: domain.ServerDeploymentDefinition{Mode: domain.ServerDeploymentModeGuided, ProfileKey: "run-local", ServerRoot: `D:\scum-stale`, Revision: 1}}
+	if err := svc.store.ServerInstances().Create(instance); err != nil {
+		t.Fatalf("create stale running server: %v", err)
+	}
+	registered := registerGeneratedRunForStatusTest(t, svc, instance, plugin.ID)
+	jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: instance.ID})
+	if err != nil || len(jobs) != 1 || jobs[0].Capability != domain.LifecycleCapabilityStatus || jobs[0].TargetKey != "actions/status.json" {
+		t.Fatalf("expected one status reconciliation job, jobs=%+v err=%v", jobs, err)
+	}
+	claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, Capabilities: []string{domain.LifecycleCapabilityStatus}, Capacity: domain.RunCapacity{MaxJobs: 1}})
+	if err != nil || !claim.HasJob || claim.Job.JobID != jobs[0].ID {
+		t.Fatalf("claim status reconciliation: claim=%+v err=%v", claim, err)
+	}
+	if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "process status queried"}, Message: "process status queried", ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "not-started", ExitClassification: "not-started", AuditSummary: "bounded process state"}}); err != nil {
+		t.Fatalf("complete status reconciliation: %v", err)
+	}
+	stored, err := svc.GetServerInstance(instance.ID)
+	if err != nil || stored.State != domain.ServerInstanceStateStopped {
+		t.Fatalf("Run-reported not-started should correct stale running state, server=%+v err=%v", stored, err)
+	}
+}
+
+func TestCoreServiceGeneratedRunRegistrationSkipsStatusWhenLifecycleJobActive(t *testing.T) {
+	svc := newTestCoreService()
+	plugin := createGeneratedRunStatusPlugin(t, svc)
+	instance := domain.ServerInstance{ID: "managed-active-start", PluginID: plugin.ID, PluginVersion: plugin.Version, RunEndpointID: dedicatedRunEndpointID("managed-active-start"), Name: "Managed Active Start", State: domain.ServerInstanceStateRunning, ConfigVersion: 1, Deployment: domain.ServerDeploymentDefinition{Mode: domain.ServerDeploymentModeGuided, ProfileKey: "run-local", ServerRoot: `D:\scum-active`, Revision: 1}}
+	if err := svc.store.ServerInstances().Create(instance); err != nil {
+		t.Fatalf("create active-start server: %v", err)
+	}
+	if err := svc.store.RunEndpoints().Create(domain.RunEndpoint{ID: instance.RunEndpointID, DisplayName: "Managed Active Start Run", Version: "0.1.0", Platform: "windows", Architecture: "amd64", Status: domain.RunEndpointStatusOnline, Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus}, Capacity: domain.RunCapacity{MaxJobs: 1}, LastHeartbeatAt: fixedTime}); err != nil {
+		t.Fatalf("create active-start endpoint: %v", err)
+	}
+	if _, err := svc.dispatchLifecycleJob(instance, domain.ServerLifecycleActionStart, "already-active-start"); err != nil {
+		t.Fatalf("queue active start job: %v", err)
+	}
+	registerGeneratedRunForStatusTest(t, svc, instance, plugin.ID)
+	jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: instance.ID})
+	if err != nil || len(jobs) != 1 || jobs[0].Capability != domain.LifecycleCapabilityStart {
+		t.Fatalf("active lifecycle job should suppress status reconciliation, jobs=%+v err=%v", jobs, err)
+	}
+}
+
+func createGeneratedRunStatusPlugin(t *testing.T, svc *CoreService) domain.GamePlugin {
+	t.Helper()
+	plugin := scumDeploymentTestPlugin()
+	plugin.Name = "SCUM"
+	plugin.Version = "0.1.1"
+	plugin.ServerType = "scum"
+	plugin.Status = domain.GamePluginStatusInstalled
+	plugin.SupportedOS = []string{"windows"}
+	plugin.RequiredRunCapabilities = []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus}
+	plugin.DeclaredPermissions = []string{"server.run.distribution"}
+	plugin.LifecycleActions = domain.PluginLifecycleActions{Install: "actions/install.json", Start: "actions/start.json", Stop: "actions/stop.json", Status: "actions/status.json"}
+	plugin.RuntimeProfiles.LifecycleProfiles = []domain.RuntimeLifecycleProfile{{Key: "run-local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus}, ActionRefs: domain.PluginLifecycleActions{Install: "actions/install.json", Start: "actions/start.json", Stop: "actions/stop.json", Status: "actions/status.json"}, Platforms: []string{"windows"}}}
+	requireManualSCUMRuntimeBindings(&plugin, "run-local")
+	if err := svc.store.GamePlugins().Create(plugin); err != nil {
+		t.Fatalf("create status plugin fixture: %v", err)
+	}
+	return plugin
+}
+
+func registerGeneratedRunForStatusTest(t *testing.T, svc *CoreService, instance domain.ServerInstance, pluginID string) domain.RunControlHelloResult {
+	t.Helper()
+	key, plainKey, err := svc.ensureActiveComponentKey(instance.ID, domain.DistributionComponentRun, "")
+	if err != nil {
+		t.Fatalf("get generated Run key: %v", err)
+	}
+	hello := validRunControlHello()
+	hello.RunEndpointID = instance.RunEndpointID
+	hello.RegistrationToken = plainKey
+	hello.ServerInstanceID = instance.ID
+	hello.PluginID = pluginID
+	hello.ComponentKind = domain.DistributionComponentRun
+	hello.KeyGeneration = key.Generation
+	hello.Platform = "windows"
+	hello.Architecture = "amd64"
+	hello.CapabilityReport.Capabilities = []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus}
+	registered, err := svc.RegisterRunHello(hello)
+	if err != nil || !registered.Accepted {
+		t.Fatalf("register generated Run: result=%+v err=%v", registered, err)
+	}
+	return registered
+}
+
 func registerDedicatedRunForTest(t *testing.T, svc *CoreService, instance domain.ServerInstance, pluginID string) {
 	t.Helper()
 	key, plainKey, err := svc.ensureActiveComponentKey(instance.ID, domain.DistributionComponentRun, "")
@@ -225,6 +225,7 @@ func TestCoreServiceGuidedPluginLifecycleSuccessDoesNotRequireExecutionReceipt(t
 	if _, err := svc.CompleteRunJob(domain.RunJobResult{
 		RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt,
 		State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "done"}, Message: "done", ResultRef: "artifact://jobs/guided-success/lifecycle-result",
+		ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running", AuditSummary: "bounded process state"},
 	}); err != nil {
 		t.Fatalf("guided plugin lifecycle success without receipt should be terminal: %v", err)
 	}
@@ -50,7 +50,7 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
 		}
 		return svc.recordAuditEvent("run:"+job.RunEndpointID, "config.write.result", "server-instance", instance.ID, domain.AuditResultSuccess, job.ExecutionResult.AuditSummary)
 	}
-	nextState, ok := lifecycleProjectedState(job.Capability, job.State)
+	nextState, ok := lifecycleProjectedState(job.Capability, job.State, job.ExecutionResult)
 	if !ok || job.ServerInstanceID == "" {
 		return nil
 	}
@@ -100,10 +100,16 @@ func (svc *CoreService) projectServerDeploymentProgress(job domain.Job, stamp ti
 	return svc.store.ServerInstances().Update(instance)
 }

-func lifecycleProjectedState(capability string, jobState domain.JobState) (domain.ServerInstanceState, bool) {
-	if capability != domain.LifecycleCapabilityInstall && capability != domain.LifecycleCapabilityStart && capability != domain.LifecycleCapabilityStop {
+func lifecycleProjectedState(capability string, jobState domain.JobState, result domain.JobExecutionResult) (domain.ServerInstanceState, bool) {
+	if capability != domain.LifecycleCapabilityInstall && capability != domain.LifecycleCapabilityStart && capability != domain.LifecycleCapabilityStop && capability != domain.LifecycleCapabilityStatus {
 		return "", false
 	}
+	if capability == domain.LifecycleCapabilityStatus {
+		if jobState != domain.JobStateSucceeded {
+			return "", false
+		}
+		return lifecycleStateFromProcessResult(result)
+	}
 	if jobState == domain.JobStateFailed || jobState == domain.JobStateCancelled {
 		return domain.ServerInstanceStateFailed, true
 	}
@@ -114,10 +120,31 @@ func lifecycleProjectedState(capability string, jobState domain.JobState) (domai
 	case domain.LifecycleCapabilityInstall:
 		return domain.ServerInstanceStateReady, true
 	case domain.LifecycleCapabilityStart:
-		return domain.ServerInstanceStateRunning, true
+		return lifecycleStateFromProcessResult(result)
 	case domain.LifecycleCapabilityStop:
+		if state, ok := lifecycleStateFromProcessResult(result); ok {
+			return state, true
+		}
 		return domain.ServerInstanceStateStopped, true
 	default:
 		return "", false
 	}
 }
+
+func lifecycleStateFromProcessResult(result domain.JobExecutionResult) (domain.ServerInstanceState, bool) {
+	switch strings.TrimSpace(result.ProcessState) {
+	case "running":
+		return domain.ServerInstanceStateRunning, true
+	case "stopped", "not-started":
+		return domain.ServerInstanceStateStopped, true
+	case "exited":
+		switch strings.TrimSpace(result.ExitClassification) {
+		case "requested-stop", "forced-stop", "already-stopped":
+			return domain.ServerInstanceStateStopped, true
+		default:
+			return domain.ServerInstanceStateFailed, true
+		}
+	default:
+		return "", false
+	}
+}
@@ -88,6 +88,21 @@ func TestCoreServiceServerLifecycleWorkflows(t *testing.T) {
 	}
 }

+func TestLifecycleProjectedStateUsesRunProcessFacts(t *testing.T) {
+	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStart, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "stopped"}); !ok || state == domain.ServerInstanceStateRunning {
+		t.Fatalf("start success without running process fact must not mark running, state=%q ok=%v", state, ok)
+	}
+	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStatus, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "not-started"}); !ok || state != domain.ServerInstanceStateStopped {
+		t.Fatalf("status not-started should project stopped, state=%q ok=%v", state, ok)
+	}
+	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStatus, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "exited", ExitClassification: "unexpected-exit"}); !ok || state != domain.ServerInstanceStateFailed {
+		t.Fatalf("unexpected exit should project failed, state=%q ok=%v", state, ok)
+	}
+	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStop, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "not-started"}); !ok || state != domain.ServerInstanceStateStopped {
+		t.Fatalf("stop not-started should project stopped, state=%q ok=%v", state, ok)
+	}
+}
+
 func TestCoreServiceCreatesTargetBoundDraftAndRequiresDedicatedRunRegistration(t *testing.T) {
 	svc, _ := newLifecycleRunService(t)
 	plugin := createLifecyclePlugin(t, svc)
@@ -430,15 +445,25 @@ func claimAndCompleteLifecycleJobForServer(t *testing.T, svc *CoreService, sessi
 	if serverInstanceID != "" && claim.Job.ServerInstanceID != serverInstanceID {
 		t.Fatalf("expected claimed lifecycle job for %s, got %+v", serverInstanceID, claim.Job)
 	}
+	executionResult := domain.JobExecutionResult{}
+	if state == domain.JobStateSucceeded {
+		switch capability {
+		case domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStatus:
+			executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "running", AuditSummary: "bounded process state"}
+		case domain.LifecycleCapabilityStop:
+			executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop", AuditSummary: "bounded process state"}
+		}
+	}
 	if _, err := svc.CompleteRunJob(domain.RunJobResult{
-		RunEndpointID: "run-local",
-		SessionToken:  sessionToken,
-		JobID:         claim.Job.JobID,
-		LeaseToken:    claim.Job.LeaseToken,
-		Attempt:       claim.Job.Attempt,
-		State:         state,
-		Progress:      domain.RunJobProgressReport{Percent: 100, Message: string(state)},
-		Message:       string(state),
+		RunEndpointID:   "run-local",
+		SessionToken:    sessionToken,
+		JobID:           claim.Job.JobID,
+		LeaseToken:      claim.Job.LeaseToken,
+		Attempt:         claim.Job.Attempt,
+		State:           state,
+		Progress:        domain.RunJobProgressReport{Percent: 100, Message: string(state)},
+		Message:         string(state),
+		ExecutionResult: executionResult,
 	}); err != nil {
 		t.Fatalf("complete lifecycle job %s: %v", capability, err)
 	}