browser/platform/service/server_lifecycle_test.go

package service

import (
	"strings"
	"testing"
	"time"

	"browser.local/platform/domain"
)

func TestCoreServiceServerLifecycleWorkflows(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)

	created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{
		ID:             "server-1",
		PluginID:       "server.scum",
		RunEndpointID:  "run-local",
		Name:           "SCUM #1",
		IdempotencyKey: "idem-create",
		ProfileKey:     "local",
	})
	if err != nil {
		t.Fatalf("create lifecycle workflow: %v", err)
	}
	if created.Action != domain.ServerLifecycleActionCreate || created.Instance.State != domain.ServerInstanceStateInstalling || created.Job.Capability != domain.LifecycleCapabilityInstall {
		t.Fatalf("expected install workflow result, got %+v", created)
	}
	if created.Job.ExecutionInput.PluginID != "server.scum" || created.Job.ExecutionInput.WorkspaceScope != "local" || created.Job.ExecutionInput.LifecycleOperation != "install" {
		t.Fatalf("expected install job to carry plugin/profile metadata, got %+v", created.Job.ExecutionInput)
	}

	claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, err := svc.GetServerInstance("server-1")
	if err != nil {
		t.Fatalf("get ready instance: %v", err)
	}
	if ready.State != domain.ServerInstanceStateReady {
		t.Fatalf("expected install result to mark ready, got %+v", ready)
	}

	started, err := svc.StartServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      "server-1",
		ExpectedConfigVersion: ready.ConfigVersion,
		IdempotencyKey:        "idem-start",
	})
	if err != nil {
		t.Fatalf("start lifecycle workflow: %v", err)
	}
	if started.Action != domain.ServerLifecycleActionStart || started.Job.Capability != domain.LifecycleCapabilityStart {
		t.Fatalf("expected start workflow result, got %+v", started)
	}
	if started.Job.ExecutionInput.PluginID != "server.scum" || started.Job.ExecutionInput.WorkspaceScope != "local" || started.Job.ExecutionInput.LifecycleOperation != "start" {
		t.Fatalf("expected start job to carry plugin/profile metadata, got %+v", started.Job.ExecutionInput)
	}
	if len(started.Job.ExecutionInput.LogSources) != 2 || started.Job.ExecutionInput.LogSources[0].StreamKey != "scum.console.stdout" || started.Job.ExecutionInput.LogSources[1].StreamKey != "scum.console.stderr" {
		t.Fatalf("expected start job to carry plugin-declared process log sources, got %+v", started.Job.ExecutionInput.LogSources)
	}
	claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	running, err := svc.GetServerInstance("server-1")
	if err != nil {
		t.Fatalf("get running instance: %v", err)
	}
	if running.State != domain.ServerInstanceStateRunning {
		t.Fatalf("expected start result to mark running, got %+v", running)
	}

	stopped, err := svc.StopServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      "server-1",
		ExpectedConfigVersion: running.ConfigVersion,
		IdempotencyKey:        "idem-stop",
	})
	if err != nil {
		t.Fatalf("stop lifecycle workflow: %v", err)
	}
	if stopped.Action != domain.ServerLifecycleActionStop || stopped.Job.Capability != domain.LifecycleCapabilityStop {
		t.Fatalf("expected stop workflow result, got %+v", stopped)
	}
	if stopped.Job.ExecutionInput.PluginID != "server.scum" || stopped.Job.ExecutionInput.WorkspaceScope != "local" || stopped.Job.ExecutionInput.LifecycleOperation != "stop" {
		t.Fatalf("expected stop job to carry plugin/profile metadata, got %+v", stopped.Job.ExecutionInput)
	}
	claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)
	final, err := svc.GetServerInstance("server-1")
	if err != nil {
		t.Fatalf("get stopped instance: %v", err)
	}
	if final.State != domain.ServerInstanceStateStopped {
		t.Fatalf("expected stop result to mark stopped, got %+v", final)
	}
}

func TestLifecycleProjectedStateUsesRunProcessFacts(t *testing.T) {
	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStart, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "stopped"}); !ok || state == domain.ServerInstanceStateRunning {
		t.Fatalf("start success without running process fact must not mark running, state=%q ok=%v", state, ok)
	}
	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStatus, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "not-started"}); !ok || state != domain.ServerInstanceStateStopped {
		t.Fatalf("status not-started should project stopped, state=%q ok=%v", state, ok)
	}
	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStatus, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "exited", ExitClassification: "unexpected-exit"}); !ok || state != domain.ServerInstanceStateFailed {
		t.Fatalf("unexpected exit should project failed, state=%q ok=%v", state, ok)
	}
	if state, ok := lifecycleProjectedState(domain.LifecycleCapabilityStop, domain.JobStateSucceeded, domain.JobExecutionResult{Kind: "process", ProcessState: "not-started"}); !ok || state != domain.ServerInstanceStateStopped {
		t.Fatalf("stop not-started should project stopped, state=%q ok=%v", state, ok)
	}
}

func TestLifecycleJobResultsPublishProcessStateEvents(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-state-events", PluginID: "server.scum", RunEndpointID: "run-local", Name: "State Events", IdempotencyKey: "state-events-create", ProfileKey: "local"})
	if err != nil {
		t.Fatalf("create lifecycle server: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, created.Instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, err := svc.GetServerInstance(created.Instance.ID)
	if err != nil {
		t.Fatalf("get ready server: %v", err)
	}
	if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "state-events-start"}); err != nil {
		t.Fatalf("dispatch start: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, ready.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)

	subscription, err := svc.SubscribeLogEvents(ready.ID)
	if err != nil {
		t.Fatalf("subscribe state events: %v", err)
	}
	defer subscription.Close()
	running, err := svc.GetServerInstance(ready.ID)
	if err != nil {
		t.Fatalf("get running server: %v", err)
	}
	if _, err := svc.StopServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: running.ID, ExpectedConfigVersion: running.ConfigVersion, IdempotencyKey: "state-events-stop"}); err != nil {
		t.Fatalf("dispatch stop: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, running.ID, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)
	assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateStopped)

	stopped, err := svc.GetServerInstance(running.ID)
	if err != nil {
		t.Fatalf("get stopped server: %v", err)
	}
	if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: stopped.ID, ExpectedConfigVersion: stopped.ConfigVersion, IdempotencyKey: "state-events-restart"}); err != nil {
		t.Fatalf("dispatch restart: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, stopped.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateRunning)
}

func assertLogProcessStateEvent(t *testing.T, subscription LogEventSubscription, want domain.ServerInstanceState) {
	t.Helper()
	select {
	case event := <-subscription.Events:
		if event.Kind != LogEventSubscriptionEventProcessState || event.ProcessState != want {
			t.Fatalf("unexpected process state event: %+v", event)
		}
	case <-time.After(time.Second):
		t.Fatalf("expected process state event %q", want)
	}
}

func TestCoreServiceCreatesUnboundDraftAndAttachesRunFromHeartbeat(t *testing.T) {
	svc, _ := newLifecycleRunService(t)
	plugin := createLifecyclePlugin(t, svc)

	draft, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-dedicated", PluginID: plugin.ID, Name: "Dedicated SCUM", IdempotencyKey: "dedicated-draft"})
	if err != nil {
		t.Fatalf("create target-bound draft: %v", err)
	}
	if draft.Instance.State != domain.ServerInstanceStateDraft || draft.Job.ID != "" || draft.Instance.RunEndpointID != "" {
		t.Fatalf("expected draft without a reserved Run identity, got %+v", draft)
	}

	key, plainKey, err := svc.ensureActiveComponentKey(draft.Instance.ID, domain.DistributionComponentRun, "")
	if err != nil {
		t.Fatalf("create Run key: %v", err)
	}
	hello := validRunControlHello()
	hello.ServerInstanceID = draft.Instance.ID
	hello.PluginID = plugin.ID
	hello.ComponentKind = domain.DistributionComponentRun
	hello.KeyGeneration = key.Generation
	hello.RegistrationToken = plainKey
	hello.RunEndpointID = "run-local"
	hello.DisplayName = "Automatic SCUM Run"
	if registered, err := svc.RegisterRunHello(hello); err != nil || !registered.Accepted {
		t.Fatalf("register automatic Run heartbeat: result=%+v err=%v", registered, err)
	}
	attached, err := svc.GetServerInstance(draft.Instance.ID)
	if err != nil || attached.RunEndpointID != "run-local" {
		t.Fatalf("expected heartbeat to attach Run endpoint, instance=%+v err=%v", attached, err)
	}
}

func TestCoreServiceFreezesReadyDLLExtensionIntoWindowsStartJob(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	setLifecycleEndpointTarget(t, svc, "windows", "amd64")
	plugin := createLifecyclePlugin(t, svc)
	attachReadyLifecycleDLLExtension(t, svc, &plugin)

	created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-dll", PluginID: plugin.ID, RunEndpointID: "run-local", Name: "SCUM DLL", IdempotencyKey: "idem-dll-create", ProfileKey: "local"})
	if err != nil {
		t.Fatalf("create DLL lifecycle server: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, created.Instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, err := svc.GetServerInstance(created.Instance.ID)
	if err != nil {
		t.Fatalf("get ready DLL server: %v", err)
	}

	started, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "idem-dll-start"})
	if err != nil {
		t.Fatalf("start DLL lifecycle server: %v", err)
	}
	if len(started.Job.ExecutionInput.DLLExtensions) != 1 {
		t.Fatalf("expected one frozen DLL plan, got %+v", started.Job.ExecutionInput)
	}
	if got := started.Job.ExecutionInput.DLLExtensions[0]; got.Key != "scum-simple-rcon" || got.Checksum != "sha256:"+strings.Repeat("a", 64) || got.ReleaseURL != "https://cdn.npc0.com/scum_simple_rcon_ue4s.dll" {
		t.Fatalf("unexpected frozen DLL plan: %+v", got)
	}

	plugin.RuntimeProfiles.DLLExtensions[0].Checksum = "sha256:" + strings.Repeat("c", 64)
	if err := svc.store.GamePlugins().Update(plugin); err != nil {
		t.Fatalf("update plugin after start fence: %v", err)
	}
	claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capabilities: []string{domain.LifecycleCapabilityStart}, Capacity: domain.RunCapacity{MaxJobs: 1}})
	if err != nil || !claim.HasJob || len(claim.Job.ExecutionInput.DLLExtensions) != 1 {
		t.Fatalf("claim frozen start job: claim=%+v err=%v", claim, err)
	}
	if got := claim.Job.ExecutionInput.DLLExtensions[0].Checksum; got != "sha256:"+strings.Repeat("a", 64) {
		t.Fatalf("queued start job was not fenced to the original DLL pin: %s", got)
	}
}

func TestCoreServiceRejectsLinuxDLLExtensionStartBeforeDispatch(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	setLifecycleEndpointTarget(t, svc, "windows", "amd64")
	plugin := createLifecyclePlugin(t, svc)
	attachReadyLifecycleDLLExtension(t, svc, &plugin)
	created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-dll-linux", PluginID: plugin.ID, RunEndpointID: "run-local", Name: "SCUM DLL Linux", IdempotencyKey: "idem-dll-linux-create", ProfileKey: "local"})
	if err != nil {
		t.Fatalf("create DLL lifecycle server: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, created.Instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, _ := svc.GetServerInstance(created.Instance.ID)
	setLifecycleEndpointTarget(t, svc, "linux", "amd64")

	_, err = svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "idem-dll-linux-start"})
	if err == nil || !strings.Contains(err.Error(), "unsupported_extension_platform") {
		t.Fatalf("expected explicit Linux DLL rejection, got %v", err)
	}
}

func TestCoreServiceServerLifecycleRejectsInvalidCommands(t *testing.T) {
	svc := newTestCoreService()
	plugin, endpoint := createPluginAndRunEndpoint(t, svc)
	instance, err := svc.CreateServerInstance(domain.ServerInstance{
		ID:            "server-ready",
		PluginID:      plugin.ID,
		RunEndpointID: endpoint.ID,
		Name:          "Ready Server",
		State:         domain.ServerInstanceStateReady,
	})
	if err != nil {
		t.Fatalf("create ready server: %v", err)
	}
	createCompleteRuntimeBinding(t, svc, instance, "local")

	_, err = svc.StartServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      instance.ID,
		ExpectedConfigVersion: instance.ConfigVersion + 1,
		IdempotencyKey:        "idem-stale",
	})
	if err == nil || !strings.Contains(err.Error(), "expectedConfigVersion") {
		t.Fatalf("expected stale config rejection, got %v", err)
	}

	_, err = svc.StopServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      instance.ID,
		ExpectedConfigVersion: instance.ConfigVersion,
		IdempotencyKey:        "idem-stop-invalid",
	})
	if err == nil || !strings.Contains(err.Error(), "cannot stop") {
		t.Fatalf("expected invalid stop state rejection, got %v", err)
	}

	weakEndpoint := endpoint
	weakEndpoint.ID = "run-no-stop"
	weakEndpoint.Capabilities = []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, "logs.read"}
	if _, err := svc.CreateRunEndpoint(weakEndpoint); err != nil {
		t.Fatalf("create weak endpoint: %v", err)
	}
	running, err := svc.CreateServerInstance(domain.ServerInstance{
		ID:            "server-running",
		PluginID:      plugin.ID,
		RunEndpointID: weakEndpoint.ID,
		Name:          "Running Server",
		State:         domain.ServerInstanceStateRunning,
	})
	if err == nil {
		createCompleteRuntimeBinding(t, svc, running, "local")
		_, err = svc.StopServerInstance(domain.ServerLifecycleCommand{
			ServerInstanceID:      running.ID,
			ExpectedConfigVersion: running.ConfigVersion,
			IdempotencyKey:        "idem-stop-missing-capability",
		})
	}
	if err == nil || !strings.Contains(err.Error(), domain.LifecycleCapabilityStop) {
		t.Fatalf("expected missing stop capability rejection, got %v", err)
	}
}

func TestCoreServiceLifecycleFailureProjectsFailedState(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	if _, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{
		ID:             "server-1",
		PluginID:       "server.scum",
		RunEndpointID:  "run-local",
		Name:           "SCUM #1",
		IdempotencyKey: "idem-create",
		ProfileKey:     "local",
	}); err != nil {
		t.Fatalf("create lifecycle workflow: %v", err)
	}

	claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityInstall, domain.JobStateFailed)
	instance, err := svc.GetServerInstance("server-1")
	if err != nil {
		t.Fatalf("get failed instance: %v", err)
	}
	if instance.State != domain.ServerInstanceStateFailed {
		t.Fatalf("expected failed install result to mark failed, got %+v", instance)
	}

	started, err := svc.StartServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      instance.ID,
		ExpectedConfigVersion: instance.ConfigVersion,
		IdempotencyKey:        "idem-start-after-failure",
	})
	if err != nil || started.Job.Capability != domain.LifecycleCapabilityStart || started.Job.TargetKey != "actions/start.json" {
		t.Fatalf("failed server should allow explicit restart dispatch, result=%+v err=%v", started, err)
	}
	claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	recovered, err := svc.GetServerInstance("server-1")
	if err != nil {
		t.Fatalf("get recovered instance: %v", err)
	}
	if recovered.State != domain.ServerInstanceStateRunning {
		t.Fatalf("expected explicit restart to recover failed instance, got %+v", recovered)
	}
}

func TestCoreServicePluginLifecycleManagesMultipleInstancesIndependently(t *testing.T) {
	svc, sessionToken := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)

	for _, id := range []string{"server-alpha", "server-beta"} {
		if _, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{
			ID:             id,
			PluginID:       "server.scum",
			RunEndpointID:  "run-local",
			Name:           id,
			IdempotencyKey: "idem-create-" + id,
			ProfileKey:     "local",
		}); err != nil {
			t.Fatalf("create %s: %v", id, err)
		}
		claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, id, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	}

	alpha, err := svc.GetServerInstance("server-alpha")
	if err != nil {
		t.Fatalf("get alpha: %v", err)
	}
	beta, err := svc.GetServerInstance("server-beta")
	if err != nil {
		t.Fatalf("get beta: %v", err)
	}
	if alpha.State != domain.ServerInstanceStateReady || beta.State != domain.ServerInstanceStateReady || alpha.ID == beta.ID || alpha.PluginID != beta.PluginID {
		t.Fatalf("expected distinct ready sibling instances, alpha=%+v beta=%+v", alpha, beta)
	}

	if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      alpha.ID,
		ExpectedConfigVersion: alpha.ConfigVersion,
		IdempotencyKey:        "idem-start-alpha",
	}); err != nil {
		t.Fatalf("start alpha: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, alpha.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)

	alpha, _ = svc.GetServerInstance("server-alpha")
	beta, _ = svc.GetServerInstance("server-beta")
	if alpha.State != domain.ServerInstanceStateRunning || beta.State != domain.ServerInstanceStateReady {
		t.Fatalf("expected alpha running and beta unchanged, alpha=%+v beta=%+v", alpha, beta)
	}

	if _, err := svc.StopServerInstance(domain.ServerLifecycleCommand{
		ServerInstanceID:      alpha.ID,
		ExpectedConfigVersion: alpha.ConfigVersion,
		IdempotencyKey:        "idem-stop-alpha",
	}); err != nil {
		t.Fatalf("stop alpha: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, alpha.ID, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)

	alpha, _ = svc.GetServerInstance("server-alpha")
	beta, _ = svc.GetServerInstance("server-beta")
	if alpha.State != domain.ServerInstanceStateStopped || beta.State != domain.ServerInstanceStateReady {
		t.Fatalf("expected alpha stopped and beta still unchanged, alpha=%+v beta=%+v", alpha, beta)
	}
}

func newLifecycleRunService(t *testing.T) (*CoreService, string) {
	t.Helper()
	svc := newTestCoreService()
	helloRequest := validRunControlHello()
	helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities,
		domain.LifecycleCapabilityInstall,
		domain.LifecycleCapabilityStart,
		domain.LifecycleCapabilityStop,
		"logs.read",
		"files.read",
	)
	helloRequest.CapabilityReport.Fingerprint = "cap-lifecycle"
	hello, err := svc.RegisterRunHello(helloRequest)
	if err != nil {
		t.Fatalf("register run hello: %v", err)
	}
	return svc, hello.SessionToken
}

func createLifecyclePlugin(t *testing.T, svc *CoreService) domain.GamePlugin {
	t.Helper()
	plugin, err := svc.CreateGamePlugin(domain.GamePlugin{
		ID:                      "server.scum",
		Name:                    "SCUM",
		Version:                 "1.0.0",
		ServerType:              "scum",
		ManifestRef:             "artifact://manifests/server.scum/1.0.0",
		CreateFormSchemaRef:     "artifact://schemas/server.scum/create-form/1.0.0",
		RequiredRunCapabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, "logs.read"},
		LifecycleActions: domain.PluginLifecycleActions{
			Install: "actions/install.json",
			Start:   "actions/start.json",
			Stop:    "actions/stop.json",
		},
		Permissions: domain.PluginPermissions{Jobs: true, Logs: true},
		RuntimeProfiles: domain.GamePluginRuntimeProfiles{
			LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop}}},
			LogSources:        []domain.RuntimeLogSource{{Key: "scum-console-stdout", Kind: "process.stdout", TargetKey: "scum/server-process", StreamKey: "scum.console.stdout", CursorKind: "sequence", RetentionDays: 30}, {Key: "scum-console-stderr", Kind: "process.stderr", TargetKey: "scum/server-process", StreamKey: "scum.console.stderr", CursorKind: "sequence", RetentionDays: 30}},
		},
	})
	if err != nil {
		t.Fatalf("create lifecycle plugin: %v", err)
	}
	return plugin
}

func attachReadyLifecycleDLLExtension(t *testing.T, svc *CoreService, plugin *domain.GamePlugin) {
	t.Helper()
	plugin.RuntimeProfiles.LifecycleProfiles[0].Platforms = []string{"windows"}
	plugin.RuntimeProfiles.LifecycleProfiles[0].DLLExtensionRefs = []string{"scum-simple-rcon"}
	plugin.RuntimeProfiles.DLLExtensions = []domain.RuntimeDLLExtensionProfile{{
		Key: "scum-simple-rcon", DisplayName: "SCUM Simple RCON", Kind: "ue4ss-dll", Activation: "server-start", Version: "0.1.0", ReleaseState: "ready",
		ReleaseURL: "https://cdn.npc0.com/scum_simple_rcon_ue4s.dll", Checksum: "sha256:" + strings.Repeat("a", 64), SizeBytes: 1024,
		TargetKey: "ue4ss/scum-simple-rcon", ModKey: "scum_simple_rcon", DLLRef: "ue4ss/Mods/scum_simple_rcon/dlls/main.dll",
		TargetExecutableChecksum: "sha256:" + strings.Repeat("b", 64), UE4SSABI: "ue4ss-3.0", SupportedTargets: []domain.RuntimeTarget{{OS: "windows", Arch: "amd64"}}, UpdateOnStart: true, RCONPort: 27015,
	}}
	if err := svc.store.GamePlugins().Update(*plugin); err != nil {
		t.Fatalf("attach ready DLL extension: %v", err)
	}
}

func setLifecycleEndpointTarget(t *testing.T, svc *CoreService, platform string, architecture string) {
	t.Helper()
	endpoint, err := svc.store.RunEndpoints().Get("run-local")
	if err != nil {
		t.Fatalf("get lifecycle endpoint: %v", err)
	}
	endpoint.Platform = platform
	endpoint.Architecture = architecture
	if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
		t.Fatalf("set lifecycle endpoint target: %v", err)
	}
}

func claimAndCompleteLifecycleJob(t *testing.T, svc *CoreService, sessionToken string, capability string, state domain.JobState) {
	t.Helper()
	claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, "", capability, state)
}

func claimAndCompleteLifecycleJobForServer(t *testing.T, svc *CoreService, sessionToken string, serverInstanceID string, capability string, state domain.JobState) {
	t.Helper()
	claim, err := svc.ClaimRunJob(domain.RunJobClaim{
		RunEndpointID: "run-local",
		SessionToken:  sessionToken,
		Capabilities:  []string{capability},
		Capacity:      domain.RunCapacity{MaxJobs: 4},
	})
	if err != nil {
		t.Fatalf("claim lifecycle job %s: %v", capability, err)
	}
	if !claim.HasJob || claim.Job.Capability != capability {
		t.Fatalf("expected claimed lifecycle job %s, got %+v", capability, claim)
	}
	if claim.Job.ExecutionInput.PluginID == "" || claim.Job.ExecutionInput.WorkspaceScope == "" {
		t.Fatalf("expected lifecycle claim to carry plugin/profile metadata, got %+v", claim.Job.ExecutionInput)
	}
	if serverInstanceID != "" && claim.Job.ServerInstanceID != serverInstanceID {
		t.Fatalf("expected claimed lifecycle job for %s, got %+v", serverInstanceID, claim.Job)
	}
	executionResult := domain.JobExecutionResult{}
	if state == domain.JobStateSucceeded {
		switch capability {
		case domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStatus:
			executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "running", Summary: "bounded process state"}
		case domain.LifecycleCapabilityStop:
			executionResult = domain.JobExecutionResult{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop", Summary: "bounded process state"}
		}
	}
	if _, err := svc.CompleteRunJob(domain.RunJobResult{
		RunEndpointID:   "run-local",
		SessionToken:    sessionToken,
		JobID:           claim.Job.JobID,
		LeaseToken:      claim.Job.LeaseToken,
		Attempt:         claim.Job.Attempt,
		State:           state,
		Progress:        domain.RunJobProgressReport{Percent: 100, Message: string(state)},
		Message:         string(state),
		ExecutionResult: executionResult,
	}); err != nil {
		t.Fatalf("complete lifecycle job %s: %v", capability, err)
	}
}

func TestCoreServiceRestartLifecycleQueuesDeclaredStopThenStart(t *testing.T) {
	svc, runSession := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	operator := createServiceUserAndLogin(t, svc, domain.User{ID: "lifecycle-operator", DisplayName: "Lifecycle Operator", Email: "lifecycle-operator@example.test", Roles: []string{"platform-admin"}, PasswordHash: "secret-password"})
	instance := runningLifecycleInstanceForTest(t, svc, runSession, "server-restart")

	restarted, err := svc.RestartServerInstanceForSession(operator, domain.ServerLifecycleCommand{
		ServerInstanceID:      instance.ID,
		ExpectedConfigVersion: instance.ConfigVersion,
		IdempotencyKey:        "idem-restart",
	})
	if err != nil {
		t.Fatalf("restart lifecycle workflow: %v", err)
	}
	if restarted.Action != domain.ServerLifecycleActionRestart || restarted.Job.Capability != domain.LifecycleCapabilityStop || restarted.Job.TargetKey != "actions/stop.json" {
		t.Fatalf("expected restart to queue the plugin-declared stop action, got %+v", restarted)
	}
	if restarted.Job.ExecutionInput.LifecycleOperation != string(domain.ServerLifecycleActionRestart) {
		t.Fatalf("expected restart intent on the queued job, got %+v", restarted.Job.ExecutionInput)
	}
	if _, ok := queuedLifecycleJob(t, svc, instance.ID, domain.LifecycleCapabilityStart); ok {
		t.Fatal("restart must not queue a start job before the declared stop result lands")
	}

	claimAndCompleteLifecycleJobForServer(t, svc, runSession, instance.ID, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)
	stopped, err := svc.GetServerInstance(instance.ID)
	if err != nil {
		t.Fatalf("get stopped instance: %v", err)
	}
	if stopped.State != domain.ServerInstanceStateStopped {
		t.Fatalf("expected graceful stop result to project stopped, got %+v", stopped)
	}

	startJob, ok := queuedLifecycleJob(t, svc, instance.ID, domain.LifecycleCapabilityStart)
	if !ok {
		t.Fatal("expected a follow-up start job after the graceful stop succeeded")
	}
	if startJob.TargetKey != "actions/start.json" || startJob.ExecutionInput.LifecycleOperation != string(domain.ServerLifecycleActionStart) || startJob.IdempotencyKey != "idem-restart:start" {
		t.Fatalf("unexpected follow-up start job: %+v", startJob)
	}

	claimAndCompleteLifecycleJobForServer(t, svc, runSession, instance.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	running, err := svc.GetServerInstance(instance.ID)
	if err != nil {
		t.Fatalf("get restarted instance: %v", err)
	}
	if running.State != domain.ServerInstanceStateRunning {
		t.Fatalf("expected restart to return the server to running, got %+v", running)
	}
}

func TestCoreServiceUpdateLifecycleQueuesInstallThenStart(t *testing.T) {
	svc, runSession := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	operator := createServiceUserAndLogin(t, svc, domain.User{ID: "update-operator", DisplayName: "Update Operator", Email: "update-operator@example.test", Roles: []string{"platform-admin"}, PasswordHash: "secret-password"})
	instance := runningLifecycleInstanceForTest(t, svc, runSession, "server-update")

	updated, err := svc.UpdateServerGameForSession(operator, domain.ServerLifecycleCommand{
		ServerInstanceID:      instance.ID,
		ExpectedConfigVersion: instance.ConfigVersion,
		IdempotencyKey:        "idem-update",
	})
	if err != nil {
		t.Fatalf("update lifecycle workflow: %v", err)
	}
	if updated.Action != domain.ServerLifecycleActionUpdate || updated.Job.Capability != domain.LifecycleCapabilityInstall || updated.Job.TargetKey != "actions/install.json" {
		t.Fatalf("expected update to queue the plugin-declared install action, got %+v", updated)
	}
	if updated.Job.ExecutionInput.LifecycleOperation != string(domain.ServerLifecycleActionUpdate) {
		t.Fatalf("expected update intent on the queued job, got %+v", updated.Job.ExecutionInput)
	}

	claimAndCompleteLifecycleJobForServer(t, svc, runSession, instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	startJob, ok := queuedLifecycleJob(t, svc, instance.ID, domain.LifecycleCapabilityStart)
	if !ok {
		t.Fatal("expected a follow-up start job after the game update succeeded")
	}
	if startJob.IdempotencyKey != "idem-update:start" {
		t.Fatalf("unexpected follow-up start job: %+v", startJob)
	}

	claimAndCompleteLifecycleJobForServer(t, svc, runSession, instance.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	running, err := svc.GetServerInstance(instance.ID)
	if err != nil {
		t.Fatalf("get updated instance: %v", err)
	}
	if running.State != domain.ServerInstanceStateRunning {
		t.Fatalf("expected update to start the server again, got %+v", running)
	}
}

func TestCoreServiceRestartAndUpdateRespectLifecycleState(t *testing.T) {
	svc, runSession := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	operator := createServiceUserAndLogin(t, svc, domain.User{ID: "state-operator", DisplayName: "State Operator", Email: "state-operator@example.test", Roles: []string{"platform-admin"}, PasswordHash: "secret-password"})
	if _, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-installing", PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM installing", IdempotencyKey: "installing-create", ProfileKey: "local"}); err != nil {
		t.Fatalf("create installing instance: %v", err)
	}
	installing, err := svc.GetServerInstance("server-installing")
	if err != nil {
		t.Fatalf("get installing instance: %v", err)
	}
	if _, err := svc.RestartServerInstanceForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: installing.ID, ExpectedConfigVersion: installing.ConfigVersion, IdempotencyKey: "idem-restart-installing"}); err == nil || !strings.Contains(err.Error(), "cannot restart") {
		t.Fatalf("expected restart state rejection, got %v", err)
	}
	if _, err := svc.UpdateServerGameForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: installing.ID, ExpectedConfigVersion: installing.ConfigVersion, IdempotencyKey: "idem-update-installing"}); err == nil || !strings.Contains(err.Error(), "cannot update") {
		t.Fatalf("expected update state rejection, got %v", err)
	}

	claimAndCompleteLifecycleJobForServer(t, svc, runSession, installing.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, err := svc.GetServerInstance(installing.ID)
	if err != nil {
		t.Fatalf("get ready instance: %v", err)
	}
	if _, err := svc.RestartServerInstanceForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "idem-restart-ready"}); err == nil || !strings.Contains(err.Error(), "cannot restart") {
		t.Fatalf("expected ready state restart rejection, got %v", err)
	}
	if _, err := svc.UpdateServerGameForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "idem-update-ready"}); err != nil {
		t.Fatalf("ready server should accept a game update dispatch: %v", err)
	}
}

func TestCoreServiceFailedGracefulStopDoesNotStartServer(t *testing.T) {
	svc, runSession := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	operator := createServiceUserAndLogin(t, svc, domain.User{ID: "failure-operator", DisplayName: "Failure Operator", Email: "failure-operator@example.test", Roles: []string{"platform-admin"}, PasswordHash: "secret-password"})
	instance := runningLifecycleInstanceForTest(t, svc, runSession, "server-restart-failure")

	if _, err := svc.RestartServerInstanceForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "idem-restart-failure"}); err != nil {
		t.Fatalf("restart lifecycle workflow: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, runSession, instance.ID, domain.LifecycleCapabilityStop, domain.JobStateFailed)
	if _, ok := queuedLifecycleJob(t, svc, instance.ID, domain.LifecycleCapabilityStart); ok {
		t.Fatal("a failed graceful stop must not queue a start job")
	}
	failed, err := svc.GetServerInstance(instance.ID)
	if err != nil {
		t.Fatalf("get failed instance: %v", err)
	}
	if failed.State != domain.ServerInstanceStateFailed {
		t.Fatalf("expected failed stop to project failed, got %+v", failed)
	}
}

func TestCoreServiceRestartAndUpdateRequireEndpointCapabilities(t *testing.T) {
	svc, runSession := newLifecycleRunService(t)
	createLifecyclePlugin(t, svc)
	operator := createServiceUserAndLogin(t, svc, domain.User{ID: "capability-operator", DisplayName: "Capability Operator", Email: "capability-operator@example.test", Roles: []string{"platform-admin"}, PasswordHash: "secret-password"})
	instance := runningLifecycleInstanceForTest(t, svc, runSession, "server-capability")

	endpoint, err := svc.store.RunEndpoints().Get("run-local")
	if err != nil {
		t.Fatalf("get lifecycle endpoint: %v", err)
	}
	remaining := make([]string, 0, len(endpoint.Capabilities))
	for _, capability := range endpoint.Capabilities {
		if capability != domain.LifecycleCapabilityStop && capability != domain.LifecycleCapabilityInstall {
			remaining = append(remaining, capability)
		}
	}
	endpoint.Capabilities = remaining
	if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
		t.Fatalf("remove lifecycle capabilities: %v", err)
	}

	if _, err := svc.RestartServerInstanceForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "idem-restart-capability"}); err == nil || !strings.Contains(err.Error(), domain.LifecycleCapabilityStop) {
		t.Fatalf("expected missing stop capability rejection, got %v", err)
	}
	if _, err := svc.UpdateServerGameForSession(operator, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "idem-update-capability"}); err == nil || !strings.Contains(err.Error(), domain.LifecycleCapabilityInstall) {
		t.Fatalf("expected missing install capability rejection, got %v", err)
	}
}

func runningLifecycleInstanceForTest(t *testing.T, svc *CoreService, runSession string, id string) domain.ServerInstance {
	t.Helper()
	if _, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: id, PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM " + id, IdempotencyKey: id + "-create", ProfileKey: "local"}); err != nil {
		t.Fatalf("create lifecycle workflow: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, runSession, id, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
	ready, err := svc.GetServerInstance(id)
	if err != nil {
		t.Fatalf("get ready instance: %v", err)
	}
	if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: id, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: id + "-start"}); err != nil {
		t.Fatalf("start lifecycle workflow: %v", err)
	}
	claimAndCompleteLifecycleJobForServer(t, svc, runSession, id, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
	running, err := svc.GetServerInstance(id)
	if err != nil {
		t.Fatalf("get running instance: %v", err)
	}
	if running.State != domain.ServerInstanceStateRunning {
		t.Fatalf("expected running instance, got %+v", running)
	}
	return running
}

func queuedLifecycleJob(t *testing.T, svc *CoreService, serverInstanceID string, capability string) (domain.Job, bool) {
	t.Helper()
	jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: serverInstanceID, State: domain.JobStateQueued})
	if err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	for _, job := range jobs {
		if job.Capability == capability {
			return job, true
		}
	}
	return domain.Job{}, false
}