352 lines
16 KiB
Go
352 lines
16 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
func TestMaterializeWorkspaceSeedUsesLifecycleProfileWhenRunComponentKeyIsEmpty(t *testing.T) {
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "game.scum"
|
|
cfg.ComponentKind = "run"
|
|
cfg.ComponentKey = ""
|
|
plan := protocol.RunAutonomousLifecyclePlan{
|
|
SchemaVersion: "1",
|
|
ServerInstanceID: cfg.ServerInstanceID,
|
|
PluginID: "game.scum",
|
|
PluginVersion: "1.0.0",
|
|
RunEndpointID: cfg.RunEndpointID,
|
|
ProfileKey: "run-local",
|
|
TargetOS: runtime.GOOS,
|
|
TargetArch: runtime.GOARCH,
|
|
TargetRelease: "run-dist-test",
|
|
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
|
}
|
|
planPayload, err := json.Marshal(plan)
|
|
if err != nil {
|
|
t.Fatalf("marshal plan: %v", err)
|
|
}
|
|
seedPayload, err := json.Marshal([]workspaceSeedFile{
|
|
{Path: "actions/start.json", Content: `{"version":1,"action":"start","mode":"supervised","executableKey":"bin/game-server"}`, Mode: 0o600},
|
|
{Path: "bin/game-server", Content: "plugin-owned executable", Mode: 0o700},
|
|
{Path: autonomousLifecyclePlanKey, Content: string(planPayload), Mode: 0o600},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal seed: %v", err)
|
|
}
|
|
cfg.WorkspaceSeed = base64.StdEncoding.EncodeToString(seedPayload)
|
|
|
|
if err := MaterializeWorkspaceSeed(cfg); err != nil {
|
|
t.Fatalf("materialize workspace seed: %v", err)
|
|
}
|
|
profileScope, err := NewWorkspaceResolver(cfg.WorkspaceRoot).Scope(cfg.ServerInstanceID, "run-local")
|
|
if err != nil {
|
|
t.Fatalf("resolve profile scope: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(profileScope, autonomousLifecyclePlanKey)); err != nil {
|
|
t.Fatalf("expected lifecycle plan under profile scope: %v", err)
|
|
}
|
|
loaded, scope, ok, err := LoadAutonomousLifecyclePlan(cfg)
|
|
if err != nil || !ok || loaded.ProfileKey != "run-local" || scope != profileScope {
|
|
t.Fatalf("expected lifecycle plan to load from profile scope, ok=%t scope=%q plan=%+v err=%v", ok, scope, loaded, err)
|
|
}
|
|
}
|
|
|
|
func TestWorkerRunsAutonomousBootstrapFromSeededPlan(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "server.scum"
|
|
cfg.ComponentKind = "run"
|
|
cfg.ComponentKey = "run-local"
|
|
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
|
SchemaVersion: "1",
|
|
ServerInstanceID: cfg.ServerInstanceID,
|
|
PluginID: cfg.PluginID,
|
|
PluginVersion: "1.0.0",
|
|
RunEndpointID: cfg.RunEndpointID,
|
|
ProfileKey: cfg.ComponentKey,
|
|
TargetOS: runtime.GOOS,
|
|
TargetArch: runtime.GOARCH,
|
|
TargetRelease: "run-dist-test",
|
|
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
|
LogSources: []protocol.RuntimeLogSourcePlan{{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}},
|
|
Deployment: &protocol.RunAutonomousDeployment{SchemaVersion: "1", Mode: "guided", ProfileKey: cfg.ComponentKey, ServerRoot: "D:/game-server", CreateInputs: map[string]string{"gamePort": "7779", "maxPlayers": "128"}, Revision: 2},
|
|
})
|
|
managed := &recordingManagedSupervisor{}
|
|
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
|
t.Fatalf("run autonomous lifecycle: %v", err)
|
|
}
|
|
|
|
if len(client.claimRequests) != 0 || len(client.ackRequests) != 0 || len(client.resultRequests) != 0 || len(client.lifecycleReports) != 1 {
|
|
t.Fatalf("autonomous bootstrap must report without platform job assignment: claims=%d acks=%d results=%d reports=%d", len(client.claimRequests), len(client.ackRequests), len(client.resultRequests), len(client.lifecycleReports))
|
|
}
|
|
if client.lifecycleReports[0].Capability != protocol.RunCapabilityProcessStart || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" {
|
|
t.Fatalf("expected autonomous process lifecycle report, got %+v", client.lifecycleReports[0])
|
|
}
|
|
if !managed.started || managed.identity.ServerInstanceID != cfg.ServerInstanceID || managed.identity.RunEndpointID != cfg.RunEndpointID {
|
|
t.Fatalf("expected managed process start from autonomous plan, managed=%+v", managed)
|
|
}
|
|
if managed.identity.StdoutStreamKey != "game.console.stdout" {
|
|
t.Fatalf("expected declared process stdout stream, identity=%+v", managed.identity)
|
|
}
|
|
if managed.command.Env["SERVER_ROOT"] != "D:/game-server" || managed.command.Env["SERVER_CREATE_GAMEPORT"] != "7779" || managed.command.Env["SERVER_CREATE_MAXPLAYERS"] != "128" {
|
|
t.Fatalf("expected deployment env from autonomous plan, env=%+v", managed.command.Env)
|
|
}
|
|
}
|
|
|
|
func TestWorkerContinuesAutonomousBootstrapWhenLifecycleReportFails(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
client.lifecycleReportErr = errors.New("platform request failed: status=401 path=/api/v1/run/lifecycle/report code=unauthorized")
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "game.example"
|
|
cfg.ComponentKind = "run"
|
|
cfg.ComponentKey = "run-local"
|
|
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
|
SchemaVersion: "1",
|
|
ServerInstanceID: cfg.ServerInstanceID,
|
|
PluginID: cfg.PluginID,
|
|
PluginVersion: "1.0.0",
|
|
RunEndpointID: cfg.RunEndpointID,
|
|
ProfileKey: cfg.ComponentKey,
|
|
TargetOS: runtime.GOOS,
|
|
TargetArch: runtime.GOARCH,
|
|
TargetRelease: "run-dist-test",
|
|
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
|
})
|
|
managed := &recordingManagedSupervisor{}
|
|
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
|
t.Fatalf("run autonomous lifecycle with report failure: %v", err)
|
|
}
|
|
|
|
if !managed.started || len(client.lifecycleReports) != 1 {
|
|
t.Fatalf("expected bootstrap to continue after lifecycle report failure, managed=%+v reports=%d", managed, len(client.lifecycleReports))
|
|
}
|
|
}
|
|
|
|
func TestWorkerDefersUnmatchedRequiredDependencyToBootstrap(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "game.example"
|
|
cfg.ComponentKind = "run"
|
|
cfg.ComponentKey = "run-local"
|
|
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
|
SchemaVersion: "1",
|
|
ServerInstanceID: cfg.ServerInstanceID,
|
|
PluginID: cfg.PluginID,
|
|
PluginVersion: "1.0.0",
|
|
RunEndpointID: cfg.RunEndpointID,
|
|
ProfileKey: cfg.ComponentKey,
|
|
TargetOS: runtime.GOOS,
|
|
TargetArch: runtime.GOARCH,
|
|
TargetRelease: "run-dist-test",
|
|
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
|
DependencyProbes: []protocol.DependencyProbe{{Key: "game-runtime", Kind: "command.version", TargetKey: "game-runtime", Required: true, Platforms: []string{runtime.GOOS}}},
|
|
InstallPlans: []protocol.DependencyInstallPlan{{Key: "install-game-server", Title: "Install game server", Platforms: []string{runtime.GOOS}, Steps: []protocol.DependencyInstallStep{{Type: "package", TargetKey: "game-server", PackageManager: "apt", PackageName: "game-server-runtime"}}}},
|
|
})
|
|
managed := &recordingManagedSupervisor{}
|
|
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithDependencyCommandRunner(missingCommandRunner{}))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
|
t.Fatalf("run autonomous lifecycle: %v", err)
|
|
}
|
|
|
|
if !managed.started || len(client.lifecycleReports) != 1 {
|
|
t.Fatalf("expected bootstrap to run despite unmatched dependency install plan, managed=%+v reports=%d", managed, len(client.lifecycleReports))
|
|
}
|
|
}
|
|
|
|
func TestLoadAutonomousLifecyclePlanRejectsMismatchedIdentity(t *testing.T) {
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "game.example"
|
|
cfg.ComponentKind = "run"
|
|
cfg.ComponentKey = "run-local"
|
|
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
|
SchemaVersion: "1",
|
|
ServerInstanceID: cfg.ServerInstanceID,
|
|
PluginID: "game.other",
|
|
PluginVersion: "1.0.0",
|
|
RunEndpointID: cfg.RunEndpointID,
|
|
ProfileKey: cfg.ComponentKey,
|
|
TargetOS: runtime.GOOS,
|
|
TargetArch: runtime.GOARCH,
|
|
TargetRelease: "run-dist-test",
|
|
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
|
})
|
|
worker, err := NewWorker(cfg, newFakeWorkerClient())
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
worker.state.SessionToken = "session-token"
|
|
|
|
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err == nil {
|
|
t.Fatal("expected mismatched autonomous plan identity to fail")
|
|
}
|
|
}
|
|
|
|
func TestGenericWorkerReportsRunningAndExitObservationsForAllEndpointServers(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
cfg := workerTestConfig(t)
|
|
cfg.ServerInstanceID = ""
|
|
cfg.ComponentKind = ""
|
|
startedAt := time.Now().UTC().Add(-time.Minute)
|
|
managed := &observationManagedSupervisor{items: []ProcessIdentity{
|
|
{Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "running", ObservationSeq: 1, UpdatedAt: startedAt},
|
|
{Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "running", ObservationSeq: 4, UpdatedAt: startedAt},
|
|
{Scope: "scope-other", ServerInstanceID: "server-other", RunEndpointID: "run-other", PID: 303, StartedAt: startedAt, CommandFingerprint: "sha256:other", State: "running", ObservationSeq: 1, UpdatedAt: startedAt},
|
|
}}
|
|
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
|
if err != nil {
|
|
t.Fatalf("new generic worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register generic worker: %v", err)
|
|
}
|
|
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil {
|
|
t.Fatalf("report running observations: %v", err)
|
|
}
|
|
if len(client.lifecycleReports) != 2 || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" || client.lifecycleReports[1].ExecutionResult.ProcessState != "running" {
|
|
t.Fatalf("generic worker did not report both matching running observations: %+v", client.lifecycleReports)
|
|
}
|
|
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil || len(client.lifecycleReports) != 2 {
|
|
t.Fatalf("unchanged observations were not deduplicated: reports=%+v err=%v", client.lifecycleReports, err)
|
|
}
|
|
managed.setItems([]ProcessIdentity{
|
|
{Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "exited", ExitCode: 1, ExitClassification: "unexpected-exit", ObservationSeq: 2, UpdatedAt: time.Now().UTC()},
|
|
{Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "exited", ExitCode: 0, ExitClassification: "clean-exit", ObservationSeq: 5, UpdatedAt: time.Now().UTC()},
|
|
})
|
|
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil {
|
|
t.Fatalf("report exit observations: %v", err)
|
|
}
|
|
if len(client.lifecycleReports) != 4 || client.lifecycleReports[2].ExecutionResult.ProcessState != "exited" || client.lifecycleReports[3].ExecutionResult.ProcessState != "exited" {
|
|
t.Fatalf("generic worker did not report matching exit observations: %+v", client.lifecycleReports)
|
|
}
|
|
}
|
|
|
|
func writeAutonomousPlanFixture(t *testing.T, root string, serverInstanceID string, profileKey string, plan protocol.RunAutonomousLifecyclePlan) {
|
|
t.Helper()
|
|
scope, err := NewWorkspaceResolver(root).Scope(serverInstanceID, profileKey)
|
|
if err != nil {
|
|
t.Fatalf("resolve scope: %v", err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o700); err != nil {
|
|
t.Fatalf("create actions: %v", err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o700); err != nil {
|
|
t.Fatalf("create bin: %v", err)
|
|
}
|
|
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{"version": 1, "action": "start", "mode": "supervised", "executableKey": "bin/game-server"})
|
|
if err := os.WriteFile(filepath.Join(scope, "bin", "game-server"), []byte("plugin-owned executable"), 0o700); err != nil {
|
|
t.Fatalf("write executable: %v", err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(scope, ".platform"), 0o700); err != nil {
|
|
t.Fatalf("create platform dir: %v", err)
|
|
}
|
|
body, err := json.Marshal(plan)
|
|
if err != nil {
|
|
t.Fatalf("marshal plan: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(scope, autonomousLifecyclePlanKey), body, 0o600); err != nil {
|
|
t.Fatalf("write plan: %v", err)
|
|
}
|
|
}
|
|
|
|
type recordingManagedSupervisor struct {
|
|
started bool
|
|
command ProcessCommand
|
|
identity ProcessIdentity
|
|
}
|
|
|
|
type observationManagedSupervisor struct {
|
|
mu sync.Mutex
|
|
items []ProcessIdentity
|
|
}
|
|
|
|
func (supervisor *observationManagedSupervisor) Start(_ context.Context, _ ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) {
|
|
return identity, nil
|
|
}
|
|
|
|
func (supervisor *observationManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
|
return identity, nil
|
|
}
|
|
|
|
func (supervisor *observationManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
|
return identity
|
|
}
|
|
|
|
func (supervisor *observationManagedSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
|
|
|
func (supervisor *observationManagedSupervisor) ManagedProcessObservations() []ProcessIdentity {
|
|
supervisor.mu.Lock()
|
|
defer supervisor.mu.Unlock()
|
|
return append([]ProcessIdentity(nil), supervisor.items...)
|
|
}
|
|
|
|
func (supervisor *observationManagedSupervisor) setItems(items []ProcessIdentity) {
|
|
supervisor.mu.Lock()
|
|
defer supervisor.mu.Unlock()
|
|
supervisor.items = append([]ProcessIdentity(nil), items...)
|
|
}
|
|
|
|
func (supervisor *recordingManagedSupervisor) Start(_ context.Context, command ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) {
|
|
supervisor.started = true
|
|
supervisor.command = command
|
|
identity.PID = 42
|
|
identity.State = "running"
|
|
supervisor.identity = identity
|
|
return identity, nil
|
|
}
|
|
|
|
func (supervisor *recordingManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
|
identity.State = "stopped"
|
|
return identity, nil
|
|
}
|
|
|
|
func (supervisor *recordingManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
|
return identity
|
|
}
|
|
|
|
func (supervisor *recordingManagedSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
|
|
|
type missingCommandRunner struct{}
|
|
|
|
func (missingCommandRunner) Run(context.Context, ProcessCommand) (ProcessResult, error) {
|
|
return ProcessResult{ExitCode: 1}, os.ErrNotExist
|
|
}
|