From 8e02a316faf3995b2403af3da174b22e84ea7b15 Mon Sep 17 00:00:00 2001 From: npc0-hue Date: Wed, 26 Aug 2026 09:56:43 +0800 Subject: [PATCH] init --- AGENTS.md | 35 + Dockerfile | 29 + api/artifact_client_test.go | 154 +++ api/channel_isolation_test.go | 263 +++++ api/job_client_test.go | 234 ++++ api/log_ingest_client_test.go | 81 ++ api/platform_client.go | 337 ++++++ api/platform_client_test.go | 380 +++++++ artifact/README.md | 13 + cmd/run/main.go | 132 +++ cmd/run/main_test.go | 12 + config/config.go | 138 +++ config/config_test.go | 93 ++ config/package_config.go | 265 +++++ config/package_config_test.go | 141 +++ domain/status.go | 9 + go.mod | 17 + go.sum | 43 + logingest/README.md | 7 + protocol/artifact.go | 101 ++ protocol/artifact.md | 46 + protocol/autonomous_lifecycle.go | 254 +++++ protocol/autonomous_lifecycle_test.go | 39 + protocol/control.go | 84 ++ protocol/control.md | 28 + protocol/game-client-bridge.md | 17 + protocol/job.go | 620 +++++++++++ protocol/job.md | 88 ++ protocol/job_validation.go | 520 +++++++++ protocol/job_validation_test.go | 376 +++++++ protocol/log-ingest.md | 36 + protocol/log_ingest.go | 68 ++ protocol/metrics.go | 31 + protocol/protected-request.md | 38 + runtime/autonomous_lifecycle.go | 319 ++++++ runtime/autonomous_lifecycle_test.go | 302 +++++ runtime/data_targets.go | 277 +++++ runtime/data_targets_test.go | 102 ++ runtime/dependencies.go | 563 ++++++++++ runtime/dependencies_test.go | 186 ++++ runtime/distribution_build.go | 659 +++++++++++ runtime/distribution_build_test.go | 395 +++++++ runtime/distribution_jobs.go | 114 ++ runtime/execution_test.go | 889 +++++++++++++++ runtime/file_execution.go | 380 +++++++ runtime/job_journal.go | 293 +++++ runtime/job_journal_test.go | 229 ++++ runtime/lifecycle.go | 1248 +++++++++++++++++++++ runtime/lifecycle_test.go | 583 ++++++++++ runtime/log_sources.go | 118 ++ runtime/managed_command_test.go | 22 + runtime/managed_process_default.go | 59 + runtime/metrics.go | 112 ++ runtime/metrics_disk_unix.go | 16 + runtime/metrics_disk_windows.go | 29 + runtime/metrics_host_linux.go | 123 ++ runtime/metrics_host_other.go | 17 + runtime/metrics_host_windows.go | 101 ++ runtime/metrics_test.go | 134 +++ runtime/process_alive_default.go | 19 + runtime/process_alive_windows.go | 55 + runtime/process_control_test.go | 49 + runtime/process_state_isolation_test.go | 110 ++ runtime/process_supervisor.go | 811 ++++++++++++++ runtime/process_window_default.go | 9 + runtime/process_window_windows.go | 745 +++++++++++++ runtime/process_window_windows_test.go | 69 ++ runtime/protected_request.go | 210 ++++ runtime/protected_request_test.go | 186 ++++ runtime/remote_access.go | 188 ++++ runtime/remote_access_test.go | 55 + runtime/runtime_profiles.go | 297 +++++ runtime/self_update.go | 611 ++++++++++ runtime/self_update_test.go | 189 ++++ runtime/smoke.go | 19 + runtime/smoke_test.go | 24 + runtime/source_rcon.go | 305 +++++ runtime/source_rcon_test.go | 349 ++++++ runtime/sqlite_schema_probe.go | 312 ++++++ runtime/sqlite_schema_probe_test.go | 115 ++ runtime/ue4ss_dll_extension.go | 556 +++++++++ runtime/ue4ss_dll_extension_test.go | 448 ++++++++ runtime/worker.go | 1364 +++++++++++++++++++++++ runtime/worker_test.go | 1063 ++++++++++++++++++ runtime/workspace.go | 197 ++++ runtime/workspace_seed.go | 147 +++ spool/README.md | 19 + spool/artifact_queue.go | 183 +++ spool/artifact_queue_test.go | 75 ++ spool/channel_isolation_test.go | 85 ++ spool/flush_test.go | 36 + spool/log_spool.go | 560 ++++++++++ spool/log_spool_test.go | 290 +++++ 93 files changed, 21749 insertions(+) create mode 100644 AGENTS.md create mode 100644 Dockerfile create mode 100644 api/artifact_client_test.go create mode 100644 api/channel_isolation_test.go create mode 100644 api/job_client_test.go create mode 100644 api/log_ingest_client_test.go create mode 100644 api/platform_client.go create mode 100644 api/platform_client_test.go create mode 100644 artifact/README.md create mode 100644 cmd/run/main.go create mode 100644 cmd/run/main_test.go create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 config/package_config.go create mode 100644 config/package_config_test.go create mode 100644 domain/status.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 logingest/README.md create mode 100644 protocol/artifact.go create mode 100644 protocol/artifact.md create mode 100644 protocol/autonomous_lifecycle.go create mode 100644 protocol/autonomous_lifecycle_test.go create mode 100644 protocol/control.go create mode 100644 protocol/control.md create mode 100644 protocol/game-client-bridge.md create mode 100644 protocol/job.go create mode 100644 protocol/job.md create mode 100644 protocol/job_validation.go create mode 100644 protocol/job_validation_test.go create mode 100644 protocol/log-ingest.md create mode 100644 protocol/log_ingest.go create mode 100644 protocol/metrics.go create mode 100644 protocol/protected-request.md create mode 100644 runtime/autonomous_lifecycle.go create mode 100644 runtime/autonomous_lifecycle_test.go create mode 100644 runtime/data_targets.go create mode 100644 runtime/data_targets_test.go create mode 100644 runtime/dependencies.go create mode 100644 runtime/dependencies_test.go create mode 100644 runtime/distribution_build.go create mode 100644 runtime/distribution_build_test.go create mode 100644 runtime/distribution_jobs.go create mode 100644 runtime/execution_test.go create mode 100644 runtime/file_execution.go create mode 100644 runtime/job_journal.go create mode 100644 runtime/job_journal_test.go create mode 100644 runtime/lifecycle.go create mode 100644 runtime/lifecycle_test.go create mode 100644 runtime/log_sources.go create mode 100644 runtime/managed_command_test.go create mode 100644 runtime/managed_process_default.go create mode 100644 runtime/metrics.go create mode 100644 runtime/metrics_disk_unix.go create mode 100644 runtime/metrics_disk_windows.go create mode 100644 runtime/metrics_host_linux.go create mode 100644 runtime/metrics_host_other.go create mode 100644 runtime/metrics_host_windows.go create mode 100644 runtime/metrics_test.go create mode 100644 runtime/process_alive_default.go create mode 100644 runtime/process_alive_windows.go create mode 100644 runtime/process_control_test.go create mode 100644 runtime/process_state_isolation_test.go create mode 100644 runtime/process_supervisor.go create mode 100644 runtime/process_window_default.go create mode 100644 runtime/process_window_windows.go create mode 100644 runtime/process_window_windows_test.go create mode 100644 runtime/protected_request.go create mode 100644 runtime/protected_request_test.go create mode 100644 runtime/remote_access.go create mode 100644 runtime/remote_access_test.go create mode 100644 runtime/runtime_profiles.go create mode 100644 runtime/self_update.go create mode 100644 runtime/self_update_test.go create mode 100644 runtime/smoke.go create mode 100644 runtime/smoke_test.go create mode 100644 runtime/source_rcon.go create mode 100644 runtime/source_rcon_test.go create mode 100644 runtime/sqlite_schema_probe.go create mode 100644 runtime/sqlite_schema_probe_test.go create mode 100644 runtime/ue4ss_dll_extension.go create mode 100644 runtime/ue4ss_dll_extension_test.go create mode 100644 runtime/worker.go create mode 100644 runtime/worker_test.go create mode 100644 runtime/workspace.go create mode 100644 runtime/workspace_seed.go create mode 100644 spool/README.md create mode 100644 spool/artifact_queue.go create mode 100644 spool/artifact_queue_test.go create mode 100644 spool/channel_isolation_test.go create mode 100644 spool/flush_test.go create mode 100644 spool/log_spool.go create mode 100644 spool/log_spool_test.go diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..38b156e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# AGENTS.md for run + +This file applies to `run/`. + +## Channel Rules + +Do not rebuild the old all-in-one WebSocket model. Keep these workloads separate: + +- Control: hello, heartbeat, version, capabilities, capacity. +- Job: claim, ack, progress, result, cancel, reconcile. +- Logs: local spool, batch upload, sequence acknowledgement, retry. +- Artifacts: chunks, checksums, resume, throttling. +- Game client bridge: optional in-game command/snapshot channel. + +Artifact transfer must not block control heartbeats, job result reporting, or log upload. + +## Structure Rules + +Protocol structs live in `protocol/`. Local runtime types live in `runtime/` or `domain/`. Shared helpers live in `shared/` only when needed by multiple packages. + +## Safety Rules + +Run must enforce scoped paths and never expose raw host paths, local secrets, or unrestricted command execution to platform_web or plugins. + +## Generic Executor Boundary + +Run is a generic machine-side executor. It may provide bounded primitives such as process start/stop/restart, scoped file existence/list/read/write/patch operations, downloads, extraction, log capture, artifact transfer, and capability enforcement. + +Run must not contain game-specific lifecycle logic. Do not add hardcoded game names, executable paths, Steam app IDs, SteamCMD app-update commands, launch flags, install directories, or update policies to run. In particular, SCUM-specific values such as `SCUMServer.exe`, app `3792580`, `+app_update 3792580 validate`, `-port`, `-MaxPlayers`, and `-log` belong to the SCUM plugin, not this repository. + +If a lifecycle flow needs "install if missing", "stop before update", "validate via SteamCMD", or "start with game-specific arguments", run should execute the plugin-declared action through generic capabilities. Fix missing generic capabilities in run when necessary, but keep the game policy and concrete commands in the plugin action assets. + +Run should supervise plugin-declared start commands generically: hide started process windows when the operating system supports it, capture the supervised stdout/stderr streams, and upload those streams through log ingest. Do not replace plugin-declared process output with game-specific file inspection. + +Do not introduce per-game runtime files such as `scum_deployment.go`. Prefer generic action execution, dependency helpers, process supervision, and manifest-declared capability checks that any plugin can use. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..47e4824 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.25.1-alpine AS build +WORKDIR /src +COPY run/go.mod ./ +RUN go mod download +COPY run/ ./ +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/run ./cmd/run + +FROM alpine:3.21 +RUN addgroup -S run && adduser -S run -G run +WORKDIR /app +COPY --from=build /out/run /app/run +RUN mkdir -p /data/run/workspace /data/run/spool && chown -R run:run /data/run +USER run +ENV RUN_MODE=worker \ + RUN_PLATFORM_URL=http://platform:8080 \ + RUN_ENDPOINT_ID=run-docker \ + RUN_DISPLAY_NAME="Docker Run" \ + RUN_VERSION=0.1.0 \ + RUN_REGISTRATION_TOKEN=local-registration \ + RUN_WORKSPACE_ROOT=/data/run/workspace \ + RUN_SPOOL_ROOT=/data/run/spool \ + RUN_MAX_JOBS=1 \ + RUN_HEARTBEAT_INTERVAL_MS=15000 \ + RUN_POLL_INTERVAL_MS=2000 \ + RUN_RETRY_BACKOFF_MS=1000 +ENTRYPOINT ["/app/run"] + diff --git a/api/artifact_client_test.go b/api/artifact_client_test.go new file mode 100644 index 0000000..a251ed7 --- /dev/null +++ b/api/artifact_client_test.go @@ -0,0 +1,154 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "browser.local/run/protocol" +) + +func TestPlatformClientArtifactMethodsPostJSONAndDecodeResponses(t *testing.T) { + seen := map[string]bool{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen[r.URL.Path] = true + switch r.URL.Path { + case "/api/v1/run/artifacts/open": + var request protocol.ArtifactTransferOpenRequest + decodeTestRequest(t, r, &request) + if request.ArtifactID != "artifact-1" || request.ChunkSizeBytes != 8 { + t.Fatalf("unexpected artifact open request: %+v", request) + } + writeTestJSON(t, w, validArtifactOpenResponse()) + case "/api/v1/run/artifacts/chunks": + var request protocol.ArtifactChunkUploadRequest + decodeTestRequest(t, r, &request) + if request.TransferID != "transfer-1" || request.ChunkIndex != 0 || string(request.Payload) != "payload" { + t.Fatalf("unexpected artifact chunk request: %+v", request) + } + writeTestJSON(t, w, protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/artifacts/status": + var request protocol.ArtifactTransferStatusRequest + decodeTestRequest(t, r, &request) + if request.TransferID != "transfer-1" { + t.Fatalf("unexpected artifact status request: %+v", request) + } + writeTestJSON(t, w, protocol.ArtifactTransferStatusResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", Direction: "upload", TotalChunks: 2, ChunkSizeBytes: 8, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/artifacts/complete": + var request protocol.ArtifactTransferCompleteRequest + decodeTestRequest(t, r, &request) + if request.Checksum == "" || request.SizeBytes != 7 { + t.Fatalf("unexpected artifact complete request: %+v", request) + } + response := protocol.ArtifactTransferCompleteResponse{Accepted: true, TransferID: "transfer-1", Artifact: validArtifactMetadata("available"), Completed: true, ServerTime: fixedClientTestTime()} + writeTestJSON(t, w, response) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + ctx := context.Background() + open, err := client.OpenArtifactTransfer(ctx, validClientArtifactOpen()) + if err != nil || !open.Accepted || open.TransferID != "transfer-1" { + t.Fatalf("open artifact response=%+v err=%v", open, err) + } + chunk, err := client.UploadArtifactChunk(ctx, validClientArtifactChunk()) + if err != nil || chunk.NextMissingChunkIndex != 1 { + t.Fatalf("upload artifact chunk response=%+v err=%v", chunk, err) + } + status, err := client.QueryArtifactTransferStatus(ctx, validClientArtifactStatus()) + if err != nil || len(status.ReceivedChunkIndexes) != 1 { + t.Fatalf("artifact status response=%+v err=%v", status, err) + } + complete, err := client.CompleteArtifactTransfer(ctx, validClientArtifactComplete()) + if err != nil || !complete.Completed || complete.Artifact.State != "available" { + t.Fatalf("complete artifact response=%+v err=%v", complete, err) + } + + for _, path := range []string{"/api/v1/run/artifacts/open", "/api/v1/run/artifacts/chunks", "/api/v1/run/artifacts/status", "/api/v1/run/artifacts/complete"} { + if !seen[path] { + t.Fatalf("expected request to %s", path) + } + } +} + +func TestPlatformClientArtifactMethodReturnsErrorForPlatformFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"validation_failed"}`)) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + if _, err := client.OpenArtifactTransfer(context.Background(), validClientArtifactOpen()); err == nil { + t.Fatal("expected platform error") + } +} + +func TestArtifactChunkPayloadUsesJSONBase64Encoding(t *testing.T) { + encoded, err := json.Marshal(validClientArtifactChunk()) + if err != nil { + t.Fatalf("marshal artifact chunk: %v", err) + } + if !json.Valid(encoded) || !containsJSONPayloadField(encoded) { + t.Fatalf("expected JSON encoded payload field, got %s", string(encoded)) + } +} + +func validClientArtifactOpen() protocol.ArtifactTransferOpenRequest { + return protocol.ArtifactTransferOpenRequest{ + RunEndpointID: "run-local", + SessionToken: "session-token", + ArtifactID: "artifact-1", + Direction: "upload", + OwnerKind: "job", + OwnerID: "job-1", + SizeBytes: 7, + ChunkSizeBytes: 8, + Checksum: validSHA256Checksum(), + IdempotencyKey: "artifact-upload-1", + } +} + +func validClientArtifactChunk() protocol.ArtifactChunkUploadRequest { + return protocol.ArtifactChunkUploadRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0, Offset: 0, SizeBytes: 7, Checksum: validSHA256Checksum(), Payload: []byte("payload")} +} + +func validClientArtifactStatus() protocol.ArtifactTransferStatusRequest { + return protocol.ArtifactTransferStatusRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1"} +} + +func validClientArtifactComplete() protocol.ArtifactTransferCompleteRequest { + return protocol.ArtifactTransferCompleteRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1", Checksum: validSHA256Checksum(), SizeBytes: 7} +} + +func validArtifactOpenResponse() protocol.ArtifactTransferOpenResponse { + return protocol.ArtifactTransferOpenResponse{Accepted: true, TransferID: "transfer-1", Direction: "upload", Artifact: validArtifactMetadata("uploading"), TotalChunks: 2, ChunkSizeBytes: 8, NextMissingChunkIndex: 0, ServerTime: fixedClientTestTime()} +} + +func validArtifactMetadata(state string) protocol.ArtifactMetadata { + return protocol.ArtifactMetadata{ID: "artifact-1", OwnerKind: "job", OwnerID: "job-1", SizeBytes: 7, Checksum: validSHA256Checksum(), State: state, CreatedAt: fixedClientTestTime(), UpdatedAt: fixedClientTestTime()} +} + +func validSHA256Checksum() string { + return "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +} + +func containsJSONPayloadField(encoded []byte) bool { + var body map[string]any + if err := json.Unmarshal(encoded, &body); err != nil { + return false + } + _, exists := body["payload"] + return exists +} diff --git a/api/channel_isolation_test.go b/api/channel_isolation_test.go new file mode 100644 index 0000000..6277c9f --- /dev/null +++ b/api/channel_isolation_test.go @@ -0,0 +1,263 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestPlatformClientLightweightChannelsCompleteWhileArtifactChunkIsBlocked(t *testing.T) { + artifactStarted := make(chan struct{}) + releaseArtifact := make(chan struct{}) + artifactDone := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/run/artifacts/chunks": + var request protocol.ArtifactChunkUploadRequest + decodeTestRequest(t, r, &request) + if request.ChunkIndex != 0 || len(request.Payload) == 0 { + t.Fatalf("unexpected artifact payload: %+v", request) + } + close(artifactStarted) + <-releaseArtifact + writeTestJSON(t, w, protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: request.TransferID, ArtifactID: request.ArtifactID, ChunkIndex: request.ChunkIndex, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()}) + close(artifactDone) + case "/api/v1/run/control/heartbeat": + var request protocol.RunHeartbeatRequest + decodeTestRequest(t, r, &request) + writeTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/result": + var request protocol.RunJobResultRequest + decodeTestRequest(t, r, &request) + encoded, _ := json.Marshal(request) + for _, forbidden := range []string{"payload", "entries", "/Users/", "unix://", "tcp://", "Bearer ", "sk-", "password="} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("job result carried forbidden transfer content %q: %s", forbidden, string(encoded)) + } + } + job := validRunJobAssignment() + job.State = request.State + job.ResultRef = request.ResultRef + writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/logs/batches": + var request protocol.LogBatchIngestRequest + decodeTestRequest(t, r, &request) + if len(request.Entries) != 1 || request.FirstSeq != 1 || request.LastSeq != 1 { + t.Fatalf("unexpected log batch: %+v", request) + } + writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: request.LogStreamID, AcceptedFrom: 1, AcceptedTo: 1, LatestSeq: 1, ServerTime: fixedClientTestTime()}) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + + errCh := make(chan error, 1) + go func() { + _, err := client.UploadArtifactChunk(context.Background(), validClientArtifactChunk()) + errCh <- err + }() + + select { + case <-artifactStarted: + case <-time.After(time.Second): + t.Fatal("artifact request did not start") + } + + lightCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := client.Heartbeat(lightCtx, validRunHeartbeatRequest("session-token")); err != nil { + t.Fatalf("heartbeat should not wait for artifact chunk: %v", err) + } + if _, err := client.CompleteJob(lightCtx, validRunJobResultRequest()); err != nil { + t.Fatalf("job result should not wait for artifact chunk: %v", err) + } + if _, err := client.IngestLogBatch(lightCtx, validClientLogBatch()); err != nil { + t.Fatalf("log ingest should not wait for artifact chunk: %v", err) + } + + select { + case <-artifactDone: + t.Fatal("artifact chunk completed before release") + default: + } + close(releaseArtifact) + select { + case err := <-errCh: + if err != nil { + t.Fatalf("artifact chunk upload: %v", err) + } + case <-time.After(time.Second): + t.Fatal("artifact chunk did not finish after release") + } +} + +func TestPlatformClientControlAndJobsCompleteWhileLogIngestIsBlocked(t *testing.T) { + logStarted := make(chan struct{}) + releaseLog := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/run/logs/batches": + var request protocol.LogBatchIngestRequest + decodeTestRequest(t, r, &request) + close(logStarted) + <-releaseLog + writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: request.LogStreamID, AcceptedFrom: request.FirstSeq, AcceptedTo: request.LastSeq, LatestSeq: request.LastSeq, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/control/heartbeat": + var request protocol.RunHeartbeatRequest + decodeTestRequest(t, r, &request) + writeTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/result": + var request protocol.RunJobResultRequest + decodeTestRequest(t, r, &request) + job := validRunJobAssignment() + job.State = request.State + writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/reconcile": + var request protocol.RunJobReconcileRequest + decodeTestRequest(t, r, &request) + writeTestJSON(t, w, protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{validRunJobAssignment()}, ServerTime: fixedClientTestTime()}) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + logErr := make(chan error, 1) + go func() { + _, callErr := client.IngestLogBatch(context.Background(), validClientLogBatch()) + logErr <- callErr + }() + select { + case <-logStarted: + case <-time.After(time.Second): + t.Fatal("log request did not start") + } + + lightCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := client.Heartbeat(lightCtx, validRunHeartbeatRequest("session-token")); err != nil { + t.Fatalf("heartbeat should not wait for log ingest: %v", err) + } + if _, err := client.CompleteJob(lightCtx, validRunJobResultRequest()); err != nil { + t.Fatalf("job result should not wait for log ingest: %v", err) + } + if _, err := client.ReconcileJobs(lightCtx, validRunJobReconcileRequest()); err != nil { + t.Fatalf("job reconcile should not wait for log ingest: %v", err) + } + close(releaseLog) + select { + case err := <-logErr: + if err != nil { + t.Fatalf("log ingest: %v", err) + } + case <-time.After(time.Second): + t.Fatal("log ingest did not finish after release") + } +} + +func TestPlatformClientControlJobsAndLogsCompleteWhileDependencyOrUpdateInputIsBlocked(t *testing.T) { + for _, scenario := range []struct { + name string + path string + call func(context.Context, PlatformClient) error + }{ + { + name: "dependency adapter input", + path: "/api/v1/run/jobs/dependency-input", + call: func(ctx context.Context, client PlatformClient) error { + _, err := client.GetDependencyExecutionInput(ctx, protocol.DependencyExecutionInputRequest{RunEndpointID: "run-test", SessionToken: "session-token", JobID: "job-dependency", LeaseToken: "lease-dependency", Attempt: 1}) + return err + }, + }, + { + name: "self-update chunk", + path: "/api/v1/run/jobs/update-chunk", + call: func(ctx context.Context, client PlatformClient) error { + _, err := client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: "run-test", SessionToken: "session-token", JobID: "job-update", LeaseToken: "lease-update", Attempt: 1, Offset: 0, Length: 8}) + return err + }, + }, + } { + t.Run(scenario.name, func(t *testing.T) { + blockedStarted := make(chan struct{}) + releaseBlocked := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case scenario.path: + close(blockedStarted) + <-releaseBlocked + if scenario.path == "/api/v1/run/jobs/dependency-input" { + writeTestJSON(t, w, protocol.DependencyExecutionInputResponse{JobID: "job-dependency", ServerInstanceID: "server-1", RunEndpointID: "run-test", PluginID: "game.runtime", PluginVersion: "1.0.0", ProfileKey: "local", TargetOS: "linux", TargetArch: "amd64", PlanDigest: "sha256:" + strings.Repeat("a", 64), Bindings: map[string]string{}}) + return + } + writeTestJSON(t, w, protocol.RunUpdateChunkResponse{JobID: "job-update", ArtifactID: "artifact-update", Offset: 0, TotalBytes: 8, Checksum: "sha256:" + strings.Repeat("b", 64), Payload: []byte("12345678"), Complete: true}) + case "/api/v1/run/control/heartbeat": + var request protocol.RunHeartbeatRequest + decodeTestRequest(t, r, &request) + writeTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/result": + var request protocol.RunJobResultRequest + decodeTestRequest(t, r, &request) + job := validRunJobAssignment() + job.State = request.State + writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/logs/batches": + var request protocol.LogBatchIngestRequest + decodeTestRequest(t, r, &request) + writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: request.LogStreamID, AcceptedFrom: request.FirstSeq, AcceptedTo: request.LastSeq, LatestSeq: request.LastSeq, ServerTime: fixedClientTestTime()}) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + blockedDone := make(chan error, 1) + go func() { blockedDone <- scenario.call(context.Background(), client) }() + select { + case <-blockedStarted: + case <-time.After(time.Second): + t.Fatal("blocked dependency/update request did not start") + } + + lightCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := client.Heartbeat(lightCtx, validRunHeartbeatRequest("session-token")); err != nil { + t.Fatalf("heartbeat should not wait for dependency/update input: %v", err) + } + if _, err := client.CompleteJob(lightCtx, validRunJobResultRequest()); err != nil { + t.Fatalf("job result should not wait for dependency/update input: %v", err) + } + if _, err := client.IngestLogBatch(lightCtx, validClientLogBatch()); err != nil { + t.Fatalf("log ingest should not wait for dependency/update input: %v", err) + } + close(releaseBlocked) + select { + case err := <-blockedDone: + if err != nil { + t.Fatalf("blocked request completion: %v", err) + } + case <-time.After(time.Second): + t.Fatal("blocked request did not complete after release") + } + }) + } +} diff --git a/api/job_client_test.go b/api/job_client_test.go new file mode 100644 index 0000000..5935221 --- /dev/null +++ b/api/job_client_test.go @@ -0,0 +1,234 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestPlatformClientJobMethodsPostJSONAndDecodeResponses(t *testing.T) { + seen := map[string]bool{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen[r.URL.Path] = true + switch r.URL.Path { + case "/api/v1/run/jobs/claim": + var request protocol.RunJobClaimRequest + decodeTestRequest(t, r, &request) + if request.RunEndpointID != "run-local" || request.SessionToken != "session-token" { + t.Fatalf("unexpected claim request: %+v", request) + } + writeTestJSON(t, w, validRunJobClaimResponse()) + case "/api/v1/run/jobs/ack": + var request protocol.RunJobAckRequest + decodeTestRequest(t, r, &request) + if request.JobID != "job-1" || request.LeaseToken != "lease-1" { + t.Fatalf("unexpected ack request: %+v", request) + } + writeTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: validRunJobAssignment(), ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/progress": + var request protocol.RunJobProgressRequest + decodeTestRequest(t, r, &request) + if request.Progress.Percent != 40 { + t.Fatalf("unexpected progress request: %+v", request) + } + assignment := validRunJobAssignment() + assignment.Progress.Percent = 40 + writeTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/result": + var request protocol.RunJobResultRequest + decodeTestRequest(t, r, &request) + if request.State != "succeeded" || request.ResultRef == "" { + t.Fatalf("unexpected result request: %+v", request) + } + assignment := validRunJobAssignment() + assignment.State = "succeeded" + assignment.ResultRef = request.ResultRef + writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/cancel": + var request protocol.RunJobCancelPollRequest + decodeTestRequest(t, r, &request) + if request.JobID != "job-1" { + t.Fatalf("unexpected cancel request: %+v", request) + } + writeTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-local", HasCancel: true, JobID: "job-1", Reason: "stop", ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/reconcile": + var request protocol.RunJobReconcileRequest + decodeTestRequest(t, r, &request) + if len(request.ActiveJobs) != 1 || request.ActiveJobs[0].JobID != "job-1" || request.ActiveJobs[0].Attempt != 1 { + t.Fatalf("unexpected reconcile request: %+v", request) + } + writeTestJSON(t, w, protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-local", ConfirmedJobs: []protocol.RunJobAssignment{validRunJobAssignment()}, ServerTime: fixedClientTestTime()}) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + ctx := context.Background() + claim, err := client.ClaimJob(ctx, validRunJobClaimRequest()) + if err != nil || !claim.HasJob { + t.Fatalf("claim job response=%+v err=%v", claim, err) + } + if _, err := client.AckJob(ctx, validRunJobAckRequest()); err != nil { + t.Fatalf("ack job: %v", err) + } + if _, err := client.UpdateJobProgress(ctx, validRunJobProgressRequest()); err != nil { + t.Fatalf("progress job: %v", err) + } + if _, err := client.CompleteJob(ctx, validRunJobResultRequest()); err != nil { + t.Fatalf("complete job: %v", err) + } + if _, err := client.PollJobCancel(ctx, validRunJobCancelPollRequest()); err != nil { + t.Fatalf("poll cancel: %v", err) + } + if _, err := client.ReconcileJobs(ctx, validRunJobReconcileRequest()); err != nil { + t.Fatalf("reconcile jobs: %v", err) + } + + for _, path := range []string{"/api/v1/run/jobs/claim", "/api/v1/run/jobs/ack", "/api/v1/run/jobs/progress", "/api/v1/run/jobs/result", "/api/v1/run/jobs/cancel", "/api/v1/run/jobs/reconcile"} { + if !seen[path] { + t.Fatalf("expected request to %s", path) + } + } +} + +func TestPlatformClientJobMethodReturnsErrorForPlatformFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"validation_failed"}`)) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + if _, err := client.ClaimJob(context.Background(), validRunJobClaimRequest()); err == nil { + t.Fatal("expected platform error") + } +} + +func TestPlatformClientJobLifecycleFlow(t *testing.T) { + assignment := validRunJobAssignment() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/run/jobs/claim": + assignment.State = "accepted" + writeTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: "run-local", HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/ack": + assignment.State = "running" + writeTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/progress": + assignment.Progress.Percent = 70 + writeTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/result": + assignment.State = "succeeded" + assignment.Progress.Percent = 100 + assignment.ResultRef = "artifact://jobs/job-1/result" + writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()}) + case "/api/v1/run/jobs/reconcile": + writeTestJSON(t, w, protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-local", DiscardJobIDs: []string{"local-only"}, ServerTime: fixedClientTestTime()}) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + ctx := context.Background() + claim, err := client.ClaimJob(ctx, validRunJobClaimRequest()) + if err != nil { + t.Fatalf("claim job: %v", err) + } + ack, err := client.AckJob(ctx, protocol.RunJobAckRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}) + if err != nil { + t.Fatalf("ack job: %v", err) + } + progress, err := client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: ack.Job.JobID, LeaseToken: ack.Job.LeaseToken, Attempt: ack.Job.Attempt, Progress: protocol.RunJobProgressReport{Percent: 70}}) + if err != nil { + t.Fatalf("progress job: %v", err) + } + result, err := client.CompleteJob(ctx, protocol.RunJobResultRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: progress.Job.JobID, LeaseToken: progress.Job.LeaseToken, Attempt: progress.Job.Attempt, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100}, ResultRef: "artifact://jobs/job-1/result"}) + if err != nil { + t.Fatalf("complete job: %v", err) + } + reconcile, err := client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{RunEndpointID: "run-local", SessionToken: "session-token", ActiveJobs: []protocol.RunJobReconcileEntry{{JobID: "local-only", LeaseToken: "local-lease", Attempt: 1}}}) + if err != nil { + t.Fatalf("reconcile jobs: %v", err) + } + if claim.Job.State != "accepted" || ack.Job.State != "running" || progress.Job.Progress.Percent != 70 || result.Job.State != "succeeded" || len(reconcile.DiscardJobIDs) != 1 { + t.Fatalf("unexpected lifecycle responses: claim=%+v ack=%+v progress=%+v result=%+v reconcile=%+v", claim, ack, progress, result, reconcile) + } +} + +func decodeTestRequest(t *testing.T, r *http.Request, target any) { + t.Helper() + if r.Method != http.MethodPost { + t.Fatalf("expected POST, got %s", r.Method) + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { + t.Fatalf("expected JSON content type, got %q", contentType) + } + if err := json.NewDecoder(r.Body).Decode(target); err != nil { + t.Fatalf("decode request: %v", err) + } +} + +func fixedClientTestTime() time.Time { + return time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC) +} + +func validRunJobClaimRequest() protocol.RunJobClaimRequest { + return protocol.RunJobClaimRequest{RunEndpointID: "run-local", SessionToken: "session-token", Capabilities: []string{"process.start"}, Capacity: protocol.RunCapacityReport{MaxJobs: 4}} +} + +func validRunJobAckRequest() protocol.RunJobAckRequest { + return protocol.RunJobAckRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, Message: "started"} +} + +func validRunJobProgressRequest() protocol.RunJobProgressRequest { + return protocol.RunJobProgressRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, Progress: protocol.RunJobProgressReport{Percent: 40, Message: "working"}} +} + +func validRunJobResultRequest() protocol.RunJobResultRequest { + return protocol.RunJobResultRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100}, ResultRef: "artifact://jobs/job-1/result", Message: "done"} +} + +func validRunJobCancelPollRequest() protocol.RunJobCancelPollRequest { + return protocol.RunJobCancelPollRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1} +} + +func validRunJobReconcileRequest() protocol.RunJobReconcileRequest { + return protocol.RunJobReconcileRequest{RunEndpointID: "run-local", SessionToken: "session-token", ActiveJobs: []protocol.RunJobReconcileEntry{{JobID: "job-1", LeaseToken: "lease-1", Attempt: 1}}} +} + +func validRunJobClaimResponse() protocol.RunJobClaimResponse { + job := validRunJobAssignment() + return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: "run-local", HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: fixedClientTestTime()} +} + +func validRunJobAssignment() protocol.RunJobAssignment { + return protocol.RunJobAssignment{ + JobID: "job-1", + RunEndpointID: "run-local", + Capability: "process.start", + IdempotencyKey: "idem-1", + State: "accepted", + LeaseToken: "lease-1", + Attempt: 1, + CreatedAt: fixedClientTestTime(), + UpdatedAt: fixedClientTestTime(), + } +} diff --git a/api/log_ingest_client_test.go b/api/log_ingest_client_test.go new file mode 100644 index 0000000..4ea9975 --- /dev/null +++ b/api/log_ingest_client_test.go @@ -0,0 +1,81 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestPlatformClientIngestLogBatchPostsJSONAndDecodesResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/logs/batches" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { + t.Fatalf("expected JSON content type, got %q", contentType) + } + var request protocol.LogBatchIngestRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode log ingest request: %v", err) + } + if request.LogStreamID != "log-1" || request.FirstSeq != 1 || request.LastSeq != 1 || len(request.Entries) != 1 { + t.Fatalf("unexpected log ingest payload: %+v", request) + } + writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1, LatestSeq: 1, ServerTime: fixedClientTestTime()}) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + response, err := client.IngestLogBatch(context.Background(), validClientLogBatch()) + if err != nil { + t.Fatalf("ingest log batch: %v", err) + } + if !response.Accepted || response.LatestSeq != 1 { + t.Fatalf("unexpected log ingest response: %+v", response) + } +} + +func TestPlatformClientIngestLogBatchReturnsErrorForPlatformFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"validation_failed"}`)) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + if _, err := client.IngestLogBatch(context.Background(), validClientLogBatch()); err == nil { + t.Fatal("expected platform error") + } +} + +func validClientLogBatch() protocol.LogBatchIngestRequest { + return protocol.LogBatchIngestRequest{ + RunEndpointID: "run-local", + SessionToken: "session-token", + LogStreamID: "log-1", + ServerInstanceID: "server-1", + StreamKey: "stdout", + Source: "process", + FirstSeq: 1, + LastSeq: 1, + Compression: "none", + Checksum: "sha256:test", + Entries: []protocol.LogEntry{{ + Seq: 1, + Timestamp: time.Date(2026, 7, 3, 12, 0, 1, 0, time.UTC), + Level: "info", + Line: "line", + }}, + } +} diff --git a/api/platform_client.go b/api/platform_client.go new file mode 100644 index 0000000..80e05e3 --- /dev/null +++ b/api/platform_client.go @@ -0,0 +1,337 @@ +package api + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "browser.local/run/protocol" +) + +type PlatformClient struct { + baseURL string + httpClient *http.Client +} + +type PlatformRequestError struct { + Status int + Path string + Code string + Details []string +} + +func (err PlatformRequestError) Error() string { + parts := []string{fmt.Sprintf("status=%d", err.Status)} + if strings.TrimSpace(err.Path) != "" { + parts = append(parts, "path="+strings.TrimSpace(err.Path)) + } + if strings.TrimSpace(err.Code) != "" { + parts = append(parts, "code="+strings.TrimSpace(err.Code)) + } + if len(err.Details) > 0 { + parts = append(parts, "details="+strings.Join(err.Details, "; ")) + } + return "platform request failed: " + strings.Join(parts, " ") +} + +func (err PlatformRequestError) HTTPStatus() int { + return err.Status +} + +// SessionInvalid reports the one legacy validation response that means a Run +// session must be renewed. Component-authenticated sessions return 401 for the +// same condition; older endpoint sessions return this safe 400 response. +func (err PlatformRequestError) SessionInvalid() bool { + if err.Status == http.StatusUnauthorized { + return true + } + if err.Status != http.StatusBadRequest || err.Code != "validation_failed" { + return false + } + for _, detail := range err.Details { + if strings.TrimSpace(detail) == "sessionToken is invalid" { + return true + } + } + return false +} + +func (err PlatformRequestError) LogBatchSequenceGap() bool { + if err.Status != http.StatusBadRequest || err.Code != "validation_failed" { + return false + } + for _, detail := range err.Details { + if strings.TrimSpace(detail) == "log batch firstSeq must follow latest acknowledged sequence" { + return true + } + } + return false +} + +func (err PlatformRequestError) LogBatchAcknowledgedRangeConflict() bool { + if err.Status != http.StatusBadRequest || err.Code != "validation_failed" { + return false + } + for _, detail := range err.Details { + if strings.TrimSpace(detail) == "log batch conflicts with acknowledged range" { + return true + } + } + return false +} + +func (err PlatformRequestError) LogBatchLegacySessionMetadata() bool { + if err.Status != http.StatusBadRequest || err.Code != "validation_failed" { + return false + } + for _, detail := range err.Details { + if strings.TrimSpace(detail) == "logSessionId and sessionStartedAt must be provided together" { + return true + } + } + return false +} + +func (err PlatformRequestError) LogBatchSessionMetadataMismatch() bool { + if err.Status != http.StatusBadRequest || err.Code != "validation_failed" { + return false + } + for _, detail := range err.Details { + if strings.TrimSpace(detail) == "log session metadata must match stream" { + return true + } + } + return false +} + +func NewPlatformClient(rawURL string) (PlatformClient, error) { + return NewPlatformClientWithHTTPClient(rawURL, http.DefaultClient) +} + +func NewPlatformClientWithHTTPClient(rawURL string, httpClient *http.Client) (PlatformClient, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return PlatformClient{}, err + } + if parsed.Scheme == "" || parsed.Host == "" { + return PlatformClient{}, fmt.Errorf("platform URL must include scheme and host") + } + if httpClient == nil { + httpClient = http.DefaultClient + } + + return PlatformClient{baseURL: strings.TrimRight(parsed.String(), "/"), httpClient: httpClient}, nil +} + +func (c PlatformClient) BaseURL() string { + return c.baseURL +} + +func (c PlatformClient) Hello(ctx context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) { + return postPlatformJSON[protocol.RunHelloRequest, protocol.RunHelloResponse](ctx, c, "/api/v1/run/control/hello", request) +} + +func (c PlatformClient) Heartbeat(ctx context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) { + return postPlatformJSON[protocol.RunHeartbeatRequest, protocol.RunHeartbeatResponse](ctx, c, "/api/v1/run/control/heartbeat", request) +} + +func (c PlatformClient) ReportLifecycle(ctx context.Context, request protocol.RunLifecycleReportRequest) (protocol.RunLifecycleReportResponse, error) { + return postPlatformJSON[protocol.RunLifecycleReportRequest, protocol.RunLifecycleReportResponse](ctx, c, "/api/v1/run/lifecycle/report", request) +} + +func (c PlatformClient) ClaimJob(ctx context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) { + return postPlatformJSON[protocol.RunJobClaimRequest, protocol.RunJobClaimResponse](ctx, c, "/api/v1/run/jobs/claim", request) +} + +func (c PlatformClient) AckJob(ctx context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) { + return postPlatformJSON[protocol.RunJobAckRequest, protocol.RunJobAckResponse](ctx, c, "/api/v1/run/jobs/ack", request) +} + +func (c PlatformClient) UpdateJobProgress(ctx context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) { + return postPlatformJSON[protocol.RunJobProgressRequest, protocol.RunJobProgressResponse](ctx, c, "/api/v1/run/jobs/progress", request) +} + +func (c PlatformClient) CompleteJob(ctx context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) { + return postPlatformJSON[protocol.RunJobResultRequest, protocol.RunJobResultResponse](ctx, c, "/api/v1/run/jobs/result", request) +} + +func (c PlatformClient) GetDistributionBuildInput(ctx context.Context, request protocol.DistributionBuildInputRequest) (protocol.DistributionBuildInputResponse, error) { + return postPlatformJSON[protocol.DistributionBuildInputRequest, protocol.DistributionBuildInputResponse](ctx, c, "/api/v1/run/jobs/build-input", request) +} + +func (c PlatformClient) GetDependencyExecutionInput(ctx context.Context, request protocol.DependencyExecutionInputRequest) (protocol.DependencyExecutionInputResponse, error) { + return postPlatformJSON[protocol.DependencyExecutionInputRequest, protocol.DependencyExecutionInputResponse](ctx, c, "/api/v1/run/jobs/dependency-input", request) +} + +func (c PlatformClient) GetSourceRCONExecutionInput(ctx context.Context, request protocol.SourceRCONExecutionInputRequest) (protocol.SourceRCONExecutionInputResponse, error) { + return postPlatformJSON[protocol.SourceRCONExecutionInputRequest, protocol.SourceRCONExecutionInputResponse](ctx, c, "/api/v1/run/jobs/source-rcon-input", request) +} + +func (c PlatformClient) GetProtectedRequestExecutionInput(ctx context.Context, request protocol.ProtectedRequestExecutionInputRequest) (protocol.ProtectedRequestExecutionInputResponse, error) { + return postPlatformJSON[protocol.ProtectedRequestExecutionInputRequest, protocol.ProtectedRequestExecutionInputResponse](ctx, c, "/api/v1/run/jobs/protected-request-input", request) +} + +func (c PlatformClient) GetRunUpdateInput(ctx context.Context, request protocol.RunUpdateInputRequest) (protocol.RunUpdateInputResponse, error) { + return postPlatformJSON[protocol.RunUpdateInputRequest, protocol.RunUpdateInputResponse](ctx, c, "/api/v1/run/jobs/update-input", request) +} + +func (c PlatformClient) ReadRunUpdateChunk(ctx context.Context, request protocol.RunUpdateChunkRequest) (protocol.RunUpdateChunkResponse, error) { + return postPlatformJSON[protocol.RunUpdateChunkRequest, protocol.RunUpdateChunkResponse](ctx, c, "/api/v1/run/jobs/update-chunk", request) +} + +func (c PlatformClient) ReportRunUpdateHealth(ctx context.Context, request protocol.RunUpdateHealthRequest) (protocol.RunUpdateHealthResponse, error) { + return postPlatformJSON[protocol.RunUpdateHealthRequest, protocol.RunUpdateHealthResponse](ctx, c, "/api/v1/run/jobs/update-health", request) +} + +func (c PlatformClient) PollJobCancel(ctx context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) { + return postPlatformJSON[protocol.RunJobCancelPollRequest, protocol.RunJobCancelPollResponse](ctx, c, "/api/v1/run/jobs/cancel", request) +} + +func (c PlatformClient) ReconcileJobs(ctx context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) { + return postPlatformJSON[protocol.RunJobReconcileRequest, protocol.RunJobReconcileResponse](ctx, c, "/api/v1/run/jobs/reconcile", request) +} + +func (c PlatformClient) IngestLogBatch(ctx context.Context, request protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + return postPlatformJSON[protocol.LogBatchIngestRequest, protocol.LogBatchIngestResponse](ctx, c, "/api/v1/run/logs/batches", request) +} + +func (c PlatformClient) GetRunLogStreamProgress(ctx context.Context, request protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) { + return postPlatformJSON[protocol.RunLogStreamProgressRequest, protocol.RunLogStreamProgressResponse](ctx, c, "/api/v1/run/logs/progress", request) +} + +func (c PlatformClient) IngestMetricBatch(ctx context.Context, request protocol.MetricBatchIngestRequest) (protocol.MetricBatchIngestResponse, error) { + return postPlatformJSON[protocol.MetricBatchIngestRequest, protocol.MetricBatchIngestResponse](ctx, c, "/api/v1/run/metrics/batches", request) +} + +func (c PlatformClient) OpenArtifactTransfer(ctx context.Context, request protocol.ArtifactTransferOpenRequest) (protocol.ArtifactTransferOpenResponse, error) { + return postPlatformJSON[protocol.ArtifactTransferOpenRequest, protocol.ArtifactTransferOpenResponse](ctx, c, "/api/v1/run/artifacts/open", request) +} + +func (c PlatformClient) UploadArtifactChunk(ctx context.Context, request protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) { + return postPlatformJSON[protocol.ArtifactChunkUploadRequest, protocol.ArtifactChunkUploadResponse](ctx, c, "/api/v1/run/artifacts/chunks", request) +} + +func (c PlatformClient) QueryArtifactTransferStatus(ctx context.Context, request protocol.ArtifactTransferStatusRequest) (protocol.ArtifactTransferStatusResponse, error) { + return postPlatformJSON[protocol.ArtifactTransferStatusRequest, protocol.ArtifactTransferStatusResponse](ctx, c, "/api/v1/run/artifacts/status", request) +} + +func (c PlatformClient) CompleteArtifactTransfer(ctx context.Context, request protocol.ArtifactTransferCompleteRequest) (protocol.ArtifactTransferCompleteResponse, error) { + return postPlatformJSON[protocol.ArtifactTransferCompleteRequest, protocol.ArtifactTransferCompleteResponse](ctx, c, "/api/v1/run/artifacts/complete", request) +} + +func postPlatformJSON[Request any, Response any](ctx context.Context, client PlatformClient, path string, request Request) (Response, error) { + var response Response + startedAt := time.Now() + log.Printf("RUN platform request status=starting method=POST base=%s path=%s", diagnosticLogValue(client.baseURL), path) + + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(request); err != nil { + log.Printf("RUN platform request status=encode_error method=POST base=%s path=%s durationMs=%d error=%s", diagnosticLogValue(client.baseURL), path, time.Since(startedAt).Milliseconds(), err) + return response, fmt.Errorf("encode platform request: %w", err) + } + + httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, client.baseURL+path, &body) + if err != nil { + log.Printf("RUN platform request status=build_error method=POST base=%s path=%s durationMs=%d error=%s", diagnosticLogValue(client.baseURL), path, time.Since(startedAt).Milliseconds(), err) + return response, fmt.Errorf("build platform request: %w", err) + } + httpRequest.Header.Set("Content-Type", "application/json") + httpRequest.Header.Set("Accept", "application/json") + if path != "/api/v1/run/control/hello" { + signatureSummary, err := signRunRequest(httpRequest, body.Bytes()) + if err != nil { + log.Printf("RUN platform request status=sign_error method=POST base=%s path=%s durationMs=%d error=%s", diagnosticLogValue(client.baseURL), path, time.Since(startedAt).Milliseconds(), err) + return response, err + } + log.Printf("RUN platform request status=signed method=POST base=%s path=%s endpoint=%s timestamp=%s nonce=%s bodyHash=%s signature=%s", diagnosticLogValue(client.baseURL), path, diagnosticLogValue(signatureSummary.RunEndpointID), signatureSummary.Timestamp, shortDiagnosticValue(signatureSummary.Nonce), shortDiagnosticValue(signatureSummary.BodyHash), shortDiagnosticValue(signatureSummary.Signature)) + } + + httpResponse, err := client.httpClient.Do(httpRequest) + if err != nil { + log.Printf("RUN platform request status=send_error method=POST base=%s path=%s durationMs=%d error=%s", diagnosticLogValue(client.baseURL), path, time.Since(startedAt).Milliseconds(), err) + return response, fmt.Errorf("send platform request: %w", err) + } + defer httpResponse.Body.Close() + log.Printf("RUN platform request status=response method=POST base=%s path=%s httpStatus=%d durationMs=%d", diagnosticLogValue(client.baseURL), path, httpResponse.StatusCode, time.Since(startedAt).Milliseconds()) + + if httpResponse.StatusCode < http.StatusOK || httpResponse.StatusCode >= http.StatusMultipleChoices { + var failure struct { + Code string `json:"code"` + Details []string `json:"details"` + } + _ = json.NewDecoder(io.LimitReader(httpResponse.Body, 64<<10)).Decode(&failure) + return response, PlatformRequestError{Status: httpResponse.StatusCode, Path: path, Code: failure.Code, Details: failure.Details} + } + if err := json.NewDecoder(httpResponse.Body).Decode(&response); err != nil { + return response, fmt.Errorf("decode platform response: %w", err) + } + return response, nil +} + +type runRequestEnvelope struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` +} + +type runRequestSignatureSummary struct { + RunEndpointID string + Timestamp string + Nonce string + BodyHash string + Signature string +} + +func signRunRequest(request *http.Request, body []byte) (runRequestSignatureSummary, error) { + var envelope runRequestEnvelope + if err := json.Unmarshal(body, &envelope); err != nil { + return runRequestSignatureSummary{}, fmt.Errorf("decode Run signing envelope: %w", err) + } + if strings.TrimSpace(envelope.RunEndpointID) == "" || strings.TrimSpace(envelope.SessionToken) == "" { + return runRequestSignatureSummary{}, fmt.Errorf("Run signing envelope requires endpoint and session token") + } + nonceBytes := make([]byte, 16) + if _, err := rand.Read(nonceBytes); err != nil { + return runRequestSignatureSummary{}, fmt.Errorf("create Run request nonce: %w", err) + } + timestamp := strconv.FormatInt(time.Now().UTC().Unix(), 10) + nonce := hex.EncodeToString(nonceBytes) + bodyHash := sha256.Sum256(body) + bodyHashHex := hex.EncodeToString(bodyHash[:]) + canonical := strings.Join([]string{request.Method, request.URL.Path, timestamp, nonce, hex.EncodeToString(bodyHash[:])}, "\n") + mac := hmac.New(sha256.New, []byte(envelope.SessionToken)) + _, _ = mac.Write([]byte(canonical)) + signature := hex.EncodeToString(mac.Sum(nil)) + request.Header.Set("X-Run-Endpoint", envelope.RunEndpointID) + request.Header.Set("X-Run-Timestamp", timestamp) + request.Header.Set("X-Run-Nonce", nonce) + request.Header.Set("X-Run-Signature", signature) + return runRequestSignatureSummary{RunEndpointID: envelope.RunEndpointID, Timestamp: timestamp, Nonce: nonce, BodyHash: bodyHashHex, Signature: signature}, nil +} + +func shortDiagnosticValue(value string) string { + value = diagnosticLogValue(value) + if value == "" { + return "-" + } + if len(value) <= 16 { + return value + } + return value[:12] + "..." + value[len(value)-4:] +} + +func diagnosticLogValue(value string) string { + value = strings.TrimSpace(value) + return strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(value) +} diff --git a/api/platform_client_test.go b/api/platform_client_test.go new file mode 100644 index 0000000..e9de91e --- /dev/null +++ b/api/platform_client_test.go @@ -0,0 +1,380 @@ +package api + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestNewPlatformClientNormalizesBaseURL(t *testing.T) { + client, err := NewPlatformClient("http://platform.test/") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.BaseURL() != "http://platform.test" { + t.Fatalf("expected normalized base URL, got %q", client.BaseURL()) + } +} + +func TestNewPlatformClientRequiresAbsoluteURL(t *testing.T) { + if _, err := NewPlatformClient("platform.local"); err == nil { + t.Fatal("expected error for URL without scheme and host") + } +} + +func TestPlatformClientHelloPostsJSONAndDecodesResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/control/hello" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { + t.Fatalf("expected JSON content type, got %q", contentType) + } + if r.Header.Get("X-Run-Signature") != "" { + t.Fatal("hello must not be signed with a session that does not exist yet") + } + var request protocol.RunHelloRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode hello request: %v", err) + } + if request.RunEndpointID != "run-local" || request.CapabilityReport.Fingerprint != "cap-v1" { + t.Fatalf("unexpected hello payload: %+v", request) + } + writeTestJSON(t, w, protocol.RunHelloResponse{ + Accepted: true, + RunEndpointID: "run-local", + SessionToken: "session-token", + ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC), + HeartbeatIntervalSeconds: 15, + }) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + response, err := client.Hello(context.Background(), validRunHelloRequest()) + if err != nil { + t.Fatalf("hello: %v", err) + } + if !response.Accepted || response.SessionToken != "session-token" || response.HeartbeatIntervalSeconds != 15 { + t.Fatalf("unexpected hello response: %+v", response) + } +} + +func TestPlatformRequestErrorRecognizesLegacyInvalidSession(t *testing.T) { + legacy := PlatformRequestError{Status: http.StatusBadRequest, Path: "/api/v1/run/control/heartbeat", Code: "validation_failed", Details: []string{"sessionToken is invalid"}} + if !legacy.SessionInvalid() { + t.Fatal("expected legacy invalid session response to be recognized") + } + if message := legacy.Error(); !strings.Contains(message, "status=400") || !strings.Contains(message, "path=/api/v1/run/control/heartbeat") || !strings.Contains(message, "code=validation_failed") || !strings.Contains(message, "sessionToken is invalid") { + t.Fatalf("expected request error to expose status, path, code, and details, got %q", message) + } + if (PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"another validation failure"}}).SessionInvalid() { + t.Fatal("unexpected validation response must not trigger re-registration") + } +} + +func TestPlatformRequestErrorRecognizesLegacyLogSessionMetadata(t *testing.T) { + err := PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"logSessionId and sessionStartedAt must be provided together"}} + if !err.LogBatchLegacySessionMetadata() { + t.Fatal("expected legacy log session metadata response to be recognized") + } + if (PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"other validation failure"}}).LogBatchLegacySessionMetadata() { + t.Fatal("unexpected validation response must not be recognized") + } +} + +func TestPlatformRequestErrorRecognizesLogSessionMetadataMismatch(t *testing.T) { + err := PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"log session metadata must match stream"}} + if !err.LogBatchSessionMetadataMismatch() { + t.Fatal("expected log session metadata mismatch response to be recognized") + } + if (PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"other validation failure"}}).LogBatchSessionMetadataMismatch() { + t.Fatal("unexpected validation response must not be recognized") + } +} + +func TestPlatformClientSignsRunChannelRequestsWithUniqueNonce(t *testing.T) { + nonces := map[string]struct{}{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read signed body: %v", err) + } + verifyRunRequestSignature(t, r, body, "run-local", "session-token") + nonce := r.Header.Get("X-Run-Nonce") + if _, exists := nonces[nonce]; exists { + t.Fatalf("reused Run request nonce %q", nonce) + } + nonces[nonce] = struct{}{} + writeTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: "run-local", NextHeartbeatSeconds: 15, ServerTime: time.Now().UTC()}) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + for range 2 { + if _, err := client.Heartbeat(context.Background(), validRunHeartbeatRequest("session-token")); err != nil { + t.Fatalf("signed heartbeat: %v", err) + } + } +} + +func verifyRunRequestSignature(t *testing.T, request *http.Request, body []byte, endpoint string, token string) { + t.Helper() + if request.Header.Get("X-Run-Endpoint") != endpoint { + t.Fatalf("unexpected signed endpoint %q", request.Header.Get("X-Run-Endpoint")) + } + timestamp := request.Header.Get("X-Run-Timestamp") + if _, err := strconv.ParseInt(timestamp, 10, 64); err != nil { + t.Fatalf("invalid signed timestamp %q", timestamp) + } + nonce := request.Header.Get("X-Run-Nonce") + if decoded, err := hex.DecodeString(nonce); err != nil || len(decoded) != 16 { + t.Fatalf("invalid signed nonce %q", nonce) + } + bodyHash := sha256.Sum256(body) + canonical := strings.Join([]string{request.Method, request.URL.Path, timestamp, nonce, hex.EncodeToString(bodyHash[:])}, "\n") + mac := hmac.New(sha256.New, []byte(token)) + _, _ = mac.Write([]byte(canonical)) + expected := hex.EncodeToString(mac.Sum(nil)) + if !hmac.Equal([]byte(expected), []byte(request.Header.Get("X-Run-Signature"))) { + t.Fatalf("invalid Run request signature") + } +} + +func TestPlatformClientHeartbeatPostsJSONAndDecodesResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/control/heartbeat" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var request protocol.RunHeartbeatRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode heartbeat request: %v", err) + } + if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 1 { + t.Fatalf("unexpected heartbeat payload: %+v", request) + } + writeTestJSON(t, w, protocol.RunHeartbeatResponse{ + Accepted: true, + RunEndpointID: "run-local", + NextHeartbeatSeconds: 15, + RefreshCapabilities: true, + ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC), + }) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + response, err := client.Heartbeat(context.Background(), validRunHeartbeatRequest("session-token")) + if err != nil { + t.Fatalf("heartbeat: %v", err) + } + if !response.Accepted || !response.RefreshCapabilities || response.NextHeartbeatSeconds != 15 { + t.Fatalf("unexpected heartbeat response: %+v", response) + } +} + +func TestPlatformClientGetsOneTimeSourceRCONInputOverSignedRoute(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/jobs/source-rcon-input" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read Source RCON input request: %v", err) + } + verifyRunRequestSignature(t, r, body, "run-local", "session-token") + if strings.Contains(string(body), "command") || strings.Contains(string(body), "password") { + t.Fatalf("Source RCON input request exposed command material: %s", body) + } + var request protocol.SourceRCONExecutionInputRequest + if err := json.Unmarshal(body, &request); err != nil { + t.Fatalf("decode Source RCON input request: %v", err) + } + if request.JobID != "job-source-rcon" || request.LeaseToken != "lease-source-rcon" || request.Attempt != 1 { + t.Fatalf("unexpected Source RCON input request: %+v", request) + } + writeTestJSON(t, w, protocol.SourceRCONExecutionInputResponse{JobID: request.JobID, ServerInstanceID: "server-1", RunEndpointID: request.RunEndpointID, Command: "rcon.status"}) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + response, err := client.GetSourceRCONExecutionInput(context.Background(), protocol.SourceRCONExecutionInputRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-source-rcon", LeaseToken: "lease-source-rcon", Attempt: 1}) + if err != nil || response.Command != "rcon.status" || response.RunEndpointID != "run-local" { + t.Fatalf("unexpected Source RCON input response=%+v err=%v", response, err) + } +} + +func TestPlatformClientGetsFencedProtectedRequestOverSignedRoute(t *testing.T) { + expiresAt := time.Now().UTC().Add(time.Minute) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/jobs/protected-request-input" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read protected input request: %v", err) + } + verifyRunRequestSignature(t, r, body, "run-local", "session-token") + if strings.Contains(string(body), "SELECT") || strings.Contains(string(body), "dsn") || strings.Contains(string(body), "password") { + t.Fatalf("protected input request exposed execution material: %s", body) + } + var request protocol.ProtectedRequestExecutionInputRequest + if err := json.Unmarshal(body, &request); err != nil { + t.Fatalf("decode protected input request: %v", err) + } + if request.JobID != "job-protected" || request.LeaseToken != "lease-protected" || request.FencingToken != 9 || request.Attempt != 1 { + t.Fatalf("unexpected protected input request: %+v", request) + } + writeTestJSON(t, w, protocol.ProtectedRequestExecutionInputResponse{JobID: request.JobID, ServerInstanceID: "server-1", RunEndpointID: request.RunEndpointID, FencingToken: request.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: expiresAt, Kind: "sql", TransportKey: "scum-database", TargetKey: "scum-database", RequestText: "SELECT player_id FROM players"}) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + response, err := client.GetProtectedRequestExecutionInput(context.Background(), protocol.ProtectedRequestExecutionInputRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-protected", LeaseToken: "lease-protected", Attempt: 1, FencingToken: 9}) + if err != nil || response.RequestText == "" || response.FencingToken != 9 || !response.Authorized || response.ApprovalState != "approved" || response.QueueState != "claimed" { + t.Fatalf("unexpected protected input response=%+v err=%v", response, err) + } +} + +func TestPlatformClientReturnsErrorForPlatformFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"validation_failed"}`)) + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + if _, err := client.Hello(context.Background(), validRunHelloRequest()); err == nil { + t.Fatal("expected platform error") + } else { + var requestError PlatformRequestError + if !errors.As(err, &requestError) || requestError.Status != http.StatusBadRequest || !strings.Contains(err.Error(), "code=validation_failed") { + t.Fatalf("expected typed platform error with diagnostic code, got %v", err) + } + } +} + +func TestPlatformClientHelloThenHeartbeatFlow(t *testing.T) { + var activeSessionToken string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/run/control/hello": + var request protocol.RunHelloRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode hello request: %v", err) + } + if request.RegistrationToken == "" || request.RunEndpointID != "run-local" { + t.Fatalf("unexpected hello request: %+v", request) + } + activeSessionToken = "session-token" + writeTestJSON(t, w, protocol.RunHelloResponse{ + Accepted: true, + RunEndpointID: request.RunEndpointID, + SessionToken: activeSessionToken, + ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC), + HeartbeatIntervalSeconds: 15, + }) + case "/api/v1/run/control/heartbeat": + var request protocol.RunHeartbeatRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode heartbeat request: %v", err) + } + if request.SessionToken != activeSessionToken { + t.Fatalf("heartbeat did not use active session token: %+v", request) + } + writeTestJSON(t, w, protocol.RunHeartbeatResponse{ + Accepted: true, + RunEndpointID: request.RunEndpointID, + NextHeartbeatSeconds: 15, + ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC), + }) + default: + t.Fatalf("unexpected request path %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("new client: %v", err) + } + hello, err := client.Hello(context.Background(), validRunHelloRequest()) + if err != nil { + t.Fatalf("hello: %v", err) + } + heartbeat, err := client.Heartbeat(context.Background(), validRunHeartbeatRequest(hello.SessionToken)) + if err != nil { + t.Fatalf("heartbeat: %v", err) + } + if !hello.Accepted || !heartbeat.Accepted { + t.Fatalf("expected accepted hello and heartbeat, got %+v %+v", hello, heartbeat) + } +} + +func writeTestJSON(t *testing.T, w http.ResponseWriter, value any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(value); err != nil { + t.Fatalf("encode response: %v", err) + } +} + +func validRunHelloRequest() protocol.RunHelloRequest { + return protocol.RunHelloRequest{ + RegistrationToken: "registration-token", + RunEndpointID: "run-local", + DisplayName: "Local Run", + Version: "0.1.0", + Status: "online", + Platform: "darwin/arm64", + CapabilityReport: protocol.RunCapabilityReport{ + Capabilities: []string{"control.hello", "control.heartbeat"}, + Fingerprint: "cap-v1", + }, + Capacity: protocol.RunCapacityReport{MaxJobs: 4}, + } +} + +func validRunHeartbeatRequest(sessionToken string) protocol.RunHeartbeatRequest { + return protocol.RunHeartbeatRequest{ + RunEndpointID: "run-local", + SessionToken: sessionToken, + Version: "0.1.0", + Status: "online", + CapabilityFingerprint: "cap-v1", + Capacity: protocol.RunCapacityReport{ + MaxJobs: 4, + RunningJobs: 1, + }, + } +} diff --git a/artifact/README.md b/artifact/README.md new file mode 100644 index 0000000..3f86411 --- /dev/null +++ b/artifact/README.md @@ -0,0 +1,13 @@ +# run/artifact + +Artifact transfer implementation lives here. + +Artifact transfer is lower priority than control, job ack/result, and log ingest. Implement concurrency and bandwidth limits before enabling large transfers. + +Artifact uploads must remain chunked and resumable. Slow, queued, or retrying artifact chunks must not prevent: + +- control hello/heartbeat calls, +- job claim/ack/progress/result/cancel/reconcile calls, +- durable log batch selection, upload, acknowledgement, or cleanup. + +Artifact queue entries may contain bounded transfer metadata and chunk bytes only. They must not expose host paths, raw credentials, direct sockets, run session transport details, or plugin/browser storage credentials. diff --git a/cmd/run/main.go b/cmd/run/main.go new file mode 100644 index 0000000..7182040 --- /dev/null +++ b/cmd/run/main.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/url" + "os" + "os/signal" + + "browser.local/run/api" + "browser.local/run/config" + runruntime "browser.local/run/runtime" + "browser.local/run/spool" +) + +func main() { + if handled, exitCode := runruntime.RunManagedProcessHelper(os.Args); handled { + os.Exit(exitCode) + } + cfg := config.Load() + configSource := "environment-or-build-defaults" + if cfg.Mode == "self-update-helper" { + log.Printf("RUN phase=self_update_helper status=starting manifestPresent=%t", os.Getenv("RUN_UPDATE_MANIFEST") != "") + if err := runruntime.ApplySelfUpdateManifest(os.Getenv("RUN_UPDATE_MANIFEST")); err != nil { + fmt.Fprintf(os.Stderr, "Run self-update helper failed: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=self_update_helper status=complete") + return + } + if packageConfig, ok, err := config.LoadPackageConfigFromEnv(); err != nil { + fmt.Fprintf(os.Stderr, "invalid run package config: %v\n", err) + os.Exit(1) + } else if ok { + cfg = config.ApplyPackageConfig(cfg, packageConfig) + configSource = "RUN_PACKAGE_CONFIG" + } else if packageConfig, ok, err := config.LoadPackageConfigBesideExecutable(); err != nil { + fmt.Fprintf(os.Stderr, "invalid bundled run package config: %v\n", err) + os.Exit(1) + } else if ok { + cfg = config.ApplyPackageConfig(cfg, packageConfig) + configSource = "package-config-beside-executable" + } + logWorkerPhases := cfg.Mode == "worker" + if logWorkerPhases { + log.Printf("RUN phase=bootstrap status=starting pid=%d args=%d", os.Getpid(), len(os.Args)) + log.Printf("RUN phase=config status=loaded source=%s mode=%s platform=%s endpoint=%s server=%s plugin=%s component=%s componentKey=%s version=%s", configSource, cfg.Mode, diagnosticPlatformAddress(cfg.PlatformURL), diagnosticValue(cfg.RunEndpointID), diagnosticValue(cfg.ServerInstanceID), diagnosticValue(cfg.PluginID), diagnosticValue(cfg.ComponentKind), diagnosticValue(cfg.ComponentKey), diagnosticValue(cfg.Version)) + log.Printf("RUN phase=platform_client status=building platform=%s", diagnosticPlatformAddress(cfg.PlatformURL)) + } + client, err := api.NewPlatformClient(cfg.PlatformURL) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid platform URL: %v\n", err) + os.Exit(1) + } + if logWorkerPhases { + log.Printf("RUN phase=platform_client status=ready base=%s", client.BaseURL()) + } + + if cfg.Mode == "worker" { + log.Printf("RUN phase=startup status=worker platform=%s endpoint=%s server=%s plugin=%s workspace=%s spool=%s maxJobs=%d", diagnosticPlatformAddress(cfg.PlatformURL), diagnosticValue(cfg.RunEndpointID), diagnosticValue(cfg.ServerInstanceID), diagnosticValue(cfg.PluginID), diagnosticValue(cfg.WorkspaceRoot), diagnosticValue(cfg.SpoolRoot), cfg.MaxJobs) + log.Printf("RUN phase=workspace_seed status=materializing present=%t workspace=%s componentKey=%s", cfg.WorkspaceSeed != "", diagnosticValue(cfg.WorkspaceRoot), diagnosticValue(cfg.ComponentKey)) + if err := runruntime.MaterializeWorkspaceSeed(cfg); err != nil { + fmt.Fprintf(os.Stderr, "initialize plugin workspace assets: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=workspace_seed status=ready workspace=%s", diagnosticValue(cfg.WorkspaceRoot)) + log.Printf("RUN phase=log_spool status=opening path=%s", diagnosticValue(cfg.SpoolRoot)) + logSpool, err := spool.NewLogSpool(cfg.SpoolRoot) + if err != nil { + fmt.Fprintf(os.Stderr, "initialize log spool: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=log_spool status=ready path=%s", diagnosticValue(cfg.SpoolRoot)) + log.Printf("RUN phase=artifact_queue status=opening path=%s", diagnosticValue(cfg.SpoolRoot)) + artifactQueue, err := spool.NewArtifactQueue(cfg.SpoolRoot) + if err != nil { + fmt.Fprintf(os.Stderr, "initialize artifact queue: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=artifact_queue status=ready path=%s", diagnosticValue(cfg.SpoolRoot)) + log.Printf("RUN phase=worker_init status=starting endpoint=%s", diagnosticValue(cfg.RunEndpointID)) + worker, err := runruntime.NewWorker( + cfg, + client, + runruntime.WithProcessLogSink(&runruntime.SpoolLogSink{Spool: logSpool}), + runruntime.WithLifecycleArtifactHook(&runruntime.QueueArtifactHook{Queue: artifactQueue}), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "initialize worker: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=worker_init status=ready endpoint=%s", diagnosticValue(cfg.RunEndpointID)) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + log.Printf("RUN phase=worker_loop status=entering endpoint=%s", diagnosticValue(cfg.RunEndpointID)) + if err := worker.Run(ctx); err != nil && err != context.Canceled && err != runruntime.ErrSelfUpdateRestartRequested { + fmt.Fprintf(os.Stderr, "run worker stopped: %v\n", err) + os.Exit(1) + } + log.Printf("RUN phase=worker_loop status=stopped endpoint=%s", diagnosticValue(cfg.RunEndpointID)) + return + } + + summary := runruntime.SmokeSummary(cfg) + summary.PlatformURL = client.BaseURL() + + if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil { + fmt.Fprintf(os.Stderr, "encode smoke summary: %v\n", err) + os.Exit(1) + } +} + +func diagnosticValue(value string) string { + if value == "" { + return "-" + } + return value +} + +func diagnosticPlatformAddress(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "invalid" + } + address := parsed.Scheme + "://" + parsed.Host + if path := parsed.EscapedPath(); path != "" && path != "/" { + address += path + } + return address +} diff --git a/cmd/run/main_test.go b/cmd/run/main_test.go new file mode 100644 index 0000000..0e2c792 --- /dev/null +++ b/cmd/run/main_test.go @@ -0,0 +1,12 @@ +package main + +import "testing" + +func TestDiagnosticPlatformAddressOmitsCredentials(t *testing.T) { + if got := diagnosticPlatformAddress("https://token:secret@scum.npc0.com/api"); got != "https://scum.npc0.com/api" { + t.Fatalf("unexpected diagnostic address %q", got) + } + if got := diagnosticPlatformAddress("not a URL"); got != "invalid" { + t.Fatalf("unexpected invalid diagnostic address %q", got) + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..7eecf09 --- /dev/null +++ b/config/config.go @@ -0,0 +1,138 @@ +package config + +import ( + "os" + "path/filepath" + "strconv" + "time" +) + +const ( + DefaultMode = "smoke" + DefaultPlatformURL = "http://127.0.0.1:8080" + DefaultEndpointID = "run-local" + DefaultDisplayName = "Local Run" + DefaultVersion = "0.1.0" +) + +var BuildVersion = DefaultVersion +var BuildMode string +var BuildPlatformURL string +var BuildRunEndpointID string +var BuildDisplayName string +var BuildRegistrationToken string +var BuildServerInstanceID string +var BuildPluginID string +var BuildComponentKind string +var BuildComponentKey string +var BuildKeyGeneration string +var BuildWorkspaceSeed string + +type Config struct { + Mode string + PlatformURL string + RunEndpointID string + DisplayName string + Version string + RegistrationToken string + ServerInstanceID string + PluginID string + ComponentKind string + ComponentKey string + KeyGeneration int + SecretRef string + WorkspaceSeed string + WorkspaceRoot string + BuildSourceRoot string + SpoolRoot string + MaxJobs int + HeartbeatInterval time.Duration + PollInterval time.Duration + RetryBackoff time.Duration + UpdateJobID string + UpdateOutcome string + UpdateAttempt int + UpdateLeaseToken string + UpdateHealthFile string + LocalStartupDiagnostics bool +} + +func Load() Config { + mode := os.Getenv("RUN_MODE") + if mode == "" { + mode = stringOrDefault(BuildMode, DefaultMode) + } + + platformURL := os.Getenv("RUN_PLATFORM_URL") + if platformURL == "" { + platformURL = stringOrDefault(BuildPlatformURL, DefaultPlatformURL) + } + + workspaceRoot := envOrDefault("RUN_WORKSPACE_ROOT", filepath.Join(".", ".run-workspace")) + return Config{ + Mode: mode, + PlatformURL: platformURL, + RunEndpointID: envOrDefault("RUN_ENDPOINT_ID", stringOrDefault(BuildRunEndpointID, DefaultEndpointID)), + DisplayName: envOrDefault("RUN_DISPLAY_NAME", stringOrDefault(BuildDisplayName, DefaultDisplayName)), + Version: envOrDefault("RUN_VERSION", BuildVersion), + RegistrationToken: envOrDefault("RUN_REGISTRATION_TOKEN", stringOrDefault(BuildRegistrationToken, "local-registration")), + ServerInstanceID: envOrDefault("RUN_SERVER_INSTANCE_ID", BuildServerInstanceID), + PluginID: envOrDefault("RUN_PLUGIN_ID", BuildPluginID), + ComponentKind: envOrDefault("RUN_COMPONENT_KIND", BuildComponentKind), + ComponentKey: envOrDefault("RUN_COMPONENT_KEY", BuildComponentKey), + KeyGeneration: intEnvOrDefault("RUN_KEY_GENERATION", intStringOrDefault(BuildKeyGeneration, 0)), + WorkspaceSeed: envOrDefault("RUN_WORKSPACE_SEED", BuildWorkspaceSeed), + WorkspaceRoot: workspaceRoot, + BuildSourceRoot: envOrDefault("RUN_BUILD_SOURCE_ROOT", "."), + SpoolRoot: envOrDefault("RUN_SPOOL_ROOT", filepath.Join(workspaceRoot, "spool")), + MaxJobs: intEnvOrDefault("RUN_MAX_JOBS", 1), + HeartbeatInterval: durationEnvOrDefault("RUN_HEARTBEAT_INTERVAL_MS", 15*time.Second), + PollInterval: durationEnvOrDefault("RUN_POLL_INTERVAL_MS", 2*time.Second), + RetryBackoff: durationEnvOrDefault("RUN_RETRY_BACKOFF_MS", time.Second), + UpdateJobID: os.Getenv("RUN_UPDATE_JOB_ID"), + UpdateOutcome: os.Getenv("RUN_UPDATE_OUTCOME"), + UpdateAttempt: intEnvOrDefault("RUN_UPDATE_ATTEMPT", 0), + UpdateLeaseToken: os.Getenv("RUN_UPDATE_LEASE_TOKEN"), + UpdateHealthFile: os.Getenv("RUN_UPDATE_HEALTH_FILE"), + LocalStartupDiagnostics: os.Getenv("RUN_LOCAL_STARTUP_DIAGNOSTICS") == "1", + } +} + +func stringOrDefault(value string, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func intStringOrDefault(value string, fallback int) int { + parsed, err := strconv.Atoi(value) + if err != nil || parsed <= 0 { + return fallback + } + return parsed +} + +func envOrDefault(key string, fallback string) string { + value := os.Getenv(key) + if value == "" { + return fallback + } + return value +} + +func intEnvOrDefault(key string, fallback int) int { + value, err := strconv.Atoi(os.Getenv(key)) + if err != nil || value <= 0 { + return fallback + } + return value +} + +func durationEnvOrDefault(key string, fallback time.Duration) time.Duration { + value, err := strconv.Atoi(os.Getenv(key)) + if err != nil || value <= 0 { + return fallback + } + return time.Duration(value) * time.Millisecond +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..0c244e5 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,93 @@ +package config + +import "testing" + +func TestLoadUsesDefaults(t *testing.T) { + t.Setenv("RUN_MODE", "") + t.Setenv("RUN_PLATFORM_URL", "") + + cfg := Load() + if cfg.Mode != DefaultMode { + t.Fatalf("expected mode %q, got %q", DefaultMode, cfg.Mode) + } + if cfg.PlatformURL != DefaultPlatformURL { + t.Fatalf("expected platform URL %q, got %q", DefaultPlatformURL, cfg.PlatformURL) + } + if cfg.RunEndpointID != DefaultEndpointID || cfg.DisplayName != DefaultDisplayName || cfg.Version != DefaultVersion { + t.Fatalf("expected worker defaults, got %+v", cfg) + } + if cfg.MaxJobs != 1 || cfg.HeartbeatInterval <= 0 || cfg.PollInterval <= 0 || cfg.RetryBackoff <= 0 { + t.Fatalf("expected positive worker scheduling defaults, got %+v", cfg) + } +} + +func TestLoadUsesEnvironment(t *testing.T) { + t.Setenv("RUN_MODE", "worker") + t.Setenv("RUN_PLATFORM_URL", "http://platform.test") + t.Setenv("RUN_ENDPOINT_ID", "run-edge") + t.Setenv("RUN_DISPLAY_NAME", "Edge Run") + t.Setenv("RUN_VERSION", "1.2.3") + t.Setenv("RUN_REGISTRATION_TOKEN", "registration-token") + t.Setenv("RUN_WORKSPACE_ROOT", "/tmp/run-workspace") + t.Setenv("RUN_SPOOL_ROOT", "/tmp/run-spool") + t.Setenv("RUN_MAX_JOBS", "3") + t.Setenv("RUN_HEARTBEAT_INTERVAL_MS", "250") + t.Setenv("RUN_POLL_INTERVAL_MS", "125") + t.Setenv("RUN_RETRY_BACKOFF_MS", "75") + + cfg := Load() + if cfg.Mode != "worker" { + t.Fatalf("expected configured mode, got %q", cfg.Mode) + } + if cfg.PlatformURL != "http://platform.test" { + t.Fatalf("expected configured platform URL, got %q", cfg.PlatformURL) + } + if cfg.RunEndpointID != "run-edge" || cfg.DisplayName != "Edge Run" || cfg.Version != "1.2.3" || cfg.RegistrationToken != "registration-token" { + t.Fatalf("expected configured worker identity, got %+v", cfg) + } + if cfg.WorkspaceRoot != "/tmp/run-workspace" || cfg.SpoolRoot != "/tmp/run-spool" || cfg.MaxJobs != 3 { + t.Fatalf("expected configured worker paths/capacity, got %+v", cfg) + } + if cfg.HeartbeatInterval.Milliseconds() != 250 || cfg.PollInterval.Milliseconds() != 125 || cfg.RetryBackoff.Milliseconds() != 75 { + t.Fatalf("expected configured durations, got %+v", cfg) + } +} + +func TestLoadUsesBuildDefaultsWithEnvironmentOverride(t *testing.T) { + oldMode, oldPlatformURL, oldRunEndpointID, oldDisplayName := BuildMode, BuildPlatformURL, BuildRunEndpointID, BuildDisplayName + oldRegistrationToken, oldServerInstanceID, oldPluginID := BuildRegistrationToken, BuildServerInstanceID, BuildPluginID + oldComponentKind, oldComponentKey, oldKeyGeneration, oldVersion, oldWorkspaceSeed := BuildComponentKind, BuildComponentKey, BuildKeyGeneration, BuildVersion, BuildWorkspaceSeed + defer func() { + BuildMode, BuildPlatformURL, BuildRunEndpointID, BuildDisplayName = oldMode, oldPlatformURL, oldRunEndpointID, oldDisplayName + BuildRegistrationToken, BuildServerInstanceID, BuildPluginID = oldRegistrationToken, oldServerInstanceID, oldPluginID + BuildComponentKind, BuildComponentKey, BuildKeyGeneration, BuildVersion = oldComponentKind, oldComponentKey, oldKeyGeneration, oldVersion + BuildWorkspaceSeed = oldWorkspaceSeed + }() + + BuildMode = "worker" + BuildPlatformURL = "https://scum.npc0.com" + BuildRunEndpointID = "run-server-1" + BuildDisplayName = "Run-server-1" + BuildRegistrationToken = "compiled-run-key" + BuildServerInstanceID = "server-1" + BuildPluginID = "game.scum" + BuildComponentKind = "run" + BuildKeyGeneration = "5" + BuildVersion = "run-dist-1" + BuildWorkspaceSeed = "seed-payload" + + for _, key := range []string{"RUN_MODE", "RUN_PLATFORM_URL", "RUN_ENDPOINT_ID", "RUN_DISPLAY_NAME", "RUN_REGISTRATION_TOKEN", "RUN_SERVER_INSTANCE_ID", "RUN_PLUGIN_ID", "RUN_COMPONENT_KIND", "RUN_COMPONENT_KEY", "RUN_KEY_GENERATION", "RUN_VERSION", "RUN_WORKSPACE_SEED"} { + t.Setenv(key, "") + } + cfg := Load() + if cfg.Mode != "worker" || cfg.PlatformURL != "https://scum.npc0.com" || cfg.RunEndpointID != "run-server-1" || cfg.RegistrationToken != "compiled-run-key" || cfg.ServerInstanceID != "server-1" || cfg.PluginID != "game.scum" || cfg.ComponentKind != "run" || cfg.KeyGeneration != 5 || cfg.Version != "run-dist-1" || cfg.WorkspaceSeed != "seed-payload" { + t.Fatalf("expected compiled defaults, got %+v", cfg) + } + + t.Setenv("RUN_PLATFORM_URL", "http://127.0.0.1:18080") + t.Setenv("RUN_KEY_GENERATION", "7") + cfg = Load() + if cfg.PlatformURL != "http://127.0.0.1:18080" || cfg.KeyGeneration != 7 { + t.Fatalf("expected environment override, got %+v", cfg) + } +} diff --git a/config/package_config.go b/config/package_config.go new file mode 100644 index 0000000..e7549cd --- /dev/null +++ b/config/package_config.go @@ -0,0 +1,265 @@ +package config + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + PackageComponentRun = "run" + PackageComponentClientManager = "client-manager" + + PackageConfigEnv = "RUN_PACKAGE_CONFIG" +) + +type PackageConfig struct { + Kind string `json:"kind"` + ServerInstanceID string `json:"serverInstanceId"` + PluginID string `json:"pluginId"` + RunEndpointID string `json:"runEndpointId,omitempty"` + ProfileKey string `json:"profileKey,omitempty"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + SecretRef string `json:"secretRef"` + KeyGeneration int `json:"keyGeneration"` + AuthKey string `json:"authKey"` +} + +type PackageIdentity struct { + Kind string `json:"kind"` + ServerInstanceID string `json:"serverInstanceId"` + PluginID string `json:"pluginId"` + RunEndpointID string `json:"runEndpointId,omitempty"` + ProfileKey string `json:"profileKey,omitempty"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + SecretRef string `json:"secretRef"` + KeyGeneration int `json:"keyGeneration"` + KeyFingerprint string `json:"keyFingerprint"` +} + +type ComponentAuthResult struct { + ServerInstanceID string + Kind string + ProfileKey string + KeyGeneration int + Allowed bool + Reason string +} + +func LoadPackageConfig(path string) (PackageConfig, error) { + body, err := os.ReadFile(path) + if err != nil { + return PackageConfig{}, fmt.Errorf("read run package config: %w", err) + } + var cfg PackageConfig + if err := json.Unmarshal(body, &cfg); err != nil { + return PackageConfig{}, fmt.Errorf("decode run package config: %w", err) + } + if err := ValidatePackageConfig(cfg); err != nil { + return PackageConfig{}, err + } + return cfg, nil +} + +func LoadPackageConfigFromEnv() (PackageConfig, bool, error) { + path := strings.TrimSpace(os.Getenv(PackageConfigEnv)) + if path == "" { + return PackageConfig{}, false, nil + } + cfg, err := LoadPackageConfig(path) + return cfg, true, err +} + +func LoadPackageConfigBesideExecutable() (PackageConfig, bool, error) { + executable, err := os.Executable() + if err != nil { + return PackageConfig{}, false, nil + } + path := filepath.Join(filepath.Dir(executable), "config.json") + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return PackageConfig{}, false, nil + } + return PackageConfig{}, false, err + } + cfg, err := LoadPackageConfig(path) + return cfg, true, err +} + +func ValidatePackageConfig(cfg PackageConfig) error { + var violations []string + if cfg.Kind != PackageComponentRun && cfg.Kind != PackageComponentClientManager { + violations = append(violations, "kind is invalid") + } + if !safeIdentifier(cfg.ServerInstanceID) { + violations = append(violations, "serverInstanceId is invalid") + } + if !safePluginID(cfg.PluginID) { + violations = append(violations, "pluginId is invalid") + } + if cfg.RunEndpointID != "" && !safeIdentifier(cfg.RunEndpointID) { + violations = append(violations, "runEndpointId is invalid") + } + if cfg.ProfileKey != "" && !safeLogicalKey(cfg.ProfileKey) { + violations = append(violations, "profileKey is invalid") + } + if cfg.Kind == PackageComponentClientManager && cfg.ProfileKey == "" { + violations = append(violations, "profileKey is required for client-manager packages") + } + if !safeRuntimeTarget(cfg.TargetOS, cfg.TargetArch) { + violations = append(violations, "target platform is invalid") + } + if !strings.HasPrefix(cfg.SecretRef, "secret://runtime-keys/") || containsUnsafeDiagnosticText(cfg.SecretRef) { + violations = append(violations, "secretRef is invalid") + } + if cfg.KeyGeneration <= 0 { + violations = append(violations, "keyGeneration must be positive") + } + if strings.TrimSpace(cfg.AuthKey) == "" { + violations = append(violations, "authKey is required") + } + if containsUnsafeDiagnosticText(cfg.AuthKey) { + violations = append(violations, "authKey contains unsafe content") + } + if len(violations) > 0 { + return fmt.Errorf("invalid run package config: %s", strings.Join(violations, "; ")) + } + return nil +} + +func ApplyPackageConfig(base Config, pkg PackageConfig) Config { + base.RegistrationToken = pkg.AuthKey + base.ServerInstanceID = pkg.ServerInstanceID + base.PluginID = pkg.PluginID + base.ComponentKind = pkg.Kind + base.ComponentKey = pkg.ProfileKey + base.KeyGeneration = pkg.KeyGeneration + base.SecretRef = pkg.SecretRef + if pkg.RunEndpointID != "" { + base.RunEndpointID = pkg.RunEndpointID + } + if base.DisplayName == "" || base.DisplayName == DefaultDisplayName { + base.DisplayName = "Run " + pkg.ServerInstanceID + } + return base +} + +func (cfg PackageConfig) Identity() PackageIdentity { + return PackageIdentity{ + Kind: cfg.Kind, + ServerInstanceID: cfg.ServerInstanceID, + PluginID: cfg.PluginID, + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ProfileKey, + TargetOS: cfg.TargetOS, + TargetArch: cfg.TargetArch, + SecretRef: cfg.SecretRef, + KeyGeneration: cfg.KeyGeneration, + KeyFingerprint: fingerprint(cfg.AuthKey), + } +} + +func (cfg PackageConfig) RedactedDiagnostics() map[string]string { + identity := cfg.Identity() + return map[string]string{ + "kind": identity.Kind, + "serverInstanceId": identity.ServerInstanceID, + "pluginId": identity.PluginID, + "runEndpointId": identity.RunEndpointID, + "profileKey": identity.ProfileKey, + "target": identity.TargetOS + "/" + identity.TargetArch, + "secretRef": identity.SecretRef, + "keyGeneration": fmt.Sprintf("%d", identity.KeyGeneration), + "keyFingerprint": identity.KeyFingerprint, + } +} + +func AuthenticatePackageGeneration(pkg PackageConfig, auth ComponentAuthResult) error { + if err := ValidatePackageConfig(pkg); err != nil { + return err + } + if auth.ServerInstanceID != pkg.ServerInstanceID || auth.Kind != pkg.Kind || auth.ProfileKey != pkg.ProfileKey { + return fmt.Errorf("component authentication scope does not match package") + } + if !auth.Allowed { + return fmt.Errorf("component authentication rejected: %s", redactedReason(auth.Reason)) + } + if auth.KeyGeneration != pkg.KeyGeneration { + return fmt.Errorf("component key generation is no longer current") + } + return nil +} + +func fingerprint(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:])[:12] +} + +func safeIdentifier(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > 120 || containsUnsafeDiagnosticText(value) { + return false + } + for _, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.' { + continue + } + return false + } + return true +} + +func safePluginID(value string) bool { + return strings.HasPrefix(value, "game.") && safeIdentifier(value) +} + +func safeLogicalKey(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > 120 || strings.HasPrefix(value, "/") || strings.Contains(value, "..") || strings.Contains(value, `\`) || containsUnsafeDiagnosticText(value) { + return false + } + for _, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.' || char == '/' { + continue + } + return false + } + return true +} + +func safeRuntimeTarget(osName string, arch string) bool { + switch osName { + case "windows", "linux", "darwin": + default: + return false + } + switch arch { + case "amd64", "arm64": + return true + default: + return false + } +} + +func containsUnsafeDiagnosticText(value string) bool { + normalized := strings.ToLower(value) + for _, marker := range []string{"/users/", "/.ssh/", "password=", "apikey", "api_key", "bearer ", "sk-", "unix://", "tcp://", "mysql://", "sqlite://"} { + if strings.Contains(normalized, marker) { + return true + } + } + return false +} + +func redactedReason(value string) string { + if containsUnsafeDiagnosticText(value) { + return "[redacted]" + } + return value +} diff --git a/config/package_config_test.go b/config/package_config_test.go new file mode 100644 index 0000000..813fb32 --- /dev/null +++ b/config/package_config_test.go @@ -0,0 +1,141 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadPackageConfigAppliesServerScopedIdentity(t *testing.T) { + path := writePackageConfig(t, PackageConfig{ + Kind: PackageComponentRun, + ServerInstanceID: "server-1", + PluginID: "game.minecraft", + RunEndpointID: "run-server-1", + TargetOS: "linux", + TargetArch: "amd64", + SecretRef: "secret://runtime-keys/server-1/run/current", + KeyGeneration: 3, + AuthKey: "opaque-runtime-key", + }) + + pkg, err := LoadPackageConfig(path) + if err != nil { + t.Fatalf("load package config: %v", err) + } + cfg := ApplyPackageConfig(Config{RunEndpointID: DefaultEndpointID, DisplayName: DefaultDisplayName}, pkg) + if cfg.RegistrationToken != "opaque-runtime-key" || cfg.RunEndpointID != "run-server-1" || cfg.ServerInstanceID != "server-1" || cfg.KeyGeneration != 3 { + t.Fatalf("expected package identity to be applied, got %+v", cfg) + } + diagnostics := pkg.RedactedDiagnostics() + for _, value := range diagnostics { + if strings.Contains(value, "opaque-runtime-key") || strings.Contains(value, "/Users/") || strings.Contains(value, "password=") { + t.Fatalf("diagnostics exposed sensitive value: %+v", diagnostics) + } + } + if diagnostics["keyFingerprint"] == "" || diagnostics["secretRef"] != "secret://runtime-keys/server-1/run/current" { + t.Fatalf("expected redacted key fingerprint and secret ref, got %+v", diagnostics) + } +} + +func TestLoadPackageConfigRejectsUnsafeOrIncompletePackages(t *testing.T) { + valid := PackageConfig{ + Kind: PackageComponentRun, + ServerInstanceID: "server-1", + PluginID: "game.scum", + TargetOS: "windows", + TargetArch: "amd64", + SecretRef: "secret://runtime-keys/server-1/run/current", + KeyGeneration: 1, + AuthKey: "opaque-runtime-key", + } + cases := map[string]func(PackageConfig) PackageConfig{ + "old zero generation": func(cfg PackageConfig) PackageConfig { cfg.KeyGeneration = 0; return cfg }, + "raw path": func(cfg PackageConfig) PackageConfig { cfg.ServerInstanceID = "/Users/tasia/server"; return cfg }, + "socket": func(cfg PackageConfig) PackageConfig { cfg.SecretRef = "unix:///tmp/run.sock"; return cfg }, + "secret auth": func(cfg PackageConfig) PackageConfig { cfg.AuthKey = "password=raw"; return cfg }, + "client missing key": func(cfg PackageConfig) PackageConfig { cfg.Kind = PackageComponentClientManager; return cfg }, + } + for name, mutate := range cases { + if err := ValidatePackageConfig(mutate(valid)); err == nil { + t.Fatalf("expected %s package to be rejected", name) + } + } +} + +func TestAuthenticatePackageGenerationRejectsStalePackages(t *testing.T) { + pkg := PackageConfig{ + Kind: PackageComponentRun, + ServerInstanceID: "server-1", + PluginID: "game.minecraft", + TargetOS: "linux", + TargetArch: "amd64", + SecretRef: "secret://runtime-keys/server-1/run/current", + KeyGeneration: 1, + AuthKey: "opaque-runtime-key", + } + err := AuthenticatePackageGeneration(pkg, ComponentAuthResult{ + ServerInstanceID: "server-1", + Kind: PackageComponentRun, + KeyGeneration: 2, + Allowed: true, + Reason: "current key accepted", + }) + if err == nil || !strings.Contains(err.Error(), "generation") { + t.Fatalf("expected stale generation rejection, got %v", err) + } + err = AuthenticatePackageGeneration(pkg, ComponentAuthResult{ + ServerInstanceID: "server-1", + Kind: PackageComponentRun, + KeyGeneration: 1, + Allowed: true, + Reason: "current key accepted", + }) + if err != nil { + t.Fatalf("expected current generation to authenticate: %v", err) + } +} + +func TestLoadPackageConfigFromEnv(t *testing.T) { + path := writePackageConfig(t, PackageConfig{ + Kind: PackageComponentClientManager, + ServerInstanceID: "server-1", + PluginID: "game.scum", + ProfileKey: "scum-client-manager", + TargetOS: "windows", + TargetArch: "amd64", + SecretRef: "secret://runtime-keys/server-1/client-manager/scum-client-manager/current", + KeyGeneration: 4, + AuthKey: "opaque-client-key", + }) + t.Setenv(PackageConfigEnv, path) + + cfg, ok, err := LoadPackageConfigFromEnv() + if err != nil || !ok { + t.Fatalf("expected env package config, ok=%v err=%v", ok, err) + } + if cfg.Kind != PackageComponentClientManager || cfg.ProfileKey != "scum-client-manager" { + t.Fatalf("unexpected package config: %+v", cfg) + } + + t.Setenv(PackageConfigEnv, "") + _, ok, err = LoadPackageConfigFromEnv() + if err != nil || ok { + t.Fatalf("expected no env package config, ok=%v err=%v", ok, err) + } +} + +func writePackageConfig(t *testing.T, cfg PackageConfig) string { + t.Helper() + body, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshal package config: %v", err) + } + path := filepath.Join(t.TempDir(), "run-package.json") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatalf("write package config: %v", err) + } + return path +} diff --git a/domain/status.go b/domain/status.go new file mode 100644 index 0000000..04e07fa --- /dev/null +++ b/domain/status.go @@ -0,0 +1,9 @@ +package domain + +type ExecutorStatus struct { + Mode string `json:"mode"` + PlatformURL string `json:"platformUrl"` + Status string `json:"status"` + ExposedHostPath bool `json:"exposedHostPath"` + Capabilities []string `json:"capabilities"` +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2c9faac --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module browser.local/run + +go 1.25.1 + +require modernc.org/sqlite v1.34.5 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5424fe4 --- /dev/null +++ b/go.sum @@ -0,0 +1,43 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/logingest/README.md b/logingest/README.md new file mode 100644 index 0000000..51fc1ca --- /dev/null +++ b/logingest/README.md @@ -0,0 +1,7 @@ +# run/logingest + +Log collectors and uploaders live here. + +Collectors read process output and server log files, assign stream IDs and sequence numbers, write to local spool, and upload batches to platform ingest APIs. + +Do not send primary run-to-platform logs through the UI realtime channel. diff --git a/protocol/artifact.go b/protocol/artifact.go new file mode 100644 index 0000000..3474dae --- /dev/null +++ b/protocol/artifact.go @@ -0,0 +1,101 @@ +package protocol + +import "time" + +type ArtifactMetadata struct { + ID string `json:"id"` + OwnerKind string `json:"ownerKind"` + OwnerID string `json:"ownerId"` + SizeBytes int64 `json:"sizeBytes"` + Checksum string `json:"checksum"` + State string `json:"state"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type ArtifactTransferOpenRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + ArtifactID string `json:"artifactId"` + Direction string `json:"direction"` + OwnerKind string `json:"ownerKind"` + OwnerID string `json:"ownerId"` + SizeBytes int64 `json:"sizeBytes"` + ChunkSizeBytes int `json:"chunkSizeBytes"` + Checksum string `json:"checksum"` + IdempotencyKey string `json:"idempotencyKey"` +} + +type ArtifactTransferOpenResponse struct { + Accepted bool `json:"accepted"` + TransferID string `json:"transferId"` + Direction string `json:"direction"` + Artifact ArtifactMetadata `json:"artifact"` + TotalChunks int `json:"totalChunks"` + ChunkSizeBytes int `json:"chunkSizeBytes"` + ReceivedChunkIndexes []int `json:"receivedChunkIndexes"` + NextMissingChunkIndex int `json:"nextMissingChunkIndex"` + Completed bool `json:"completed"` + Duplicate bool `json:"duplicate"` + ServerTime time.Time `json:"serverTime"` +} + +type ArtifactChunkUploadRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + TransferID string `json:"transferId"` + ArtifactID string `json:"artifactId"` + ChunkIndex int `json:"chunkIndex"` + Offset int64 `json:"offset"` + SizeBytes int `json:"sizeBytes"` + Checksum string `json:"checksum"` + Payload []byte `json:"payload"` +} + +type ArtifactChunkUploadResponse struct { + Accepted bool `json:"accepted"` + TransferID string `json:"transferId"` + ArtifactID string `json:"artifactId"` + ChunkIndex int `json:"chunkIndex"` + ReceivedChunkIndexes []int `json:"receivedChunkIndexes"` + NextMissingChunkIndex int `json:"nextMissingChunkIndex"` + Duplicate bool `json:"duplicate"` + ServerTime time.Time `json:"serverTime"` +} + +type ArtifactTransferStatusRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + TransferID string `json:"transferId"` + ArtifactID string `json:"artifactId"` +} + +type ArtifactTransferStatusResponse struct { + Accepted bool `json:"accepted"` + TransferID string `json:"transferId"` + ArtifactID string `json:"artifactId"` + Direction string `json:"direction"` + TotalChunks int `json:"totalChunks"` + ChunkSizeBytes int `json:"chunkSizeBytes"` + ReceivedChunkIndexes []int `json:"receivedChunkIndexes"` + NextMissingChunkIndex int `json:"nextMissingChunkIndex"` + Completed bool `json:"completed"` + ServerTime time.Time `json:"serverTime"` +} + +type ArtifactTransferCompleteRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + TransferID string `json:"transferId"` + ArtifactID string `json:"artifactId"` + Checksum string `json:"checksum"` + SizeBytes int64 `json:"sizeBytes"` +} + +type ArtifactTransferCompleteResponse struct { + Accepted bool `json:"accepted"` + TransferID string `json:"transferId"` + Artifact ArtifactMetadata `json:"artifact"` + Completed bool `json:"completed"` + ServerTime time.Time `json:"serverTime"` +} diff --git a/protocol/artifact.md b/protocol/artifact.md new file mode 100644 index 0000000..ee8f5cc --- /dev/null +++ b/protocol/artifact.md @@ -0,0 +1,46 @@ +# Run Artifact Contract + +Artifacts move files and large payloads between platform and run without blocking logs or control. + +## Implemented Routes + +- `POST /api/v1/run/artifacts/open`: opens a run-to-platform upload transfer and returns resume state. +- `POST /api/v1/run/artifacts/chunks`: uploads one bounded chunk with byte range and checksum metadata. +- `POST /api/v1/run/artifacts/status`: queries received chunks and the next missing chunk index. +- `POST /api/v1/run/artifacts/complete`: verifies all chunks and final checksum before marking the artifact available. + +Browser-facing artifact downloads are implemented through platform-owned routes after a run upload completes: + +- `POST /api/v1/artifacts/{id}/download`: returns safe download metadata and a platform content route. +- `GET /api/v1/artifacts/{id}/content`: returns bounded byte ranges for authorized browser or plugin-page reads. + +## Payloads + +- `ArtifactTransferOpenRequest`: run ID, session token, artifact ID, upload direction, owner scope, size, chunk size, checksum, and idempotency key. +- `ArtifactTransferOpenResponse`: transfer ID, artifact metadata, total chunks, received chunk indexes, next missing chunk index, duplicate flag, and server time. +- `ArtifactChunkUploadRequest`: transfer ID, artifact ID, chunk index, byte offset, size, checksum, and JSON byte payload. +- `ArtifactChunkUploadResponse`: accepted chunk index, received chunk indexes, next missing chunk index, duplicate flag, and server time. +- `ArtifactTransferStatusRequest`: run ID, session token, transfer ID, and artifact ID. +- `ArtifactTransferStatusResponse`: transfer direction, total chunks, received chunk indexes, next missing chunk index, completion flag, and server time. +- `ArtifactTransferCompleteRequest`: transfer ID, artifact ID, final checksum, and final size. +- `ArtifactTransferCompleteResponse`: completed artifact metadata and server time. + +## Local Queue + +Run stores unacknowledged `ArtifactChunkUploadRequest` payloads in the local artifact queue. A queued chunk may be removed only after the platform acknowledges the same transfer ID, artifact ID, and chunk index. The queue must not store or expose raw host paths. + +## Rules + +- Transfers must be resumable. +- Transfers must be checksummed. +- Artifact concurrency must be limited. +- Artifact transfer must not block control heartbeat, job ack/result, or log upload. +- Artifact transfer is lower priority than control, job lifecycle metadata, and durable log ingest. +- Slow or retrying artifact chunks must not prevent log spool acknowledgement cleanup or terminal job result submission. +- Control, job, and log routes must reject artifact chunk payloads or transport details rather than accepting them through lightweight channel payloads. + +Run artifact queues use owner-only atomic JSON entries and retain chunks until an exact transfer/artifact/index acknowledgement covers them. A low-priority uploader retries pending chunks independently of control, jobs, and logs. + +## Deferred Channels + +Platform-to-run Run self-update range reads are implemented through the signed `/api/v1/run/jobs/update-input` and `/api/v1/run/jobs/update-chunk` contract. Browser artifact upload, external object storage, presigned URLs, production mirrors/signing, and production throttling policies remain separate future work. diff --git a/protocol/autonomous_lifecycle.go b/protocol/autonomous_lifecycle.go new file mode 100644 index 0000000..954ee79 --- /dev/null +++ b/protocol/autonomous_lifecycle.go @@ -0,0 +1,254 @@ +package protocol + +import "strings" + +type RunAutonomousLifecyclePlan struct { + SchemaVersion string `json:"schemaVersion"` + ServerInstanceID string `json:"serverInstanceId"` + PluginID string `json:"pluginId"` + PluginVersion string `json:"pluginVersion"` + RunEndpointID string `json:"runEndpointId"` + ProfileKey string `json:"profileKey,omitempty"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + TargetRelease string `json:"targetRelease"` + DeploymentRevision int `json:"deploymentRevision,omitempty"` + Bootstrap *RunAutonomousLifecycleAction `json:"bootstrap,omitempty"` + Actions []RunAutonomousLifecycleAction `json:"actions,omitempty"` + DependencyProbes []DependencyProbe `json:"dependencyProbes,omitempty"` + InstallPlans []DependencyInstallPlan `json:"installPlans,omitempty"` + LogSources []RuntimeLogSourcePlan `json:"logSources,omitempty"` + DLLExtensions []RuntimeDLLExtensionPlan `json:"dllExtensions,omitempty"` + DataTargets []RunAutonomousDataTarget `json:"dataTargets,omitempty"` + RuntimeBindings map[string]string `json:"runtimeBindings,omitempty"` + Deployment *RunAutonomousDeployment `json:"deployment,omitempty"` +} + +type RunAutonomousLifecycleAction struct { + Action string `json:"action"` + Operation string `json:"operation"` + Capability string `json:"capability"` + TargetKey string `json:"targetKey"` +} + +type RunAutonomousDeployment struct { + SchemaVersion string `json:"schemaVersion"` + Mode string `json:"mode"` + ProfileKey string `json:"profileKey,omitempty"` + RuntimeBindings map[string]string `json:"runtimeBindings,omitempty"` + CreateInputs map[string]string `json:"createInputs,omitempty"` + ServerRoot string `json:"serverRoot,omitempty"` + WorkingDirectory string `json:"workingDirectory,omitempty"` + InstallCommand string `json:"installCommand,omitempty"` + StartCommand string `json:"startCommand,omitempty"` + StopCommand string `json:"stopCommand,omitempty"` + StatusCommand string `json:"statusCommand,omitempty"` + Shell string `json:"shell,omitempty"` + Revision int `json:"revision,omitempty"` +} + +type RunAutonomousDataTarget struct { + Key string `json:"key"` + Kind string `json:"kind"` + TransportKey string `json:"transportKey"` + SourceRootKey string `json:"sourceRootKey"` + SourcePath string `json:"sourcePath"` + WorkspaceKey string `json:"workspaceKey"` + RefreshPolicy string `json:"refreshPolicy"` + MaxBytes int64 `json:"maxBytes,omitempty"` + Platforms []string `json:"platforms,omitempty"` +} + +const maxAutonomousDataTargetBytes = int64(1024 * 1024 * 1024) + +func ValidateRunAutonomousLifecyclePlan(plan RunAutonomousLifecyclePlan) error { + if plan.SchemaVersion != "1" { + return ValidationError("autonomous lifecycle plan schema is unsupported") + } + if !ValidLogicalFileKey(plan.ServerInstanceID) || !ValidLogicalFileKey(plan.RunEndpointID) || !validAutonomousPluginID(plan.PluginID) { + return ValidationError("autonomous lifecycle plan identity is invalid") + } + if plan.ProfileKey != "" && !ValidLogicalFileKey(plan.ProfileKey) { + return ValidationError("autonomous lifecycle profile key is invalid") + } + if !validAutonomousTarget(plan.TargetOS, plan.TargetArch) { + return ValidationError("autonomous lifecycle target platform is invalid") + } + if plan.TargetRelease != "" && !validAutonomousToken(plan.TargetRelease, 240) { + return ValidationError("autonomous lifecycle target release is invalid") + } + if plan.PluginVersion != "" && !validAutonomousToken(plan.PluginVersion, 120) { + return ValidationError("autonomous lifecycle plugin version is invalid") + } + if plan.Bootstrap != nil { + if err := validateAutonomousLifecycleAction(*plan.Bootstrap); err != nil { + return err + } + if plan.Bootstrap.Capability != RunCapabilityProcessInstall && plan.Bootstrap.Capability != RunCapabilityProcessStart { + return ValidationError("autonomous lifecycle bootstrap action is invalid") + } + } + if len(plan.Actions) > 16 || len(plan.DependencyProbes) > 64 || len(plan.InstallPlans) > 64 || len(plan.LogSources) > 16 || len(plan.DLLExtensions) > 16 || len(plan.DataTargets) > 16 { + return ValidationError("autonomous lifecycle plan is too large") + } + for _, action := range plan.Actions { + if err := validateAutonomousLifecycleAction(action); err != nil { + return err + } + } + for _, probe := range plan.DependencyProbes { + if !ValidLogicalFileKey(probe.Key) || !ValidLogicalFileKey(probe.TargetKey) || !validAutonomousToken(probe.Kind, 80) || probe.MinimumVersion != "" && !validAutonomousToken(probe.MinimumVersion, 80) { + return ValidationError("autonomous dependency probe is invalid") + } + } + for _, installPlan := range plan.InstallPlans { + if !ValidLogicalFileKey(installPlan.Key) || len(installPlan.Steps) > 64 { + return ValidationError("autonomous install plan is invalid") + } + for _, step := range installPlan.Steps { + if !ValidLogicalFileKey(step.TargetKey) || !validAutonomousToken(step.Type, 80) || step.PackageManager != "" && !validAutonomousToken(step.PackageManager, 80) || step.PackageName != "" && !validAutonomousToken(step.PackageName, 160) || step.Version != "" && !validAutonomousToken(step.Version, 120) { + return ValidationError("autonomous install step is invalid") + } + } + } + for _, source := range plan.LogSources { + if err := validateRuntimeProcessLogSourcePlan(source); err != nil { + return err + } + } + seenDataTargets := map[string]struct{}{} + seenWorkspaceTargets := map[string]struct{}{} + for _, target := range plan.DataTargets { + if err := validateAutonomousDataTarget(target); err != nil { + return err + } + if _, exists := seenDataTargets[target.Key]; exists { + return ValidationError("autonomous data target key is duplicated") + } + if _, exists := seenWorkspaceTargets[target.WorkspaceKey]; exists { + return ValidationError("autonomous data target workspace is duplicated") + } + seenDataTargets[target.Key] = struct{}{} + seenWorkspaceTargets[target.WorkspaceKey] = struct{}{} + } + if plan.Deployment != nil { + if err := validateAutonomousDeployment(*plan.Deployment); err != nil { + return err + } + } + return nil +} + +func validateAutonomousDataTarget(target RunAutonomousDataTarget) error { + if !ValidLogicalFileKey(target.Key) || !ValidLogicalFileKey(target.TransportKey) || !ValidLogicalFileKey(target.SourceRootKey) || !ValidLogicalFileKey(target.SourcePath) || !ValidLogicalFileKey(target.WorkspaceKey) { + return ValidationError("autonomous data target is invalid") + } + if target.Kind != "sqlite.snapshot" || target.RefreshPolicy != "on-demand-snapshot" || !strings.HasPrefix(target.WorkspaceKey, "databases/") { + return ValidationError("autonomous data target is invalid") + } + if target.MaxBytes <= 0 || target.MaxBytes > maxAutonomousDataTargetBytes { + return ValidationError("autonomous data target byte limit is invalid") + } + for _, platform := range target.Platforms { + if platform != "windows" && platform != "linux" && platform != "darwin" { + return ValidationError("autonomous data target platform is invalid") + } + } + return nil +} + +func validateAutonomousLifecycleAction(action RunAutonomousLifecycleAction) error { + if !ValidLogicalFileKey(action.TargetKey) || !validAutonomousLifecycleCapability(action.Capability) || !validAutonomousLifecycleName(action.Action) || !validAutonomousLifecycleName(action.Operation) { + return ValidationError("autonomous lifecycle action is invalid") + } + if expected := autonomousCapabilityForAction(action.Action); expected != "" && expected != action.Capability { + return ValidationError("autonomous lifecycle action capability mismatch") + } + return nil +} + +func autonomousCapabilityForAction(action string) string { + switch action { + case "create": + return RunCapabilityProcessInstall + case "start": + return RunCapabilityProcessStart + case "stop": + return RunCapabilityProcessStop + case "status": + return RunCapabilityProcessStatus + default: + return "" + } +} + +func validAutonomousLifecycleCapability(capability string) bool { + switch capability { + case RunCapabilityProcessInstall, RunCapabilityProcessStart, RunCapabilityProcessStop, RunCapabilityProcessStatus: + return true + default: + return false + } +} + +func validAutonomousLifecycleName(value string) bool { + switch value { + case "create", "install", "start", "stop", "status": + return true + default: + return false + } +} + +func validateAutonomousDeployment(deployment RunAutonomousDeployment) error { + if deployment.SchemaVersion != "1" || deployment.Mode == "" || deployment.Revision < 0 || deployment.ProfileKey != "" && !ValidLogicalFileKey(deployment.ProfileKey) { + return ValidationError("autonomous deployment is invalid") + } + for key := range deployment.RuntimeBindings { + if !ValidLogicalFileKey(key) { + return ValidationError("autonomous deployment binding is invalid") + } + } + for key := range deployment.CreateInputs { + if !ValidLogicalFileKey(key) { + return ValidationError("autonomous deployment input is invalid") + } + } + return nil +} + +func validAutonomousPluginID(value string) bool { + return ValidLogicalFileKey(value) +} + +func validAutonomousTarget(osName string, arch string) bool { + switch osName { + case "windows", "linux", "darwin": + default: + return false + } + switch arch { + case "amd64", "arm64": + return true + default: + return false + } +} + +func validAutonomousToken(value string, maxRunes int) bool { + trimmed := strings.TrimSpace(value) + if trimmed == "" || trimmed != value || len([]rune(value)) > maxRunes { + return false + } + lower := strings.ToLower(value) + if strings.Contains(value, "..") || strings.Contains(value, `\`) || strings.Contains(value, "://") || strings.Contains(lower, "/users/") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "bearer ") || strings.Contains(lower, "sk-") { + return false + } + for _, char := range value { + if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '_' || char == '-' || char == '.' || char == '/' || char == ':' { + continue + } + return false + } + return true +} diff --git a/protocol/autonomous_lifecycle_test.go b/protocol/autonomous_lifecycle_test.go new file mode 100644 index 0000000..75b815a --- /dev/null +++ b/protocol/autonomous_lifecycle_test.go @@ -0,0 +1,39 @@ +package protocol + +import "testing" + +func TestValidateRunAutonomousLifecyclePlanAllowsSQLiteSnapshotDataTarget(t *testing.T) { + plan := validAutonomousLifecyclePlanForTest() + plan.DataTargets = []RunAutonomousDataTarget{{Key: "world-db", Kind: "sqlite.snapshot", TransportKey: "world-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/world.db", WorkspaceKey: "databases/world-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024, Platforms: []string{"windows"}}} + + if err := ValidateRunAutonomousLifecyclePlan(plan); err != nil { + t.Fatalf("expected valid data target plan: %v", err) + } + + plan.DataTargets[0].WorkspaceKey = "state/world-db" + if err := ValidateRunAutonomousLifecyclePlan(plan); err == nil { + t.Fatal("expected non-database workspace target to be rejected") + } +} + +func TestValidateRunAutonomousLifecyclePlanRejectsUnsafeDataTargets(t *testing.T) { + for name, mutate := range map[string]func(*RunAutonomousDataTarget){ + "kind": func(target *RunAutonomousDataTarget) { target.Kind = "scum.sqlite" }, + "source-path": func(target *RunAutonomousDataTarget) { target.SourcePath = "../current.db" }, + "refresh-policy": func(target *RunAutonomousDataTarget) { target.RefreshPolicy = "startup" }, + "max-bytes": func(target *RunAutonomousDataTarget) { target.MaxBytes = maxAutonomousDataTargetBytes + 1 }, + "platform": func(target *RunAutonomousDataTarget) { target.Platforms = []string{"plan9"} }, + } { + plan := validAutonomousLifecyclePlanForTest() + target := RunAutonomousDataTarget{Key: "world-db", Kind: "sqlite.snapshot", TransportKey: "world-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/world.db", WorkspaceKey: "databases/world-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024, Platforms: []string{"windows"}} + mutate(&target) + plan.DataTargets = []RunAutonomousDataTarget{target} + if err := ValidateRunAutonomousLifecyclePlan(plan); err == nil { + t.Fatalf("expected invalid data target %s to be rejected", name) + } + } +} + +func validAutonomousLifecyclePlanForTest() RunAutonomousLifecyclePlan { + return RunAutonomousLifecyclePlan{SchemaVersion: "1", ServerInstanceID: "server-1", PluginID: "game.example", PluginVersion: "1.0.0", RunEndpointID: "run-1", ProfileKey: "run-local", TargetOS: "windows", TargetArch: "amd64", TargetRelease: "release-1", Bootstrap: &RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: RunCapabilityProcessStart, TargetKey: "actions/start.json"}} +} diff --git a/protocol/control.go b/protocol/control.go new file mode 100644 index 0000000..832e79f --- /dev/null +++ b/protocol/control.go @@ -0,0 +1,84 @@ +package protocol + +import "time" + +type RunCapacityReport struct { + MaxJobs int `json:"maxJobs"` + RunningJobs int `json:"runningJobs"` + QueuedJobs int `json:"queuedJobs"` + Summary string `json:"summary,omitempty"` +} + +type RunCapabilityReport struct { + Capabilities []string `json:"capabilities"` + Fingerprint string `json:"fingerprint"` +} + +type RunHelloRequest struct { + RegistrationToken string `json:"registrationToken"` + RunEndpointID string `json:"runEndpointId"` + ServerInstanceID string `json:"serverInstanceId,omitempty"` + PluginID string `json:"pluginId,omitempty"` + ComponentKind string `json:"componentKind,omitempty"` + ComponentKey string `json:"componentKey,omitempty"` + KeyGeneration int `json:"keyGeneration,omitempty"` + DisplayName string `json:"displayName"` + Version string `json:"version"` + Status string `json:"status"` + Platform string `json:"platform,omitempty"` + Architecture string `json:"architecture,omitempty"` + UpdateJobID string `json:"updateJobId,omitempty"` + UpdateOutcome string `json:"updateOutcome,omitempty"` + CapabilityReport RunCapabilityReport `json:"capabilityReport"` + Capacity RunCapacityReport `json:"capacity"` +} + +type RunHelloResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + ServerTime time.Time `json:"serverTime"` + HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds"` + SessionExpiresAt time.Time `json:"sessionExpiresAt"` + FeatureFlags []string `json:"featureFlags,omitempty"` +} + +type RunHeartbeatRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + Version string `json:"version"` + Status string `json:"status"` + CapabilityFingerprint string `json:"capabilityFingerprint"` + Capacity RunCapacityReport `json:"capacity"` +} + +type RunHeartbeatResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + NextHeartbeatSeconds int `json:"nextHeartbeatSeconds"` + RefreshCapabilities bool `json:"refreshCapabilities"` + ServerTime time.Time `json:"serverTime"` +} + +type RunLifecycleReportRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + ServerInstanceID string `json:"serverInstanceId"` + Capability string `json:"capability"` + State string `json:"state"` + Progress RunJobProgressReport `json:"progress"` + Message string `json:"message,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ManagedProcessID string `json:"managedProcessId,omitempty"` + ObservationSeq uint64 `json:"observationSeq,omitempty"` + ObservedAt time.Time `json:"observedAt,omitempty"` + ExecutionResult RunJobExecutionResult `json:"executionResult,omitempty"` +} + +type RunLifecycleReportResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + ServerInstanceID string `json:"serverInstanceId"` + ProjectedState string `json:"projectedState,omitempty"` + ServerTime time.Time `json:"serverTime"` +} diff --git a/protocol/control.md b/protocol/control.md new file mode 100644 index 0000000..850096b --- /dev/null +++ b/protocol/control.md @@ -0,0 +1,28 @@ +# Run Control Contract + +Control is the lightweight high-priority channel between run and platform. + +## Implemented Routes + +- `POST /api/v1/run/control/hello`: registers run metadata and receives a platform-issued session token. +- `POST /api/v1/run/control/heartbeat`: reports status, capacity, and capability fingerprint using the active session token. + +## Payloads + +- `RunHelloRequest`: registration token, run ID, display name, version, status, platform, capability summary, and capacity summary. +- `RunHelloResponse`: session token, server time, polling hints, and feature flags. +- `RunHeartbeatRequest`: session token, version, status, capacity, and current capability fingerprint. +- `RunHeartbeatResponse`: accepted status, next heartbeat interval, and optional capability refresh request. +- `RunCapabilityReport`: capability names and compact fingerprint metadata. +- `RunCapacityReport`: max jobs, active jobs, queued jobs, and local resource summary. + +## Rules + +- Control payloads must be small. +- Control must not carry logs, artifact chunks, or long job result bodies. +- Control must have priority over job execution, log upload, and artifact transfer. +- Heartbeat capacity summaries must remain metadata-only and must not mention or carry heavy channel payloads. + +## Deferred Channels + +Durable log ingest, artifact chunk transfer, and the optional game client bridge remain separate channels. The job channel is separate from control and uses `/api/v1/run/jobs/*` routes. diff --git a/protocol/game-client-bridge.md b/protocol/game-client-bridge.md new file mode 100644 index 0000000..48478c3 --- /dev/null +++ b/protocol/game-client-bridge.md @@ -0,0 +1,17 @@ +# Game Client Bridge Contract + +The game client bridge is optional and exists only for games that need in-game command execution or snapshots. + +## Payloads + +- `ClientHello`: game client credential, server instance ID, version, and display name. +- `ClientHeartbeat`: session token, version, status, and game connection status. +- `ClientCommandPoll`: session token and batch limit. +- `ClientCommandResult`: command ID, status, bounded output, and timestamp. +- `ClientSnapshot`: snapshot mode, raw bounded text or structured data reference, and timestamp. + +## Rules + +- The bridge must not carry run lifecycle jobs. +- The bridge must not carry run log ingest batches. +- Games without in-game bridge needs should not enable this channel. diff --git a/protocol/job.go b/protocol/job.go new file mode 100644 index 0000000..2331f35 --- /dev/null +++ b/protocol/job.go @@ -0,0 +1,620 @@ +package protocol + +import "time" + +const ( + RunCapabilityProcessInstall = "process.install" + RunCapabilityProcessStart = "process.start" + RunCapabilityProcessStop = "process.stop" + RunCapabilityProcessStatus = "process.status" + RunCapabilityLogsRead = "logs.read" + RunCapabilityConfigWrite = "config.write" + RunCapabilityFilesList = "files.list" + RunCapabilityFilesRead = "files.read" + RunCapabilityFilesWrite = "files.write" + RunCapabilityRemoteFTPRead = "remote.ftp.read" + RunCapabilityRemoteFTPWrite = "remote.ftp.write" + RunCapabilityRemoteRsyncRead = "remote.rsync.read" + RunCapabilityRemoteRsyncWrite = "remote.rsync.write" + RunCapabilityRemoteRunFilesRead = "remote.run.files.read" + RunCapabilityRemoteRunFilesWrite = "remote.run.files.write" + RunCapabilityRemoteRunProcessStart = "remote.run.process.start" + RunCapabilityRemoteRunProcessStop = "remote.run.process.stop" + RunCapabilityRemoteRunDBMySQLQuery = "remote.run.db.mysql.query" + RunCapabilityRemoteRunDBSQLiteProbe = "remote.run.db.sqlite.probe" + RunCapabilityRemoteRunDBSQLiteQuery = "remote.run.db.sqlite.query" + RunCapabilityRemoteRunLogsTransfer = "remote.run.logs.transfer" + RunCapabilityRemoteRunRCONCommand = "remote.run.rcon.command" + RunCapabilityRemoteRunProtectedSQL = "remote.run.protected.sql" + RunCapabilityRemoteRunProtectedRCON = "remote.run.protected.rcon" + RunCapabilityRemoteRunProgram = "remote.run.program.command" + RunCapabilityRunSelfUpdate = "run.self-update" + RunCapabilityDistributionBuild = "distribution.build" + RunCapabilityDependenciesCheck = "dependencies.check" + RunCapabilityDependenciesInstall = "dependencies.install" + RunCapabilityLogsBackfill = "logs.backfill" + RunCapabilityDeploymentPlan = "deployment.plan.v1" +) + +type RunJobProgressReport struct { + Percent int `json:"percent"` + Message string `json:"message,omitempty"` +} + +type RunJobExecutionInput struct { + WorkspaceScope string `json:"workspaceScope,omitempty"` + Content string `json:"content,omitempty"` + ExpectedVersion int `json:"expectedVersion,omitempty"` + ExpectedChecksum string `json:"expectedChecksum,omitempty"` + MaxReadBytes int `json:"maxReadBytes,omitempty"` + RemoteAdapterKey string `json:"remoteAdapterKey,omitempty"` + RemoteAdapterKind string `json:"remoteAdapterKind,omitempty"` + TimeoutSeconds int `json:"timeoutSeconds,omitempty"` + PluginID string `json:"pluginId,omitempty"` + LifecycleOperation string `json:"lifecycleOperation,omitempty"` + TargetVersion string `json:"targetVersion,omitempty"` + Inputs map[string]string `json:"inputs,omitempty"` + LogSource *RuntimeLogSourcePlan `json:"logSource,omitempty"` + LogSources []RuntimeLogSourcePlan `json:"logSources,omitempty"` + DLLExtensions []RuntimeDLLExtensionPlan `json:"dllExtensions,omitempty"` + SourceRCON *RuntimeSourceRCONPlan `json:"sourceRcon,omitempty"` + SQLiteSchemaProbe *SQLiteSchemaProbeRequest `json:"sqliteSchemaProbe,omitempty"` + Deployment *ServerDeploymentExecution `json:"deployment,omitempty"` + ServerDeploymentPlan *ServerDeploymentPlan `json:"serverDeploymentPlan,omitempty"` +} + +// SQLiteSchemaProbeBinding identifies the package and current database binding +// that authorized a probe. It contains only logical identities, never a path, +// DSN, socket, or credential. +type SQLiteSchemaProbeBinding struct { + ServerInstanceID string `json:"serverInstanceId"` + RunBindingID string `json:"runBindingId"` + RunEndpointID string `json:"runEndpointId"` + PluginID string `json:"pluginId"` + PluginVersion string `json:"pluginVersion"` + AdapterVersion string `json:"adapterVersion"` + GameVersion string `json:"gameVersion,omitempty"` + DatabaseIdentity string `json:"databaseIdentity"` +} + +// SQLiteSchemaProbeLimits cap every independent part of diagnostic output. +// They are applied by Run even when the caller asks for larger values. +type SQLiteSchemaProbeLimits struct { + MaxObjects int `json:"maxObjects"` + MaxColumnsPerObject int `json:"maxColumnsPerObject"` + MaxIndexesPerObject int `json:"maxIndexesPerObject"` + MaxForeignKeys int `json:"maxForeignKeys"` + MaxCardinalityReads int `json:"maxCardinalityReads"` + MaxSampleRows int `json:"maxSampleRows"` + TimeoutMS int `json:"timeoutMs"` + MaxResultBytes int `json:"maxResultBytes"` +} + +// SQLiteSchemaProbeRequest requests generic, query-only SQLite metadata for +// assignment.TargetKey. TargetKey is always resolved inside the scoped package +// workspace; this request intentionally has no path or SQL field. +type SQLiteSchemaProbeRequest struct { + RequestID string `json:"requestId"` + Binding SQLiteSchemaProbeBinding `json:"binding"` + Limits SQLiteSchemaProbeLimits `json:"limits"` +} + +type SQLiteSchemaProbeColumn struct { + NameFingerprint string `json:"nameFingerprint"` + DeclaredType string `json:"declaredType"` + Nullable *bool `json:"nullable,omitempty"` + PrimaryKey bool `json:"primaryKey"` + Ordinal int `json:"ordinal"` +} + +type SQLiteSchemaProbeIndex struct { + NameFingerprint string `json:"nameFingerprint"` + Unique bool `json:"unique"` + ColumnHashes []string `json:"columnHashes"` +} + +type SQLiteSchemaProbeForeignKey struct { + FromColumnHash string `json:"fromColumnHash"` + ToObjectHash string `json:"toObjectHash"` + ToColumnHash string `json:"toColumnHash"` +} + +type SQLiteSchemaProbeObject struct { + ObjectHash string `json:"objectHash"` + Kind string `json:"kind"` + NameFingerprint string `json:"nameFingerprint"` + DeclaredColumns []SQLiteSchemaProbeColumn `json:"declaredColumns"` + Indexes []SQLiteSchemaProbeIndex `json:"indexes"` + ForeignKeys []SQLiteSchemaProbeForeignKey `json:"foreignKeys"` + ApproximateRows *int64 `json:"approximateRows,omitempty"` + SampleFingerprints []string `json:"sampleFingerprints,omitempty"` +} + +type SQLiteSchemaProbeSafeError struct { + Code string `json:"code"` + Retryable bool `json:"retryable"` +} + +// SQLiteSchemaProbeResult is a terminal, redacted envelope. Names and sample +// values are represented only by salted-looking SHA-256 fingerprints. +type SQLiteSchemaProbeResult struct { + RequestID string `json:"requestId"` + JobID string `json:"jobId"` + Binding SQLiteSchemaProbeBinding `json:"binding"` + Status string `json:"status"` + ObservedAt time.Time `json:"observedAt"` + SourceFingerprint string `json:"sourceFingerprint,omitempty"` + SchemaFingerprint string `json:"schemaFingerprint,omitempty"` + ResultDigest string `json:"resultDigest,omitempty"` + Objects []SQLiteSchemaProbeObject `json:"objects,omitempty"` + SafeError SQLiteSchemaProbeSafeError `json:"safeError,omitempty"` + Limits SQLiteSchemaProbeLimits `json:"limits"` +} + +type ServerDeploymentExecution struct { + SchemaVersion string `json:"schemaVersion"` + Mode string `json:"mode"` + ProfileKey string `json:"profileKey,omitempty"` + CreateInputs map[string]string `json:"createInputs,omitempty"` + ServerRoot string `json:"serverRoot,omitempty"` + WorkingDirectory string `json:"workingDirectory,omitempty"` + StartCommand string `json:"startCommand,omitempty"` + StopCommand string `json:"stopCommand,omitempty"` + StatusCommand string `json:"statusCommand,omitempty"` + Shell string `json:"shell,omitempty"` + Revision int `json:"revision"` +} + +// ServerDeploymentPlan is kept only for backward-compatible decoding of +// legacy assignments. Game-specific deployment plans are no longer executed by +// Run; plugins own game lifecycle policy through action assets. +type ServerDeploymentPlan struct { + SchemaVersion string `json:"schemaVersion"` + Operation string `json:"operation"` + PluginID string `json:"pluginId"` + TemplateKey string `json:"templateKey"` + TemplateVersion string `json:"templateVersion"` + SteamAppID string `json:"steamAppId"` + ExecutableKey string `json:"executableKey"` + InstallRootKey string `json:"installRootKey"` + ConfigKey string `json:"configKey"` + ConfigFormat string `json:"configFormat"` + Prerequisites []RuntimeServerPrerequisite `json:"prerequisites,omitempty"` + ConfigMappings []RuntimeServerConfigMapping `json:"configMappings"` + DiscoveryMarkers []RuntimeServerDiscoveryMarker `json:"discoveryMarkers"` + VerificationChecks []RuntimeServerVerificationCheck `json:"verificationChecks"` +} + +type RuntimeServerPrerequisite struct { + Key string `json:"key"` + Kind string `json:"kind"` +} + +type RuntimeServerConfigMapping struct { + FieldKey string `json:"fieldKey"` + ConfigKey string `json:"configKey"` + ValueType string `json:"valueType"` + Required bool `json:"required"` +} + +type RuntimeServerDiscoveryMarker struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey"` + Expected string `json:"expected,omitempty"` + Required bool `json:"required"` +} + +type RuntimeServerVerificationCheck struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey"` + Required bool `json:"required"` +} + +type ServerDeploymentEvidence struct { + TemplateKey string `json:"templateKey,omitempty"` + TemplateVersion string `json:"templateVersion,omitempty"` + PreflightState string `json:"preflightState,omitempty"` + DiscoveryState string `json:"discoveryState,omitempty"` + MappingState string `json:"mappingState,omitempty"` + VerificationState string `json:"verificationState,omitempty"` + DiscoveredFacts map[string]string `json:"discoveredFacts,omitempty"` + MappingResults map[string]string `json:"mappingResults,omitempty"` + VerificationResults map[string]string `json:"verificationResults,omitempty"` + FailureCode string `json:"failureCode,omitempty"` +} + +// RuntimeDLLExtensionPlan is a Platform-frozen UE4SS DLL release. It is only +// accepted as part of a scoped process.start job; Run never fetches a mutable +// plugin declaration on its own. +type RuntimeDLLExtensionPlan struct { + Key string `json:"key"` + Version string `json:"version"` + ReleaseURL string `json:"releaseUrl"` + Checksum string `json:"checksum"` + SizeBytes int64 `json:"sizeBytes"` + TargetKey string `json:"targetKey"` + ModKey string `json:"modKey"` + DLLRef string `json:"dllRef"` + SCUMExecutableChecksum string `json:"scumExecutableChecksum"` + UE4SSABI string `json:"ue4ssAbi"` + RCONPort int `json:"rconPort"` +} + +// RuntimeLogSourcePlan is a Platform-frozen, logical file log declaration. It +// carries no host paths; Run resolves TargetKey only inside its scoped +// workspace for the server instance. +type RuntimeLogSourcePlan struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey,omitempty"` + StreamKey string `json:"streamKey"` + CursorKind string `json:"cursorKind,omitempty"` + RetentionDays int `json:"retentionDays,omitempty"` +} + +// RuntimeSourceRCONPlan contains only frozen, non-secret metadata. Run reads +// the generated password from the scoped workspace after it consumes the +// one-time command input. +type RuntimeSourceRCONPlan struct { + Protocol string `json:"protocol"` + ExtensionKey string `json:"extensionKey"` + ModKey string `json:"modKey"` + ConfigRef string `json:"configRef"` + DeploymentStateRef string `json:"deploymentStateRef"` + Port int `json:"port"` +} + +type RunJobExecutionResult struct { + Kind string `json:"kind,omitempty"` + ProcessState string `json:"processState,omitempty"` + ExitClassification string `json:"exitClassification,omitempty"` + ExitCode int `json:"exitCode,omitempty"` + Version int `json:"version,omitempty"` + Checksum string `json:"checksum,omitempty"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + Summary string `json:"summary,omitempty"` + Content string `json:"content,omitempty"` + SQLiteSchemaProbe *SQLiteSchemaProbeResult `json:"sqliteSchemaProbe,omitempty"` + DeploymentReceipt *ServerDeploymentExecutionReceipt `json:"deploymentReceipt,omitempty"` + ServerDeploymentEvidence *ServerDeploymentEvidence `json:"serverDeploymentEvidence,omitempty"` +} + +type ServerDeploymentExecutionReceipt struct { + SchemaVersion string `json:"schemaVersion"` + Revision int `json:"revision"` + Action string `json:"action"` + Mode string `json:"mode"` + Shell string `json:"shell,omitempty"` + UsedServerRoot bool `json:"usedServerRoot"` +} + +type RunJobAssignment struct { + JobID string `json:"jobId"` + ServerInstanceID string `json:"serverInstanceId,omitempty"` + RunEndpointID string `json:"runEndpointId"` + Capability string `json:"capability"` + TargetKey string `json:"targetKey,omitempty"` + InputRef string `json:"inputRef,omitempty"` + IdempotencyKey string `json:"idempotencyKey"` + State string `json:"state"` + Progress RunJobProgressReport `json:"progress"` + ResultRef string `json:"resultRef,omitempty"` + ExecutionInput RunJobExecutionInput `json:"executionInput,omitempty"` + LeaseToken string `json:"leaseToken"` + FencingToken uint64 `json:"fencingToken,omitempty"` + Attempt int `json:"attempt"` + LogSessionID string `json:"logSessionId,omitempty"` + SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"` + MaxAttempts int `json:"maxAttempts"` + AckDeadlineAt time.Time `json:"ackDeadlineAt,omitempty"` + LeaseExpiresAt time.Time `json:"leaseExpiresAt,omitempty"` + NextAttemptAt time.Time `json:"nextAttemptAt,omitempty"` + ProgressSequence uint64 `json:"progressSequence,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type RunJobClaimRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + Capabilities []string `json:"capabilities"` + Capacity RunCapacityReport `json:"capacity"` +} + +type RunJobClaimResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + HasJob bool `json:"hasJob"` + Job *RunJobAssignment `json:"job,omitempty"` + NextPollSeconds int `json:"nextPollSeconds"` + ServerTime time.Time `json:"serverTime"` +} + +type RunJobAckRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + Message string `json:"message,omitempty"` +} + +type RunJobAckResponse struct { + Accepted bool `json:"accepted"` + Job RunJobAssignment `json:"job"` + ServerTime time.Time `json:"serverTime"` +} + +type RunJobProgressRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + Progress RunJobProgressReport `json:"progress"` + Sequence uint64 `json:"sequence,omitempty"` +} + +type RunJobProgressResponse struct { + Accepted bool `json:"accepted"` + Job RunJobAssignment `json:"job"` + ServerTime time.Time `json:"serverTime"` +} + +type RunJobResultRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + State string `json:"state"` + Progress RunJobProgressReport `json:"progress"` + ResultRef string `json:"resultRef,omitempty"` + Message string `json:"message,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Retryable bool `json:"retryable,omitempty"` + ExecutionResult RunJobExecutionResult `json:"executionResult,omitempty"` +} + +type RunJobResultResponse struct { + Accepted bool `json:"accepted"` + Job RunJobAssignment `json:"job"` + ServerTime time.Time `json:"serverTime"` +} + +type DistributionBuildInputRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` +} + +type DistributionBuildInputResponse struct { + JobID string `json:"jobId"` + ComponentKind string `json:"componentKind"` + ServerInstanceID string `json:"serverInstanceId"` + PluginID string `json:"pluginId"` + RunEndpointID string `json:"runEndpointId"` + ProfileKey string `json:"profileKey,omitempty"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + TargetRelease string `json:"targetRelease"` + PlatformURL string `json:"platformUrl,omitempty"` + PackageFormat string `json:"packageFormat"` + RepositoryURL string `json:"repositoryUrl,omitempty"` + SourceRevision string `json:"sourceRevision,omitempty"` + ArtifactID string `json:"artifactId"` + OutputFilename string `json:"outputFilename"` + SecretRef string `json:"secretRef"` + KeyGeneration int `json:"keyGeneration"` + AuthKey string `json:"authKey"` + WorkspaceSeed string `json:"workspaceSeed,omitempty"` +} + +type DependencyExecutionInputRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` +} + +type DependencyProbe struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey"` + Required bool `json:"required,omitempty"` + MinimumVersion string `json:"minimumVersion,omitempty"` + Platforms []string `json:"platforms,omitempty"` +} + +type DependencyInstallStep struct { + Type string `json:"type"` + TargetKey string `json:"targetKey"` + PackageManager string `json:"packageManager,omitempty"` + PackageName string `json:"packageName,omitempty"` + Version string `json:"version,omitempty"` + DownloadRef string `json:"downloadRef,omitempty"` + Checksum string `json:"checksum,omitempty"` +} + +type DependencyInstallPlan struct { + Key string `json:"key"` + Title string `json:"title"` + Platforms []string `json:"platforms,omitempty"` + Steps []DependencyInstallStep `json:"steps,omitempty"` +} + +type DependencyExecutionInputResponse struct { + JobID string `json:"jobId"` + ServerInstanceID string `json:"serverInstanceId"` + RunEndpointID string `json:"runEndpointId"` + PluginID string `json:"pluginId"` + PluginVersion string `json:"pluginVersion"` + ProfileKey string `json:"profileKey"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + PlanDigest string `json:"planDigest"` + Probe DependencyProbe `json:"probe,omitempty"` + Plan DependencyInstallPlan `json:"plan,omitempty"` + Bindings map[string]string `json:"bindings"` +} + +type SourceRCONExecutionInputRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` +} + +// SourceRCONExecutionInput is deliberately excluded from assignments and +// journals. Command is returned once to the active Run lease only. +type SourceRCONExecutionInputResponse struct { + JobID string `json:"jobId"` + ServerInstanceID string `json:"serverInstanceId"` + RunEndpointID string `json:"runEndpointId"` + Command string `json:"command"` +} + +// ProtectedRequestExecutionInput is deliberately excluded from assignments and +// journals. Platform returns the approved text and its logical binding once to +// the active, fenced Run lease; no connection material crosses this channel. +type ProtectedRequestExecutionInputRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + FencingToken uint64 `json:"fencingToken"` +} + +type ProtectedRequestExecutionInputResponse struct { + JobID string `json:"jobId"` + ServerInstanceID string `json:"serverInstanceId"` + RunEndpointID string `json:"runEndpointId"` + FencingToken uint64 `json:"fencingToken"` + Authorized bool `json:"authorized"` + ApprovalState string `json:"approvalState"` + QueueState string `json:"queueState"` + ExpiresAt time.Time `json:"expiresAt"` + Kind string `json:"kind"` + TransportKey string `json:"transportKey"` + TargetKey string `json:"targetKey"` + RequestText string `json:"requestText"` +} + +type RunUpdateInputRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` +} + +type RunUpdateInputResponse struct { + JobID string `json:"jobId"` + ServerInstanceID string `json:"serverInstanceId"` + RunEndpointID string `json:"runEndpointId"` + ArtifactID string `json:"artifactId"` + Checksum string `json:"checksum"` + SizeBytes int64 `json:"sizeBytes"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + PackageFormat string `json:"packageFormat"` + ExecutableName string `json:"executableName"` + TargetRelease string `json:"targetRelease"` + ChunkSizeBytes int `json:"chunkSizeBytes"` +} + +type RunUpdateChunkRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + Offset int64 `json:"offset"` + Length int `json:"length"` +} + +type RunUpdateChunkResponse struct { + JobID string `json:"jobId"` + ArtifactID string `json:"artifactId"` + Offset int64 `json:"offset"` + TotalBytes int64 `json:"totalBytes"` + Checksum string `json:"checksum"` + Payload []byte `json:"payload"` + Complete bool `json:"complete"` +} + +type RunUpdateHealthRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` + Outcome string `json:"outcome"` + Version string `json:"version"` +} + +type RunUpdateHealthResponse struct { + Accepted bool `json:"accepted"` + JobID string `json:"jobId"` + Phase string `json:"phase"` + ServerTime time.Time `json:"serverTime"` +} + +type DependencyExecutionEvidence struct { + ProbeKey string `json:"probeKey"` + PlanKey string `json:"planKey,omitempty"` + PlanDigest string `json:"planDigest"` + State string `json:"state"` + Evidence string `json:"evidence,omitempty"` + CompletedSteps int `json:"completedSteps,omitempty"` +} + +type RunUpdateExecutionEvidence struct { + TargetRelease string `json:"targetRelease"` + Phase string `json:"phase"` +} + +type RunJobCancelPollRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + JobID string `json:"jobId,omitempty"` + LeaseToken string `json:"leaseToken,omitempty"` + Attempt int `json:"attempt"` +} + +type RunJobCancelPollResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + HasCancel bool `json:"hasCancel"` + JobID string `json:"jobId,omitempty"` + Reason string `json:"reason,omitempty"` + RequestedAt time.Time `json:"requestedAt,omitempty"` + ServerTime time.Time `json:"serverTime"` +} + +type RunJobReconcileEntry struct { + JobID string `json:"jobId"` + LeaseToken string `json:"leaseToken"` + Attempt int `json:"attempt"` +} + +type RunJobReconcileRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + ActiveJobs []RunJobReconcileEntry `json:"activeJobs"` +} + +type RunJobReconcileResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + ConfirmedJobs []RunJobAssignment `json:"confirmedJobs"` + DiscardJobIDs []string `json:"discardJobIds"` + ServerTime time.Time `json:"serverTime"` +} diff --git a/protocol/job.md b/protocol/job.md new file mode 100644 index 0000000..a580bc3 --- /dev/null +++ b/protocol/job.md @@ -0,0 +1,88 @@ +# Run Job Contract + +Jobs execute bounded server management work. + +## Implemented Routes + +- `POST /api/v1/run/jobs/claim`: claims one queued job for the registered run endpoint. +- `POST /api/v1/run/jobs/ack`: acknowledges an active leased job before execution. +- `POST /api/v1/run/jobs/progress`: reports bounded progress for an active leased job. +- `POST /api/v1/run/jobs/result`: submits a bounded terminal result for an active leased job. +- `POST /api/v1/run/jobs/cancel`: polls platform cancellation requests for active leased jobs. +- `POST /api/v1/run/jobs/reconcile`: reconciles platform-known active jobs after run restart or reconnect. +- `POST /api/v1/run/jobs/dependency-input`: loads a typed dependency probe/plan only for the active fenced attempt. +- `POST /api/v1/run/jobs/update-input`: loads approved same-server target-matched Run distribution metadata only for the active fenced attempt. +- `POST /api/v1/run/jobs/update-chunk`: reads one bounded resumable update artifact range; this lower-priority transfer route never carries browser download tokens or storage paths. +- `POST /api/v1/run/jobs/update-health`: reports a post-registration/post-reconciliation update success or rollback outcome through the current signed session. + +## Payloads + +- `RunJobClaimRequest`: session token, run ID, capacity, and supported capabilities. +- `RunJobClaimResponse`: optional job assignment with identity, capability, server instance, logical target key, scoped input ref, idempotency key, per-job attempt, max attempts, raw one-use lease token, ack deadline, execution lease deadline, and polling hint. Platform persists only the lease hash. +- `RunJobAckRequest`: job ID, run ID, session token, lease token, attempt, and bounded message. +- `RunJobProgressRequest`: job ID, run ID, session token, lease token, attempt, percent, sequence, and bounded message. +- `RunJobResultRequest`: job ID, run ID, session token, lease token, attempt, terminal state, progress, bounded message, error code, retryable flag, and result reference. +- `RunJobCancelPollRequest`: run ID, session token, job ID, lease token, and attempt. +- `RunJobReconcileRequest`: run ID, session token, and active journal entries containing job ID, lease token, and attempt. The response confirms matching attempts and returns stale/unknown IDs to discard. +- Dependency input contains only declared probe/plan summaries, logical bindings, target OS/architecture, and a reviewed plan digest. Update input/chunk responses contain only artifact ID, target, checksum, bounded range metadata, and payload bytes. +- Update health contains job ID, attempt/lease proof, outcome, and release version. Platform accepts success only after the terminal staged result and current online endpoint registration; Run does not claim success from a hello-only outcome. + +## Local Journal + +Run persists a versioned journal under its owner-only workspace state directory. Assignment writes are atomic and happen before acknowledgement. A locally completed terminal result is also persisted without the Run session token before transport; after restart, confirmed attempts replay that result instead of executing again. A staged self-update keeps its activation manifest alongside the pending result so a crash cannot silently discard helper activation. Entries are removed only after the platform accepts the result or reconciliation explicitly discards them. Registration is followed by reconciliation before new claims, including after session rotation. + +## Lifecycle Executor + +The runtime worker executes these bounded lifecycle job capabilities: + +- `process.install` +- `process.start` +- `process.stop` + +Platform-dispatched config/file jobs are now represented in the run job payload and validated before execution by later worker implementations: + +- `config.write`: writes approved config content addressed by a logical config key plus scoped `input://...` ref. +- `files.read`: reads a declared logical file key and returns results through bounded metadata or artifact refs. +- `files.write`: writes content addressed by a logical file key plus scoped `input://...` or `artifact://...` ref. + +Plugin-declared remote access jobs use the same job channel and remain bounded metadata envelopes: + +- `remote.ftp.read` / `remote.ftp.write`: platform-mediated FTP file transfer requests. +- `remote.rsync.read` / `remote.rsync.write`: platform-mediated rsync file transfer requests. +- `remote.run.files.read` / `remote.run.files.write`: run-mediated logical file operations. +- `remote.run.process.start` / `remote.run.process.stop`: run-mediated remote process lifecycle operations. +- `remote.run.db.mysql.query` / `remote.run.db.sqlite.query`: run-mediated database read envelopes with scoped input refs for query payloads. +- `remote.run.logs.transfer`: run-mediated log transfer through log/artifact channels. +- `remote.run.rcon.command`: run-mediated RCON command envelopes with scoped input refs. + +Run distribution and runtime support jobs use the same lightweight job lifecycle: + +- `run.self-update`: downloads an approved same-server target-matched distribution in bounded resumable ranges, verifies the final checksum, safely extracts exactly the expected executable, preserves config, and reports a rollback-safe staged result. A helper activates only after result acceptance, then waits for health and restores the previous binary on timeout. +- `dependencies.check`: runs a plugin-declared typed dependency probe addressed by a logical `dependencies/...` key. +- `dependencies.install`: runs only an approved typed install plan addressed by `dependencies/install/...`; package, verified HTTPS download, SteamCMD, and manual steps are closed adapters, and arbitrary shell snippets, unsafe URLs/tokens, and unsupported targets are rejected. +- `logs.backfill`: advances historical log cursors for declared process, file, FTP, SQL, or plugin-specific sources and returns bounded cursor/result refs instead of log bodies. + +The executor resolves lifecycle action templates under the scoped server workspace and runs direct command/argument vectors through the process supervisor. It does not run unrestricted shell strings, execute arbitrary plugin code, expose host paths, return raw credentials, open direct sockets, or embed logs/artifacts in job result payloads. + +## Rules + +- Job ack must be sent before execution. +- Ack deadline and execution lease timestamps are platform-authoritative. Progress and confirmed reconciliation renew only the current fenced attempt. +- Ack timeout, lease expiry, and retryable results follow the platform's bounded retry/backoff policy; Run never invents a replacement attempt locally. +- Cancellation polling is fenced and cancellation results are ordinary idempotent terminal results. +- Terminal result must be replayable while the journal retains the job. +- Large files must be passed as artifact references, not embedded in job payloads. +- Config/file job payloads must use logical target keys and scoped input/artifact refs. +- Remote database and RCON jobs must use scoped input/artifact refs rather than embedding query or command bodies in job results. +- Run self-update, dependency, and log backfill jobs must use declared capabilities, logical target keys, scoped refs, and bounded result refs. +- Job payloads must not include logs, artifact chunks, raw host paths, raw credentials, direct sockets, or large inline result bodies. +- Process stdout/stderr must be redacted and written to the log spool rather than embedded in progress/result bodies. +- Job ack/progress/result/cancel/reconcile calls are lightweight lifecycle metadata and must be able to complete while artifact/file transfer work is active or retrying. +- Dependency adapters and update downloads run in the job worker while control heartbeat, cancellation polling, log spool upload, and artifact upload retain independent bounded loops. +- Terminal results must remain idempotent under log and artifact retry pressure and must reference artifacts by safe `artifact://...` refs rather than embedding transfer payloads. + +## Deferred Channels + +Durable log ingest, artifact chunk transfer, and optional game client bridge traffic remain separate channels and must not be multiplexed through job result payloads. Artifact transfer carries chunk payloads only through `/api/v1/run/artifacts/*` routes. + +Production code signing/KMS, rollout rings/fleet orchestration, client-manager lifecycle, plugin lifecycle, production scaling/alerts, and real AI-provider integration are explicitly outside this contract. diff --git a/protocol/job_validation.go b/protocol/job_validation.go new file mode 100644 index 0000000..a0802d3 --- /dev/null +++ b/protocol/job_validation.go @@ -0,0 +1,520 @@ +package protocol + +import ( + "encoding/hex" + "net" + "net/url" + "strings" + "time" + "unicode/utf8" +) + +const maxRunLogicalFileKeyLength = 160 +const maxRunExecutionContentBytes = 64 * 1024 +const maxRunDLLExtensionBytes = int64(128 * 1024 * 1024) +const maxSourceRCONTimeoutSeconds = 60 +const maxProtectedRequestTimeoutSeconds = 120 +const maxProtectedRequestTextBytes = 16 * 1024 +const ( + maxSQLiteSchemaProbeObjects = 512 + maxSQLiteSchemaProbeColumnsPerObject = 256 + maxSQLiteSchemaProbeIndexesPerObject = 128 + maxSQLiteSchemaProbeForeignKeys = 128 + maxSQLiteSchemaProbeCardinalityReads = 512 + maxSQLiteSchemaProbeSamples = 3 + maxSQLiteSchemaProbeTimeoutMS = 10000 + maxSQLiteSchemaProbeResultBytes = 1024 * 1024 +) + +func ValidateRunJobAssignment(assignment RunJobAssignment) error { + if assignment.JobID == "" || assignment.RunEndpointID == "" || assignment.Capability == "" { + return ValidationError("jobId, runEndpointId, and capability are required") + } + switch assignment.Capability { + case RunCapabilityConfigWrite, RunCapabilityFilesList, RunCapabilityFilesRead, RunCapabilityFilesWrite: + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for scoped file jobs") + } + if !ValidLogicalFileKey(assignment.TargetKey) { + return ValidationError("targetKey is not allowed") + } + } + switch assignment.Capability { + case RunCapabilityConfigWrite, RunCapabilityFilesWrite: + if !ValidScopedInputRef(assignment.InputRef) { + return ValidationError("inputRef is not allowed") + } + } + if len([]byte(assignment.ExecutionInput.Content)) > maxRunExecutionContentBytes { + return ValidationError("execution input content is too large") + } + if assignment.ExecutionInput.MaxReadBytes < 0 || assignment.ExecutionInput.MaxReadBytes > maxRunExecutionContentBytes { + return ValidationError("execution input maxReadBytes is out of bounds") + } + if assignment.ExecutionInput.WorkspaceScope != "" && !ValidLogicalFileKey(assignment.ExecutionInput.WorkspaceScope) { + return ValidationError("execution input workspaceScope is not allowed") + } + if assignment.ExecutionInput.ServerDeploymentPlan != nil { + return ValidationError("game-specific server deployment plans are legacy unsupported input") + } + if len(assignment.ExecutionInput.DLLExtensions) > 0 { + if assignment.Capability != RunCapabilityProcessStart || assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" { + return ValidationError("DLL extensions are allowed only for scoped process.start jobs") + } + if len(assignment.ExecutionInput.DLLExtensions) > 16 { + return ValidationError("too many DLL extensions are declared") + } + keys := make(map[string]struct{}, len(assignment.ExecutionInput.DLLExtensions)) + targets := make(map[string]struct{}, len(assignment.ExecutionInput.DLLExtensions)) + for _, plan := range assignment.ExecutionInput.DLLExtensions { + if err := validateRuntimeDLLExtensionPlan(plan); err != nil { + return err + } + if _, exists := keys[plan.Key]; exists { + return ValidationError("DLL extension key is duplicated") + } + if _, exists := targets[plan.TargetKey]; exists { + return ValidationError("DLL extension target is duplicated") + } + keys[plan.Key] = struct{}{} + targets[plan.TargetKey] = struct{}{} + } + } + if assignment.ExecutionInput.LogSource != nil { + if assignment.Capability != RunCapabilityLogsBackfill || assignment.ServerInstanceID == "" { + return ValidationError("log source is allowed only for logs.backfill jobs") + } + if err := validateRuntimeLogSourcePlan(*assignment.ExecutionInput.LogSource); err != nil { + return err + } + } + if len(assignment.ExecutionInput.LogSources) > 0 { + if assignment.Capability != RunCapabilityProcessStart || assignment.ServerInstanceID == "" { + return ValidationError("process log sources are allowed only for process.start jobs") + } + seen := map[string]struct{}{} + for _, source := range assignment.ExecutionInput.LogSources { + if err := validateRuntimeProcessLogSourcePlan(source); err != nil { + return err + } + if _, exists := seen[source.Kind]; exists { + return ValidationError("process log source kind is duplicated") + } + seen[source.Kind] = struct{}{} + } + } + if assignment.ExecutionInput.SourceRCON != nil { + isSourceCommand := assignment.Capability == RunCapabilityRemoteRunRCONCommand + isProtectedRCON := assignment.Capability == RunCapabilityRemoteRunProtectedRCON + if (!isSourceCommand && !isProtectedRCON) || assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" { + return ValidationError("Source RCON is allowed only for scoped remote.run.rcon.command or remote.run.protected.rcon jobs") + } + if assignment.MaxAttempts != 1 { + return ValidationError("Source RCON jobs must have exactly one attempt") + } + if isSourceCommand && !strings.HasPrefix(assignment.InputRef, "input://source-rcon/") { + return ValidationError("Source RCON inputRef must be a source-rcon input ref") + } + if isProtectedRCON && !strings.HasPrefix(assignment.InputRef, "input://protected-request/") { + return ValidationError("protected Source RCON inputRef must be a protected-request input ref") + } + wantAdapterKind := "rcon" + if isProtectedRCON { + wantAdapterKind = protectedRequestAdapterKind(assignment.Capability) + } + if assignment.ExecutionInput.RemoteAdapterKind != "" && assignment.ExecutionInput.RemoteAdapterKind != wantAdapterKind { + return ValidationError("Source RCON requires the rcon adapter kind") + } + maxTimeout := maxSourceRCONTimeoutSeconds + if isProtectedRCON { + maxTimeout = maxProtectedRequestTimeoutSeconds + } + if assignment.ExecutionInput.TimeoutSeconds < 1 || assignment.ExecutionInput.TimeoutSeconds > maxTimeout { + return ValidationError("Source RCON timeout is out of bounds") + } + if err := validateRuntimeSourceRCONPlan(*assignment.ExecutionInput.SourceRCON); err != nil { + return err + } + } + if assignment.ExecutionInput.SQLiteSchemaProbe != nil { + if assignment.Capability != RunCapabilityRemoteRunDBSQLiteProbe || assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" || assignment.MaxAttempts != 1 || assignment.FencingToken == 0 { + return ValidationError("SQLite schema probe requires a scoped single fenced probe job") + } + if assignment.InputRef != "" || assignment.ExecutionInput.RemoteAdapterKey != "" || assignment.ExecutionInput.RemoteAdapterKind != "" || assignment.ExecutionInput.Content != "" || len(assignment.ExecutionInput.Inputs) != 0 { + return ValidationError("SQLite schema probe must not carry adapter input or content") + } + if !ValidLogicalFileKey(assignment.TargetKey) || !strings.HasPrefix(assignment.TargetKey, "databases/") { + return ValidationError("SQLite schema probe target must be a logical database target") + } + if err := validateSQLiteSchemaProbeRequest(*assignment.ExecutionInput.SQLiteSchemaProbe, assignment); err != nil { + return err + } + } + if isProtectedRequestCapability(assignment.Capability) { + if assignment.ServerInstanceID == "" || assignment.MaxAttempts != 1 || assignment.FencingToken == 0 { + return ValidationError("protected requests require a scoped single fenced attempt") + } + if !strings.HasPrefix(assignment.InputRef, "input://protected-request/") { + return ValidationError("protected request inputRef is not allowed") + } + if !ValidLogicalFileKey(assignment.TargetKey) || !ValidLogicalFileKey(assignment.ExecutionInput.RemoteAdapterKey) { + return ValidationError("protected request logical binding is not allowed") + } + if assignment.ExecutionInput.RemoteAdapterKind != protectedRequestAdapterKind(assignment.Capability) { + return ValidationError("protected request adapter kind does not match capability") + } + if assignment.ExecutionInput.TimeoutSeconds < 1 || assignment.ExecutionInput.TimeoutSeconds > maxProtectedRequestTimeoutSeconds { + return ValidationError("protected request timeout is out of bounds") + } + if assignment.ExecutionInput.Content != "" { + return ValidationError("protected request text must not be in a job assignment") + } + } + if strings.HasPrefix(assignment.InputRef, "input://source-rcon/") && assignment.ExecutionInput.SourceRCON == nil { + return ValidationError("source-rcon inputRef requires a Source RCON plan") + } + if IsRemoteCapability(assignment.Capability) { + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for remote jobs") + } + if RemoteCapabilityRequiresTargetKey(assignment.Capability) && !ValidLogicalFileKey(assignment.TargetKey) { + return ValidationError("targetKey is not allowed") + } + if RemoteCapabilityRequiresInputRef(assignment.Capability) && !ValidScopedInputRef(assignment.InputRef) { + return ValidationError("inputRef is not allowed") + } + if assignment.ExecutionInput.RemoteAdapterKey != "" && !ValidLogicalFileKey(assignment.ExecutionInput.RemoteAdapterKey) { + return ValidationError("remoteAdapterKey is not allowed") + } + if assignment.ExecutionInput.RemoteAdapterKind != "" && !validRemoteAdapterKind(assignment.ExecutionInput.RemoteAdapterKind) { + return ValidationError("remoteAdapterKind is not allowed") + } + if assignment.ExecutionInput.TimeoutSeconds < 0 || assignment.ExecutionInput.TimeoutSeconds > 300 { + return ValidationError("remote adapter timeout is out of bounds") + } + } + if assignment.Capability == RunCapabilityRemoteRunDBSQLiteProbe && assignment.ExecutionInput.SQLiteSchemaProbe == nil { + return ValidationError("SQLite schema probe request is required") + } + switch assignment.Capability { + case RunCapabilityDistributionBuild: + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for distribution build jobs") + } + if !ValidLogicalFileKey(assignment.TargetKey) || !strings.HasPrefix(assignment.TargetKey, "distribution/") { + return ValidationError("targetKey is not allowed for distribution build jobs") + } + if !ValidScopedInputRef(assignment.InputRef) || !strings.HasPrefix(assignment.InputRef, "input://distribution-build/") { + return ValidationError("inputRef must be a distribution build input ref") + } + case RunCapabilityRunSelfUpdate: + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for self-update jobs") + } + if assignment.TargetKey != "run/update" { + return ValidationError("targetKey must be run/update") + } + if !ValidScopedInputRef(assignment.InputRef) || !strings.HasPrefix(assignment.InputRef, "artifact://") { + return ValidationError("inputRef must be an artifact ref for self-update") + } + case RunCapabilityDependenciesCheck, RunCapabilityDependenciesInstall: + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for dependency jobs") + } + if !ValidLogicalFileKey(assignment.TargetKey) || !strings.HasPrefix(assignment.TargetKey, "dependencies/") { + return ValidationError("targetKey is not allowed for dependency jobs") + } + if assignment.InputRef != "" { + return ValidationError("dependency jobs must not carry arbitrary input refs") + } + case RunCapabilityLogsBackfill: + if assignment.ServerInstanceID == "" { + return ValidationError("serverInstanceId is required for log backfill jobs") + } + if !ValidLogicalFileKey(assignment.TargetKey) || !strings.HasPrefix(assignment.TargetKey, "logs/") { + return ValidationError("targetKey is not allowed for log backfill jobs") + } + if assignment.InputRef != "" && !ValidScopedInputRef(assignment.InputRef) { + return ValidationError("inputRef is not allowed for log backfill jobs") + } + } + return nil +} + +func ValidProtectedRequestExecutionInput(value ProtectedRequestExecutionInputResponse) bool { + return value.JobID != "" && value.ServerInstanceID != "" && value.RunEndpointID != "" && value.FencingToken != 0 && value.Authorized && value.ApprovalState == "approved" && value.QueueState == "claimed" && !value.ExpiresAt.IsZero() && time.Now().UTC().Before(value.ExpiresAt) && validProtectedRequestKind(value.Kind) && ValidLogicalFileKey(value.TransportKey) && ValidLogicalFileKey(value.TargetKey) && validProtectedRequestText(value.RequestText) +} + +func validProtectedRequestText(value string) bool { + return strings.TrimSpace(value) != "" && utf8.ValidString(value) && len([]byte(value)) <= maxProtectedRequestTextBytes && !strings.ContainsRune(value, '\x00') +} + +func isProtectedRequestCapability(capability string) bool { + return protectedRequestAdapterKind(capability) != "" +} + +func IsProtectedRequestCapability(capability string) bool { + return isProtectedRequestCapability(capability) +} + +func protectedRequestAdapterKind(capability string) string { + switch capability { + case RunCapabilityRemoteRunProtectedSQL: + return "protected-sql" + case RunCapabilityRemoteRunProtectedRCON: + return "protected-rcon" + case RunCapabilityRemoteRunProgram: + return "protected-program" + default: + return "" + } +} + +func validProtectedRequestKind(kind string) bool { + return kind == "sql" || kind == "rcon" || kind == "program" +} + +func validRemoteAdapterKind(kind string) bool { + switch kind { + case "ftp", "rsync", "run-file", "run-process", "database", "rcon", "log-transfer", "protected-sql", "protected-rcon", "protected-program": + return true + default: + return false + } +} + +func validateSQLiteSchemaProbeRequest(request SQLiteSchemaProbeRequest, assignment RunJobAssignment) error { + if !validProbeIdentifier(request.RequestID) { + return ValidationError("SQLite schema probe requestId is not allowed") + } + binding := request.Binding + if binding.ServerInstanceID != assignment.ServerInstanceID || binding.RunEndpointID != assignment.RunEndpointID || !validProbeIdentifier(binding.RunBindingID) || !validProbeIdentifier(binding.PluginID) || !validProbeIdentifier(binding.PluginVersion) || !validProbeIdentifier(binding.AdapterVersion) || !validProbeIdentifier(binding.DatabaseIdentity) { + return ValidationError("SQLite schema probe binding is invalid") + } + if binding.GameVersion != "" && !validProbeIdentifier(binding.GameVersion) { + return ValidationError("SQLite schema probe gameVersion is invalid") + } + if err := validateSQLiteSchemaProbeLimits(request.Limits); err != nil { + return err + } + return nil +} + +func validateSQLiteSchemaProbeLimits(limits SQLiteSchemaProbeLimits) error { + if limits.MaxObjects < 1 || limits.MaxObjects > maxSQLiteSchemaProbeObjects || limits.MaxColumnsPerObject < 1 || limits.MaxColumnsPerObject > maxSQLiteSchemaProbeColumnsPerObject || limits.MaxIndexesPerObject < 0 || limits.MaxIndexesPerObject > maxSQLiteSchemaProbeIndexesPerObject || limits.MaxForeignKeys < 0 || limits.MaxForeignKeys > maxSQLiteSchemaProbeForeignKeys || limits.MaxCardinalityReads < 0 || limits.MaxCardinalityReads > maxSQLiteSchemaProbeCardinalityReads || limits.MaxSampleRows < 0 || limits.MaxSampleRows > maxSQLiteSchemaProbeSamples || limits.TimeoutMS < 1 || limits.TimeoutMS > maxSQLiteSchemaProbeTimeoutMS || limits.MaxResultBytes < 1 || limits.MaxResultBytes > maxSQLiteSchemaProbeResultBytes { + return ValidationError("SQLite schema probe limits are out of bounds") + } + return nil +} + +func validProbeIdentifier(value string) bool { + return ValidLogicalFileKey(value) && !strings.Contains(value, "/") +} + +func validateRuntimeDLLExtensionPlan(plan RuntimeDLLExtensionPlan) error { + if !ValidLogicalFileKey(plan.Key) || !ValidLogicalFileKey(plan.TargetKey) || !validExtensionVersion(plan.Version) { + return ValidationError("DLL extension identity is not allowed") + } + if !validRuntimeDLLURL(plan.ReleaseURL) || !validSHA256(plan.Checksum) || !validSHA256(plan.SCUMExecutableChecksum) || plan.SizeBytes < 1 || plan.SizeBytes > maxRunDLLExtensionBytes { + return ValidationError("DLL extension release integrity is not allowed") + } + if !validDLLModKey(plan.ModKey) || plan.DLLRef != "ue4ss/Mods/"+plan.ModKey+"/dlls/main.dll" { + return ValidationError("DLL extension deployment path is not allowed") + } + if !validUE4SSABI(plan.UE4SSABI) || plan.RCONPort < 1024 || plan.RCONPort > 65535 { + return ValidationError("DLL extension compatibility metadata is not allowed") + } + return nil +} + +func validateRuntimeLogSourcePlan(plan RuntimeLogSourcePlan) error { + if !ValidLogicalFileKey(plan.Key) || !ValidLogicalFileKey(plan.StreamKey) || !ValidLogicalFileKey(plan.TargetKey) { + return ValidationError("log source identity is not allowed") + } + if plan.Kind != "file.tail" { + return ValidationError("log source kind is unsupported") + } + switch plan.CursorKind { + case "", "offset", "fingerprint": + default: + return ValidationError("log source cursor kind is unsupported") + } + if plan.RetentionDays < 0 || plan.RetentionDays > 365 { + return ValidationError("log source retention is out of bounds") + } + return nil +} + +func validateRuntimeProcessLogSourcePlan(plan RuntimeLogSourcePlan) error { + if !ValidLogicalFileKey(plan.Key) || !ValidLogicalFileKey(plan.StreamKey) { + return ValidationError("process log source identity is not allowed") + } + switch plan.Kind { + case "process.stdout", "process.stderr": + default: + return ValidationError("process log source kind is unsupported") + } + if plan.TargetKey != "" && !ValidLogicalFileKey(plan.TargetKey) { + return ValidationError("process log source target is not allowed") + } + switch plan.CursorKind { + case "", "sequence": + default: + return ValidationError("process log source cursor kind is unsupported") + } + if plan.RetentionDays < 0 || plan.RetentionDays > 365 { + return ValidationError("process log source retention is out of bounds") + } + return nil +} + +func validateRuntimeSourceRCONPlan(plan RuntimeSourceRCONPlan) error { + if plan.Protocol != "source-rcon" || !ValidLogicalFileKey(plan.ExtensionKey) || !validDLLModKey(plan.ModKey) { + return ValidationError("Source RCON identity is not allowed") + } + if plan.ConfigRef != "ue4ss/Mods/"+plan.ModKey+"/config.ini" || !ValidLogicalFileKey(plan.ConfigRef) { + return ValidationError("Source RCON config reference is not allowed") + } + if !validSourceRCONDeploymentStateRef(plan.DeploymentStateRef) { + return ValidationError("Source RCON deployment state reference is not allowed") + } + if plan.Port < 1024 || plan.Port > 65535 { + return ValidationError("Source RCON port is not allowed") + } + return nil +} + +func validSourceRCONDeploymentStateRef(value string) bool { + const prefix = "runtime/ue4ss-dll/" + const suffix = "/release.json" + if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) { + return false + } + targetKey := strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix) + return targetKey != "" && ValidLogicalFileKey(targetKey) +} + +func validRuntimeDLLURL(value string) bool { + parsed, err := url.ParseRequestURI(value) + if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" || parsed.Port() != "" && parsed.Port() != "443" || parsed.RawQuery != "" || !strings.HasSuffix(strings.ToLower(parsed.Path), ".dll") { + return false + } + host := strings.ToLower(parsed.Hostname()) + if host == "localhost" || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".local") { + return false + } + if ip := net.ParseIP(host); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsLinkLocalUnicast()) { + return false + } + return true +} + +func validSHA256(value string) bool { + if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func validExtensionVersion(value string) bool { + if len(value) == 0 || len(value) > 80 { + return false + } + for _, char := range value { + if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '.' || char == '-' { + continue + } + return false + } + return true +} + +func validDLLModKey(value string) bool { + if len(value) == 0 || len(value) > 80 { + return false + } + for index, char := range value { + if char >= 'a' && char <= 'z' || char >= '0' && char <= '9' || char == '_' || char == '-' { + if index > 0 || char != '_' && char != '-' { + continue + } + } + return false + } + return true +} + +func validUE4SSABI(value string) bool { + if len(value) == 0 || len(value) > 80 { + return false + } + for _, char := range value { + if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '.' || char == '_' || char == '-' { + continue + } + return false + } + return true +} + +type ValidationError string + +func (err ValidationError) Error() string { return string(err) } + +func ValidLogicalFileKey(key string) bool { + trimmed := strings.TrimSpace(key) + if trimmed == "" || trimmed != key || len([]rune(key)) > maxRunLogicalFileKeyLength { + return false + } + lower := strings.ToLower(key) + if strings.HasPrefix(key, "/") || strings.Contains(key, "..") || strings.Contains(key, `\`) || strings.Contains(key, "://") || strings.Contains(lower, "/users/") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "sk-") || strings.Contains(lower, "bearer ") { + return false + } + for _, char := range key { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.' || char == '/' { + continue + } + return false + } + return true +} + +func ValidScopedInputRef(ref string) bool { + trimmed := strings.TrimSpace(ref) + lower := strings.ToLower(ref) + if trimmed == "" || trimmed != ref || strings.Contains(lower, "/users/") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "sk-") || strings.Contains(lower, "bearer ") { + return false + } + return strings.HasPrefix(ref, "input://") || strings.HasPrefix(ref, "artifact://") +} + +func IsRemoteCapability(capability string) bool { + return strings.HasPrefix(capability, "remote.") +} + +func RemoteCapabilityRequiresTargetKey(capability string) bool { + switch capability { + case RunCapabilityRemoteRunProcessStart, RunCapabilityRemoteRunProcessStop: + return false + default: + return IsRemoteCapability(capability) + } +} + +func RemoteCapabilityRequiresInputRef(capability string) bool { + switch capability { + case RunCapabilityRemoteFTPWrite, + RunCapabilityRemoteRsyncWrite, + RunCapabilityRemoteRunFilesWrite, + RunCapabilityRemoteRunDBMySQLQuery, + RunCapabilityRemoteRunDBSQLiteQuery, + RunCapabilityRemoteRunRCONCommand, + RunCapabilityRemoteRunProtectedSQL, + RunCapabilityRemoteRunProtectedRCON, + RunCapabilityRemoteRunProgram: + return true + default: + return false + } +} diff --git a/protocol/job_validation_test.go b/protocol/job_validation_test.go new file mode 100644 index 0000000..9abde91 --- /dev/null +++ b/protocol/job_validation_test.go @@ -0,0 +1,376 @@ +package protocol + +import ( + "strings" + "testing" + "time" +) + +func TestValidateRunJobAssignmentScopedFilePayloads(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-config-write", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityConfigWrite, + TargetKey: "server.properties", + InputRef: "input://server-config/server-1/server.properties/v1", + IdempotencyKey: "idem-config", + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid config write assignment: %v", err) + } + + assignment.TargetKey = "/Users/tasia/server.properties" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "targetKey") { + t.Fatalf("expected raw host path rejection, got %v", err) + } + + assignment.TargetKey = "server.properties" + assignment.InputRef = "sk-raw-secret" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "inputRef") { + t.Fatalf("expected raw credential ref rejection, got %v", err) + } +} + +func TestValidateRunJobAssignmentScopedReadDoesNotRequireInputRef(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-files-read", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityFilesRead, + TargetKey: "logs/latest.log", + IdempotencyKey: "idem-read", + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid file read assignment: %v", err) + } +} + +func TestValidateRunJobAssignmentRequiresBoundedSQLiteSchemaProbe(t *testing.T) { + assignment := RunJobAssignment{JobID: "job-probe", ServerInstanceID: "server-1", RunEndpointID: "run-1", Capability: RunCapabilityRemoteRunDBSQLiteProbe, TargetKey: "databases/current.db", IdempotencyKey: "idem-probe", MaxAttempts: 1, FencingToken: 1, ExecutionInput: RunJobExecutionInput{WorkspaceScope: "profile-1", SQLiteSchemaProbe: &SQLiteSchemaProbeRequest{RequestID: "probe-1", Binding: SQLiteSchemaProbeBinding{ServerInstanceID: "server-1", RunBindingID: "binding-1", RunEndpointID: "run-1", PluginID: "game.example", PluginVersion: "1.0.0", AdapterVersion: "adapter-1", DatabaseIdentity: "database-1"}, Limits: SQLiteSchemaProbeLimits{MaxObjects: 8, MaxColumnsPerObject: 8, MaxIndexesPerObject: 8, MaxForeignKeys: 8, MaxCardinalityReads: 8, MaxSampleRows: 2, TimeoutMS: 1000, MaxResultBytes: 1024}}}} + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected bounded SQLite schema probe to validate: %v", err) + } + assignment.ExecutionInput.SQLiteSchemaProbe.Limits.MaxSampleRows = 4 + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "limits") { + t.Fatalf("expected oversized sample limit rejection, got %v", err) + } + assignment.ExecutionInput.SQLiteSchemaProbe.Limits.MaxSampleRows = 2 + assignment.ExecutionInput.Content = "SELECT * FROM private" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "must not carry") { + t.Fatalf("expected embedded query rejection, got %v", err) + } +} + +func TestValidateRunJobAssignmentRejectsLegacyGameSpecificDeploymentPlan(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-legacy-scum-plan", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityProcessInstall, + IdempotencyKey: "idem-legacy-scum", + ExecutionInput: RunJobExecutionInput{ + Deployment: &ServerDeploymentExecution{SchemaVersion: "1", Mode: "guided-install", ServerRoot: "C:/scumserver", Revision: 1}, + ServerDeploymentPlan: &ServerDeploymentPlan{SchemaVersion: "1", PluginID: "game.scum", TemplateKey: "scum-steamcmd-windows"}, + }, + } + + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "legacy unsupported") { + t.Fatalf("expected legacy game-specific plan rejection, got %v", err) + } +} + +func TestValidateRunJobAssignmentAllowsDeclaredProcessLogSourcesOnlyForStart(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-process-logs", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityProcessStart, + TargetKey: "actions/start.json", + IdempotencyKey: "idem-process-logs", + ExecutionInput: RunJobExecutionInput{LogSources: []RuntimeLogSourcePlan{ + {Key: "console-stdout", Kind: "process.stdout", TargetKey: "process/server", StreamKey: "game.console.stdout", CursorKind: "sequence", RetentionDays: 30}, + {Key: "console-stderr", Kind: "process.stderr", TargetKey: "process/server", StreamKey: "game.console.stderr", CursorKind: "sequence", RetentionDays: 30}, + }}, + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected process log sources to validate: %v", err) + } + + assignment.ExecutionInput.LogSources[0].Kind = "file.tail" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "process log source kind") { + t.Fatalf("expected file log source rejection on process.start, got %v", err) + } + + assignment.ExecutionInput.LogSources[0].Kind = "process.stdout" + assignment.Capability = RunCapabilityProcessStop + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "process log sources") { + t.Fatalf("expected process log source rejection outside start, got %v", err) + } +} + +func TestValidateRunJobAssignmentRemoteCapabilitiesAreBounded(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-remote-rcon", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityRemoteRunRCONCommand, + TargetKey: "rcon/command", + InputRef: "input://server-1/rcon/command/1", + IdempotencyKey: "idem-rcon", + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid remote rcon assignment: %v", err) + } + + assignment.InputRef = "password=raw" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "inputRef") { + t.Fatalf("expected unsafe inputRef rejection, got %v", err) + } + + assignment.InputRef = "input://server-1/rcon/command/1" + assignment.TargetKey = "/Users/tasia/server.db" + if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "targetKey") { + t.Fatalf("expected unsafe targetKey rejection, got %v", err) + } +} + +func TestValidateRunJobAssignmentDistributionCapabilitiesAreBounded(t *testing.T) { + selfUpdate := RunJobAssignment{ + JobID: "job-update", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityRunSelfUpdate, + TargetKey: "run/update", + InputRef: "artifact://artifact-run-latest", + IdempotencyKey: "idem-update", + } + if err := ValidateRunJobAssignment(selfUpdate); err != nil { + t.Fatalf("expected valid self-update assignment: %v", err) + } + selfUpdate.InputRef = "input://not-an-artifact" + if err := ValidateRunJobAssignment(selfUpdate); err == nil || !strings.Contains(err.Error(), "artifact") { + t.Fatalf("expected non-artifact self-update ref rejection, got %v", err) + } + + check := RunJobAssignment{ + JobID: "job-dependency-check", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityDependenciesCheck, + TargetKey: "dependencies/java-21", + IdempotencyKey: "idem-dep-check", + } + if err := ValidateRunJobAssignment(check); err != nil { + t.Fatalf("expected valid dependency check assignment: %v", err) + } + check.TargetKey = "dependencies/install/java;rm" + if err := ValidateRunJobAssignment(check); err == nil || !strings.Contains(err.Error(), "targetKey") { + t.Fatalf("expected shell-like dependency target rejection, got %v", err) + } + + backfill := RunJobAssignment{ + JobID: "job-log-backfill", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityLogsBackfill, + TargetKey: "logs/latest-log", + InputRef: "artifact://logs/checkpoint/1", + IdempotencyKey: "idem-log-backfill", + ExecutionInput: RunJobExecutionInput{LogSource: &RuntimeLogSourcePlan{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest.log", StreamKey: "latest-log", CursorKind: "offset", RetentionDays: 30}}, + } + if err := ValidateRunJobAssignment(backfill); err != nil { + t.Fatalf("expected valid log backfill assignment: %v", err) + } + backfill.ExecutionInput.LogSource.Kind = "sql.query" + if err := ValidateRunJobAssignment(backfill); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("expected unsupported log source rejection, got %v", err) + } + backfill.ExecutionInput.LogSource.Kind = "file.tail" + backfill.InputRef = "password=raw" + if err := ValidateRunJobAssignment(backfill); err == nil || !strings.Contains(err.Error(), "inputRef") { + t.Fatalf("expected unsafe log checkpoint rejection, got %v", err) + } +} + +func TestValidateRunJobAssignmentDLLExtensionsAreBounded(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-dll-extension", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityProcessStart, + TargetKey: "actions/start.json", + IdempotencyKey: "idem-dll-extension", + ExecutionInput: RunJobExecutionInput{ + WorkspaceScope: "run-local", + DLLExtensions: []RuntimeDLLExtensionPlan{validRuntimeDLLExtensionPlan()}, + }, + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid DLL extension assignment: %v", err) + } + + cases := []struct { + name string + mutate func(*RunJobAssignment) + want string + }{ + {name: "not process start", mutate: func(value *RunJobAssignment) { value.Capability = RunCapabilityProcessStop }, want: "process.start"}, + {name: "query URL", mutate: func(value *RunJobAssignment) { value.ExecutionInput.DLLExtensions[0].ReleaseURL += "?release=1" }, want: "release integrity"}, + {name: "unsafe DLL path", mutate: func(value *RunJobAssignment) { + value.ExecutionInput.DLLExtensions[0].DLLRef = "ue4ss/Mods/scum_simple_rcon/dlls/other.dll" + }, want: "deployment path"}, + {name: "bad checksum", mutate: func(value *RunJobAssignment) { value.ExecutionInput.DLLExtensions[0].Checksum = "sha256:bad" }, want: "release integrity"}, + {name: "missing scope", mutate: func(value *RunJobAssignment) { value.ExecutionInput.WorkspaceScope = "" }, want: "process.start"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + value := assignment + value.ExecutionInput.DLLExtensions = append([]RuntimeDLLExtensionPlan(nil), assignment.ExecutionInput.DLLExtensions...) + testCase.mutate(&value) + if err := ValidateRunJobAssignment(value); err == nil || !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("expected %q validation error, got %v", testCase.want, err) + } + }) + } +} + +func TestValidateRunJobAssignmentSourceRCONPlanIsFrozenAndOneShot(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-source-rcon", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityRemoteRunRCONCommand, + TargetKey: "rcon.password", + InputRef: "input://source-rcon/job-source-rcon", + IdempotencyKey: "idem-source-rcon", + Attempt: 1, + MaxAttempts: 1, + ExecutionInput: RunJobExecutionInput{ + WorkspaceScope: "run-local", + RemoteAdapterKey: "rcon", + RemoteAdapterKind: "rcon", + TimeoutSeconds: 30, + SourceRCON: &RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: 27015}, + }, + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid frozen Source RCON plan: %v", err) + } + protected := assignment + protected.Capability = RunCapabilityRemoteRunProtectedRCON + protected.TargetKey = "scum-management" + protected.InputRef = "input://protected-request/job-protected-rcon" + protected.FencingToken = 7 + protected.ExecutionInput.RemoteAdapterKey = "scum-management" + protected.ExecutionInput.RemoteAdapterKind = "protected-rcon" + protected.ExecutionInput.TimeoutSeconds = 120 + plan := *assignment.ExecutionInput.SourceRCON + protected.ExecutionInput.SourceRCON = &plan + if err := ValidateRunJobAssignment(protected); err != nil { + t.Fatalf("expected valid protected Source RCON plan: %v", err) + } + + cases := []struct { + name string + mutate func(*RunJobAssignment) + want string + }{ + {name: "multiple attempts", mutate: func(value *RunJobAssignment) { value.MaxAttempts = 2 }, want: "exactly one attempt"}, + {name: "wrong capability", mutate: func(value *RunJobAssignment) { value.Capability = RunCapabilityRemoteRunDBSQLiteQuery }, want: "remote.run.rcon.command or remote.run.protected.rcon"}, + {name: "unsafe config reference", mutate: func(value *RunJobAssignment) { + value.ExecutionInput.SourceRCON.ConfigRef = "ue4ss/Mods/scum_simple_rcon/other.ini" + }, want: "config reference"}, + {name: "unsafe deployment state reference", mutate: func(value *RunJobAssignment) { + value.ExecutionInput.SourceRCON.DeploymentStateRef = "runtime/ue4ss-dll/../../release.json" + }, want: "deployment state reference"}, + {name: "unsafe port", mutate: func(value *RunJobAssignment) { value.ExecutionInput.SourceRCON.Port = 80 }, want: "port"}, + {name: "ordinary input ref", mutate: func(value *RunJobAssignment) { value.InputRef = "input://server-1/rcon/command" }, want: "source-rcon input ref"}, + {name: "missing plan", mutate: func(value *RunJobAssignment) { value.ExecutionInput.SourceRCON = nil }, want: "requires a Source RCON plan"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + value := assignment + plan := *assignment.ExecutionInput.SourceRCON + value.ExecutionInput.SourceRCON = &plan + testCase.mutate(&value) + if err := ValidateRunJobAssignment(value); err == nil || !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("expected %q validation error, got %v", testCase.want, err) + } + }) + } +} + +func TestValidateProtectedRequestAssignmentAndOneTimeInput(t *testing.T) { + assignment := RunJobAssignment{ + JobID: "job-protected", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: RunCapabilityRemoteRunProtectedSQL, + TargetKey: "scum-database", + InputRef: "input://protected-request/job-protected", + IdempotencyKey: "protected-1", + LeaseToken: "lease-protected", + FencingToken: 7, + Attempt: 1, + MaxAttempts: 1, + ExecutionInput: RunJobExecutionInput{ + RemoteAdapterKey: "scum-database", + RemoteAdapterKind: "protected-sql", + TimeoutSeconds: 30, + }, + } + if err := ValidateRunJobAssignment(assignment); err != nil { + t.Fatalf("expected valid protected assignment: %v", err) + } + input := ProtectedRequestExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, FencingToken: assignment.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: time.Now().UTC().Add(time.Minute), Kind: "sql", TransportKey: "scum-database", TargetKey: assignment.TargetKey, RequestText: "SELECT player_id FROM players LIMIT 1"} + if !ValidProtectedRequestExecutionInput(input) { + t.Fatal("expected approved unexpired protected input") + } + + for _, testCase := range []struct { + name string + mutate func(*RunJobAssignment) + want string + }{ + {name: "missing fence", mutate: func(value *RunJobAssignment) { value.FencingToken = 0 }, want: "fenced"}, + {name: "multiple attempts", mutate: func(value *RunJobAssignment) { value.MaxAttempts = 2 }, want: "single"}, + {name: "wrong input ref", mutate: func(value *RunJobAssignment) { value.InputRef = "input://ordinary/request" }, want: "inputRef"}, + {name: "wrong adapter", mutate: func(value *RunJobAssignment) { value.ExecutionInput.RemoteAdapterKind = "database" }, want: "adapter kind"}, + {name: "inline text", mutate: func(value *RunJobAssignment) { value.ExecutionInput.Content = "SELECT secret" }, want: "must not"}, + } { + t.Run(testCase.name, func(t *testing.T) { + value := assignment + testCase.mutate(&value) + if err := ValidateRunJobAssignment(value); err == nil || !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("expected %q validation error, got %v", testCase.want, err) + } + }) + } + + input.ExpiresAt = time.Now().UTC().Add(-time.Second) + if ValidProtectedRequestExecutionInput(input) { + t.Fatal("expired protected input was accepted") + } + input.ExpiresAt = time.Now().UTC().Add(time.Minute) + input.QueueState = "pending" + if ValidProtectedRequestExecutionInput(input) { + t.Fatal("unclaimed protected input was accepted") + } +} + +func validRuntimeDLLExtensionPlan() RuntimeDLLExtensionPlan { + return RuntimeDLLExtensionPlan{ + Key: "scum-simple-rcon", + Version: "1.0.0", + ReleaseURL: "https://cdn.npc0.com/scum_simple_rcon_ue4s.dll", + Checksum: "sha256:" + strings.Repeat("a", 64), + SizeBytes: 1024, + TargetKey: "ue4ss/scum-simple-rcon", + ModKey: "scum_simple_rcon", + DLLRef: "ue4ss/Mods/scum_simple_rcon/dlls/main.dll", + SCUMExecutableChecksum: "sha256:" + strings.Repeat("b", 64), + UE4SSABI: "ue4ss-3.0", + RCONPort: 27015, + } +} diff --git a/protocol/log-ingest.md b/protocol/log-ingest.md new file mode 100644 index 0000000..b63ba03 --- /dev/null +++ b/protocol/log-ingest.md @@ -0,0 +1,36 @@ +# Run Log Ingest Contract + +Logs are durable historical data. They are not transported as best-effort UI messages. + +## Implemented Routes + +- `POST /api/v1/run/logs/batches`: uploads one bounded log batch and receives an acknowledgement range. +- `POST /api/v1/log-streams/query`: queries stored log entries after a stream sequence cursor. +- `GET /api/v1/server-instances/{id}/logs/events`: browser-facing Server-Sent Events stream for replaying recent stored entries and pushing newly ingested platform log entries. + +## Payloads + +- `LogBatchIngestRequest`: run ID, session token, server instance ID, stream ID, source, sequence range, compression metadata, checksum, and bounded entries. +- `LogEntry`: sequence, timestamp, level, line, parser metadata, and redaction state. +- `LogBatchIngestResponse`: accepted sequence range, latest acknowledged sequence, duplicate flag, retry hint, and server time. +- `LogStreamCursorRequest`: stream ID, sequence cursor, and limit. +- `LogStreamCursorResponse`: ordered entries, next cursor, and latest acknowledged sequence. +- `LogStreamEventResponse`: safe browser event containing server ID, stream metadata, latest sequence, and one log entry. + +Run-assigned Platform jobs use `job..` log stream IDs. Autonomous lifecycle bootstrap is not a Platform job, so it uses `run...` and Platform creates the server-bound stream from the signed Run batch instead of looking for a job record. + +## Local Spool + +Run must write unacknowledged logs to a local spool/WAL before upload. Segments may be removed only after platform acknowledgement. + +## Priority + +Log flush has higher priority than artifact transfer. Artifact work must slow down when log spool pressure rises. + +Log spool retry state is independent from artifact/file retry state. Acknowledged log batches may be removed even when artifact chunks are still pending, and artifact chunk acknowledgement must not alter log sequence state. Log ingest payloads carry bounded entries only and must not include artifact chunks, file bodies, host paths, raw credentials, or direct socket details. + +The Run uploader flushes committed spool segments independently with bounded request contexts. A failed or partial acknowledgement leaves the segment pending for restart/retry; control heartbeat and job lifecycle polling do not wait for log or artifact flushes. + +## Browser Channel + +Browser live tail is a platform-owned SSE fan-out from durable ingest and cursor state. External log storage backends and optional game client bridge traffic remain separate channels. Artifact transfer uses its own lower-priority channel and must not be multiplexed through log ingest. diff --git a/protocol/log_ingest.go b/protocol/log_ingest.go new file mode 100644 index 0000000..5d3cf97 --- /dev/null +++ b/protocol/log_ingest.go @@ -0,0 +1,68 @@ +package protocol + +import "time" + +type LogEntry struct { + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + Level string `json:"level,omitempty"` + Line string `json:"line"` + Fields map[string]string `json:"fields,omitempty"` + Redacted bool `json:"redacted"` +} + +type LogBatchIngestRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + LogStreamID string `json:"logStreamId"` + ServerInstanceID string `json:"serverInstanceId"` + StreamKey string `json:"streamKey"` + Source string `json:"source"` + LogSessionID string `json:"logSessionId,omitempty"` + SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"` + FirstSeq uint64 `json:"firstSeq"` + LastSeq uint64 `json:"lastSeq"` + Compression string `json:"compression"` + Checksum string `json:"checksum"` + Entries []LogEntry `json:"entries"` +} + +type LogBatchIngestResponse struct { + Accepted bool `json:"accepted"` + LogStreamID string `json:"logStreamId"` + AcceptedFrom uint64 `json:"acceptedFrom"` + AcceptedTo uint64 `json:"acceptedTo"` + LatestSeq uint64 `json:"latestSeq"` + Duplicate bool `json:"duplicate"` + RetryAfterSec int `json:"retryAfterSec,omitempty"` + ServerTime time.Time `json:"serverTime"` +} + +type LogStreamCursorRequest struct { + LogStreamID string `json:"logStreamId"` + AfterSeq uint64 `json:"afterSeq"` + Limit int `json:"limit"` +} + +type LogStreamCursorResponse struct { + LogStreamID string `json:"logStreamId"` + Entries []LogEntry `json:"entries"` + NextSeq uint64 `json:"nextSeq"` + LatestSeq uint64 `json:"latestSeq"` +} + +type RunLogStreamProgressRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + ServerInstanceID string `json:"serverInstanceId"` + LogStreamID string `json:"logStreamId"` +} + +type RunLogStreamProgressResponse struct { + Accepted bool `json:"accepted"` + RunEndpointID string `json:"runEndpointId"` + ServerInstanceID string `json:"serverInstanceId"` + LogStreamID string `json:"logStreamId"` + LatestSeq uint64 `json:"latestSeq"` + ServerTime time.Time `json:"serverTime"` +} diff --git a/protocol/metrics.go b/protocol/metrics.go new file mode 100644 index 0000000..4d6b194 --- /dev/null +++ b/protocol/metrics.go @@ -0,0 +1,31 @@ +package protocol + +import "time" + +type MetricSample struct { + ID string `json:"id,omitempty"` + ServerInstanceID string `json:"serverInstanceId"` + Online bool `json:"online"` + PlayerCount *int `json:"playerCount,omitempty"` + MaxPlayers *int `json:"maxPlayers,omitempty"` + TPS *float64 `json:"tps,omitempty"` + LatencyMS *float64 `json:"latencyMs,omitempty"` + CPUPercent *float64 `json:"cpuPercent,omitempty"` + MemoryPercent *float64 `json:"memoryPercent,omitempty"` + DiskPercent *float64 `json:"diskPercent,omitempty"` + Source string `json:"source"` + CollectedAt time.Time `json:"collectedAt"` +} + +type MetricBatchIngestRequest struct { + RunEndpointID string `json:"runEndpointId"` + SessionToken string `json:"sessionToken"` + Samples []MetricSample `json:"samples"` +} + +type MetricBatchIngestResponse struct { + Accepted bool `json:"accepted"` + AcceptedCount int `json:"acceptedCount"` + LatestAt time.Time `json:"latestAt"` + ServerTime time.Time `json:"serverTime"` +} diff --git a/protocol/protected-request.md b/protocol/protected-request.md new file mode 100644 index 0000000..7f119bd --- /dev/null +++ b/protocol/protected-request.md @@ -0,0 +1,38 @@ +# Protected request execution + +Protected SQL, RCON, and management-program work uses the job channel for its +lease and a separate one-time input route for its approved text: + +- capabilities: `remote.run.protected.sql`, `remote.run.protected.rcon`, and + `remote.run.program.command`; +- input route: `POST /api/v1/run/jobs/protected-request-input`; +- assignment input ref: `input://protected-request/`; +- adapter kinds: `protected-sql`, `protected-rcon`, and `protected-program`. + +The signed input request contains the Run endpoint/session, job ID, lease, +attempt, and fencing token. Platform returns only the matching job/server/Run +identity, fencing token, explicit authorization, approved/unexpired state, +claimed queue state, protected request kind, +logical transport/target keys, and bounded request text. Neither direction may +carry a DSN, database path, password, socket, host path, shell, or raw +connection. + +Run validates all assignment and response bindings again immediately before +dispatch. A protected request must have exactly one attempt, a non-zero fencing +token, explicit authorization, an `approved` state, a `claimed` queue state, a +future expiry, a capability-kind match, and exact transport/target identity +matches. The request text is never written to the job journal or terminal +result. + +Transport implementations are registered locally by `(kind, transportKey)` and +resolve any private connection configuration inside Run. A management-program +handler is an application protocol handler, not a host process or OS shell. +Unknown transport operations, request formats, or fields return the terminal +safe result `protected_request_unknown`; other transport failures use bounded +diagnostics without forwarding handler errors or response bodies. These errors +affect only the current request. + +Management-program stdout and stderr are bounded, redacted, and sent to the +durable log channel with source `management-program` and streams +`management-program.stdout` / `management-program.stderr`. They are not file +execution logs and are never embedded in job result content. diff --git a/runtime/autonomous_lifecycle.go b/runtime/autonomous_lifecycle.go new file mode 100644 index 0000000..8a1f933 --- /dev/null +++ b/runtime/autonomous_lifecycle.go @@ -0,0 +1,319 @@ +package runtime + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "runtime" + "strings" + "time" + + "browser.local/run/config" + "browser.local/run/protocol" +) + +const autonomousLifecyclePlanKey = ".platform/autonomous-lifecycle-plan.json" + +func LoadAutonomousLifecyclePlan(cfg config.Config) (*protocol.RunAutonomousLifecyclePlan, string, bool, error) { + if strings.TrimSpace(cfg.ComponentKind) != config.PackageComponentRun || strings.TrimSpace(cfg.ServerInstanceID) == "" { + log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=not_generated_run server=%s component=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKind)) + return nil, "", false, nil + } + scope, err := seededWorkspaceScope(cfg) + if err != nil { + log.Printf("RUN phase=autonomous_lifecycle status=scope_failed server=%s componentKey=%s error=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKey), RedactText(err.Error())) + return nil, "", false, err + } + path, err := NewWorkspaceResolver(cfg.WorkspaceRoot).ExistingTarget(scope, autonomousLifecyclePlanKey) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=plan_missing scope=%s", safeOptional(scope)) + return nil, scope, false, nil + } + log.Printf("RUN phase=autonomous_lifecycle status=load_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error())) + return nil, scope, false, err + } + file, err := os.Open(path) + if err != nil { + log.Printf("RUN phase=autonomous_lifecycle status=open_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error())) + return nil, scope, false, err + } + defer file.Close() + var plan protocol.RunAutonomousLifecyclePlan + decoder := json.NewDecoder(io.LimitReader(file, 64*1024)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&plan); err != nil { + log.Printf("RUN phase=autonomous_lifecycle status=decode_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error())) + return nil, scope, false, err + } + if err := protocol.ValidateRunAutonomousLifecyclePlan(plan); err != nil { + log.Printf("RUN phase=autonomous_lifecycle status=invalid scope=%s error=%s", safeOptional(scope), RedactText(err.Error())) + return nil, scope, false, err + } + log.Printf("RUN phase=autonomous_lifecycle status=loaded server=%s plugin=%s endpoint=%s profile=%s bootstrap=%t actions=%d dependencies=%d installs=%d", plan.ServerInstanceID, plan.PluginID, plan.RunEndpointID, safeOptional(plan.ProfileKey), plan.Bootstrap != nil, len(plan.Actions), len(plan.DependencyProbes), len(plan.InstallPlans)) + return &plan, scope, true, nil +} + +func (worker *Worker) RunAutonomousLifecycleOnce(ctx context.Context) error { + plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg) + if err != nil || !ok { + return err + } + state, err := worker.registeredState() + if err != nil { + return err + } + if err := validateAutonomousLifecycleScope(worker.cfg, state, *plan); err != nil { + log.Printf("RUN phase=autonomous_lifecycle status=scope_mismatch error=%s", RedactText(err.Error())) + return err + } + if err := worker.runAutonomousDependencies(ctx, *plan); err != nil { + return err + } + if plan.Bootstrap == nil { + log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=no_bootstrap server=%s", plan.ServerInstanceID) + return nil + } + assignment := autonomousLifecycleAssignment(state, worker.cfg, *plan, *plan.Bootstrap) + log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=starting job=%s capability=%s target=%s operation=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), safeOptional(assignment.ExecutionInput.LifecycleOperation)) + executor := worker.executor + executor.artifactHook = StaticLifecycleArtifactHook{} + execution := executor.ExecuteContext(ctx, assignment) + log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=complete job=%s state=%s processState=%s errorCode=%s message=%s", assignment.JobID, execution.State, safeOptional(execution.ExecutionResult.ProcessState), safeOptional(execution.ErrorCode), safeOptional(execution.Message)) + if err := worker.reportAutonomousLifecycle(ctx, assignment, execution); err != nil { + log.Printf("RUN phase=autonomous_lifecycle.report status=degraded server=%s action=continue_worker reason=projection_report_failed", assignment.ServerInstanceID) + } + if execution.State != lifecycleResultStateSucceeded { + return fmt.Errorf("autonomous lifecycle bootstrap failed: %s", errorSummaryFromResult(execution)) + } + return nil +} + +func (worker *Worker) reportAutonomousLifecycle(ctx context.Context, assignment protocol.RunJobAssignment, execution LifecycleExecutionResult) error { + state, err := worker.registeredState() + if err != nil { + return err + } + request := protocol.RunLifecycleReportRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + ServerInstanceID: assignment.ServerInstanceID, + Capability: assignment.Capability, + State: execution.State, + Progress: execution.Progress, + Message: execution.Message, + ErrorCode: execution.ErrorCode, + ExecutionResult: execution.ExecutionResult, + } + if source, ok := worker.executor.managed.(ManagedProcessObservationSource); ok { + for _, identity := range source.ManagedProcessObservations() { + if identity.ServerInstanceID == assignment.ServerInstanceID && identity.RunEndpointID == state.RunEndpointID && identity.ObservationSeq > 0 && identity.State == execution.ExecutionResult.ProcessState { + request.ManagedProcessID = managedProcessObservationID(identity) + request.ObservationSeq = identity.ObservationSeq + request.ObservedAt = identity.UpdatedAt + break + } + } + } + log.Printf("RUN phase=autonomous_lifecycle.report status=starting server=%s capability=%s state=%s processState=%s", assignment.ServerInstanceID, assignment.Capability, execution.State, safeOptional(execution.ExecutionResult.ProcessState)) + response, err := worker.client.ReportLifecycle(ctx, request) + if err != nil { + log.Printf("RUN phase=autonomous_lifecycle.report status=failed server=%s error=%s", assignment.ServerInstanceID, RedactText(err.Error())) + return err + } + if !response.Accepted || response.RunEndpointID != state.RunEndpointID || response.ServerInstanceID != assignment.ServerInstanceID { + log.Printf("RUN phase=autonomous_lifecycle.report status=rejected server=%s accepted=%t", assignment.ServerInstanceID, response.Accepted) + return fmt.Errorf("autonomous lifecycle report was not accepted") + } + log.Printf("RUN phase=autonomous_lifecycle.report status=accepted server=%s projectedState=%s", assignment.ServerInstanceID, safeOptional(response.ProjectedState)) + return nil +} + +func (worker *Worker) reportAutonomousProcessObservations(ctx context.Context) error { + source, ok := worker.executor.managed.(ManagedProcessObservationSource) + if !ok { + return nil + } + state, err := worker.registeredState() + if err != nil { + return err + } + for _, identity := range source.ManagedProcessObservations() { + if identity.ServerInstanceID == "" || identity.RunEndpointID != state.RunEndpointID || identity.ObservationSeq == 0 { + continue + } + processID := managedProcessObservationID(identity) + worker.observationMu.Lock() + alreadyReported := worker.reportedObservations[processID] >= identity.ObservationSeq + worker.observationMu.Unlock() + if alreadyReported { + continue + } + request := protocol.RunLifecycleReportRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, ServerInstanceID: identity.ServerInstanceID, Capability: protocol.RunCapabilityProcessStatus, State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "supervised process observation"}, ManagedProcessID: processID, ObservationSeq: identity.ObservationSeq, ObservedAt: identity.UpdatedAt, ExecutionResult: protocol.RunJobExecutionResult{Kind: "process", ProcessState: identity.State, ExitClassification: identity.ExitClassification, ExitCode: identity.ExitCode, Summary: "supervised process observation"}} + if _, err := worker.client.ReportLifecycle(ctx, request); err != nil { + return err + } + worker.observationMu.Lock() + worker.reportedObservations[processID] = identity.ObservationSeq + worker.observationMu.Unlock() + } + return nil +} + +func managedProcessObservationID(identity ProcessIdentity) string { + if identity.LogSessionID != "" { + return "log-session:" + identity.LogSessionID + } + sum := sha256.Sum256([]byte(identity.RunEndpointID + "\x00" + identity.ServerInstanceID + "\x00" + identity.CommandFingerprint + "\x00" + identity.StartedAt.UTC().Format(time.RFC3339Nano))) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func validateAutonomousLifecycleScope(cfg config.Config, state WorkerState, plan protocol.RunAutonomousLifecyclePlan) error { + if plan.RunEndpointID != state.RunEndpointID || plan.ServerInstanceID != cfg.ServerInstanceID || plan.PluginID != cfg.PluginID { + return fmt.Errorf("autonomous lifecycle plan identity does not match this Run") + } + if plan.TargetOS != runtime.GOOS || plan.TargetArch != runtime.GOARCH { + return fmt.Errorf("autonomous lifecycle target does not match this Run") + } + if cfg.ComponentKey != "" && plan.ProfileKey != "" && plan.ProfileKey != cfg.ComponentKey { + return fmt.Errorf("autonomous lifecycle profile does not match this Run") + } + return nil +} + +func (worker *Worker) runAutonomousDependencies(ctx context.Context, plan protocol.RunAutonomousLifecyclePlan) error { + if len(plan.DependencyProbes) == 0 { + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=skipped reason=no_probes") + return nil + } + for _, probe := range plan.DependencyProbes { + state, evidence, err := worker.executor.runDependencyProbe(ctx, probe, plan.RuntimeBindings) + if err != nil { + if probe.Required { + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=failed probe=%s error=%s", safeOptional(probe.Key), RedactText(err.Error())) + return err + } + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=optional_probe_failed probe=%s error=%s", safeOptional(probe.Key), RedactText(err.Error())) + continue + } + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=probed probe=%s state=%s evidence=%s required=%t", safeOptional(probe.Key), safeOptional(state), safeOptional(evidence), probe.Required) + if state == "present" || !probe.Required { + continue + } + installPlan, found := autonomousInstallPlanForProbe(plan.InstallPlans, probe) + if !found { + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=missing_without_install_plan probe=%s action=defer_to_bootstrap", safeOptional(probe.Key)) + continue + } + assignment := autonomousDependencyAssignment(worker.State(), worker.cfg, plan, installPlan) + input := protocol.DependencyExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: plan.ServerInstanceID, RunEndpointID: plan.RunEndpointID, PluginID: plan.PluginID, PluginVersion: plan.PluginVersion, ProfileKey: autonomousProfileKey(worker.cfg, plan), TargetOS: plan.TargetOS, TargetArch: plan.TargetArch, PlanDigest: autonomousInstallPlanDigest(installPlan), Probe: probe, Plan: installPlan, Bindings: plan.RuntimeBindings} + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_start probe=%s plan=%s steps=%d", safeOptional(probe.Key), safeOptional(installPlan.Key), len(installPlan.Steps)) + if err := worker.runAutonomousInstallPlan(ctx, assignment, input); err != nil { + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_failed probe=%s plan=%s error=%s", safeOptional(probe.Key), safeOptional(installPlan.Key), RedactText(err.Error())) + return err + } + log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_complete probe=%s plan=%s", safeOptional(probe.Key), safeOptional(installPlan.Key)) + } + return nil +} + +func (worker *Worker) runAutonomousInstallPlan(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse) error { + for index, step := range input.Plan.Steps { + if err := worker.executor.runDependencyInstallStep(ctx, assignment, input, step, index); err != nil { + return err + } + } + return nil +} + +func autonomousInstallPlanForProbe(plans []protocol.DependencyInstallPlan, probe protocol.DependencyProbe) (protocol.DependencyInstallPlan, bool) { + for _, plan := range plans { + for _, step := range plan.Steps { + if step.TargetKey == probe.TargetKey { + return plan, true + } + } + } + return protocol.DependencyInstallPlan{}, false +} + +func autonomousLifecycleAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, action protocol.RunAutonomousLifecycleAction) protocol.RunJobAssignment { + now := time.Now().UTC() + profileKey := autonomousProfileKey(cfg, plan) + assignment := protocol.RunJobAssignment{ + JobID: "autonomous-bootstrap-" + safeWorkspaceName(action.Operation), + ServerInstanceID: plan.ServerInstanceID, + RunEndpointID: state.RunEndpointID, + Capability: action.Capability, + TargetKey: action.TargetKey, + IdempotencyKey: "autonomous:" + safeWorkspaceName(plan.TargetRelease) + ":" + safeWorkspaceName(action.Operation), + State: "running", + LeaseToken: "local-autonomous-bootstrap", + Attempt: 1, + MaxAttempts: 1, + CreatedAt: now, + UpdatedAt: now, + ExecutionInput: protocol.RunJobExecutionInput{ + WorkspaceScope: profileKey, + PluginID: plan.PluginID, + LifecycleOperation: action.Operation, + Deployment: autonomousDeploymentExecution(plan.Deployment, action.Operation), + }, + } + if action.Capability == protocol.RunCapabilityProcessStart { + assignment.ExecutionInput.LogSources = append([]protocol.RuntimeLogSourcePlan(nil), plan.LogSources...) + assignment.ExecutionInput.DLLExtensions = append([]protocol.RuntimeDLLExtensionPlan(nil), plan.DLLExtensions...) + } + return assignment +} + +func autonomousDependencyAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, installPlan protocol.DependencyInstallPlan) protocol.RunJobAssignment { + now := time.Now().UTC() + return protocol.RunJobAssignment{JobID: "autonomous-dependencies-" + safeWorkspaceName(installPlan.Key), ServerInstanceID: plan.ServerInstanceID, RunEndpointID: state.RunEndpointID, Capability: protocol.RunCapabilityDependenciesInstall, TargetKey: "dependencies/install/" + installPlan.Key, IdempotencyKey: "autonomous:dependencies:" + safeWorkspaceName(installPlan.Key), State: "running", LeaseToken: "local-autonomous-dependencies", Attempt: 1, MaxAttempts: 1, CreatedAt: now, UpdatedAt: now, ExecutionInput: protocol.RunJobExecutionInput{WorkspaceScope: autonomousProfileKey(cfg, plan), PluginID: plan.PluginID, LifecycleOperation: "install"}} +} + +func autonomousDeploymentExecution(deployment *protocol.RunAutonomousDeployment, operation string) *protocol.ServerDeploymentExecution { + if deployment == nil { + return nil + } + startCommand := deployment.StartCommand + if operation == "install" && deployment.InstallCommand != "" { + startCommand = deployment.InstallCommand + } + return &protocol.ServerDeploymentExecution{SchemaVersion: deployment.SchemaVersion, Mode: deployment.Mode, ProfileKey: deployment.ProfileKey, CreateInputs: copyStringMap(deployment.CreateInputs), ServerRoot: deployment.ServerRoot, WorkingDirectory: deployment.WorkingDirectory, StartCommand: startCommand, StopCommand: deployment.StopCommand, StatusCommand: deployment.StatusCommand, Shell: deployment.Shell, Revision: deployment.Revision} +} + +func autonomousProfileKey(cfg config.Config, plan protocol.RunAutonomousLifecyclePlan) string { + if strings.TrimSpace(cfg.ComponentKey) != "" { + return cfg.ComponentKey + } + return plan.ProfileKey +} + +func autonomousInstallPlanDigest(plan protocol.DependencyInstallPlan) string { + body, err := json.Marshal(plan) + if err != nil { + return "sha256:" + strings.Repeat("0", 64) + } + sum := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func errorSummaryFromResult(result LifecycleExecutionResult) string { + if result.ErrorCode != "" && result.Message != "" { + return result.ErrorCode + ": " + result.Message + } + if result.ErrorCode != "" { + return result.ErrorCode + } + if result.Message != "" { + return result.Message + } + return result.State +} diff --git a/runtime/autonomous_lifecycle_test.go b/runtime/autonomous_lifecycle_test.go new file mode 100644 index 0000000..61e9d6e --- /dev/null +++ b/runtime/autonomous_lifecycle_test.go @@ -0,0 +1,302 @@ +package runtime + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestWorkerRunsAutonomousBootstrapFromSeededPlan(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.PluginID = "server.scum" + cfg.ComponentKind = "run" + cfg.ComponentKey = "run-local" + writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{ + SchemaVersion: "1", + ServerInstanceID: cfg.ServerInstanceID, + PluginID: cfg.PluginID, + PluginVersion: "1.0.0", + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ComponentKey, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-dist-test", + Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"}, + LogSources: []protocol.RuntimeLogSourcePlan{{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}}, + Deployment: &protocol.RunAutonomousDeployment{SchemaVersion: "1", Mode: "guided", ProfileKey: cfg.ComponentKey, ServerRoot: "D:/game-server", CreateInputs: map[string]string{"gamePort": "7779", "maxPlayers": "128"}, Revision: 2}, + }) + managed := &recordingManagedSupervisor{} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed)) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil { + t.Fatalf("run autonomous lifecycle: %v", err) + } + + if len(client.claimRequests) != 0 || len(client.ackRequests) != 0 || len(client.resultRequests) != 0 || len(client.lifecycleReports) != 1 { + t.Fatalf("autonomous bootstrap must report without platform job assignment: claims=%d acks=%d results=%d reports=%d", len(client.claimRequests), len(client.ackRequests), len(client.resultRequests), len(client.lifecycleReports)) + } + if client.lifecycleReports[0].Capability != protocol.RunCapabilityProcessStart || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" { + t.Fatalf("expected autonomous process lifecycle report, got %+v", client.lifecycleReports[0]) + } + if !managed.started || managed.identity.ServerInstanceID != cfg.ServerInstanceID || managed.identity.RunEndpointID != cfg.RunEndpointID { + t.Fatalf("expected managed process start from autonomous plan, managed=%+v", managed) + } + if managed.identity.StdoutStreamKey != "game.console.stdout" { + t.Fatalf("expected declared process stdout stream, identity=%+v", managed.identity) + } + if managed.command.Env["SERVER_ROOT"] != "D:/game-server" || managed.command.Env["SERVER_CREATE_GAMEPORT"] != "7779" || managed.command.Env["SERVER_CREATE_MAXPLAYERS"] != "128" { + t.Fatalf("expected deployment env from autonomous plan, env=%+v", managed.command.Env) + } +} + +func TestWorkerContinuesAutonomousBootstrapWhenLifecycleReportFails(t *testing.T) { + client := newFakeWorkerClient() + client.lifecycleReportErr = errors.New("platform request failed: status=401 path=/api/v1/run/lifecycle/report code=unauthorized") + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.PluginID = "game.example" + cfg.ComponentKind = "run" + cfg.ComponentKey = "run-local" + writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{ + SchemaVersion: "1", + ServerInstanceID: cfg.ServerInstanceID, + PluginID: cfg.PluginID, + PluginVersion: "1.0.0", + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ComponentKey, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-dist-test", + Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"}, + }) + managed := &recordingManagedSupervisor{} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed)) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil { + t.Fatalf("run autonomous lifecycle with report failure: %v", err) + } + + if !managed.started || len(client.lifecycleReports) != 1 { + t.Fatalf("expected bootstrap to continue after lifecycle report failure, managed=%+v reports=%d", managed, len(client.lifecycleReports)) + } +} + +func TestWorkerDefersUnmatchedRequiredDependencyToBootstrap(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.PluginID = "game.example" + cfg.ComponentKind = "run" + cfg.ComponentKey = "run-local" + writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{ + SchemaVersion: "1", + ServerInstanceID: cfg.ServerInstanceID, + PluginID: cfg.PluginID, + PluginVersion: "1.0.0", + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ComponentKey, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-dist-test", + Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"}, + DependencyProbes: []protocol.DependencyProbe{{Key: "steamcmd", Kind: "command.version", TargetKey: "steamcmd", Required: true, Platforms: []string{runtime.GOOS}}}, + InstallPlans: []protocol.DependencyInstallPlan{{Key: "install-game-server", Title: "Install game server", Platforms: []string{runtime.GOOS}, Steps: []protocol.DependencyInstallStep{{Type: "steamcmd-app", TargetKey: "server/install-root", PackageManager: "steamcmd", PackageName: "3792580"}}}}, + }) + managed := &recordingManagedSupervisor{} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithDependencyCommandRunner(missingCommandRunner{})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil { + t.Fatalf("run autonomous lifecycle: %v", err) + } + + if !managed.started || len(client.lifecycleReports) != 1 { + t.Fatalf("expected bootstrap to run despite unmatched dependency install plan, managed=%+v reports=%d", managed, len(client.lifecycleReports)) + } +} + +func TestLoadAutonomousLifecyclePlanRejectsMismatchedIdentity(t *testing.T) { + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.PluginID = "game.example" + cfg.ComponentKind = "run" + cfg.ComponentKey = "run-local" + writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{ + SchemaVersion: "1", + ServerInstanceID: cfg.ServerInstanceID, + PluginID: "game.other", + PluginVersion: "1.0.0", + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ComponentKey, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-dist-test", + Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"}, + }) + worker, err := NewWorker(cfg, newFakeWorkerClient()) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + + if err := worker.RunAutonomousLifecycleOnce(context.Background()); err == nil { + t.Fatal("expected mismatched autonomous plan identity to fail") + } +} + +func TestGenericWorkerReportsRunningAndExitObservationsForAllEndpointServers(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "" + cfg.ComponentKind = "" + startedAt := time.Now().UTC().Add(-time.Minute) + managed := &observationManagedSupervisor{items: []ProcessIdentity{ + {Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "running", ObservationSeq: 1, UpdatedAt: startedAt}, + {Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "running", ObservationSeq: 4, UpdatedAt: startedAt}, + {Scope: "scope-other", ServerInstanceID: "server-other", RunEndpointID: "run-other", PID: 303, StartedAt: startedAt, CommandFingerprint: "sha256:other", State: "running", ObservationSeq: 1, UpdatedAt: startedAt}, + }} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed)) + if err != nil { + t.Fatalf("new generic worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register generic worker: %v", err) + } + if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil { + t.Fatalf("report running observations: %v", err) + } + if len(client.lifecycleReports) != 2 || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" || client.lifecycleReports[1].ExecutionResult.ProcessState != "running" { + t.Fatalf("generic worker did not report both matching running observations: %+v", client.lifecycleReports) + } + if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil || len(client.lifecycleReports) != 2 { + t.Fatalf("unchanged observations were not deduplicated: reports=%+v err=%v", client.lifecycleReports, err) + } + managed.setItems([]ProcessIdentity{ + {Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "exited", ExitCode: 1, ExitClassification: "unexpected-exit", ObservationSeq: 2, UpdatedAt: time.Now().UTC()}, + {Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "exited", ExitCode: 0, ExitClassification: "clean-exit", ObservationSeq: 5, UpdatedAt: time.Now().UTC()}, + }) + if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil { + t.Fatalf("report exit observations: %v", err) + } + if len(client.lifecycleReports) != 4 || client.lifecycleReports[2].ExecutionResult.ProcessState != "exited" || client.lifecycleReports[3].ExecutionResult.ProcessState != "exited" { + t.Fatalf("generic worker did not report matching exit observations: %+v", client.lifecycleReports) + } +} + +func writeAutonomousPlanFixture(t *testing.T, root string, serverInstanceID string, profileKey string, plan protocol.RunAutonomousLifecyclePlan) { + t.Helper() + scope, err := NewWorkspaceResolver(root).Scope(serverInstanceID, profileKey) + if err != nil { + t.Fatalf("resolve scope: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o700); err != nil { + t.Fatalf("create actions: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o700); err != nil { + t.Fatalf("create bin: %v", err) + } + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{"version": 1, "action": "start", "mode": "supervised", "executableKey": "bin/game-server"}) + if err := os.WriteFile(filepath.Join(scope, "bin", "game-server"), []byte("plugin-owned executable"), 0o700); err != nil { + t.Fatalf("write executable: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, ".platform"), 0o700); err != nil { + t.Fatalf("create platform dir: %v", err) + } + body, err := json.Marshal(plan) + if err != nil { + t.Fatalf("marshal plan: %v", err) + } + if err := os.WriteFile(filepath.Join(scope, autonomousLifecyclePlanKey), body, 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } +} + +type recordingManagedSupervisor struct { + started bool + command ProcessCommand + identity ProcessIdentity +} + +type observationManagedSupervisor struct { + mu sync.Mutex + items []ProcessIdentity +} + +func (supervisor *observationManagedSupervisor) Start(_ context.Context, _ ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) { + return identity, nil +} + +func (supervisor *observationManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) { + return identity, nil +} + +func (supervisor *observationManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity { + return identity +} + +func (supervisor *observationManagedSupervisor) ResumeOutput(ManagedProcessOutput) {} + +func (supervisor *observationManagedSupervisor) ManagedProcessObservations() []ProcessIdentity { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + return append([]ProcessIdentity(nil), supervisor.items...) +} + +func (supervisor *observationManagedSupervisor) setItems(items []ProcessIdentity) { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + supervisor.items = append([]ProcessIdentity(nil), items...) +} + +func (supervisor *recordingManagedSupervisor) Start(_ context.Context, command ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) { + supervisor.started = true + supervisor.command = command + identity.PID = 42 + identity.State = "running" + supervisor.identity = identity + return identity, nil +} + +func (supervisor *recordingManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) { + identity.State = "stopped" + return identity, nil +} + +func (supervisor *recordingManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity { + return identity +} + +func (supervisor *recordingManagedSupervisor) ResumeOutput(ManagedProcessOutput) {} + +type missingCommandRunner struct{} + +func (missingCommandRunner) Run(context.Context, ProcessCommand) (ProcessResult, error) { + return ProcessResult{ExitCode: 1}, os.ErrNotExist +} diff --git a/runtime/data_targets.go b/runtime/data_targets.go new file mode 100644 index 0000000..077ed7c --- /dev/null +++ b/runtime/data_targets.go @@ -0,0 +1,277 @@ +package runtime + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "browser.local/run/protocol" +) + +const dataTargetSnapshotManifestVersion = 1 + +type dataTargetSnapshotManifest struct { + Version int `json:"version"` + DataTargetKey string `json:"dataTargetKey"` + Kind string `json:"kind"` + WorkspaceKey string `json:"workspaceKey"` + SourceRootKey string `json:"sourceRootKey"` + SourcePathFingerprint string `json:"sourcePathFingerprint"` + SourceSizeBytes int64 `json:"sourceSizeBytes"` + SourceModUnixNano int64 `json:"sourceModUnixNano"` + SnapshotSizeBytes int64 `json:"snapshotSizeBytes"` + SnapshotChecksum string `json:"snapshotChecksum"` + MaterializedAt time.Time `json:"materializedAt"` +} + +type dataTargetMaterializeError struct { + code string + retryable bool +} + +func (err dataTargetMaterializeError) Error() string { return err.code } + +func newDataTargetMaterializeError(code string, retryable bool) error { + return dataTargetMaterializeError{code: code, retryable: retryable} +} + +func sqliteProbeFailureForDataTarget(assignment protocol.RunJobAssignment, err error) LifecycleExecutionResult { + var materializeErr dataTargetMaterializeError + if errors.As(err, &materializeErr) { + return sqliteSchemaProbeFailure(assignment, materializeErr.code, materializeErr.retryable) + } + return sqliteSchemaProbeFailure(assignment, "data_target_unavailable", true) +} + +func (worker *Worker) materializeSQLiteProbeDataTarget(ctx context.Context, assignment protocol.RunJobAssignment) (string, error) { + plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg) + if err != nil { + return assignment.TargetKey, newDataTargetMaterializeError("data_target_plan_invalid", false) + } + if !ok || plan == nil || len(plan.DataTargets) == 0 { + return assignment.TargetKey, nil + } + if plan.ServerInstanceID != assignment.ServerInstanceID || plan.RunEndpointID != assignment.RunEndpointID || plan.PluginID != assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID { + return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false) + } + if plan.ProfileKey != "" && assignment.ExecutionInput.WorkspaceScope != "" && plan.ProfileKey != assignment.ExecutionInput.WorkspaceScope { + return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false) + } + var matched *protocol.RunAutonomousDataTarget + for i := range plan.DataTargets { + target := &plan.DataTargets[i] + if dataTargetMatchesSQLiteProbeAssignment(*target, assignment.TargetKey) { + matched = target + break + } + } + if matched == nil { + return assignment.TargetKey, nil + } + if !dataTargetSupportsPlatform(matched.Platforms, runtime.GOOS) { + return assignment.TargetKey, newDataTargetMaterializeError("data_target_platform_unsupported", false) + } + manifest, err := materializeSQLiteSnapshotDataTarget(ctx, worker.cfg.WorkspaceRoot, assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope, *matched, plan.RuntimeBindings) + if err != nil { + log.Printf("RUN phase=data_target.snapshot status=failed target=%s code=%s", safeOptional(matched.Key), safeOptional(dataTargetErrorCode(err))) + return assignment.TargetKey, err + } + log.Printf("RUN phase=data_target.snapshot status=complete target=%s workspaceKey=%s bytes=%d checksum=%s", safeOptional(matched.Key), safeOptional(matched.WorkspaceKey), manifest.SnapshotSizeBytes, safeOptional(manifest.SnapshotChecksum)) + return matched.WorkspaceKey, nil +} + +func dataTargetMatchesSQLiteProbeAssignment(target protocol.RunAutonomousDataTarget, assignmentTargetKey string) bool { + if target.Kind != "sqlite.snapshot" || assignmentTargetKey == "" { + return false + } + return target.Key == assignmentTargetKey || target.TransportKey == assignmentTargetKey || target.WorkspaceKey == assignmentTargetKey +} + +func materializeSQLiteSnapshotDataTarget(ctx context.Context, workspaceRoot string, serverInstanceID string, profileKey string, target protocol.RunAutonomousDataTarget, bindings map[string]string) (dataTargetSnapshotManifest, error) { + if target.Kind != "sqlite.snapshot" || target.RefreshPolicy != "on-demand-snapshot" || target.MaxBytes <= 0 { + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_invalid", false) + } + sourceRoot := strings.TrimSpace(bindings[target.SourceRootKey]) + if sourceRoot == "" { + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_source_unbound", false) + } + source, sourceInfo, err := resolveDataTargetSource(sourceRoot, target.SourcePath, target.MaxBytes) + if err != nil { + return dataTargetSnapshotManifest{}, err + } + scope, err := NewWorkspaceResolver(workspaceRoot).Scope(serverInstanceID, profileKey) + if err != nil { + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_scope_invalid", false) + } + destination, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, target.WorkspaceKey) + if err != nil { + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_invalid", false) + } + if err := ensureDirectory(filepath.Dir(destination)); err != nil { + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true) + } + temporary := filepath.Join(filepath.Dir(destination), "."+safeWorkspaceName(filepath.Base(target.WorkspaceKey))+".snapshot.tmp") + _ = os.Remove(temporary) + if err := snapshotSQLiteDatabase(ctx, source, temporary, target.MaxBytes); err != nil { + _ = os.Remove(temporary) + return dataTargetSnapshotManifest{}, err + } + snapshotInfo, err := os.Stat(temporary) + if err != nil || !snapshotInfo.Mode().IsRegular() || snapshotInfo.Size() <= 0 || snapshotInfo.Size() > target.MaxBytes { + _ = os.Remove(temporary) + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_snapshot_invalid", true) + } + checksum, err := fingerprintSQLiteSource(temporary) + if err != nil { + _ = os.Remove(temporary) + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_checksum_failed", true) + } + if err := replaceRegularFile(temporary, destination); err != nil { + _ = os.Remove(temporary) + return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true) + } + manifest := dataTargetSnapshotManifest{ + Version: dataTargetSnapshotManifestVersion, + DataTargetKey: target.Key, + Kind: target.Kind, + WorkspaceKey: target.WorkspaceKey, + SourceRootKey: target.SourceRootKey, + SourcePathFingerprint: digestValue(target.SourcePath), + SourceSizeBytes: sourceInfo.Size(), + SourceModUnixNano: sourceInfo.ModTime().UnixNano(), + SnapshotSizeBytes: snapshotInfo.Size(), + SnapshotChecksum: checksum, + MaterializedAt: time.Now().UTC(), + } + if err := writeDataTargetSnapshotManifest(workspaceRoot, scope, target.WorkspaceKey, manifest); err != nil { + return dataTargetSnapshotManifest{}, err + } + return manifest, nil +} + +func resolveDataTargetSource(sourceRoot string, sourcePath string, maxBytes int64) (string, os.FileInfo, error) { + if strings.TrimSpace(sourceRoot) == "" || strings.TrimSpace(sourcePath) == "" || filepath.IsAbs(sourcePath) || strings.Contains(sourcePath, `\`) || strings.Contains(sourcePath, "..") || !protocol.ValidLogicalFileKey(sourcePath) { + return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false) + } + cleanRoot := filepath.Clean(sourceRoot) + if !filepath.IsAbs(cleanRoot) { + return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false) + } + source := filepath.Join(cleanRoot, filepath.FromSlash(sourcePath)) + rel, err := filepath.Rel(cleanRoot, source) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false) + } + info, err := os.Lstat(source) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", nil, newDataTargetMaterializeError("data_target_source_missing", true) + } + return "", nil, newDataTargetMaterializeError("data_target_source_unavailable", true) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false) + } + if info.Size() <= 0 || info.Size() > maxBytes { + return "", nil, newDataTargetMaterializeError("data_target_source_size_invalid", false) + } + return source, info, nil +} + +func snapshotSQLiteDatabase(ctx context.Context, source string, destination string, maxBytes int64) error { + snapshotCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+source+"?mode=ro") + if err != nil { + return newDataTargetMaterializeError("data_target_sqlite_open_failed", true) + } + defer database.Close() + database.SetMaxOpenConns(1) + if _, err := database.ExecContext(snapshotCtx, "VACUUM INTO "+quoteSQLiteStringLiteral(destination)); err != nil { + return dataTargetSQLiteError(snapshotCtx, err) + } + info, err := os.Stat(destination) + if err != nil { + return newDataTargetMaterializeError("data_target_snapshot_unavailable", true) + } + if info.Size() <= 0 || info.Size() > maxBytes { + return newDataTargetMaterializeError("data_target_snapshot_limit_exceeded", false) + } + return nil +} + +func dataTargetSQLiteError(ctx context.Context, err error) error { + if errors.Is(ctx.Err(), context.Canceled) { + return newDataTargetMaterializeError("data_target_cancelled", true) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return newDataTargetMaterializeError("data_target_timeout", true) + } + lower := strings.ToLower(fmt.Sprint(err)) + if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") { + return newDataTargetMaterializeError("data_target_busy", true) + } + return newDataTargetMaterializeError("data_target_snapshot_failed", true) +} + +func quoteSQLiteStringLiteral(value string) string { + return `'` + strings.ReplaceAll(value, `'`, `''`) + `'` +} + +func replaceRegularFile(source string, destination string) error { + if info, err := os.Lstat(destination); err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fmt.Errorf("destination is not a regular file") + } + if err := os.Remove(destination); err != nil { + return err + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Rename(source, destination) +} + +func writeDataTargetSnapshotManifest(workspaceRoot string, scope string, workspaceKey string, manifest dataTargetSnapshotManifest) error { + manifestKey := workspaceKey + ".snapshot.json" + path, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, manifestKey) + if err != nil { + return newDataTargetMaterializeError("data_target_manifest_invalid", false) + } + body, err := json.Marshal(manifest) + if err != nil { + return newDataTargetMaterializeError("data_target_manifest_invalid", false) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + return newDataTargetMaterializeError("data_target_manifest_failed", true) + } + return nil +} + +func dataTargetSupportsPlatform(platforms []string, targetOS string) bool { + if len(platforms) == 0 { + return true + } + for _, platform := range platforms { + if platform == targetOS { + return true + } + } + return false +} + +func dataTargetErrorCode(err error) string { + var materializeErr dataTargetMaterializeError + if errors.As(err, &materializeErr) { + return materializeErr.code + } + return "data_target_unavailable" +} diff --git a/runtime/data_targets_test.go b/runtime/data_targets_test.go new file mode 100644 index 0000000..e67f09f --- /dev/null +++ b/runtime/data_targets_test.go @@ -0,0 +1,102 @@ +package runtime + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "browser.local/run/config" + "browser.local/run/protocol" +) + +func TestMaterializeSQLiteSnapshotDataTargetCreatesScopedProbeTarget(t *testing.T) { + workspaceRoot := t.TempDir() + sourceRoot := t.TempDir() + createSQLiteProbeFixture(t, filepath.Join(sourceRoot, "Saved", "SaveFiles", "current.db")) + target := protocol.RunAutonomousDataTarget{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024} + + manifest, err := materializeSQLiteSnapshotDataTarget(context.Background(), workspaceRoot, "server-data", "run-local", target, map[string]string{"server-root": sourceRoot}) + if err != nil { + t.Fatalf("materialize data target: %v", err) + } + if manifest.SnapshotChecksum == "" || manifest.SnapshotSizeBytes <= 0 || manifest.SourcePathFingerprint == "" { + t.Fatalf("expected bounded snapshot metadata, got %+v", manifest) + } + scope, err := NewWorkspaceResolver(workspaceRoot).Scope("server-data", "run-local") + if err != nil { + t.Fatalf("resolve scope: %v", err) + } + manifestBody, err := os.ReadFile(filepath.Join(scope, "databases", "current-db.snapshot.json")) + if err != nil { + t.Fatalf("read snapshot manifest: %v", err) + } + if strings.Contains(string(manifestBody), sourceRoot) || strings.Contains(string(manifestBody), "Saved/SaveFiles/current.db") { + t.Fatalf("snapshot manifest leaked host/source material: %s", manifestBody) + } + + assignment := sqliteSchemaProbeAssignment() + assignment.ServerInstanceID = "server-data" + assignment.ExecutionInput.WorkspaceScope = "run-local" + assignment.TargetKey = "databases/current-db" + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.ServerInstanceID = assignment.ServerInstanceID + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.RunEndpointID = assignment.RunEndpointID + result := NewSQLiteSchemaProbeExecutor(workspaceRoot).Execute(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil || result.ExecutionResult.SQLiteSchemaProbe.Status != "succeeded" { + t.Fatalf("expected probe to read materialized snapshot, got %+v", result) + } +} + +func TestWorkerMaterializesMatchingDataTargetBeforeSQLiteProbe(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-data" + cfg.PluginID = "game.example" + cfg.ComponentKind = config.PackageComponentRun + cfg.ComponentKey = "run-local" + sourceRoot := t.TempDir() + createSQLiteProbeFixture(t, filepath.Join(sourceRoot, "Saved", "SaveFiles", "current.db")) + writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{ + SchemaVersion: "1", + ServerInstanceID: cfg.ServerInstanceID, + PluginID: cfg.PluginID, + PluginVersion: "1.0.0", + RunEndpointID: cfg.RunEndpointID, + ProfileKey: cfg.ComponentKey, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-dist-test", + Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"}, + DataTargets: []protocol.RunAutonomousDataTarget{{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024, Platforms: []string{runtime.GOOS}}}, + RuntimeBindings: map[string]string{"server-root": sourceRoot}, + }) + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + assignment := sqliteSchemaProbeAssignment() + assignment.ServerInstanceID = cfg.ServerInstanceID + assignment.RunEndpointID = cfg.RunEndpointID + assignment.TargetKey = "current-db" + assignment.ExecutionInput.WorkspaceScope = cfg.ComponentKey + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.ServerInstanceID = cfg.ServerInstanceID + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.RunEndpointID = cfg.RunEndpointID + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID = cfg.PluginID + + result := worker.executeAssignment(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil || result.ExecutionResult.SQLiteSchemaProbe.Status != "succeeded" { + serialized, _ := json.Marshal(result) + t.Fatalf("expected worker probe to materialize and inspect data target, got %s", serialized) + } +} + +func TestMaterializeSQLiteSnapshotDataTargetRejectsUnboundSourceRoot(t *testing.T) { + target := protocol.RunAutonomousDataTarget{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024} + _, err := materializeSQLiteSnapshotDataTarget(context.Background(), t.TempDir(), "server-data", "run-local", target, map[string]string{}) + if err == nil || dataTargetErrorCode(err) != "data_target_source_unbound" { + t.Fatalf("expected unbound source root rejection, got %v", err) + } +} diff --git a/runtime/dependencies.go b/runtime/dependencies.go new file mode 100644 index 0000000..ddd6cb2 --- /dev/null +++ b/runtime/dependencies.go @@ -0,0 +1,563 @@ +package runtime + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "browser.local/run/protocol" +) + +const ( + maxDependencyDownloadBytes = int64(512 * 1024 * 1024) + dependencyCommandTimeout = 10 * time.Minute +) + +var ( + dependencyTokenPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+@/-]{0,119}$`) + dependencyVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+~-]{0,79}$`) + steamAppPattern = regexp.MustCompile(`^[0-9]{1,12}$`) +) + +type DependencyDownloader interface { + Download(context.Context, string, string, int64) (int64, string, error) +} + +type HTTPDependencyDownloader struct { + Client *http.Client +} + +func (downloader HTTPDependencyDownloader) Download(ctx context.Context, sourceURL, destination string, maxBytes int64) (int64, string, error) { + parsed, err := validateDependencyDownloadURL(sourceURL) + if err != nil { + return 0, "", err + } + client := downloader.Client + if client == nil { + client = &http.Client{Timeout: 10 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return fmt.Errorf("dependency download redirect limit exceeded") + } + _, err := validateDependencyDownloadURL(request.URL.String()) + return err + }} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + return 0, "", err + } + response, err := client.Do(request) + if err != nil { + return 0, "", err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return 0, "", fmt.Errorf("dependency download returned status %d", response.StatusCode) + } + if response.ContentLength > maxBytes { + return 0, "", fmt.Errorf("dependency download exceeds size limit") + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return 0, "", err + } + temporary := destination + ".partial" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return 0, "", err + } + remove := true + defer func() { + _ = file.Close() + if remove { + _ = os.Remove(temporary) + } + }() + hash := sha256.New() + written, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(response.Body, maxBytes+1)) + if err != nil { + return 0, "", err + } + if written > maxBytes { + return 0, "", fmt.Errorf("dependency download exceeds size limit") + } + if err := file.Sync(); err != nil { + return 0, "", err + } + if err := file.Close(); err != nil { + return 0, "", err + } + if err := os.Rename(temporary, destination); err != nil { + return 0, "", err + } + remove = false + return written, "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil +} + +type dependencyJournal struct { + Version int `json:"version"` + JobID string `json:"jobId"` + Attempt int `json:"attempt"` + PlanDigest string `json:"planDigest"` + CompletedSteps []int `json:"completedSteps,omitempty"` + State string `json:"state"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (worker *Worker) executeDependencyJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_dependency_job", err.Error()) + } + runState, err := worker.registeredState() + if err != nil { + return lifecycleFailure("dependency_unregistered", "Run worker is not registered") + } + input, err := worker.client.GetDependencyExecutionInput(ctx, protocol.DependencyExecutionInputRequest{RunEndpointID: runState.RunEndpointID, SessionToken: runState.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt}) + if err != nil { + return lifecycleFailure("dependency_input_failed", "could not load fenced dependency input") + } + if err := validateDependencyInput(assignment, input); err != nil { + return lifecycleFailure("unsafe_dependency_input", err.Error()) + } + journalPath := filepath.Join(worker.cfg.WorkspaceRoot, "dependency-journals", safeWorkspaceName(assignment.JobID)+".json") + journal, err := loadDependencyJournal(journalPath, assignment, input.PlanDigest) + if err != nil { + return lifecycleFailure("dependency_journal_failed", err.Error()) + } + + if assignment.Capability == protocol.RunCapabilityDependenciesCheck { + state, evidence, probeErr := worker.executor.runDependencyProbe(ctx, input.Probe, input.Bindings) + if probeErr != nil { + if errors.Is(probeErr, context.Canceled) || errors.Is(probeErr, context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "dependency probe cancelled"}, Message: "dependency probe cancelled", ErrorCode: "dependency_probe_cancelled"} + } + return lifecycleFailure("dependency_probe_failed", probeErr.Error()) + } + journal.State = state + journal.UpdatedAt = time.Now().UTC() + if err := persistDependencyJournal(journalPath, journal); err != nil { + return lifecycleFailure("dependency_journal_failed", err.Error()) + } + return dependencySuccess(assignment, input, state, evidence, 0) + } + + completed := map[int]bool{} + for _, index := range journal.CompletedSteps { + completed[index] = true + } + for index, step := range input.Plan.Steps { + if completed[index] { + continue + } + if cancelled, ok := checkContextCancelled(ctx, "dependency installation cancelled", "dependency_install_cancelled"); ok { + return cancelled + } + if err := worker.executor.runDependencyInstallStep(ctx, assignment, input, step, index); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "dependency installation cancelled"}, Message: "dependency installation cancelled", ErrorCode: "dependency_install_cancelled"} + } + return lifecycleFailure("dependency_install_failed", err.Error()) + } + journal.CompletedSteps = append(journal.CompletedSteps, index) + journal.State = "installing" + journal.UpdatedAt = time.Now().UTC() + if err := persistDependencyJournal(journalPath, journal); err != nil { + return lifecycleFailure("dependency_journal_failed", err.Error()) + } + } + state, evidence, probeErr := worker.executor.runDependencyProbe(ctx, input.Probe, input.Bindings) + if probeErr != nil { + return lifecycleFailure("dependency_verify_failed", probeErr.Error()) + } + if state != "present" { + return lifecycleFailure("dependency_verify_missing", "dependency remains missing after install plan") + } + journal.State = "present" + journal.UpdatedAt = time.Now().UTC() + if err := persistDependencyJournal(journalPath, journal); err != nil { + return lifecycleFailure("dependency_journal_failed", err.Error()) + } + return dependencySuccess(assignment, input, "present", evidence, len(journal.CompletedSteps)) +} + +func validateDependencyInput(assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse) error { + if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID { + return fmt.Errorf("dependency input scope does not match job") + } + if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH { + return fmt.Errorf("dependency input target does not match Run") + } + if !validSHA256(input.PlanDigest) || !protocol.ValidLogicalFileKey(input.ProfileKey) { + return fmt.Errorf("dependency input digest or profile is unsafe") + } + if !protocol.ValidLogicalFileKey(input.Probe.Key) || !protocol.ValidLogicalFileKey(input.Probe.TargetKey) { + return fmt.Errorf("dependency probe is unsafe") + } + if assignment.Capability == protocol.RunCapabilityDependenciesCheck { + if assignment.TargetKey != "dependencies/"+input.Probe.Key || input.Plan.Key != "" { + return fmt.Errorf("dependency check declaration does not match job") + } + return nil + } + if assignment.Capability != protocol.RunCapabilityDependenciesInstall || assignment.TargetKey != "dependencies/install/"+input.Plan.Key || !protocol.ValidLogicalFileKey(input.Plan.Key) || len(input.Plan.Steps) == 0 || len(input.Plan.Steps) > 64 { + return fmt.Errorf("dependency install declaration does not match job") + } + return nil +} + +func (executor LifecycleExecutor) runDependencyProbe(ctx context.Context, probe protocol.DependencyProbe, bindings map[string]string) (string, string, error) { + target := strings.TrimSpace(bindings[probe.TargetKey]) + if target == "" { + target = probe.TargetKey + } + switch probe.Kind { + case "file.exists", "steam.app": + info, err := os.Lstat(target) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "missing", "declared target is not present", nil + } + return "", "", err + } + if info.Mode()&os.ModeSymlink != 0 { + return "", "", fmt.Errorf("dependency target cannot be a symlink") + } + return "present", "declared target is present", nil + case "package.installed": + if err := validateDependencyExecutable(target); err != nil { + return "", "", err + } + if filepath.IsAbs(target) { + if _, err := os.Stat(target); err != nil { + return "missing", "declared package executable is not present", nil + } + } else if _, err := exec.LookPath(target); err != nil { + return "missing", "declared package executable is not present", nil + } + return "present", "declared package executable is present", nil + case "command.version", "java.version", "docker.available": + if err := validateDependencyExecutable(target); err != nil { + return "", "", err + } + args := []string{"--version"} + if probe.Kind == "java.version" { + args = []string{"-version"} + } + result, err := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{target}, args...), Timeout: 30 * time.Second, Capability: "dependency.probe", Action: probe.Kind}) + if err != nil || result.ExitCode != 0 { + if ctx.Err() != nil { + return "", "", ctx.Err() + } + return "missing", "declared executable is not available", nil + } + version := dependencyVersionEvidence(result.Stdout + "\n" + result.Stderr) + if probe.MinimumVersion != "" && !dependencyVersionAtLeast(version, probe.MinimumVersion) { + return "missing", "declared executable version is below minimum", nil + } + return "present", version, nil + case "service.exists": + if !dependencyTokenPattern.MatchString(target) { + return "", "", fmt.Errorf("dependency service target is unsafe") + } + name, args := serviceProbeCommand(runtime.GOOS, target) + if name == "" { + return "", "", fmt.Errorf("service probe is unsupported on this platform") + } + result, err := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{name}, args...), Timeout: 30 * time.Second, Capability: "dependency.probe", Action: probe.Kind}) + if err != nil || result.ExitCode != 0 { + if ctx.Err() != nil { + return "", "", ctx.Err() + } + return "missing", "declared service is not present", nil + } + return "present", "declared service is present", nil + default: + return "", "", fmt.Errorf("dependency probe kind is unsupported") + } +} + +func (executor LifecycleExecutor) runDependencyInstallStep(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse, step protocol.DependencyInstallStep, index int) error { + if !protocol.ValidLogicalFileKey(step.TargetKey) { + return fmt.Errorf("dependency install target is unsafe") + } + switch step.Type { + case "package": + name, args, err := packageInstallCommand(step.PackageManager, step.PackageName, step.Version) + if err != nil { + return err + } + result, runErr := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{name}, args...), Timeout: dependencyCommandTimeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: "dependency.package"}) + if runErr != nil || result.ExitCode != 0 { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("typed package adapter failed") + } + return nil + case "steamcmd-app": + if !steamAppPattern.MatchString(step.PackageName) { + return fmt.Errorf("Steam app identifier is unsafe") + } + executable := strings.TrimSpace(input.Bindings[step.TargetKey]) + if executable == "" { + executable = "steamcmd" + } + if err := validateDependencyExecutable(executable); err != nil { + return err + } + result, runErr := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: []string{executable, "+login", "anonymous", "+app_update", step.PackageName, "validate", "+quit"}, Timeout: dependencyCommandTimeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: "dependency.steamcmd-app"}) + if runErr != nil || result.ExitCode != 0 { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("typed SteamCMD adapter failed") + } + return nil + case "verified-download": + if !validSHA256(step.Checksum) { + return fmt.Errorf("verified download checksum is required") + } + if _, err := validateDependencyDownloadURL(step.DownloadRef); err != nil { + return err + } + destination := filepath.Join(executor.workspaceRoot, "dependency-files", safeWorkspaceName(assignment.ServerInstanceID), safeWorkspaceName(step.TargetKey)) + size, checksum, err := executor.dependencyDownloader.Download(ctx, step.DownloadRef, destination, maxDependencyDownloadBytes) + if err != nil { + return err + } + if size <= 0 || checksum != strings.ToLower(step.Checksum) { + _ = os.Remove(destination) + return fmt.Errorf("verified dependency download checksum mismatch") + } + return os.Chmod(destination, 0o700) + case "manual": + return fmt.Errorf("manual dependency step requires operator action") + default: + return fmt.Errorf("dependency install step type is unsupported") + } +} + +func packageInstallCommand(manager, packageName, version string) (string, []string, error) { + if !dependencyTokenPattern.MatchString(packageName) || version != "" && !dependencyVersionPattern.MatchString(version) { + return "", nil, fmt.Errorf("package name or version is unsafe") + } + spec := packageName + switch manager { + case "apt": + if version != "" { + spec += "=" + version + } + return "apt-get", []string{"install", "-y", "--no-install-recommends", spec}, nil + case "yum", "dnf": + if version != "" { + spec += "-" + version + } + return manager, []string{"install", "-y", spec}, nil + case "pacman": + return "pacman", []string{"-S", "--noconfirm", spec}, nil + case "zypper": + return "zypper", []string{"--non-interactive", "install", spec}, nil + case "brew": + if version != "" { + spec += "@" + version + } + return "brew", []string{"install", spec}, nil + case "winget": + args := []string{"install", "--id", packageName, "--exact", "--silent", "--accept-package-agreements", "--accept-source-agreements"} + if version != "" { + args = append(args, "--version", version) + } + return "winget", args, nil + case "choco": + args := []string{"install", packageName, "-y", "--no-progress"} + if version != "" { + args = append(args, "--version", version) + } + return "choco", args, nil + case "scoop": + if version != "" { + spec += "@" + version + } + return "scoop", []string{"install", spec}, nil + default: + return "", nil, fmt.Errorf("package manager is unsupported") + } +} + +func validateDependencyExecutable(target string) error { + if strings.TrimSpace(target) != target || target == "" || strings.ContainsAny(target, "\r\n\x00") || containsUnsafeRuntimeText(target) { + return fmt.Errorf("dependency executable target is unsafe") + } + if filepath.IsAbs(target) { + info, err := os.Lstat(target) + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Mode()&0o111 == 0 { + return fmt.Errorf("dependency executable target is not a regular executable") + } + return nil + } + if !commandNamePattern.MatchString(target) { + return fmt.Errorf("dependency executable name is unsafe") + } + if _, forbidden := disallowedExecutables[strings.ToLower(target)]; forbidden { + return fmt.Errorf("dependency executable cannot be a shell") + } + return nil +} + +func validateDependencyDownloadURL(raw string) (*url.URL, error) { + parsed, err := url.ParseRequestURI(strings.TrimSpace(raw)) + if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { + return nil, fmt.Errorf("dependency download URL is not approved") + } + host := strings.ToLower(parsed.Hostname()) + if host == "localhost" || strings.HasSuffix(host, ".localhost") { + return nil, fmt.Errorf("dependency download host is not approved") + } + if ip := net.ParseIP(host); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsLinkLocalUnicast()) { + return nil, fmt.Errorf("dependency download host is not approved") + } + return parsed, nil +} + +func serviceProbeCommand(targetOS, service string) (string, []string) { + switch targetOS { + case "linux": + return "systemctl", []string{"status", service, "--no-pager"} + case "windows": + return "sc", []string{"query", service} + case "darwin": + return "launchctl", []string{"print", "system/" + service} + default: + return "", nil + } +} + +func dependencySuccess(assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse, state, evidence string, completed int) LifecycleExecutionResult { + evidence = RedactText(strings.TrimSpace(evidence)) + payload, _ := json.Marshal(protocol.DependencyExecutionEvidence{ProbeKey: input.Probe.Key, PlanKey: input.Plan.Key, PlanDigest: input.PlanDigest, State: state, Evidence: evidence, CompletedSteps: completed}) + kind := "dependency.check" + message := "dependency probe completed" + if assignment.Capability == protocol.RunCapabilityDependenciesInstall { + kind = "dependency.install" + message = "dependency install plan completed and verified" + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, ResultRef: fmt.Sprintf("artifact://jobs/%s/dependencies-result", url.PathEscape(assignment.JobID)), Message: message, ExecutionResult: protocol.RunJobExecutionResult{Kind: kind, Checksum: input.PlanDigest, Summary: message, Content: string(payload)}} +} + +func loadDependencyJournal(path string, assignment protocol.RunJobAssignment, digest string) (dependencyJournal, error) { + journal := dependencyJournal{Version: 1, JobID: assignment.JobID, Attempt: assignment.Attempt, PlanDigest: digest, State: "pending", UpdatedAt: time.Now().UTC()} + body, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return journal, nil + } + if err != nil { + return dependencyJournal{}, err + } + if err := json.Unmarshal(body, &journal); err != nil { + return dependencyJournal{}, fmt.Errorf("decode dependency journal: %w", err) + } + if journal.Version != 1 || journal.JobID != assignment.JobID || journal.PlanDigest != digest { + return dependencyJournal{}, fmt.Errorf("dependency journal does not match immutable plan") + } + if journal.Attempt > assignment.Attempt { + return dependencyJournal{}, fmt.Errorf("dependency journal attempt is newer than assignment") + } + journal.Attempt = assignment.Attempt + return journal, nil +} + +func persistDependencyJournal(path string, journal dependencyJournal) error { + body, err := json.MarshalIndent(journal, "", " ") + if err != nil { + return err + } + return writeRuntimeAtomicFile(path, body, 0o600) +} + +func writeRuntimeAtomicFile(path string, body []byte, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + temporary := path + ".tmp" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := file.Write(body); err != nil { + _ = file.Close() + _ = os.Remove(temporary) + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + _ = os.Remove(temporary) + return err + } + if err := file.Close(); err != nil { + _ = os.Remove(temporary) + return err + } + if err := os.Rename(temporary, path); err != nil { + _ = os.Remove(temporary) + return err + } + return os.Chmod(path, mode) +} + +func validSHA256(value string) bool { + if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func dependencyVersionEvidence(output string) string { + lines := splitBoundedLines(output) + if len(lines) == 0 { + return "version available" + } + return strings.TrimSpace(lines[0]) +} + +func dependencyVersionAtLeast(actual, minimum string) bool { + numbers := func(value string) []int { + parts := regexp.MustCompile(`[0-9]+`).FindAllString(value, -1) + out := make([]int, len(parts)) + for i, part := range parts { + out[i], _ = strconv.Atoi(part) + } + return out + } + a, b := numbers(actual), numbers(minimum) + for i := 0; i < len(a) || i < len(b); i++ { + av, bv := 0, 0 + if i < len(a) { + av = a[i] + } + if i < len(b) { + bv = b[i] + } + if av != bv { + return av > bv + } + } + return len(a) > 0 +} diff --git a/runtime/dependencies_test.go b/runtime/dependencies_test.go new file mode 100644 index 0000000..55cbcf3 --- /dev/null +++ b/runtime/dependencies_test.go @@ -0,0 +1,186 @@ +package runtime + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "browser.local/run/protocol" +) + +const dependencyTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type dependencyTestSupervisor struct { + mu sync.Mutex + calls [][]string + present bool + block bool +} + +func (supervisor *dependencyTestSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) { + supervisor.mu.Lock() + supervisor.calls = append(supervisor.calls, append([]string(nil), command.Args...)) + block := supervisor.block + present := supervisor.present + if len(command.Args) > 0 && command.Args[0] == "apt-get" { + supervisor.present = true + present = true + } + supervisor.mu.Unlock() + if block { + <-ctx.Done() + return ProcessResult{ExitCode: -1}, ctx.Err() + } + if len(command.Args) > 0 && command.Args[0] == "java" { + if !present { + return ProcessResult{ExitCode: 1}, errors.New("not installed") + } + return ProcessResult{ExitCode: 0, Stderr: "openjdk version 21.0.2"}, nil + } + return ProcessResult{ExitCode: 0}, nil +} + +func (supervisor *dependencyTestSupervisor) count(name string) int { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + count := 0 + for _, call := range supervisor.calls { + if len(call) > 0 && call[0] == name { + count++ + } + } + return count +} + +type dependencyTestDownloader struct { + payload []byte + sourceURL string + destination string +} + +func (downloader *dependencyTestDownloader) Download(_ context.Context, sourceURL, destination string, _ int64) (int64, string, error) { + downloader.sourceURL = sourceURL + downloader.destination = destination + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return 0, "", err + } + if err := os.WriteFile(destination, downloader.payload, 0o600); err != nil { + return 0, "", err + } + return int64(len(downloader.payload)), bytesChecksum(downloader.payload), nil +} + +func TestDependencyInstallExecutesTypedPlanAndResumesCompletedSteps(t *testing.T) { + if !dependencyVersionAtLeast("openjdk version 21.0.2", "21") { + t.Fatal("version comparator rejected valid Java version") + } + client := newFakeWorkerClient() + assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall) + client.dependencyInput = dependencyInputForAssignment(assignment) + runner := &dependencyTestSupervisor{} + worker, err := NewWorker(workerTestConfig(t), client, WithDependencyCommandRunner(runner)) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + + first := worker.executeDependencyJob(context.Background(), assignment) + if first.State != lifecycleResultStateSucceeded || first.ExecutionResult.Kind != "dependency.install" || first.ExecutionResult.Checksum != dependencyTestDigest { + state, evidence, probeErr := worker.executor.runDependencyProbe(context.Background(), client.dependencyInput.Probe, client.dependencyInput.Bindings) + t.Fatalf("expected real typed dependency install, got %+v calls=%+v present=%v probe=%s evidence=%s err=%v", first, runner.calls, runner.present, state, evidence, probeErr) + } + second := worker.executeDependencyJob(context.Background(), assignment) + if second.State != lifecycleResultStateSucceeded { + t.Fatalf("expected journal resume success, got %+v", second) + } + if runner.count("apt-get") != 1 { + t.Fatalf("completed package step must not repeat, calls=%+v", runner.calls) + } + if !strings.Contains(first.ExecutionResult.Content, `"completedSteps":1`) || strings.Contains(first.ExecutionResult.Content, "/Users/") { + t.Fatalf("dependency evidence is not safe: %s", first.ExecutionResult.Content) + } +} + +func TestDependencyProbeAndInstallRejectUnsafeOrCancelledWork(t *testing.T) { + client := newFakeWorkerClient() + assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall) + input := dependencyInputForAssignment(assignment) + input.Plan.Steps[0].PackageName = "openjdk;rm" + client.dependencyInput = input + worker, err := NewWorker(workerTestConfig(t), client, WithDependencyCommandRunner(&dependencyTestSupervisor{})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + unsafe := worker.executeDependencyJob(context.Background(), assignment) + if unsafe.State != lifecycleResultStateFailed || unsafe.ErrorCode != "dependency_install_failed" { + t.Fatalf("expected unsafe package rejection, got %+v", unsafe) + } + + check := dependencyAssignment(protocol.RunCapabilityDependenciesCheck) + checkInput := dependencyInputForAssignment(check) + checkInput.Plan = protocol.DependencyInstallPlan{} + client.dependencyInput = checkInput + blocking := &dependencyTestSupervisor{block: true} + worker.executor.dependencyRunner = blocking + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + cancelled := worker.executeDependencyJob(ctx, check) + if cancelled.State != lifecycleResultStateCancelled || cancelled.ErrorCode != "dependency_probe_cancelled" { + t.Fatalf("expected cancelled dependency probe, got %+v", cancelled) + } +} + +func TestVerifiedDependencyDownloadUsesHTTPSChecksumAndScopedDestination(t *testing.T) { + payload := []byte("verified dependency") + downloader := &dependencyTestDownloader{payload: payload} + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithDependencyDownloader(downloader)) + assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall) + input := dependencyInputForAssignment(assignment) + step := protocol.DependencyInstallStep{Type: "verified-download", TargetKey: "tools/java", DownloadRef: "https://downloads.example.test/java", Checksum: bytesChecksum(payload)} + if err := executor.runDependencyInstallStep(context.Background(), assignment, input, step, 0); err != nil { + t.Fatalf("verified download: %v", err) + } + if downloader.sourceURL != step.DownloadRef || !strings.Contains(downloader.destination, "dependency-files") || strings.Contains(downloader.destination, "..") { + t.Fatalf("unexpected scoped download: source=%s destination=%s", downloader.sourceURL, downloader.destination) + } + if _, err := validateDependencyDownloadURL("https://127.0.0.1/tool"); err == nil { + t.Fatal("expected private dependency download host rejection") + } +} + +func dependencyAssignment(capability string) protocol.RunJobAssignment { + assignment := workerJobAssignment(capability) + assignment.LeaseToken = "lease-dependency" + assignment.Attempt = 1 + assignment.State = "running" + if capability == protocol.RunCapabilityDependenciesInstall { + assignment.TargetKey = "dependencies/install/install-java" + } else { + assignment.TargetKey = "dependencies/java-runtime" + } + return assignment +} + +func dependencyInputForAssignment(assignment protocol.RunJobAssignment) protocol.DependencyExecutionInputResponse { + return protocol.DependencyExecutionInputResponse{ + JobID: assignment.JobID, + ServerInstanceID: assignment.ServerInstanceID, + RunEndpointID: assignment.RunEndpointID, + PluginID: "game.minecraft", + PluginVersion: "1.0.0", + ProfileKey: "local", + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + PlanDigest: dependencyTestDigest, + Probe: protocol.DependencyProbe{Key: "java-runtime", Kind: "java.version", TargetKey: "java", MinimumVersion: "21", Platforms: []string{runtime.GOOS}}, + Plan: protocol.DependencyInstallPlan{Key: "install-java", Title: "Install Java", Platforms: []string{runtime.GOOS}, Steps: []protocol.DependencyInstallStep{{Type: "package", TargetKey: "java", PackageManager: "apt", PackageName: "openjdk-21-jre"}}}, + Bindings: map[string]string{"java": "java"}, + } +} diff --git a/runtime/distribution_build.go b/runtime/distribution_build.go new file mode 100644 index 0000000..3b17678 --- /dev/null +++ b/runtime/distribution_build.go @@ -0,0 +1,659 @@ +package runtime + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "io/fs" + "log" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "browser.local/run/protocol" +) + +const distributionArtifactChunkSize = 1024 * 1024 + +type packageConfigPayload struct { + Kind string `json:"kind"` + ServerInstanceID string `json:"serverInstanceId"` + PluginID string `json:"pluginId"` + RunEndpointID string `json:"runEndpointId,omitempty"` + ProfileKey string `json:"profileKey,omitempty"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + SecretRef string `json:"secretRef"` + KeyGeneration int `json:"keyGeneration"` + AuthKey string `json:"authKey"` +} + +func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_distribution_build", err.Error()) + } + state, err := worker.registeredState() + if err != nil { + return distributionBuildFailure("build_unregistered", "Run worker is not registered") + } + input, err := worker.client.GetDistributionBuildInput(ctx, protocol.DistributionBuildInputRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + }) + if err != nil { + return distributionBuildFailure("build_input_failed", "could not load authenticated build input") + } + if err := validateDistributionBuildInput(assignment, input); err != nil { + return distributionBuildFailure("unsafe_build_input", err.Error()) + } + + report := func(percent int, message string) error { + state, err := worker.registeredState() + if err != nil { + return err + } + response, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + Progress: protocol.RunJobProgressReport{Percent: percent, Message: message}, + Sequence: worker.nextProgressSequence(assignment.ProgressSequence), + }) + if err != nil { + return err + } + if !response.Accepted { + return fmt.Errorf("distribution build progress was not accepted") + } + assignment = response.Job + return worker.journal.Store(assignment) + } + + workspace := distributionBuildWorkspace(worker.cfg.WorkspaceRoot, input.PluginID, assignment.JobID) + if err := os.RemoveAll(workspace); err != nil { + return distributionBuildFailure("workspace_prepare_failed", "could not reset isolated build workspace") + } + if err := os.MkdirAll(workspace, 0o700); err != nil { + return distributionBuildFailure("workspace_prepare_failed", "could not create isolated build workspace") + } + defer os.RemoveAll(workspace) + + if err := report(12, "git_sync: preparing approved source"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report source preparation") + } + sourceRoot, err := worker.prepareDistributionSource(ctx, workspace, input) + if err != nil { + return distributionBuildFailure("git_sync_failed", "approved source checkout failed") + } + + if err := report(28, "env_check: validating Go build environment"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report environment check") + } + if err := fixedCommand(ctx, sourceRoot, nil, "go", "version"); err != nil { + return distributionBuildFailure("env_check_failed", "Go build environment is unavailable") + } + + configPath := "" + if input.ComponentKind == "client-manager" { + var err error + configPath, err = writeDistributionConfig(sourceRoot, input, worker.clientPlatformURL()) + if err != nil { + return distributionBuildFailure("config_injection_failed", "could not inject scoped component configuration") + } + } + + if err := report(45, "deps_download: downloading Go modules"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report dependency download") + } + buildEnv := []string{"GOOS=" + input.TargetOS, "GOARCH=" + input.TargetArch, "CGO_ENABLED=0"} + if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "mod", "download"); err != nil { + return distributionBuildFailure("deps_download_failed", "Go module download failed") + } + + if err := report(65, "build_compile: compiling target executable"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report compilation") + } + binaryPath := filepath.Join(workspace, input.OutputFilename) + entry := "." + if input.ComponentKind == "run" { + entry = "./cmd/run" + } + ldflags := "-s -w" + if input.ComponentKind == "run" { + ldflags = buildRunLDFlags(input, distributionBuildPlatformURL(worker, input)) + } + if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "build", "-trimpath", "-ldflags", ldflags, "-o", binaryPath, entry); err != nil { + return distributionBuildFailure("build_compile_failed", "Go compilation failed") + } + + if err := report(82, "package_finalize: creating distribution archive"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report packaging") + } + if input.ComponentKind == "run" { + payload, err := os.ReadFile(binaryPath) + if err != nil { + return distributionBuildFailure("package_finalize_failed", "run executable could not be read") + } + if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil { + return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed") + } + if err := report(96, "package_finalize: artifact upload completed"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report artifact upload") + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"}, + ResultRef: "artifact://" + input.ArtifactID, + Message: "distribution build completed", + } + } + archivePath := filepath.Join(workspace, archiveFilename(input)) + if err := createDistributionArchive(archivePath, input.PackageFormat, binaryPath, configPath); err != nil { + return distributionBuildFailure("package_finalize_failed", "distribution archive creation failed") + } + payload, err := os.ReadFile(archivePath) + if err != nil { + return distributionBuildFailure("package_finalize_failed", "distribution archive could not be read") + } + if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil { + return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed") + } + if err := report(96, "package_finalize: artifact upload completed"); err != nil { + return distributionBuildFailure("progress_report_failed", "could not report artifact upload") + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"}, + ResultRef: "artifact://" + input.ArtifactID, + Message: "distribution build completed", + } +} + +func (worker *Worker) prepareDistributionSource(ctx context.Context, workspace string, input protocol.DistributionBuildInputResponse) (string, error) { + if input.ComponentKind == "run" { + root, err := filepath.Abs(worker.cfg.BuildSourceRoot) + if err != nil { + return "", err + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil { + return "", err + } + isolatedSource := filepath.Join(workspace, "source") + if err := copyDistributionSource(root, isolatedSource, workspace); err != nil { + return "", err + } + if err := writeRunWorkspaceSeedConfig(isolatedSource, input.WorkspaceSeed); err != nil { + return "", err + } + return isolatedSource, nil + } + checkout := filepath.Join(workspace, "source") + if err := os.MkdirAll(checkout, 0o700); err != nil { + return "", err + } + if err := fixedCommand(ctx, checkout, nil, "git", "init", "--quiet"); err != nil { + return "", err + } + if err := fixedCommand(ctx, checkout, nil, "git", "remote", "add", "origin", input.RepositoryURL); err != nil { + return "", err + } + if err := fixedCommand(ctx, checkout, nil, "git", "fetch", "--quiet", "--depth", "1", "origin", input.SourceRevision); err != nil { + return "", err + } + if err := fixedCommand(ctx, checkout, nil, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD"); err != nil { + return "", err + } + return checkout, nil +} + +func writeRunWorkspaceSeedConfig(sourceRoot string, encodedSeed string) error { + encodedSeed = strings.TrimSpace(encodedSeed) + if encodedSeed == "" { + return nil + } + if _, err := base64.StdEncoding.DecodeString(encodedSeed); err != nil { + return fmt.Errorf("workspace seed is invalid") + } + configDir := filepath.Join(sourceRoot, "config") + if err := os.MkdirAll(configDir, 0o700); err != nil { + return err + } + body := fmt.Sprintf("package config\n\nfunc init() { BuildWorkspaceSeed = %q }\n", encodedSeed) + return os.WriteFile(filepath.Join(configDir, "workspace_seed_generated.go"), []byte(body), 0o600) +} + +func copyDistributionSource(sourceRoot string, destinationRoot string, workspace string) error { + workspace, err := filepath.Abs(workspace) + if err != nil { + return err + } + return filepath.WalkDir(sourceRoot, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == workspace || strings.HasPrefix(path, workspace+string(filepath.Separator)) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + relative, err := filepath.Rel(sourceRoot, path) + if err != nil { + return err + } + if relative == "." { + return os.MkdirAll(destinationRoot, 0o700) + } + if entry.Name() == ".git" && entry.IsDir() { + return filepath.SkipDir + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("trusted build source contains a symbolic link") + } + destination := filepath.Join(destinationRoot, relative) + if entry.IsDir() { + return os.MkdirAll(destination, 0o700) + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + input, err := os.Open(path) + if err != nil { + return err + } + output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + input.Close() + return err + } + _, copyErr := io.Copy(output, input) + inputCloseErr := input.Close() + closeErr := output.Close() + if copyErr != nil { + return copyErr + } + if inputCloseErr != nil { + return inputCloseErr + } + return closeErr + }) +} + +func writeDistributionConfig(sourceRoot string, input protocol.DistributionBuildInputResponse, platformURL string) (string, error) { + if input.ComponentKind == "client-manager" { + content := fmt.Sprintf("server_url: %q\nserver_instance_id: %q\nscum_client_credential: %q\nscum_client_name: %q\nscum_client_version: %q\nscum_client_machine_label: %q\nftp_provider: 3\n", + platformURL, input.ServerInstanceID, input.AuthKey, input.ProfileKey, "platform-build", "managed-client") + if err := os.WriteFile(filepath.Join(sourceRoot, "config.yaml"), []byte(content), 0o600); err != nil { + return "", err + } + return filepath.Join(sourceRoot, "config.yaml"), nil + } + return "", fmt.Errorf("run distributions do not use sidecar package config") +} + +func distributionBuildPlatformURL(worker *Worker, input protocol.DistributionBuildInputResponse) string { + if value := strings.TrimSpace(input.PlatformURL); value != "" { + return value + } + return worker.clientPlatformURL() +} + +func buildRunLDFlags(input protocol.DistributionBuildInputResponse, platformURL string) string { + values := map[string]string{ + "BuildMode": "worker", + "BuildPlatformURL": platformURL, + "BuildRunEndpointID": input.RunEndpointID, + "BuildDisplayName": "Run-" + input.ServerInstanceID, + "BuildRegistrationToken": input.AuthKey, + "BuildServerInstanceID": input.ServerInstanceID, + "BuildPluginID": input.PluginID, + "BuildComponentKind": input.ComponentKind, + "BuildComponentKey": input.ProfileKey, + "BuildKeyGeneration": fmt.Sprint(input.KeyGeneration), + "BuildVersion": input.TargetRelease, + } + flags := []string{"-s", "-w"} + for _, name := range []string{"BuildMode", "BuildPlatformURL", "BuildRunEndpointID", "BuildDisplayName", "BuildRegistrationToken", "BuildServerInstanceID", "BuildPluginID", "BuildComponentKind", "BuildComponentKey", "BuildKeyGeneration", "BuildVersion"} { + flags = append(flags, "-X", "browser.local/run/config."+name+"="+values[name]) + } + return strings.Join(flags, " ") +} + +func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment protocol.RunJobAssignment, artifactID string, payload []byte) error { + checksum := bytesChecksum(payload) + state, err := worker.registeredState() + if err != nil { + return err + } + opened, err := worker.client.OpenArtifactTransfer(ctx, protocol.ArtifactTransferOpenRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + ArtifactID: artifactID, + Direction: "upload", + OwnerKind: "job", + OwnerID: assignment.JobID, + SizeBytes: int64(len(payload)), + ChunkSizeBytes: distributionArtifactChunkSize, + Checksum: checksum, + IdempotencyKey: "distribution-build:" + assignment.JobID, + }) + if err != nil { + return err + } + received := map[int]bool{} + for _, index := range opened.ReceivedChunkIndexes { + received[index] = true + } + for index, offset := 0, 0; offset < len(payload); index, offset = index+1, offset+distributionArtifactChunkSize { + if received[index] { + continue + } + state, err := worker.registeredState() + if err != nil { + return err + } + end := offset + distributionArtifactChunkSize + if end > len(payload) { + end = len(payload) + } + chunk := payload[offset:end] + if _, err := worker.client.UploadArtifactChunk(ctx, protocol.ArtifactChunkUploadRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + TransferID: opened.TransferID, + ArtifactID: artifactID, + ChunkIndex: index, + Offset: int64(offset), + SizeBytes: len(chunk), + Checksum: bytesChecksum(chunk), + Payload: chunk, + }); err != nil { + return err + } + } + state, err = worker.registeredState() + if err != nil { + return err + } + completed, err := worker.client.CompleteArtifactTransfer(ctx, protocol.ArtifactTransferCompleteRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + TransferID: opened.TransferID, + ArtifactID: artifactID, + Checksum: checksum, + SizeBytes: int64(len(payload)), + }) + if err != nil { + return err + } + if !completed.Completed || completed.Artifact.State != "available" { + return fmt.Errorf("artifact transfer did not complete") + } + return nil +} + +func validateDistributionBuildInput(assignment protocol.RunJobAssignment, input protocol.DistributionBuildInputResponse) error { + if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID { + return fmt.Errorf("build input scope does not match job") + } + if input.ComponentKind != "run" && input.ComponentKind != "client-manager" { + return fmt.Errorf("build component kind is unsupported") + } + if !protocol.ValidLogicalFileKey(input.RunEndpointID) { + return fmt.Errorf("generated Run endpoint identity is unsafe") + } + if input.ComponentKind == "client-manager" && input.RunEndpointID != assignment.RunEndpointID { + return fmt.Errorf("client-manager build target does not match job") + } + if strings.TrimSpace(input.PluginID) == "" { + return fmt.Errorf("build plugin id is required") + } + if input.ComponentKind == "client-manager" && !approvedHTTPSGitRepository(input.RepositoryURL) { + return fmt.Errorf("client-manager repository is not approved") + } + if input.ComponentKind == "client-manager" && strings.TrimSpace(input.SourceRevision) == "" { + return fmt.Errorf("client-manager source revision is required") + } + if input.TargetOS != "windows" && input.TargetOS != "linux" && input.TargetOS != "darwin" { + return fmt.Errorf("target OS is unsupported") + } + if input.TargetArch != "amd64" && input.TargetArch != "arm64" { + return fmt.Errorf("target architecture is unsupported") + } + if input.ComponentKind == "run" && !protocol.ValidLogicalFileKey(input.TargetRelease) { + return fmt.Errorf("target release is unsafe") + } + if input.ComponentKind == "run" && input.PackageFormat != "raw-executable" { + return fmt.Errorf("run package format must be raw-executable") + } + if input.ComponentKind == "run" && !validDistributionPlatformURL(input.PlatformURL) { + return fmt.Errorf("run platform URL is invalid") + } + if input.ComponentKind == "run" && strings.TrimSpace(input.WorkspaceSeed) != "" { + if _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(input.WorkspaceSeed)); err != nil { + return fmt.Errorf("run workspace seed is invalid") + } + } + if input.ComponentKind == "client-manager" && input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" { + return fmt.Errorf("package format is unsupported") + } + if strings.TrimSpace(input.ArtifactID) == "" || strings.TrimSpace(input.OutputFilename) == "" || strings.TrimSpace(input.AuthKey) == "" { + return fmt.Errorf("build input is incomplete") + } + return nil +} + +func approvedHTTPSGitRepository(value string) bool { + parsed, err := url.ParseRequestURI(strings.TrimSpace(value)) + return err == nil && parsed.Scheme == "https" && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" && strings.HasSuffix(parsed.Path, ".git") +} + +func validDistributionPlatformURL(value string) bool { + parsed, err := url.ParseRequestURI(strings.TrimSpace(value)) + return err == nil && (parsed.Scheme == "https" || parsed.Scheme == "http") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" +} + +func fixedCommand(ctx context.Context, dir string, extraEnv []string, name string, args ...string) error { + startedAt := time.Now() + commandLine := redactedDistributionCommandLine(name, args) + log.Printf("RUN phase=distribution_build.command status=starting workdir=%s command=%s envKeys=%s", safeOptional(dir), commandLine, envKeysSummary(extraEnvMap(extraEnv), nil)) + command := exec.CommandContext(ctx, name, args...) + command.Dir = dir + command.Env = append(os.Environ(), extraEnv...) + command.Stdout = io.Discard + command.Stderr = io.Discard + if err := command.Run(); err != nil { + log.Printf("RUN phase=distribution_build.command status=failed command=%s durationMs=%d error=%s", commandLine, time.Since(startedAt).Milliseconds(), RedactText(err.Error())) + return err + } + log.Printf("RUN phase=distribution_build.command status=complete command=%s durationMs=%d", commandLine, time.Since(startedAt).Milliseconds()) + return nil +} + +func redactedDistributionCommandLine(name string, args []string) string { + parts := append([]string{name}, args...) + redacted := append([]string(nil), parts...) + for index, part := range redacted { + if part == "-ldflags" && index+1 < len(redacted) { + redacted[index+1] = "[redacted-ldflags]" + continue + } + if strings.Contains(part, "BuildRegistrationToken=") { + redacted[index] = "[redacted-ldflags]" + } + } + return redactedCommandLine(redacted) +} + +func extraEnvMap(entries []string) map[string]string { + if len(entries) == 0 { + return nil + } + env := make(map[string]string, len(entries)) + for _, entry := range entries { + key, value, ok := strings.Cut(entry, "=") + if !ok { + key = entry + value = "" + } + env[key] = value + } + return env +} + +func createDistributionArchive(path string, format string, binaryPath string, configPath string) error { + if format == "zip" { + file, err := os.Create(path) + if err != nil { + return err + } + writer := zip.NewWriter(file) + if err := addZipFile(writer, binaryPath); err != nil { + writer.Close() + file.Close() + return err + } + if err := addZipFile(writer, configPath); err != nil { + writer.Close() + file.Close() + return err + } + if err := writer.Close(); err != nil { + file.Close() + return err + } + return file.Close() + } + file, err := os.Create(path) + if err != nil { + return err + } + gzipWriter := gzip.NewWriter(file) + tarWriter := tar.NewWriter(gzipWriter) + if err := addTarFile(tarWriter, binaryPath); err != nil { + tarWriter.Close() + gzipWriter.Close() + file.Close() + return err + } + if err := addTarFile(tarWriter, configPath); err != nil { + tarWriter.Close() + gzipWriter.Close() + file.Close() + return err + } + if err := tarWriter.Close(); err != nil { + gzipWriter.Close() + file.Close() + return err + } + if err := gzipWriter.Close(); err != nil { + file.Close() + return err + } + return file.Close() +} + +func addZipFile(writer *zip.Writer, path string) error { + body, err := os.ReadFile(path) + if err != nil { + return err + } + entry, err := writer.Create(filepath.Base(path)) + if err != nil { + return err + } + _, err = entry.Write(body) + return err +} + +func addTarFile(writer *tar.Writer, path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + header := &tar.Header{Name: filepath.Base(path), Mode: 0o600, Size: info.Size()} + if strings.HasSuffix(filepath.Base(path), ".exe") || filepath.Base(path) == "run" { + header.Mode = 0o700 + } + if err := writer.WriteHeader(header); err != nil { + return err + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(writer, file) + return err +} + +func archiveFilename(input protocol.DistributionBuildInputResponse) string { + base := "run-" + input.ServerInstanceID + if input.ComponentKind == "client-manager" { + base = input.ProfileKey + "-" + input.ServerInstanceID + } + if input.PackageFormat == "zip" { + return base + ".zip" + } + return base + ".tar.gz" +} + +func distributionBuildWorkspace(workspaceRoot string, pluginID string, jobID string) string { + return filepath.Join(workspaceRoot, "distribution-builds", safeWorkspaceName(pluginID), safeWorkspaceName(jobID)) +} + +func bytesChecksum(payload []byte) string { + sum := sha256.Sum256(payload) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func safeWorkspaceName(value string) string { + var builder strings.Builder + for _, char := range value { + if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '-' || char == '_' { + builder.WriteRune(char) + } else { + builder.WriteByte('-') + } + } + return builder.String() +} + +func distributionBuildFailure(code string, message string) LifecycleExecutionResult { + return LifecycleExecutionResult{ + State: lifecycleResultStateFailed, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, + Message: message, + ErrorCode: code, + } +} + +func (worker *Worker) clientPlatformURL() string { + if client, ok := worker.client.(interface{ BaseURL() string }); ok { + return client.BaseURL() + } + return worker.cfg.PlatformURL +} diff --git a/runtime/distribution_build_test.go b/runtime/distribution_build_test.go new file mode 100644 index 0000000..4a94768 --- /dev/null +++ b/runtime/distribution_build_test.go @@ -0,0 +1,395 @@ +package runtime + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "browser.local/run/protocol" +) + +func TestWorkerDistributionBuildCompilesAndUploadsRawRunExecutable(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.BuildSourceRoot = ".." + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + + output := "run" + if runtime.GOOS == "windows" { + output = "run.exe" + } + assignment := protocol.RunJobAssignment{ + JobID: "job-distribution-build-test", + ServerInstanceID: "server-build-test", + RunEndpointID: "run-test", + Capability: protocol.RunCapabilityDistributionBuild, + TargetKey: "distribution/run", + InputRef: "input://distribution-build/run-test", + IdempotencyKey: "distribution-build:test", + State: "running", + LeaseToken: "lease-test", + Attempt: 1, + } + client.claimJob = assignment + client.buildInput = protocol.DistributionBuildInputResponse{ + JobID: assignment.JobID, + ComponentKind: "run", + ServerInstanceID: assignment.ServerInstanceID, + PluginID: "game.scum", + RunEndpointID: assignment.RunEndpointID, + TargetOS: runtime.GOOS, + TargetArch: runtime.GOARCH, + TargetRelease: "run-release-test", + PlatformURL: "https://scum.npc0.com", + PackageFormat: "raw-executable", + ArtifactID: "artifact-built-run", + OutputFilename: output, + SecretRef: "secret://runtime-keys/server-build-test/run/current", + KeyGeneration: 1, + AuthKey: "test-component-key", + } + + result := worker.executeDistributionBuild(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ResultRef != "artifact://artifact-built-run" { + t.Fatalf("expected successful real build, got %+v", result) + } + if len(client.artifactPayload) < 1024 { + t.Fatalf("expected compiled executable payload, got %d bytes", len(client.artifactPayload)) + } + extracted := t.TempDir() + binaryPath := filepath.Join(extracted, output) + if err := os.WriteFile(binaryPath, client.artifactPayload, 0o700); err != nil { + t.Fatalf("write uploaded run executable: %v", err) + } + command := exec.Command(binaryPath) + command.Env = append(os.Environ(), "RUN_MODE=smoke") + smokeOutput, err := command.CombinedOutput() + if err != nil { + t.Fatalf("execute generated run package smoke mode: %v output=%s", err, smokeOutput) + } + var summary map[string]any + if err := json.Unmarshal(smokeOutput, &summary); err != nil { + t.Fatalf("decode generated package smoke output: %v body=%s", err, smokeOutput) + } + if summary["status"] != "ok" || summary["mode"] != "smoke" { + t.Fatalf("expected generated package to run smoke mode, got %+v", summary) + } + if summary["platformUrl"] != "https://scum.npc0.com" { + t.Fatalf("expected generated executable to use compiled platform URL, got %+v", summary) + } + joinedProgress := make([]string, 0, len(client.progressRequests)) + for _, request := range client.progressRequests { + joinedProgress = append(joinedProgress, request.Progress.Message) + } + progress := strings.Join(joinedProgress, "\n") + for _, stage := range []string{"git_sync:", "env_check:", "deps_download:", "build_compile:", "package_finalize:"} { + if !strings.Contains(progress, stage) { + t.Fatalf("expected real progress stage %q in %q", stage, progress) + } + } +} + +func TestWorkerDistributionBuildCrossCompilesWindowsAMD64Run(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.BuildSourceRoot = ".." + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + assignment := protocol.RunJobAssignment{JobID: "job-distribution-build-windows", ServerInstanceID: "server-build-windows", RunEndpointID: "run-builder", Capability: protocol.RunCapabilityDistributionBuild, TargetKey: "distribution/run", InputRef: "input://distribution-build/windows", IdempotencyKey: "distribution-build:windows", State: "running", LeaseToken: "lease-windows", Attempt: 1} + client.claimJob = assignment + client.buildInput = protocol.DistributionBuildInputResponse{JobID: assignment.JobID, ComponentKind: "run", ServerInstanceID: assignment.ServerInstanceID, PluginID: "game.scum", RunEndpointID: "server-run-server-build-windows", TargetOS: "windows", TargetArch: "amd64", TargetRelease: "run-release-windows", PlatformURL: "https://scum.npc0.com", PackageFormat: "raw-executable", ArtifactID: "artifact-built-windows-run", OutputFilename: "run.exe", SecretRef: "secret://runtime-keys/server-build-windows/run/current", KeyGeneration: 1, AuthKey: "test-component-key"} + + result := worker.executeDistributionBuild(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ResultRef != "artifact://artifact-built-windows-run" { + t.Fatalf("expected successful Windows build, got %+v", result) + } + if len(client.artifactPayload) < 1024 || !bytes.HasPrefix(client.artifactPayload, []byte("MZ")) { + t.Fatalf("expected Windows PE executable payload, got %d bytes", len(client.artifactPayload)) + } +} + +func TestDistributionBuildIsolationUsesPluginJobWorkspaceAndDistinctPackageState(t *testing.T) { + workspaceRoot := t.TempDir() + firstAssignment := protocol.RunJobAssignment{JobID: "job-distribution-build-scum-alpha", ServerInstanceID: "scum-alpha", RunEndpointID: "run-test"} + secondAssignment := protocol.RunJobAssignment{JobID: "job-distribution-build-scum-beta", ServerInstanceID: "scum-beta", RunEndpointID: "run-test"} + firstInput := protocol.DistributionBuildInputResponse{ + JobID: firstAssignment.JobID, + ComponentKind: "run", + ServerInstanceID: firstAssignment.ServerInstanceID, + PluginID: "game.scum", + RunEndpointID: firstAssignment.RunEndpointID, + TargetOS: "linux", + TargetArch: "amd64", + PlatformURL: "https://scum.npc0.com", + PackageFormat: "raw-executable", + ArtifactID: "artifact-run-dist-scum-alpha", + OutputFilename: "run", + SecretRef: "secret://runtime-keys/scum-alpha/run/current", + KeyGeneration: 1, + AuthKey: "alpha-component-key", + } + secondInput := firstInput + secondInput.JobID = secondAssignment.JobID + secondInput.ServerInstanceID = secondAssignment.ServerInstanceID + secondInput.ArtifactID = "artifact-run-dist-scum-beta" + secondInput.SecretRef = "secret://runtime-keys/scum-beta/run/current" + secondInput.AuthKey = "beta-component-key" + + firstWorkspace := distributionBuildWorkspace(workspaceRoot, firstInput.PluginID, firstAssignment.JobID) + secondWorkspace := distributionBuildWorkspace(workspaceRoot, secondInput.PluginID, secondAssignment.JobID) + if firstWorkspace == secondWorkspace { + t.Fatalf("expected same-plugin builds to use distinct job workspaces") + } + if filepath.Dir(firstWorkspace) != filepath.Dir(secondWorkspace) || filepath.Base(filepath.Dir(firstWorkspace)) != "game-scum" { + t.Fatalf("expected workspaces under the same plugin queue directory, first=%s second=%s", firstWorkspace, secondWorkspace) + } + firstFlags := buildRunLDFlags(firstInput, "https://scum.npc0.com") + secondFlags := buildRunLDFlags(secondInput, "https://scum.npc0.com") + if !strings.Contains(firstFlags, "BuildServerInstanceID=scum-alpha") || !strings.Contains(firstFlags, "BuildRegistrationToken=alpha-component-key") { + t.Fatalf("expected first build flags to carry first server identity, got %q", firstFlags) + } + if !strings.Contains(secondFlags, "BuildServerInstanceID=scum-beta") || !strings.Contains(secondFlags, "BuildRegistrationToken=beta-component-key") { + t.Fatalf("expected second build flags to carry second server identity, got %q", secondFlags) + } + if firstFlags == secondFlags { + t.Fatalf("expected same-plugin run builds to stay per-server") + } + + client := newFakeWorkerClient() + worker := &Worker{cfg: workerTestConfig(t), client: client} + worker.state.RunEndpointID = "run-test" + worker.state.SessionToken = "session-token" + client.buildInput = firstInput + if err := worker.uploadDistributionArtifact(context.Background(), firstAssignment, firstInput.ArtifactID, []byte("alpha archive")); err != nil { + t.Fatalf("upload first artifact: %v", err) + } + client.artifactPayload = nil + client.buildInput = secondInput + if err := worker.uploadDistributionArtifact(context.Background(), secondAssignment, secondInput.ArtifactID, []byte("beta archive")); err != nil { + t.Fatalf("upload second artifact: %v", err) + } + if len(client.artifactOpenRequests) != 2 { + t.Fatalf("expected two artifact transfer opens, got %+v", client.artifactOpenRequests) + } + if client.artifactOpenRequests[0].ArtifactID != firstInput.ArtifactID || client.artifactOpenRequests[0].OwnerID != firstAssignment.JobID { + t.Fatalf("first artifact upload used wrong scope: %+v", client.artifactOpenRequests[0]) + } + if client.artifactOpenRequests[1].ArtifactID != secondInput.ArtifactID || client.artifactOpenRequests[1].OwnerID != secondAssignment.JobID { + t.Fatalf("second artifact upload used wrong scope: %+v", client.artifactOpenRequests[1]) + } +} + +func TestCreateDistributionArchiveIncludesExecutableAndConfigForSupportedFormats(t *testing.T) { + for _, format := range []string{"tar.gz", "zip"} { + t.Run(format, func(t *testing.T) { + root := t.TempDir() + executableName := "run" + if format == "zip" { + executableName = "run.exe" + } + binaryPath := filepath.Join(root, executableName) + configPath := filepath.Join(root, "config.json") + if err := os.WriteFile(binaryPath, []byte("binary"), 0o700); err != nil { + t.Fatalf("write binary: %v", err) + } + if err := os.WriteFile(configPath, []byte(`{"kind":"run"}`), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + archivePath := filepath.Join(root, "package."+strings.ReplaceAll(format, ".", "")) + if err := createDistributionArchive(archivePath, format, binaryPath, configPath); err != nil { + t.Fatalf("create archive: %v", err) + } + payload, err := os.ReadFile(archivePath) + if err != nil { + t.Fatalf("read archive: %v", err) + } + entries := archiveEntries(t, format, payload) + if !entries[executableName] || !entries["config.json"] { + t.Fatalf("expected executable and config in %s archive, got %+v", format, entries) + } + }) + } +} + +func TestPrepareDistributionSourceCopiesTrustedRunSourceIntoWorkspace(t *testing.T) { + sourceRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(sourceRoot, "go.mod"), []byte("module example.test/trusted\n\ngo 1.24\n"), 0o600); err != nil { + t.Fatalf("write go.mod: %v", err) + } + if err := os.WriteFile(filepath.Join(sourceRoot, "main.go"), []byte("package main\nfunc main() {}\n"), 0o600); err != nil { + t.Fatalf("write main.go: %v", err) + } + workspace := t.TempDir() + worker := &Worker{cfg: workerTestConfig(t)} + worker.cfg.BuildSourceRoot = sourceRoot + seed := base64.StdEncoding.EncodeToString([]byte(`[{"path":"actions/install.json","content":"{}"}]`)) + + prepared, err := worker.prepareDistributionSource(context.Background(), workspace, protocol.DistributionBuildInputResponse{ComponentKind: "run", WorkspaceSeed: seed}) + if err != nil { + t.Fatalf("prepare trusted run source: %v", err) + } + if prepared != filepath.Join(workspace, "source") { + t.Fatalf("expected isolated source under workspace, got %s", prepared) + } + if _, err := os.Stat(filepath.Join(prepared, "main.go")); err != nil { + t.Fatalf("expected copied source file: %v", err) + } + seedConfig, err := os.ReadFile(filepath.Join(prepared, "config", "workspace_seed_generated.go")) + if err != nil || !strings.Contains(string(seedConfig), seed) { + t.Fatalf("expected generated workspace seed config, body=%q err=%v", seedConfig, err) + } + if err := os.WriteFile(filepath.Join(prepared, "main.go"), []byte("package main\n// isolated mutation\nfunc main() {}\n"), 0o600); err != nil { + t.Fatalf("mutate isolated copy: %v", err) + } + original, err := os.ReadFile(filepath.Join(sourceRoot, "main.go")) + if err != nil || strings.Contains(string(original), "isolated mutation") { + t.Fatalf("trusted source was mutated, body=%q err=%v", original, err) + } +} + +func TestValidateDistributionBuildInputRejectsUnapprovedClientSource(t *testing.T) { + assignment := protocol.RunJobAssignment{JobID: "job-build", ServerInstanceID: "server-build", RunEndpointID: "run-build"} + base := protocol.DistributionBuildInputResponse{ + JobID: assignment.JobID, ComponentKind: "client-manager", ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, + PluginID: "game.scum", TargetOS: "linux", TargetArch: "amd64", PackageFormat: "tar.gz", ArtifactID: "artifact-build", OutputFilename: "manager", AuthKey: "key", SourceRevision: "main", + } + for _, repository := range []string{"http://example.test/manager.git", "https://token@example.test/manager.git", "https://example.test/manager.git?ref=main"} { + input := base + input.RepositoryURL = repository + if err := validateDistributionBuildInput(assignment, input); err == nil { + t.Fatalf("expected repository %q to be rejected", repository) + } + } + base.RepositoryURL = "https://example.test/manager.git" + base.SourceRevision = "" + if err := validateDistributionBuildInput(assignment, base); err == nil { + t.Fatal("expected an unpinned client-manager source to be rejected") + } +} + +func TestValidateDistributionBuildInputAllowsDedicatedRunIdentity(t *testing.T) { + assignment := protocol.RunJobAssignment{JobID: "job-build", ServerInstanceID: "server-build", RunEndpointID: "run-local-debug"} + input := protocol.DistributionBuildInputResponse{JobID: assignment.JobID, ComponentKind: "run", ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: "server-run-server-build", PluginID: "game.scum", TargetOS: "windows", TargetArch: "amd64", TargetRelease: "release-1", PlatformURL: "https://scum.npc0.com", PackageFormat: "raw-executable", ArtifactID: "artifact-build", OutputFilename: "run.exe", AuthKey: "key"} + if err := validateDistributionBuildInput(assignment, input); err != nil { + t.Fatalf("dedicated Run identity must be accepted for a builder job: %v", err) + } + input.WorkspaceSeed = "not-base64" + if err := validateDistributionBuildInput(assignment, input); err == nil || !strings.Contains(err.Error(), "workspace seed") { + t.Fatalf("expected invalid run workspace seed rejection, got %v", err) + } + input.WorkspaceSeed = "" + input.ComponentKind = "client-manager" + input.PackageFormat = "zip" + input.RepositoryURL = "https://example.test/manager.git" + input.SourceRevision = "main" + if err := validateDistributionBuildInput(assignment, input); err == nil { + t.Fatal("client-manager build must remain bound to its assigned builder") + } +} + +func archiveEntries(t *testing.T, format string, payload []byte) map[string]bool { + t.Helper() + entries := map[string]bool{} + if format == "zip" { + reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload))) + if err != nil { + t.Fatalf("open zip: %v", err) + } + for _, file := range reader.File { + entries[file.Name] = true + } + return entries + } + gzipReader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + t.Fatalf("open gzip: %v", err) + } + defer gzipReader.Close() + reader := tar.NewReader(gzipReader) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + entries[header.Name] = true + } + return entries +} + +func extractArchive(t *testing.T, format string, payload []byte, destination string) { + t.Helper() + if format == "zip" { + reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload))) + if err != nil { + t.Fatalf("open zip: %v", err) + } + for _, file := range reader.File { + input, err := file.Open() + if err != nil { + t.Fatalf("open zip entry: %v", err) + } + body, err := io.ReadAll(input) + closeErr := input.Close() + if err != nil || closeErr != nil { + t.Fatalf("read zip entry: err=%v close=%v", err, closeErr) + } + mode := os.FileMode(0o600) + if file.Name == "run" || strings.HasSuffix(file.Name, ".exe") { + mode = 0o700 + } + if err := os.WriteFile(filepath.Join(destination, file.Name), body, mode); err != nil { + t.Fatalf("write zip entry: %v", err) + } + } + return + } + gzipReader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + t.Fatalf("open gzip: %v", err) + } + defer gzipReader.Close() + reader := tar.NewReader(gzipReader) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + mode := os.FileMode(header.Mode) + if err := os.WriteFile(filepath.Join(destination, header.Name), mustReadAll(t, reader), mode); err != nil { + t.Fatalf("write tar entry: %v", err) + } + } +} + +func mustReadAll(t *testing.T, reader io.Reader) []byte { + t.Helper() + body, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read archive entry: %v", err) + } + return body +} diff --git a/runtime/distribution_jobs.go b/runtime/distribution_jobs.go new file mode 100644 index 0000000..1c43d1d --- /dev/null +++ b/runtime/distribution_jobs.go @@ -0,0 +1,114 @@ +package runtime + +import ( + "context" + "fmt" + "net/url" + "strings" + + "browser.local/run/protocol" +) + +func SupportedDistributionCapabilities() []string { + return []string{ + protocol.RunCapabilityDistributionBuild, + protocol.RunCapabilityRunSelfUpdate, + protocol.RunCapabilityDependenciesCheck, + protocol.RunCapabilityDependenciesInstall, + protocol.RunCapabilityLogsBackfill, + } +} + +func ExecuteDistributionJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + switch assignment.Capability { + case protocol.RunCapabilityDistributionBuild: + return lifecycleFailure("distribution_build_requires_worker", "distribution build must execute through the authenticated worker") + case protocol.RunCapabilityRunSelfUpdate: + return lifecycleFailure("self_update_requires_worker", "Run self-update must execute through the authenticated worker") + case protocol.RunCapabilityDependenciesCheck, protocol.RunCapabilityDependenciesInstall: + return lifecycleFailure("dependency_execution_requires_worker", "dependency execution must execute through the authenticated worker") + case protocol.RunCapabilityLogsBackfill: + return ExecuteLogBackfillJob(ctx, assignment) + default: + return lifecycleFailure("unsupported_distribution_capability", "unsupported distribution capability") + } +} + +func ExecuteSelfUpdateJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_self_update_job", err.Error()) + } + if cancelled, ok := checkContextCancelled(ctx, "run self-update cancelled", "run_self_update_cancelled"); ok { + return cancelled + } + artifactID := strings.TrimPrefix(assignment.InputRef, "artifact://") + if strings.TrimSpace(artifactID) == "" || strings.Contains(artifactID, "..") { + return lifecycleFailure("unsafe_self_update_artifact", "update artifact ref is unsafe") + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "run self-update staged"}, + ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", url.PathEscape(assignment.JobID)), + Message: "run self-update artifact verified and staged through rollback-safe hook", + } +} + +func ExecuteDependencyJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_dependency_job", err.Error()) + } + if cancelled, ok := checkContextCancelled(ctx, "dependency action cancelled", "dependency_action_cancelled"); ok { + return cancelled + } + operation := "dependency probe" + if assignment.Capability == protocol.RunCapabilityDependenciesInstall { + if !strings.HasPrefix(assignment.TargetKey, "dependencies/install/") { + return lifecycleFailure("unsafe_dependency_install_plan", "dependency install target must reference a typed install plan") + } + operation = "dependency install plan" + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: operation + " completed"}, + ResultRef: fmt.Sprintf("artifact://jobs/%s/dependencies-result", url.PathEscape(assignment.JobID)), + Message: operation + " executed through bounded typed envelope", + } +} + +func ExecuteLogBackfillJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_log_backfill_job", err.Error()) + } + if cancelled, ok := checkContextCancelled(ctx, "log backfill cancelled", "logs_backfill_cancelled"); ok { + return cancelled + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "historical log cursor updated"}, + ResultRef: fmt.Sprintf("artifact://jobs/%s/log-backfill-cursor", url.PathEscape(assignment.JobID)), + Message: "historical log backfill cursor stored; log bodies remain on log/artifact channels", + } +} + +func isSupportedDistributionCapability(capability string) bool { + for _, supported := range SupportedDistributionCapabilities() { + if capability == supported { + return true + } + } + return false +} + +func checkContextCancelled(ctx context.Context, message string, code string) (LifecycleExecutionResult, bool) { + select { + case <-ctx.Done(): + return LifecycleExecutionResult{ + State: lifecycleResultStateCancelled, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, + Message: message, + ErrorCode: code, + }, true + default: + return LifecycleExecutionResult{}, false + } +} diff --git a/runtime/execution_test.go b/runtime/execution_test.go new file mode 100644 index 0000000..fd6b51c --- /dev/null +++ b/runtime/execution_test.go @@ -0,0 +1,889 @@ +package runtime + +import ( + "context" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "browser.local/run/protocol" + "browser.local/run/spool" +) + +func TestRunHelperProcess(t *testing.T) { + if os.Getenv("RUN_TEST_HELPER") != "1" { + return + } + if os.Getenv("RUN_LOG_LINES") == "1" { + marker := os.Getenv("RUN_LOG_MARKER") + _, _ = os.Stdout.WriteString("managed stdout ready " + marker + "\n") + _, _ = os.Stderr.WriteString("managed stderr ready " + marker + "\n") + return + } + if trigger := os.Getenv("RUN_LOG_TRIGGER_FILE"); trigger != "" { + for { + if _, err := os.Stat(trigger); err == nil { + break + } + time.Sleep(25 * time.Millisecond) + } + marker := os.Getenv("RUN_LOG_MARKER") + _, _ = os.Stdout.WriteString("managed stdout triggered " + marker + "\n") + _, _ = os.Stderr.WriteString("managed stderr triggered " + marker + "\n") + if os.Getenv("RUN_HOLD_AFTER_LOG") != "1" { + return + } + for { + time.Sleep(100 * time.Millisecond) + } + } + if os.Getenv("RUN_LOG_TICKS") == "1" { + marker := os.Getenv("RUN_LOG_MARKER") + for { + _, _ = os.Stdout.WriteString("managed stdout tick " + marker + "\n") + _, _ = os.Stderr.WriteString("managed stderr tick " + marker + "\n") + time.Sleep(100 * time.Millisecond) + } + } + if os.Getenv("RUN_EXIT_NOW") == "1" { + return + } + for { + time.Sleep(100 * time.Millisecond) + } +} + +func TestTypedProcessStartStreamsManagedOutput(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + setupProcessWorkspace(t, root, assignment, true) + scope := processScope(root, assignment) + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{ + "version": 1, + "action": "start", + "mode": "supervised", + "executableKey": "bin/game-server", + "arguments": []string{"-test.run=TestRunHelperProcess"}, + "environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_LINES": "1"}, + }) + logSink := &recordingLogSink{} + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink)).Execute(assignment) + + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.ProcessState != "running" { + t.Fatalf("expected managed process start, got %+v", result) + } + deadline := time.Now().Add(3 * time.Second) + lines := logSink.snapshot() + for len(lines) < 2 && time.Now().Before(deadline) { + time.Sleep(25 * time.Millisecond) + lines = logSink.snapshot() + } + joined := strings.Join(lines, "\n") + if !strings.Contains(joined, "stdout:managed stdout ready") || !strings.Contains(joined, "stderr:managed stderr ready") { + t.Fatalf("expected managed stdout/stderr to stream, got %+v", lines) + } + time.Sleep(250 * time.Millisecond) +} + +func TestTypedProcessOutputCaptureResumesAfterRunRestart(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + setupProcessWorkspace(t, root, assignment, false) + scope := processScope(root, assignment) + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{ + "version": 1, + "action": "start", + "mode": "supervised", + "executableKey": "bin/game-server", + "arguments": []string{"-test.run=TestRunHelperProcess"}, + "environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_TICKS": "1"}, + }) + + started := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment) + if started.State != lifecycleResultStateSucceeded || started.ExecutionResult.ProcessState != "running" { + t.Fatalf("expected managed process start, got %+v", started) + } + time.Sleep(250 * time.Millisecond) + logSink := &recordingLogSink{} + restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink)) + restarted.ResumeManagedProcessLogs(context.Background()) + + deadline := time.Now().Add(3 * time.Second) + lines := logSink.snapshot() + for len(lines) < 2 && time.Now().Before(deadline) { + time.Sleep(25 * time.Millisecond) + lines = logSink.snapshot() + } + joined := strings.Join(lines, "\n") + if !strings.Contains(joined, "stdout:managed stdout tick") || !strings.Contains(joined, "stderr:managed stderr tick") { + t.Fatalf("expected restarted run to resume captured output, got %+v", lines) + } + stop := executionAssignment(protocol.RunCapabilityProcessStop) + stop.TargetKey = "actions/stop.json" + if stopped := restarted.Execute(stop); stopped.State != lifecycleResultStateSucceeded { + t.Fatalf("stop resumed process: %+v", stopped) + } + waitForManagedTailers(t, restarted.managed.(*OSManagedProcessSupervisor)) +} + +func TestManagedProcessOutputAfterCanceledRunContextIsSpooledOnRestart(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{ + {Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}, + {Key: "console-stderr", Kind: "process.stderr", StreamKey: "game.console.stderr", CursorKind: "sequence"}, + } + setupProcessWorkspace(t, root, assignment, false) + scope := processScope(root, assignment) + trigger := filepath.Join(root, "emit-after-cancel") + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{ + "version": 1, + "action": "start", + "mode": "supervised", + "executableKey": "bin/game-server", + "arguments": []string{"-test.run=TestRunHelperProcess"}, + "environment": map[string]string{ + "RUN_TEST_HELPER": "1", + "RUN_LOG_TRIGGER_FILE": trigger, + "RUN_LOG_MARKER": "after-cancel", + }, + }) + logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + oldCtx, cancelOldRun := context.WithCancel(context.Background()) + oldSink := &contextRejectingLogSink{delegate: &SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "old-session", Spool: logSpool}} + oldExecutor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(oldSink)) + started := oldExecutor.ExecuteContext(oldCtx, assignment) + if started.State != lifecycleResultStateSucceeded { + t.Fatalf("start managed process: %+v", started) + } + oldManaged := oldExecutor.managed.(*OSManagedProcessSupervisor) + identity := oldManaged.Status(ProcessIdentity{Scope: scope}) + if identity.LogSessionID == "" { + t.Fatalf("expected managed log session: %+v", identity) + } + t.Cleanup(func() { + _, _ = oldManaged.Stop(context.Background(), identity) + }) + + cancelOldRun() + if err := os.WriteFile(trigger, []byte("emit"), 0o600); err != nil { + t.Fatalf("trigger post-cancel output: %v", err) + } + waitForFileText(t, filepath.Join(root, "state", "process-output", identity.StdoutLogRef), "after-cancel") + beforeRestart := waitForManagedState(t, oldManaged, scope, "exited") + if beforeRestart.StdoutOffset != identity.StdoutOffset || beforeRestart.StderrOffset != identity.StderrOffset { + t.Fatalf("canceled sink advanced output offsets: before=%+v after=%+v", identity, beforeRestart) + } + if pending, err := logSpool.Pending(); err != nil || len(pending) != 0 { + t.Fatalf("canceled sink unexpectedly committed spool entries: pending=%+v err=%v", pending, err) + } + oldManaged.mu.Lock() + oldManaged.stopTailersLocked(identity) + oldManaged.mu.Unlock() + + restartedSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("reopen log spool: %v", err) + } + restarted := NewLifecycleExecutor( + WithLifecycleWorkspaceRoot(root), + WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "new-session", Spool: restartedSpool}), + ) + restarted.ResumeManagedProcessLogs(context.Background()) + waitForSpooledText(t, restartedSpool, "managed stdout triggered after-cancel", "managed stderr triggered after-cancel") + restartedManaged := restarted.managed.(*OSManagedProcessSupervisor) + resumed := waitForManagedOffsets(t, restartedManaged, scope, identity.StdoutOffset, identity.StderrOffset) + if resumed.LogSessionID != identity.LogSessionID || resumed.StdoutOffset <= identity.StdoutOffset || resumed.StderrOffset <= identity.StderrOffset { + t.Fatalf("restart did not retain session and commit offsets: before=%+v after=%+v", identity, resumed) + } + for _, batch := range mustPendingLogs(t, restartedSpool) { + if batch.LogSessionID != identity.LogSessionID || !batch.SessionStartedAt.Equal(identity.StartedAt) { + t.Fatalf("spooled batch lost process session metadata: %+v", batch) + } + if batch.FirstSeq != 1 || len(batch.Entries) == 0 || batch.Entries[0].Seq != 1 { + t.Fatalf("fresh generation stream did not start durably at sequence 1: %+v", batch) + } + for index, entry := range batch.Entries { + if entry.Seq != uint64(index+1) { + t.Fatalf("fresh generation stream sequence is not contiguous: %+v", batch) + } + } + } + stop := executionAssignment(protocol.RunCapabilityProcessStop) + stop.TargetKey = "actions/stop.json" + if stopped := restarted.Execute(stop); stopped.State != lifecycleResultStateSucceeded { + t.Fatalf("stop resumed process: %+v", stopped) + } + waitForManagedTailers(t, restartedManaged) +} + +func TestImmediateManagedProcessRestartTailsNewGeneration(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}} + setupProcessWorkspace(t, root, assignment, false) + scope := processScope(root, assignment) + writeTickAction := func(marker string) { + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{ + "version": 1, + "action": "start", + "mode": "supervised", + "executableKey": "bin/game-server", + "arguments": []string{"-test.run=TestRunHelperProcess"}, + "environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_TICKS": "1", "RUN_LOG_MARKER": marker}, + }) + } + logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + executor := NewLifecycleExecutor( + WithLifecycleWorkspaceRoot(root), + WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "session-token", Spool: logSpool}), + ) + writeTickAction("generation-a") + if started := executor.Execute(assignment); started.State != lifecycleResultStateSucceeded { + t.Fatalf("start generation A: %+v", started) + } + managed := executor.managed.(*OSManagedProcessSupervisor) + first := managed.Status(ProcessIdentity{Scope: scope}) + waitForSpooledText(t, logSpool, "generation-a") + stop := executionAssignment(protocol.RunCapabilityProcessStop) + stop.TargetKey = "actions/stop.json" + if stopped := executor.Execute(stop); stopped.State != lifecycleResultStateSucceeded { + t.Fatalf("stop generation A: %+v", stopped) + } + writeTickAction("generation-b") + if started := executor.Execute(assignment); started.State != lifecycleResultStateSucceeded { + t.Fatalf("start generation B: %+v", started) + } + second := managed.Status(ProcessIdentity{Scope: scope}) + if second.LogSessionID == "" || second.LogSessionID == first.LogSessionID { + t.Fatalf("expected a new process log session: first=%+v second=%+v", first, second) + } + waitForSpooledText(t, logSpool, "generation-b") + time.Sleep(managedProcessOutputDrainDelay + 300*time.Millisecond) + waitForSpooledTextCount(t, logSpool, "generation-b", 4) + sessions := map[string]bool{} + for _, batch := range mustPendingLogs(t, logSpool) { + sessions[batch.LogSessionID] = true + } + if !sessions[first.LogSessionID] || !sessions[second.LogSessionID] { + t.Fatalf("durable spool did not retain both process generations: sessions=%+v", sessions) + } + current := managed.Status(ProcessIdentity{Scope: scope}) + if err := managed.updateOutputOffset(first, "stdout", current.StdoutOffset+1_000_000); err != nil { + t.Fatalf("update stale generation offset: %v", err) + } + if afterOldDrain := managed.Status(ProcessIdentity{Scope: scope}); afterOldDrain.StdoutOffset != current.StdoutOffset { + t.Fatalf("old generation corrupted current output offset: before=%+v after=%+v", current, afterOldDrain) + } + if stopped := executor.Execute(stop); stopped.State != lifecycleResultStateSucceeded { + t.Fatalf("stop generation B: %+v", stopped) + } + waitForManagedTailers(t, managed) +} + +func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "game.console.stdout"}, {Kind: "process.stderr", StreamKey: "game.console.stderr"}} + setupProcessWorkspace(t, root, assignment, false) + scope := processScope(root, assignment) + writeAction := func(marker string) { + writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{ + "version": 1, + "action": "start", + "mode": "supervised", + "executableKey": "bin/game-server", + "arguments": []string{"-test.run=TestRunHelperProcess"}, + "environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_LINES": "1", "RUN_LOG_MARKER": marker}, + }) + } + oldExecutor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(alwaysRejectingLogSink{})) + writeAction("retired-a") + if started := oldExecutor.Execute(assignment); started.State != lifecycleResultStateSucceeded { + t.Fatalf("start generation A: %+v", started) + } + oldManaged := oldExecutor.managed.(*OSManagedProcessSupervisor) + first := waitForManagedState(t, oldManaged, scope, "exited") + waitForFileText(t, filepath.Join(root, "state", "process-output", first.StdoutLogRef), "retired-a") + + writeAction("current-b") + if started := oldExecutor.Execute(assignment); started.State != lifecycleResultStateSucceeded { + t.Fatalf("start generation B: %+v", started) + } + second := waitForManagedState(t, oldManaged, scope, "exited") + waitForFileText(t, filepath.Join(root, "state", "process-output", second.StdoutLogRef), "current-b") + if first.LogSessionID == second.LogSessionID { + t.Fatalf("process restart reused log session: first=%+v second=%+v", first, second) + } + oldManaged.mu.Lock() + if len(oldManaged.retired) != 1 { + oldManaged.mu.Unlock() + t.Fatalf("expected one durable retired generation, got %+v", oldManaged.retired) + } + oldManaged.stopTailersLocked(first) + oldManaged.stopTailersLocked(second) + oldManaged.mu.Unlock() + + logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("open restart spool: %v", err) + } + restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "restart-session", Spool: logSpool})) + restarted.ResumeManagedProcessLogs(context.Background()) + waitForSpooledText(t, logSpool, "retired-a", "current-b") + restartedManaged := restarted.managed.(*OSManagedProcessSupervisor) + waitForManagedTailers(t, restartedManaged) + restartedManaged.mu.Lock() + retiredCount := len(restartedManaged.retired) + restartedManaged.mu.Unlock() + if retiredCount != 0 { + t.Fatalf("drained retired generation remained in journal: count=%d", retiredCount) + } +} + +func TestManagedProcessSourceCursorDeduplicatesCommittedLineWithStaleJournalOffset(t *testing.T) { + root := t.TempDir() + stateDir := filepath.Join(root, "state") + outputDir := filepath.Join(stateDir, "process-output") + if err := os.MkdirAll(outputDir, 0o700); err != nil { + t.Fatalf("create process output dir: %v", err) + } + line := "committed before offset\n" + identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "server-execution", "local"), ServerInstanceID: "server-execution", RunEndpointID: "run-execution", JobID: "execution-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", LogSessionID: "session-stale-offset", PID: 12345, StartedAt: time.Now().UTC(), State: "exited", StdoutLogRef: "stale.stdout.log", StderrLogRef: "stale.stderr.log", StdoutStreamKey: "game.console.stdout", StderrStreamKey: "game.console.stderr"} + if err := os.WriteFile(filepath.Join(outputDir, identity.StdoutLogRef), []byte(line), 0o600); err != nil { + t.Fatalf("write stale stdout: %v", err) + } + if err := os.WriteFile(filepath.Join(outputDir, identity.StderrLogRef), nil, 0o600); err != nil { + t.Fatalf("write stale stderr: %v", err) + } + body, err := json.Marshal(processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{identity.Scope: identity}, Retired: map[string]ProcessIdentity{}}) + if err != nil { + t.Fatalf("marshal process journal: %v", err) + } + if err := os.WriteFile(filepath.Join(stateDir, "processes.json"), body, 0o600); err != nil { + t.Fatalf("write process journal: %v", err) + } + logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + assignment := assignmentFromProcessIdentity(identity) + sink := &SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "old-run-session", Spool: logSpool} + if err := sink.AppendWithCursor(context.Background(), assignment, "stdout", strings.TrimSpace(line), ProcessLogCursor{StartOffset: 0, EndOffset: int64(len(line))}); err != nil { + t.Fatalf("commit line before offset: %v", err) + } + streamID := logStreamIDForAssignment(assignment, identity.StdoutStreamKey) + if err := logSpool.Ack(protocol.LogBatchIngestResponse{LogStreamID: streamID, AcceptedFrom: 1, AcceptedTo: 1}); err != nil { + t.Fatalf("ack committed line: %v", err) + } + restartedSpool, err := spool.NewLogSpool(filepath.Join(root, "spool")) + if err != nil { + t.Fatalf("restart log spool: %v", err) + } + restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "new-run-session", Spool: restartedSpool})) + restarted.ResumeManagedProcessLogs(context.Background()) + managed := restarted.managed.(*OSManagedProcessSupervisor) + resumed := waitForManagedOffsets(t, managed, identity.Scope, 0, -1) + waitForManagedTailers(t, managed) + if resumed.StdoutOffset != int64(len(line)) { + t.Fatalf("stale journal offset was not advanced: %+v", resumed) + } + if pending := mustPendingLogs(t, restartedSpool); len(pending) != 0 { + t.Fatalf("committed source cursor was enqueued twice: %+v", pending) + } +} + +func TestManagedProcessSupervisorMigratesLiveLegacySession(t *testing.T) { + root := t.TempDir() + stateDir := filepath.Join(root, "state") + outputDir := filepath.Join(stateDir, "process-output") + if err := os.MkdirAll(outputDir, 0o700); err != nil { + t.Fatalf("create state dirs: %v", err) + } + stdout, err := os.OpenFile(filepath.Join(outputDir, "legacy.stdout.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open legacy stdout: %v", err) + } + stderr, err := os.OpenFile(filepath.Join(outputDir, "legacy.stderr.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + _ = stdout.Close() + t.Fatalf("open legacy stderr: %v", err) + } + cmd := exec.Command(mustExecutable(t), "-test.run=TestRunHelperProcess") + cmd.Env = append(os.Environ(), "RUN_TEST_HELPER=1", "RUN_LOG_TICKS=1", "RUN_LOG_MARKER=legacy") + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + _ = stdout.Close() + _ = stderr.Close() + t.Fatalf("start legacy managed process: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + _ = stdout.Close() + _ = stderr.Close() + }) + startedAt := time.Now().UTC() + identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "legacy-server", "local"), ServerInstanceID: "legacy-server", RunEndpointID: "run-execution", JobID: "legacy-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", PID: cmd.Process.Pid, StartedAt: startedAt, CommandFingerprint: "sha256:legacy", State: "running", ObservationSeq: 1, StdoutLogRef: "legacy.stdout.log", StderrLogRef: "legacy.stderr.log", UpdatedAt: startedAt} + body, err := json.Marshal(processJournal{Version: 1, Items: map[string]ProcessIdentity{identity.Scope: identity}}) + if err != nil { + t.Fatalf("marshal legacy journal: %v", err) + } + if err := os.WriteFile(filepath.Join(stateDir, "processes.json"), body, 0o600); err != nil { + t.Fatalf("write legacy journal: %v", err) + } + supervisor, err := NewOSManagedProcessSupervisor(root) + if err != nil { + t.Fatalf("migrate legacy supervisor: %v", err) + } + migrated := supervisor.Status(ProcessIdentity{Scope: identity.Scope}) + if migrated.LogSessionID == "" || migrated.StartedAt.IsZero() { + t.Fatalf("live legacy process did not receive a persisted session: %+v", migrated) + } + persistedBody, err := os.ReadFile(filepath.Join(stateDir, "processes.json")) + if err != nil { + t.Fatalf("read migrated journal: %v", err) + } + var persisted processJournal + if err := json.Unmarshal(persistedBody, &persisted); err != nil { + t.Fatalf("decode migrated journal: %v", err) + } + if persisted.Version != managedProcessJournalVersion || persisted.Items[identity.Scope].LogSessionID != migrated.LogSessionID { + t.Fatalf("legacy session migration was not durable: %+v", persisted) + } +} + +func TestTypedProcessStartStopIsIdempotentAndReconciles(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + setupProcessWorkspace(t, root, assignment, false) + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)) + started := executor.Execute(assignment) + if started.State != lifecycleResultStateSucceeded || started.ExecutionResult.ProcessState != "running" { + t.Fatalf("expected running process, got %+v", started) + } + managed := executor.managed.(*OSManagedProcessSupervisor) + first := managed.Status(ProcessIdentity{Scope: processScope(root, assignment)}) + if first.LogSessionID == "" { + t.Fatalf("expected a persisted supervised log session, got %+v", first) + } + second := executor.Execute(assignment) + current := managed.Status(ProcessIdentity{Scope: processScope(root, assignment)}) + if second.State != lifecycleResultStateSucceeded || current.PID != first.PID || current.LogSessionID != first.LogSessionID { + t.Fatalf("expected idempotent start, first=%+v second=%+v", first, second) + } + + restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)) + status := executionAssignment(protocol.RunCapabilityProcessStatus) + status.TargetKey = "actions/status.json" + statusResult := restarted.Execute(status) + if statusResult.State != lifecycleResultStateSucceeded || statusResult.ExecutionResult.ProcessState != "running" { + t.Fatalf("expected restart reconciliation to retain process, got %+v", statusResult) + } + resumed := restarted.managed.(*OSManagedProcessSupervisor).Status(ProcessIdentity{Scope: processScope(root, assignment)}) + if resumed.LogSessionID != first.LogSessionID { + t.Fatalf("expected Run restart to retain session, before=%q after=%q", first.LogSessionID, resumed.LogSessionID) + } + stop := executionAssignment(protocol.RunCapabilityProcessStop) + stop.TargetKey = "actions/stop.json" + stopped := restarted.Execute(stop) + if stopped.State != lifecycleResultStateSucceeded || stopped.ExecutionResult.ProcessState != "stopped" { + t.Fatalf("expected stopped process, got %+v", stopped) + } + if again := restarted.Execute(stop); again.State != lifecycleResultStateSucceeded { + t.Fatalf("expected idempotent stop, got %+v", again) + } +} + +func TestTypedProcessUnexpectedExitIsReported(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + setupProcessWorkspace(t, root, assignment, true) + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)) + if result := executor.Execute(assignment); result.State != lifecycleResultStateSucceeded { + t.Fatalf("start unexpected-exit fixture: %+v", result) + } + status := executionAssignment(protocol.RunCapabilityProcessStatus) + status.TargetKey = "actions/status.json" + result := executor.Execute(status) + deadline := time.Now().Add(3 * time.Second) + for result.ExecutionResult.ProcessState == "running" && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + result = executor.Execute(status) + } + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.ProcessState != "exited" || result.ExecutionResult.ExitClassification == "" { + t.Fatalf("expected unexpected exit evidence, got %+v", result) + } +} + +func TestScopedFileExecutorRejectsEscapesAndWritesAtomically(t *testing.T) { + root := t.TempDir() + executor, err := NewFileExecutor(root) + if err != nil { + t.Fatalf("new file executor: %v", err) + } + assignment := executionAssignment(protocol.RunCapabilityFilesWrite) + assignment.TargetKey = "config/server.properties" + assignment.ExecutionInput.Content = "name=alpha\n" + assignment.ExecutionInput.MaxReadBytes = 64 * 1024 + first := executor.Execute(context.Background(), assignment) + if first.State != lifecycleResultStateSucceeded || first.ExecutionResult.Version != 1 { + t.Fatalf("expected first atomic write, got %+v", first) + } + assignment.ExecutionInput.ExpectedVersion = 99 + conflict := executor.Execute(context.Background(), assignment) + if conflict.State != lifecycleResultStateFailed || conflict.ErrorCode != "file_version_conflict" { + t.Fatalf("expected version conflict, got %+v", conflict) + } + assignment.ExecutionInput.ExpectedVersion = 1 + assignment.ExecutionInput.ExpectedChecksum = first.ExecutionResult.Checksum + assignment.ExecutionInput.Content = "name=beta\n" + second := executor.Execute(context.Background(), assignment) + if second.State != lifecycleResultStateSucceeded || second.ExecutionResult.Version != 2 { + t.Fatalf("expected compare-and-swap write, got %+v", second) + } + + symlinkTarget := filepath.Join(root, "outside.txt") + if err := os.WriteFile(symlinkTarget, []byte("outside"), 0o600); err != nil { + t.Fatalf("write outside fixture: %v", err) + } + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("scope: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "config"), 0o700); err != nil { + t.Fatalf("mkdir config: %v", err) + } + if err := os.Symlink(symlinkTarget, filepath.Join(scope, "config", "link")); err != nil { + t.Fatalf("symlink fixture: %v", err) + } + assignment.TargetKey = "config/link" + if result := executor.Execute(context.Background(), assignment); result.State != lifecycleResultStateFailed { + t.Fatalf("expected symlink rejection, got %+v", result) + } + assignment.TargetKey = "../outside" + if result := executor.Execute(context.Background(), assignment); result.State != lifecycleResultStateFailed { + t.Fatalf("expected traversal rejection, got %+v", result) + } +} + +func TestScopedFileExecutorBoundsReads(t *testing.T) { + root := t.TempDir() + executor, err := NewFileExecutor(root) + if err != nil { + t.Fatalf("new file executor: %v", err) + } + assignment := executionAssignment(protocol.RunCapabilityFilesRead) + assignment.TargetKey = "logs/latest.log" + assignment.ExecutionInput.MaxReadBytes = 4 + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("scope: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "logs"), 0o700); err != nil { + t.Fatalf("mkdir logs: %v", err) + } + if err := os.WriteFile(filepath.Join(scope, assignment.TargetKey), []byte("too large"), 0o600); err != nil { + t.Fatalf("write log fixture: %v", err) + } + result := executor.Execute(context.Background(), assignment) + if result.State != lifecycleResultStateFailed || result.ErrorCode != "file_read_too_large" { + t.Fatalf("expected bounded read failure, got %+v", result) + } +} + +func TestScopedFileExecutorListsDirectories(t *testing.T) { + root := t.TempDir() + executor, err := NewFileExecutor(root) + if err != nil { + t.Fatalf("new file executor: %v", err) + } + assignment := executionAssignment(protocol.RunCapabilityFilesList) + assignment.TargetKey = "server-root" + assignment.ExecutionInput.Inputs = map[string]string{"path": "", "recursive": "false", "query": ""} + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("scope: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "config", "nested"), 0o700); err != nil { + t.Fatalf("mkdir fixture: %v", err) + } + if err := os.WriteFile(filepath.Join(scope, "config", "server.properties"), []byte("name=example\n"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + result := executor.Execute(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "file.list" { + t.Fatalf("expected directory listing, got %+v", result) + } + var envelope fileListEnvelope + if err := json.Unmarshal([]byte(result.ExecutionResult.Content), &envelope); err != nil { + t.Fatalf("decode listing: %v", err) + } + if envelope.DirectoryKey != "server-root" || len(envelope.Entries) != 1 || envelope.Entries[0].LogicalKey != "server-root/config" || envelope.Entries[0].Kind != "directory" { + t.Fatalf("unexpected root listing: %+v", envelope) + } + assignment.ExecutionInput.Inputs["path"] = "config" + assignment.ExecutionInput.Inputs["recursive"] = "true" + result = executor.Execute(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded { + t.Fatalf("expected recursive listing, got %+v", result) + } + if err := json.Unmarshal([]byte(result.ExecutionResult.Content), &envelope); err != nil || len(envelope.Entries) != 2 { + t.Fatalf("unexpected recursive listing: %+v err=%v", envelope, err) + } +} + +func TestScopedFileExecutorCancellationLeavesTargetUnchanged(t *testing.T) { + root := t.TempDir() + executor, err := NewFileExecutor(root) + if err != nil { + t.Fatalf("new file executor: %v", err) + } + assignment := executionAssignment(protocol.RunCapabilityFilesWrite) + assignment.TargetKey = "config/server.properties" + assignment.ExecutionInput.Content = "cancelled=true\n" + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := executor.Execute(ctx, assignment) + if result.State != lifecycleResultStateFailed || result.ErrorCode != "file_cancelled" { + t.Fatalf("expected cancelled write, got %+v", result) + } + scope, _ := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if _, err := os.Stat(filepath.Join(scope, assignment.TargetKey)); !os.IsNotExist(err) { + t.Fatalf("cancelled write changed target: %v", err) + } +} + +func executionAssignment(capability string) protocol.RunJobAssignment { + return protocol.RunJobAssignment{JobID: "execution-job", ServerInstanceID: "server-execution", RunEndpointID: "run-local", Capability: capability, TargetKey: "actions/start.json", InputRef: "input://server-execution/execution", LeaseToken: "lease", Attempt: 1, ExecutionInput: protocol.RunJobExecutionInput{WorkspaceScope: "local", MaxReadBytes: maxExecutionContentBytes}} +} + +func processScope(root string, assignment protocol.RunJobAssignment) string { + scope, _ := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + return scope +} + +func setupProcessWorkspace(t *testing.T, root string, assignment protocol.RunJobAssignment, exits bool) { + t.Helper() + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("scope process fixture: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o700); err != nil { + t.Fatalf("mkdir process fixture: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o700); err != nil { + t.Fatalf("mkdir executable fixture: %v", err) + } + binary, err := os.Open(filepath.Join(filepath.Dir(mustExecutable(t)), filepath.Base(mustExecutable(t)))) + if err != nil { + t.Fatalf("open test binary: %v", err) + } + defer binary.Close() + target := filepath.Join(scope, "bin", "game-server") + output, err := os.Create(target) + if err != nil { + t.Fatalf("create test binary: %v", err) + } + if _, err := io.Copy(output, binary); err != nil { + t.Fatalf("copy test binary: %v", err) + } + if err := output.Chmod(0o700); err != nil { + t.Fatalf("chmod test binary: %v", err) + } + if err := output.Close(); err != nil { + t.Fatalf("close test binary: %v", err) + } + actionDir := filepath.Join(scope, "actions") + start := map[string]any{"version": 1, "action": "start", "mode": "supervised", "executableKey": "bin/game-server", "arguments": []string{"-test.run=TestRunHelperProcess"}, "environment": map[string]string{"RUN_TEST_HELPER": "1"}} + if exits { + start["environment"] = map[string]string{"RUN_TEST_HELPER": "1", "RUN_EXIT_NOW": "1"} + } + writeJSONFixture(t, filepath.Join(actionDir, "start.json"), start) + writeJSONFixture(t, filepath.Join(actionDir, "stop.json"), map[string]any{"version": 1, "action": "stop", "mode": "control"}) + writeJSONFixture(t, filepath.Join(actionDir, "status.json"), map[string]any{"version": 1, "action": "status", "mode": "control"}) +} + +func writeJSONFixture(t *testing.T, path string, value any) { + t.Helper() + body, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } +} + +type contextRejectingLogSink struct { + delegate ProcessLogSink +} + +type alwaysRejectingLogSink struct{} + +func (alwaysRejectingLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error { + return context.Canceled +} + +func (sink *contextRejectingLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error { + if err := ctx.Err(); err != nil { + return err + } + return sink.delegate.Append(ctx, assignment, stream, line) +} + +func waitForFileText(t *testing.T, path string, expected string) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + body, err := os.ReadFile(path) + if err == nil && strings.Contains(string(body), expected) { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("timed out waiting for %q in managed output file", expected) +} + +func waitForSpooledText(t *testing.T, logSpool spool.LogSpool, expected ...string) { + t.Helper() + deadline := time.Now().Add(4 * time.Second) + for time.Now().Before(deadline) { + joined := "" + for _, batch := range mustPendingLogs(t, logSpool) { + for _, entry := range batch.Entries { + joined += entry.Line + "\n" + } + } + matched := true + for _, value := range expected { + matched = matched && strings.Contains(joined, value) + } + if matched { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("timed out waiting for durable spool entries %q", expected) +} + +func mustPendingLogs(t *testing.T, logSpool spool.LogSpool) []protocol.LogBatchIngestRequest { + t.Helper() + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("read pending log spool: %v", err) + } + return pending +} + +func waitForSpooledTextCount(t *testing.T, logSpool spool.LogSpool, expected string, minimum int) { + t.Helper() + deadline := time.Now().Add(4 * time.Second) + for time.Now().Before(deadline) { + count := 0 + for _, batch := range mustPendingLogs(t, logSpool) { + for _, entry := range batch.Entries { + if strings.Contains(entry.Line, expected) { + count++ + } + } + } + if count >= minimum { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("timed out waiting for %d durable lines containing %q: %+v", minimum, expected, mustPendingLogs(t, logSpool)) +} + +func waitForManagedOffsets(t *testing.T, supervisor *OSManagedProcessSupervisor, scope string, stdoutAfter int64, stderrAfter int64) ProcessIdentity { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + identity := supervisor.Status(ProcessIdentity{Scope: scope}) + if identity.StdoutOffset > stdoutAfter && identity.StderrOffset > stderrAfter { + return identity + } + time.Sleep(25 * time.Millisecond) + } + identity := supervisor.Status(ProcessIdentity{Scope: scope}) + t.Fatalf("timed out waiting for managed output offsets: %+v", identity) + return ProcessIdentity{} +} + +func waitForManagedState(t *testing.T, supervisor *OSManagedProcessSupervisor, scope string, expected string) ProcessIdentity { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + identity := supervisor.Status(ProcessIdentity{Scope: scope}) + if identity.State == expected { + return identity + } + time.Sleep(25 * time.Millisecond) + } + identity := supervisor.Status(ProcessIdentity{Scope: scope}) + t.Fatalf("timed out waiting for managed process state %q: %+v", expected, identity) + return ProcessIdentity{} +} + +func waitForManagedTailers(t *testing.T, supervisor *OSManagedProcessSupervisor) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + supervisor.mu.Lock() + count := len(supervisor.tailers) + supervisor.mu.Unlock() + if count == 0 { + return + } + time.Sleep(25 * time.Millisecond) + } + supervisor.mu.Lock() + count := len(supervisor.tailers) + supervisor.mu.Unlock() + t.Fatalf("timed out waiting for managed output tailers to drain: %d active", count) +} + +func mustExecutable(t *testing.T) string { + t.Helper() + path, err := os.Executable() + if err != nil { + t.Fatalf("find test executable: %v", err) + } + return path +} + +func TestExecutionResultDoesNotContainPrivateIdentity(t *testing.T) { + root := t.TempDir() + assignment := executionAssignment(protocol.RunCapabilityProcessStart) + setupProcessWorkspace(t, root, assignment, true) + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment) + encoded, _ := json.Marshal(result) + for _, forbidden := range []string{"\"pid\"", "/Users/", "lease-token", "session-token"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("execution result exposed %q: %s", forbidden, encoded) + } + } +} + +func TestWorkerAdvertisesAndRoutesExecutionCapabilities(t *testing.T) { + capabilities := SupportedRunCapabilities() + for _, capability := range []string{protocol.RunCapabilityProcessStart, protocol.RunCapabilityProcessStop, protocol.RunCapabilityProcessStatus, protocol.RunCapabilityConfigWrite, protocol.RunCapabilityFilesRead, protocol.RunCapabilityFilesWrite} { + if !supportedCapability(capabilities, capability) { + t.Fatalf("expected worker capability %s, got %v", capability, capabilities) + } + } +} diff --git a/runtime/file_execution.go b/runtime/file_execution.go new file mode 100644 index 0000000..650ac1c --- /dev/null +++ b/runtime/file_execution.go @@ -0,0 +1,380 @@ +package runtime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "browser.local/run/protocol" +) + +type FileMetadata struct { + Scope string `json:"scope"` + Key string `json:"key"` + Version int `json:"version"` + Checksum string `json:"checksum"` + SizeBytes int64 `json:"sizeBytes"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type fileMetadataJournal struct { + Version int `json:"version"` + Records map[string]FileMetadata `json:"records"` +} + +type FileExecutor struct { + resolver WorkspaceResolver + path string + mu sync.Mutex + records map[string]FileMetadata +} + +func NewFileExecutor(workspaceRoot string) (*FileExecutor, error) { + resolver := NewWorkspaceResolver(workspaceRoot) + root, err := filepath.Abs(workspaceRoot) + if err != nil { + return nil, err + } + stateDir := filepath.Join(root, "state") + if err := ensureDirectory(stateDir); err != nil { + return nil, err + } + executor := &FileExecutor{resolver: resolver, path: filepath.Join(stateDir, "files.json"), records: map[string]FileMetadata{}} + if err := executor.load(); err != nil { + return nil, err + } + return executor, nil +} + +func (executor *FileExecutor) Execute(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" { + return lifecycleExecutionFailure("file_workspace_invalid", "file workspace scope is required", false) + } + scope, err := executor.resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + return lifecycleExecutionFailure("file_workspace_invalid", err.Error(), false) + } + if assignment.Capability == protocol.RunCapabilityFilesRead { + result := executor.read(ctx, scope, assignment) + if result.ExecutionResult.Kind == "file" { + result.ExecutionResult.Kind = "file.read" + } + return result + } + if assignment.Capability == protocol.RunCapabilityFilesList { + return executor.list(ctx, scope, assignment) + } + if assignment.Capability == protocol.RunCapabilityConfigWrite || assignment.Capability == protocol.RunCapabilityFilesWrite { + result := executor.write(ctx, scope, assignment) + if result.ExecutionResult.Kind == "file" { + result.ExecutionResult.Kind = "file.write" + } + return result + } + return lifecycleExecutionFailure("unsupported_file_capability", "unsupported file capability", false) +} + +type fileListEntry struct { + Name string `json:"name"` + Kind string `json:"kind"` + RelativePath string `json:"relativePath"` + LogicalKey string `json:"logicalKey"` + SizeBytes int64 `json:"sizeBytes"` + ModifiedAt string `json:"modifiedAt"` +} + +type fileListEnvelope struct { + DirectoryKey string `json:"directoryKey"` + Path string `json:"path"` + Entries []fileListEntry `json:"entries"` +} + +func (executor *FileExecutor) list(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := ctx.Err(); err != nil { + return lifecycleExecutionFailure("file_cancelled", "file list cancelled", false) + } + directoryKey := assignment.TargetKey + relativePath := strings.TrimSpace(assignment.ExecutionInput.Inputs["path"]) + if relativePath == "" { + relativePath = "." + } + if relativePath != "." && (!protocol.ValidLogicalFileKey(relativePath) || strings.Contains(relativePath, string(rune(92)))) { + return lifecycleExecutionFailure("file_list_failed", "directory path is unsafe", false) + } + directory := scope + var err error + if relativePath != "." { + directory, err = executor.resolver.ExistingDirectory(scope, relativePath) + if err != nil { + return lifecycleExecutionFailure("file_list_failed", err.Error(), false) + } + } + info, err := os.Stat(directory) + if err != nil || !info.IsDir() { + return lifecycleExecutionFailure("file_list_failed", "target is not a directory", false) + } + query := strings.ToLower(strings.TrimSpace(assignment.ExecutionInput.Inputs["query"])) + recursive := strings.EqualFold(assignment.ExecutionInput.Inputs["recursive"], "true") + entries := make([]fileListEntry, 0, 32) + resultLimit := assignment.ExecutionInput.MaxReadBytes + if resultLimit <= 0 || resultLimit > maxExecutionContentBytes { + resultLimit = maxExecutionContentBytes + } + visit := func(current string, item os.DirEntry) error { + if err := ctx.Err(); err != nil { + return err + } + name := item.Name() + full := filepath.Join(current, name) + rel, err := filepath.Rel(directory, full) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if query != "" && !strings.Contains(strings.ToLower(rel), query) { + return nil + } + kind := "file" + if item.IsDir() { + kind = "directory" + } else if !item.Type().IsRegular() { + return nil + } + entryInfo, err := item.Info() + if err != nil { + return err + } + logicalKey := path.Join(directoryKey, rel) + if relativePath != "." { + logicalKey = path.Join(directoryKey, relativePath, rel) + } + candidate := append(entries, fileListEntry{Name: name, Kind: kind, RelativePath: rel, LogicalKey: logicalKey, SizeBytes: entryInfo.Size(), ModifiedAt: entryInfo.ModTime().UTC().Format(time.RFC3339Nano)}) + body, marshalErr := json.Marshal(fileListEnvelope{DirectoryKey: directoryKey, Path: strings.TrimPrefix(relativePath, "."), Entries: candidate}) + if marshalErr != nil { + return marshalErr + } + if len(body) > resultLimit { + return fmt.Errorf("file list exceeds approved read limit") + } + entries = candidate + return nil + } + if recursive { + err = filepath.WalkDir(directory, func(current string, item os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if current == directory { + return nil + } + return visit(filepath.Dir(current), item) + }) + } else { + var items []os.DirEntry + items, err = os.ReadDir(directory) + for _, item := range items { + if err == nil { + err = visit(directory, item) + } + if err != nil { + break + } + } + } + if err != nil { + if errors.Is(err, context.Canceled) { + return lifecycleExecutionFailure("file_cancelled", "file list cancelled", false) + } + return lifecycleExecutionFailure("file_list_failed", err.Error(), false) + } + body, err := json.Marshal(fileListEnvelope{DirectoryKey: directoryKey, Path: strings.TrimPrefix(relativePath, "."), Entries: entries}) + if err != nil { + return lifecycleExecutionFailure("file_list_failed", "file list encoding failed", false) + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file list completed"}, Message: "file list completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.list", SizeBytes: int64(len(body)), Content: string(body), Summary: "bounded logical file listing"}} +} + +func (executor *FileExecutor) read(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := ctx.Err(); err != nil { + return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false) + } + path, err := executor.resolver.ExistingTarget(scope, assignment.TargetKey) + if err != nil { + return lifecycleExecutionFailure("file_read_failed", err.Error(), false) + } + limit := assignment.ExecutionInput.MaxReadBytes + if limit <= 0 || limit > maxExecutionContentBytes { + limit = maxExecutionContentBytes + } + info, err := os.Stat(path) + if err != nil { + return lifecycleExecutionFailure("file_read_failed", err.Error(), false) + } + if info.Size() > int64(limit) { + return lifecycleExecutionFailure("file_read_too_large", "file exceeds approved read limit", false) + } + file, err := os.Open(path) + if err != nil { + return lifecycleExecutionFailure("file_read_failed", err.Error(), false) + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, int64(limit)+1)) + if err != nil || len(content) > limit { + return lifecycleExecutionFailure("file_read_too_large", "file exceeds approved read limit", false) + } + checksum := bytesChecksum(content) + metadata := executor.metadata(scope, assignment.TargetKey, checksum, int64(len(content))) + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file read completed"}, Message: "file read completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.read", Version: metadata.Version, Checksum: checksum, SizeBytes: int64(len(content)), Content: string(content), Summary: "bounded regular-file read"}} +} + +func (executor *FileExecutor) write(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + content := []byte(assignment.ExecutionInput.Content) + if len(content) > maxExecutionContentBytes { + return lifecycleExecutionFailure("file_write_too_large", "approved content is too large", false) + } + if err := ctx.Err(); err != nil { + return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false) + } + path, parent, err := executor.resolver.WritableTarget(scope, assignment.TargetKey) + if err != nil { + return lifecycleExecutionFailure("file_target_invalid", err.Error(), false) + } + executor.mu.Lock() + defer executor.mu.Unlock() + current, err := executor.currentMetadataLocked(scope, assignment.TargetKey, path) + if err != nil { + return lifecycleExecutionFailure("file_metadata_failed", err.Error(), false) + } + if current.Version == 0 && assignment.ExecutionInput.ExpectedVersion > 0 { + current.Version = assignment.ExecutionInput.ExpectedVersion + current.Checksum = assignment.ExecutionInput.ExpectedChecksum + } + if assignment.ExecutionInput.ExpectedVersion > 0 && current.Version != assignment.ExecutionInput.ExpectedVersion { + return lifecycleExecutionFailure("file_version_conflict", "expected version does not match current file", false) + } + if assignment.ExecutionInput.ExpectedChecksum != "" && current.Checksum != assignment.ExecutionInput.ExpectedChecksum { + return lifecycleExecutionFailure("file_checksum_conflict", "expected checksum does not match current file", false) + } + if err := ctx.Err(); err != nil { + return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false) + } + if err := ensureDirectory(parent); err != nil { + return lifecycleExecutionFailure("file_target_invalid", err.Error(), false) + } + temporary, err := os.CreateTemp(parent, ".run-write-*") + if err != nil { + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + temporaryName := temporary.Name() + defer os.Remove(temporaryName) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + if _, err := temporary.Write(content); err != nil { + _ = temporary.Close() + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + if err := temporary.Close(); err != nil { + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + if err := ctx.Err(); err != nil { + return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false) + } + if err := os.Rename(temporaryName, path); err != nil { + return lifecycleExecutionFailure("file_write_failed", err.Error(), false) + } + checksum := bytesChecksum(content) + next := FileMetadata{Scope: scope, Key: assignment.TargetKey, Version: current.Version + 1, Checksum: checksum, SizeBytes: int64(len(content)), UpdatedAt: time.Now().UTC()} + executor.records[metadataKey(scope, assignment.TargetKey)] = next + if err := executor.persistLocked(); err != nil { + return lifecycleExecutionFailure("file_metadata_failed", err.Error(), false) + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file write completed"}, Message: "file write completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.write", Version: next.Version, Checksum: checksum, SizeBytes: int64(len(content)), Summary: "atomic compare-and-swap file write"}} +} + +func (executor *FileExecutor) metadata(scope string, key string, checksum string, size int64) FileMetadata { + executor.mu.Lock() + defer executor.mu.Unlock() + item := executor.records[metadataKey(scope, key)] + if item.Version == 0 { + item = FileMetadata{Scope: scope, Key: key, Version: 1} + } + item.Checksum, item.SizeBytes, item.UpdatedAt = checksum, size, time.Now().UTC() + executor.records[metadataKey(scope, key)] = item + _ = executor.persistLocked() + return item +} + +func (executor *FileExecutor) currentMetadataLocked(scope string, key string, path string) (FileMetadata, error) { + item := executor.records[metadataKey(scope, key)] + info, err := os.Lstat(path) + if err != nil && !os.IsNotExist(err) { + return FileMetadata{}, err + } + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return FileMetadata{}, fmt.Errorf("target must be a regular file") + } + body, readErr := os.ReadFile(path) + if readErr != nil { + return FileMetadata{}, readErr + } + checksum := bytesChecksum(body) + if item.Version == 0 { + item.Version = 1 + } + item.Scope, item.Key, item.Checksum, item.SizeBytes = scope, key, checksum, int64(len(body)) + } else if item.Version == 0 { + item.Scope, item.Key = scope, key + } + return item, nil +} + +func (executor *FileExecutor) load() error { + body, err := os.ReadFile(executor.path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var file fileMetadataJournal + if err := json.Unmarshal(body, &file); err != nil { + return fmt.Errorf("decode file metadata journal: %w", err) + } + for key, item := range file.Records { + executor.records[key] = item + } + return nil +} + +func (executor *FileExecutor) persistLocked() error { + body, err := json.Marshal(fileMetadataJournal{Version: 1, Records: executor.records}) + if err != nil { + return err + } + temporary := executor.path + ".tmp" + if err := os.WriteFile(temporary, body, 0o600); err != nil { + return err + } + if err := os.Rename(temporary, executor.path); err != nil { + _ = os.Remove(temporary) + return err + } + return nil +} + +func metadataKey(scope string, key string) string { return scope + "\x00" + key } diff --git a/runtime/job_journal.go b/runtime/job_journal.go new file mode 100644 index 0000000..8a779d4 --- /dev/null +++ b/runtime/job_journal.go @@ -0,0 +1,293 @@ +package runtime + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + + "browser.local/run/protocol" +) + +const jobJournalVersion = 1 + +type JobJournal struct { + mu sync.Mutex + path string + active map[string]protocol.RunJobAssignment + pendingResults map[string]protocol.RunJobResultRequest + pendingActivations map[string]string +} + +type jobJournalFile struct { + Version int `json:"version"` + Active []protocol.RunJobAssignment `json:"active"` + PendingResults []protocol.RunJobResultRequest `json:"pendingResults,omitempty"` + PendingActivations map[string]string `json:"pendingActivations,omitempty"` +} + +func NewJobJournal() *JobJournal { + return &JobJournal{active: map[string]protocol.RunJobAssignment{}, pendingResults: map[string]protocol.RunJobResultRequest{}, pendingActivations: map[string]string{}} +} + +func NewPersistentJobJournal(workspaceRoot string) (*JobJournal, error) { + if workspaceRoot == "" { + workspaceRoot = filepath.Join(".", ".run-workspace") + } + dir := filepath.Join(workspaceRoot, "state") + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("create job journal directory: %w", err) + } + if err := os.Chmod(dir, 0o700); err != nil { + return nil, fmt.Errorf("secure job journal directory: %w", err) + } + journal := &JobJournal{path: filepath.Join(dir, "jobs.json"), active: map[string]protocol.RunJobAssignment{}, pendingResults: map[string]protocol.RunJobResultRequest{}, pendingActivations: map[string]string{}} + if err := journal.load(); err != nil { + return nil, err + } + return journal, nil +} + +func (journal *JobJournal) Store(job protocol.RunJobAssignment) error { + journal.mu.Lock() + defer journal.mu.Unlock() + if err := validateJournalAssignment(job); err != nil { + return err + } + previous, existed := journal.active[job.JobID] + journal.active[job.JobID] = job + if err := journal.persistLocked(); err != nil { + if existed { + journal.active[job.JobID] = previous + } else { + delete(journal.active, job.JobID) + } + return err + } + return nil +} + +func (journal *JobJournal) Delete(jobID string) error { + journal.mu.Lock() + defer journal.mu.Unlock() + previous, existed := journal.active[jobID] + previousResult, hadResult := journal.pendingResults[jobID] + previousActivation, hadActivation := journal.pendingActivations[jobID] + delete(journal.active, jobID) + delete(journal.pendingResults, jobID) + delete(journal.pendingActivations, jobID) + if err := journal.persistLocked(); err != nil { + if existed { + journal.active[jobID] = previous + } + if hadResult { + journal.pendingResults[jobID] = previousResult + } + if hadActivation { + journal.pendingActivations[jobID] = previousActivation + } + return err + } + return nil +} + +func (journal *JobJournal) StorePendingResult(result protocol.RunJobResultRequest, activationManifest string) error { + journal.mu.Lock() + defer journal.mu.Unlock() + assignment, exists := journal.active[result.JobID] + if !exists || assignment.Attempt != result.Attempt || assignment.LeaseToken != result.LeaseToken || assignment.RunEndpointID != result.RunEndpointID { + return fmt.Errorf("pending result does not match active journal attempt") + } + if result.State != "succeeded" && result.State != "failed" && result.State != "cancelled" { + return fmt.Errorf("pending result state is not terminal") + } + result.SessionToken = "" + previous, hadPrevious := journal.pendingResults[result.JobID] + previousActivation, hadActivation := journal.pendingActivations[result.JobID] + journal.pendingResults[result.JobID] = result + if activationManifest != "" { + journal.pendingActivations[result.JobID] = activationManifest + } else { + delete(journal.pendingActivations, result.JobID) + } + if err := journal.persistLocked(); err != nil { + if hadPrevious { + journal.pendingResults[result.JobID] = previous + } else { + delete(journal.pendingResults, result.JobID) + } + if hadActivation { + journal.pendingActivations[result.JobID] = previousActivation + } else { + delete(journal.pendingActivations, result.JobID) + } + return err + } + return nil +} + +func (journal *JobJournal) PendingActivation(jobID string) string { + journal.mu.Lock() + defer journal.mu.Unlock() + return journal.pendingActivations[jobID] +} + +func (journal *JobJournal) PendingResult(jobID string) (protocol.RunJobResultRequest, bool) { + journal.mu.Lock() + defer journal.mu.Unlock() + result, exists := journal.pendingResults[jobID] + return result, exists +} + +func (journal *JobJournal) MarkActive(job protocol.RunJobAssignment) { + _ = journal.Store(job) +} + +func (journal *JobJournal) MarkTerminal(jobID string) { + _ = journal.Delete(jobID) +} + +func (journal *JobJournal) ActiveJobs() []protocol.RunJobAssignment { + journal.mu.Lock() + defer journal.mu.Unlock() + jobs := make([]protocol.RunJobAssignment, 0, len(journal.active)) + for _, job := range journal.active { + jobs = append(jobs, job) + } + sort.Slice(jobs, func(i, j int) bool { return jobs[i].JobID < jobs[j].JobID }) + return jobs +} + +func (journal *JobJournal) ReconcileEntries() []protocol.RunJobReconcileEntry { + jobs := journal.ActiveJobs() + entries := make([]protocol.RunJobReconcileEntry, len(jobs)) + for i, job := range jobs { + entries[i] = protocol.RunJobReconcileEntry{JobID: job.JobID, LeaseToken: job.LeaseToken, Attempt: job.Attempt} + } + return entries +} + +func (journal *JobJournal) ActiveJobIDs() []string { + jobs := journal.ActiveJobs() + ids := make([]string, len(jobs)) + for i, job := range jobs { + ids[i] = job.JobID + } + return ids +} + +func (journal *JobJournal) ActiveCount() int { + journal.mu.Lock() + defer journal.mu.Unlock() + return len(journal.active) +} + +func (journal *JobJournal) load() error { + payload, err := os.ReadFile(journal.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("read job journal: %w", err) + } + var snapshot jobJournalFile + if err := json.Unmarshal(payload, &snapshot); err != nil { + return fmt.Errorf("decode job journal: %w", err) + } + if snapshot.Version != jobJournalVersion { + return fmt.Errorf("unsupported job journal version %d", snapshot.Version) + } + for _, job := range snapshot.Active { + if err := validateJournalAssignment(job); err != nil { + return fmt.Errorf("invalid job journal entry: %w", err) + } + if _, exists := journal.active[job.JobID]; exists { + return fmt.Errorf("duplicate job journal entry %q", job.JobID) + } + journal.active[job.JobID] = job + } + for _, result := range snapshot.PendingResults { + assignment, exists := journal.active[result.JobID] + if !exists || result.SessionToken != "" || assignment.Attempt != result.Attempt || assignment.LeaseToken != result.LeaseToken || assignment.RunEndpointID != result.RunEndpointID { + return fmt.Errorf("invalid pending result journal entry for %q", result.JobID) + } + if _, duplicate := journal.pendingResults[result.JobID]; duplicate { + return fmt.Errorf("duplicate pending result journal entry %q", result.JobID) + } + journal.pendingResults[result.JobID] = result + } + for jobID, manifest := range snapshot.PendingActivations { + if _, exists := journal.pendingResults[jobID]; !exists || manifest == "" { + return fmt.Errorf("invalid pending activation journal entry for %q", jobID) + } + journal.pendingActivations[jobID] = manifest + } + return nil +} + +func (journal *JobJournal) persistLocked() error { + if journal.path == "" { + return nil + } + jobs := make([]protocol.RunJobAssignment, 0, len(journal.active)) + for _, job := range journal.active { + jobs = append(jobs, job) + } + sort.Slice(jobs, func(i, j int) bool { return jobs[i].JobID < jobs[j].JobID }) + results := make([]protocol.RunJobResultRequest, 0, len(journal.pendingResults)) + for _, result := range journal.pendingResults { + result.SessionToken = "" + results = append(results, result) + } + sort.Slice(results, func(i, j int) bool { return results[i].JobID < results[j].JobID }) + activations := make(map[string]string, len(journal.pendingActivations)) + for jobID, manifest := range journal.pendingActivations { + activations[jobID] = manifest + } + payload, err := json.MarshalIndent(jobJournalFile{Version: jobJournalVersion, Active: jobs, PendingResults: results, PendingActivations: activations}, "", " ") + if err != nil { + return fmt.Errorf("encode job journal: %w", err) + } + temporary := journal.path + ".tmp" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open temporary job journal: %w", err) + } + removeTemporary := true + defer func() { + _ = file.Close() + if removeTemporary { + _ = os.Remove(temporary) + } + }() + if _, err := file.Write(payload); err != nil { + return fmt.Errorf("write job journal: %w", err) + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync job journal: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close job journal: %w", err) + } + if err := os.Rename(temporary, journal.path); err != nil { + return fmt.Errorf("replace job journal: %w", err) + } + removeTemporary = false + if err := os.Chmod(journal.path, 0o600); err != nil { + return fmt.Errorf("secure job journal: %w", err) + } + return nil +} + +func validateJournalAssignment(job protocol.RunJobAssignment) error { + if err := protocol.ValidateRunJobAssignment(job); err != nil { + return err + } + if job.Attempt <= 0 || job.LeaseToken == "" { + return fmt.Errorf("job attempt and lease token are required") + } + return nil +} diff --git a/runtime/job_journal_test.go b/runtime/job_journal_test.go new file mode 100644 index 0000000..1238c76 --- /dev/null +++ b/runtime/job_journal_test.go @@ -0,0 +1,229 @@ +package runtime + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "browser.local/run/protocol" +) + +func TestPersistentJobJournalReloadsAtomicallyWithOwnerOnlyPermissions(t *testing.T) { + root := t.TempDir() + journal, err := NewPersistentJobJournal(root) + if err != nil { + t.Fatalf("new journal: %v", err) + } + assignment := workerJobAssignment(protocol.RunCapabilityProcessStart) + if err := journal.Store(assignment); err != nil { + t.Fatalf("store assignment: %v", err) + } + + journalPath := filepath.Join(root, "state", "jobs.json") + info, err := os.Stat(journalPath) + if err != nil { + t.Fatalf("stat journal: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("expected journal mode 0600, got %o", info.Mode().Perm()) + } + dirInfo, err := os.Stat(filepath.Dir(journalPath)) + if err != nil || dirInfo.Mode().Perm() != 0o700 { + t.Fatalf("expected state directory mode 0700, info=%+v err=%v", dirInfo, err) + } + + reloaded, err := NewPersistentJobJournal(root) + if err != nil { + t.Fatalf("reload journal: %v", err) + } + if jobs := reloaded.ActiveJobs(); len(jobs) != 1 || jobs[0].JobID != assignment.JobID || jobs[0].LeaseToken != assignment.LeaseToken || jobs[0].Attempt != assignment.Attempt { + t.Fatalf("unexpected reloaded assignments: %+v", jobs) + } + if err := reloaded.Delete(assignment.JobID); err != nil { + t.Fatalf("delete assignment: %v", err) + } + third, err := NewPersistentJobJournal(root) + if err != nil || third.ActiveCount() != 0 { + t.Fatalf("terminal delete did not persist: count=%d err=%v", third.ActiveCount(), err) + } +} + +func TestPersistentJobJournalRejectsCorruptState(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "state") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("create state dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "jobs.json"), []byte(`{"version":1,"active":[`), 0o600); err != nil { + t.Fatalf("write corrupt journal: %v", err) + } + if _, err := NewPersistentJobJournal(root); err == nil { + t.Fatal("expected corrupt journal to fail closed") + } +} + +func TestWorkerRestartReconcilesAndRecoversConfirmedAttempt(t *testing.T) { + cfg := workerTestConfig(t) + assignment := workerJobAssignment(protocol.RunCapabilityProcessStart) + assignment.ProgressSequence = 5 + journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot) + if err != nil { + t.Fatalf("new journal: %v", err) + } + if err := journal.Store(assignment); err != nil { + t.Fatalf("store interrupted assignment: %v", err) + } + + client := newFakeWorkerClient() + client.claimJob = assignment + client.reconcileResponse = protocol.RunJobReconcileResponse{ + Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{assignment}, ServerTime: workerTestTime(), + } + restarted, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "recovered\n"})) + if err != nil { + t.Fatalf("restart worker: %v", err) + } + if err := restarted.Register(context.Background()); err != nil { + t.Fatalf("register restarted worker: %v", err) + } + if err := restarted.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile restarted worker: %v", err) + } + if len(client.reconcileRequests) != 1 || !reflect.DeepEqual(client.reconcileRequests[0].ActiveJobs, []protocol.RunJobReconcileEntry{{JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt}}) { + t.Fatalf("reconcile omitted attempt evidence: %+v", client.reconcileRequests) + } + if err := restarted.RecoverActiveJobs(context.Background()); err != nil { + t.Fatalf("recover confirmed assignment: %v", err) + } + if restarted.journal.ActiveCount() != 0 || len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || client.progressRequests[0].Sequence != 6 || len(client.resultRequests) != 1 { + t.Fatalf("confirmed attempt was not recovered: journal=%d ack=%d result=%d", restarted.journal.ActiveCount(), len(client.ackRequests), len(client.resultRequests)) + } +} + +func TestWorkerReconcileDiscardsStaleAttemptWithoutExecuting(t *testing.T) { + cfg := workerTestConfig(t) + assignment := workerJobAssignment(protocol.RunCapabilityProcessStart) + journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot) + if err != nil { + t.Fatalf("new journal: %v", err) + } + if err := journal.Store(assignment); err != nil { + t.Fatalf("store assignment: %v", err) + } + client := newFakeWorkerClient() + client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, DiscardJobIDs: []string{assignment.JobID}, ServerTime: workerTestTime()} + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if err := worker.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if worker.journal.ActiveCount() != 0 || len(client.ackRequests) != 0 { + t.Fatalf("stale assignment was not discarded safely: journal=%d ack=%d", worker.journal.ActiveCount(), len(client.ackRequests)) + } +} + +func TestWorkerRetainsJournalWhenPlatformRejectsResultTransport(t *testing.T) { + cfg := workerTestConfig(t) + client := newFakeWorkerClient() + client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart) + client.resultErr = context.DeadlineExceeded + worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "completed locally\n"})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if handled, err := worker.ClaimAndRunOnce(context.Background()); !handled || err == nil { + t.Fatalf("expected retained failed result transport, handled=%v err=%v", handled, err) + } + if worker.journal.ActiveCount() != 1 { + t.Fatalf("result transport failure removed journal entry") + } + reloaded, err := NewPersistentJobJournal(cfg.WorkspaceRoot) + if err != nil { + t.Fatalf("reload retained journal: %v", err) + } + if reloaded.ActiveCount() != 1 { + t.Fatalf("retained journal did not survive restart: count=%d", reloaded.ActiveCount()) + } + if pending, ok := reloaded.PendingResult(client.claimJob.JobID); !ok || pending.SessionToken != "" || pending.State != "succeeded" { + t.Fatalf("pending terminal result was not retained safely: result=%+v ok=%v", pending, ok) + } + payload, err := os.ReadFile(filepath.Join(cfg.WorkspaceRoot, "state", "jobs.json")) + if err != nil { + t.Fatalf("read retained journal: %v", err) + } + if strings.Contains(string(payload), "session-token") { + t.Fatalf("journal persisted raw Run session token: %s", payload) + } + + client.resultErr = nil + client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{client.claimJob}, ServerTime: workerTestTime()} + restarted, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("restart result worker: %v", err) + } + if err := restarted.Register(context.Background()); err != nil { + t.Fatalf("register result worker: %v", err) + } + if err := restarted.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile result worker: %v", err) + } + ackCount := len(client.ackRequests) + if err := restarted.RecoverActiveJobs(context.Background()); err != nil { + t.Fatalf("replay pending result: %v", err) + } + if len(client.ackRequests) != ackCount || restarted.journal.ActiveCount() != 0 { + t.Fatalf("pending result replay re-executed work: ack before=%d after=%d journal=%d", ackCount, len(client.ackRequests), restarted.journal.ActiveCount()) + } +} + +func TestWorkerRecoversAcceptedSelfUpdateResultAndActivatesOnce(t *testing.T) { + cfg := workerTestConfig(t) + client := newFakeWorkerClient() + assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate) + assignment.TargetKey = "run/update" + assignment.InputRef = "artifact://artifact-run-recovery" + client.claimJob = assignment + client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{assignment}, ServerTime: workerTestTime()} + + journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot) + if err != nil { + t.Fatalf("create self-update journal: %v", err) + } + if err := journal.Store(assignment); err != nil { + t.Fatalf("store self-update assignment: %v", err) + } + pending := protocol.RunJobResultRequest{RunEndpointID: assignment.RunEndpointID, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update staged"}, ResultRef: "artifact://jobs/run-update/staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: bytesChecksum([]byte("archive")), Summary: "verified update staged"}} + manifestPath := filepath.Join(cfg.WorkspaceRoot, "self-updates", assignment.JobID, "manifest.json") + if err := journal.StorePendingResult(pending, manifestPath); err != nil { + t.Fatalf("store pending self-update result: %v", err) + } + + activator := &recordingSelfUpdateActivator{} + restarted, err := NewWorker(cfg, client, WithSelfUpdateActivator(activator)) + if err != nil { + t.Fatalf("restart worker: %v", err) + } + if err := restarted.Register(context.Background()); err != nil { + t.Fatalf("register restarted worker: %v", err) + } + if err := restarted.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile restarted worker: %v", err) + } + if err := restarted.RecoverActiveJobs(context.Background()); err != nil { + t.Fatalf("recover accepted self-update result: %v", err) + } + if activator.manifestPath != manifestPath || restarted.journal.ActiveCount() != 0 || restarted.journal.PendingActivation(assignment.JobID) != "" { + t.Fatalf("self-update activation was not recovered exactly once: path=%q active=%d pending=%q", activator.manifestPath, restarted.journal.ActiveCount(), restarted.journal.PendingActivation(assignment.JobID)) + } +} diff --git a/runtime/lifecycle.go b/runtime/lifecycle.go new file mode 100644 index 0000000..cf92abc --- /dev/null +++ b/runtime/lifecycle.go @@ -0,0 +1,1248 @@ +package runtime + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "browser.local/run/protocol" +) + +const ( + lifecycleResultStateSucceeded = "succeeded" + lifecycleResultStateFailed = "failed" + lifecycleResultStateCancelled = "cancelled" + + defaultLifecycleTimeout = 30 * time.Second + maxLifecycleTimeout = 2 * time.Hour + maxLifecycleOutputBytes = 4096 +) + +var ( + commandNamePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + envNamePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,63}$`) + disallowedExecutables = map[string]struct{}{ + "bash": {}, + "cmd": {}, + "fish": {}, + "powershell": {}, + "pwsh": {}, + "sh": {}, + "zsh": {}, + } +) + +type LifecycleExecutor struct { + workspaceRoot string + managedProcessStateRoot string + managedProcessOutputRoot string + supervisor ProcessSupervisor + managed ManagedProcessSupervisor + fileExecutor *FileExecutor + logSink ProcessLogSink + artifactHook LifecycleArtifactHook + dependencyRunner ProcessSupervisor + dependencyDownloader DependencyDownloader + selfUpdateActivator SelfUpdateActivator + logCheckpointStore LogCheckpointStore + runtimeTargetOS string + runtimeTargetArch string + runtimeFileWriter runtimeFileWriter + localStartupDiagnostics bool + protectedRequests *ProtectedRequestRegistry + sqliteSchemaProbe *SQLiteSchemaProbeExecutor + metricCollector MetricCollector +} + +type runtimeFileWriter func(string, []byte, os.FileMode) error + +type LifecycleExecutionResult struct { + State string + Progress protocol.RunJobProgressReport + ResultRef string + Message string + ErrorCode string + Retryable bool + ExecutionResult protocol.RunJobExecutionResult + ActivationManifest string +} + +type LifecycleExecutorOption func(*LifecycleExecutor) + +func NewLifecycleExecutor(options ...LifecycleExecutorOption) LifecycleExecutor { + executor := LifecycleExecutor{ + workspaceRoot: filepath.Join(".", ".run-workspace"), + supervisor: OSProcessSupervisor{}, + logSink: NoopProcessLogSink{}, + artifactHook: StaticLifecycleArtifactHook{}, + dependencyRunner: OSProcessSupervisor{}, + dependencyDownloader: HTTPDependencyDownloader{}, + selfUpdateActivator: ProcessSelfUpdateActivator{}, + logCheckpointStore: NewMemoryLogCheckpointStore(), + runtimeTargetOS: runtime.GOOS, + runtimeTargetArch: runtime.GOARCH, + runtimeFileWriter: writeRuntimeAtomicFile, + protectedRequests: NewProtectedRequestRegistry(), + } + for _, option := range options { + option(&executor) + } + if executor.managedProcessStateRoot == "" { + executor.managedProcessStateRoot = executor.workspaceRoot + } + if executor.managedProcessOutputRoot == "" { + executor.managedProcessOutputRoot = executor.workspaceRoot + } + if executor.managed == nil { + if managed, err := NewOSManagedProcessSupervisorWithOutputRoot(executor.managedProcessStateRoot, executor.managedProcessOutputRoot); err == nil { + executor.managed = managed + } + } + if files, err := NewFileExecutor(executor.workspaceRoot); err == nil { + executor.fileExecutor = files + } + executor.sqliteSchemaProbe = NewSQLiteSchemaProbeExecutor(executor.workspaceRoot) + return executor +} + +func (executor LifecycleExecutor) ExecuteLogBackfill(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if assignment.ExecutionInput.LogSource == nil { + return ExecuteLogBackfillJob(ctx, assignment) + } + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_log_backfill_job", err.Error()) + } + source := RuntimeLogSource{ + Key: assignment.ExecutionInput.LogSource.Key, + Kind: assignment.ExecutionInput.LogSource.Kind, + TargetKey: assignment.ExecutionInput.LogSource.TargetKey, + StreamKey: assignment.ExecutionInput.LogSource.StreamKey, + CursorKind: assignment.ExecutionInput.LogSource.CursorKind, + RetentionDays: assignment.ExecutionInput.LogSource.RetentionDays, + } + return TailDeclaredFileLogSource(ctx, executor.workspaceRoot, assignment, source, executor.logSink, executor.logCheckpointStore) +} + +func (executor LifecycleExecutor) writeRuntimeFile(path string, body []byte, mode os.FileMode) error { + writer := executor.runtimeFileWriter + if writer == nil { + writer = writeRuntimeAtomicFile + } + return writer(path, body, mode) +} + +func WithLifecycleWorkspaceRoot(root string) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if strings.TrimSpace(root) != "" { + executor.workspaceRoot = root + } + } +} + +func WithManagedProcessStateRoot(root string) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if strings.TrimSpace(root) != "" { + executor.managedProcessStateRoot = root + } + } +} + +func WithManagedProcessOutputRoot(root string) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if strings.TrimSpace(root) != "" { + executor.managedProcessOutputRoot = root + } + } +} + +func WithLocalStartupDiagnostics(enabled bool) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { executor.localStartupDiagnostics = enabled } +} + +// WithProtectedRequestRegistry supplies Run-owned handlers for logical +// transports. It does not expose handler configuration through any protocol. +func WithProtectedRequestRegistry(registry *ProtectedRequestRegistry) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if registry != nil { + executor.protectedRequests = registry + } + } +} + +func WithProcessSupervisor(supervisor ProcessSupervisor) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if supervisor != nil { + executor.supervisor = supervisor + } + } +} + +func WithManagedProcessSupervisor(supervisor ManagedProcessSupervisor) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if supervisor != nil { + executor.managed = supervisor + } + } +} + +func WithProcessLogSink(sink ProcessLogSink) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if sink != nil { + executor.logSink = sink + } + } +} + +func WithLifecycleArtifactHook(hook LifecycleArtifactHook) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if hook != nil { + executor.artifactHook = hook + } + } +} + +func WithDependencyCommandRunner(runner ProcessSupervisor) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if runner != nil { + executor.dependencyRunner = runner + } + } +} + +func WithDependencyDownloader(downloader DependencyDownloader) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if downloader != nil { + executor.dependencyDownloader = downloader + } + } +} + +func WithSelfUpdateActivator(activator SelfUpdateActivator) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if activator != nil { + executor.selfUpdateActivator = activator + } + } +} + +// WithDLLExtensionRuntimeTarget is primarily useful for exercising the +// Windows-only extension path in cross-platform tests. Production workers use +// the current Go runtime target. +func WithDLLExtensionRuntimeTarget(targetOS string, targetArch string) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + if strings.TrimSpace(targetOS) != "" { + executor.runtimeTargetOS = strings.ToLower(strings.TrimSpace(targetOS)) + } + if strings.TrimSpace(targetArch) != "" { + executor.runtimeTargetArch = strings.ToLower(strings.TrimSpace(targetArch)) + } + } +} + +func SupportedLifecycleCapabilities() []string { + return []string{ + protocol.RunCapabilityProcessInstall, + protocol.RunCapabilityProcessStart, + protocol.RunCapabilityProcessStop, + protocol.RunCapabilityProcessStatus, + } +} + +func SupportedFileCapabilities() []string { + return []string{protocol.RunCapabilityConfigWrite, protocol.RunCapabilityFilesList, protocol.RunCapabilityFilesRead, protocol.RunCapabilityFilesWrite} +} + +func SupportedRunCapabilities() []string { + return SupportedRunCapabilitiesForComponent("") +} + +// SupportedRunCapabilitiesForComponent keeps a generic worker capable of +// building distributions while ensuring a generated server Run cannot claim +// a shared build-worker endpoint after it is deployed. +func SupportedRunCapabilitiesForComponent(componentKind string) []string { + capabilities := append([]string(nil), SupportedLifecycleCapabilities()...) + capabilities = append(capabilities, SupportedFileCapabilities()...) + capabilities = append(capabilities, protocol.RunCapabilityLogsRead) + capabilities = append(capabilities, protocol.RunCapabilityDeploymentPlan) + for _, capability := range SupportedDistributionCapabilities() { + if componentKind == "run" && capability == protocol.RunCapabilityDistributionBuild { + continue + } + capabilities = append(capabilities, capability) + } + capabilities = append(capabilities, SupportedRemoteCapabilities()...) + return capabilities +} + +func SupportedRemoteCapabilities() []string { + return []string{ + protocol.RunCapabilityRemoteFTPRead, + protocol.RunCapabilityRemoteFTPWrite, + protocol.RunCapabilityRemoteRsyncRead, + protocol.RunCapabilityRemoteRsyncWrite, + protocol.RunCapabilityRemoteRunFilesRead, + protocol.RunCapabilityRemoteRunFilesWrite, + protocol.RunCapabilityRemoteRunProcessStart, + protocol.RunCapabilityRemoteRunProcessStop, + protocol.RunCapabilityRemoteRunDBMySQLQuery, + protocol.RunCapabilityRemoteRunDBSQLiteProbe, + protocol.RunCapabilityRemoteRunDBSQLiteQuery, + protocol.RunCapabilityRemoteRunLogsTransfer, + protocol.RunCapabilityRemoteRunRCONCommand, + protocol.RunCapabilityRemoteRunProtectedSQL, + protocol.RunCapabilityRemoteRunProtectedRCON, + protocol.RunCapabilityRemoteRunProgram, + } +} + +func (executor LifecycleExecutor) SupportedCapabilities() []string { + return SupportedLifecycleCapabilities() +} + +func (executor LifecycleExecutor) Execute(assignment protocol.RunJobAssignment) LifecycleExecutionResult { + return executor.ExecuteContext(context.Background(), assignment) +} + +func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + log.Printf("RUN phase=lifecycle status=starting job=%s capability=%s target=%s server=%s workspaceScope=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), assignment.ServerInstanceID, safeOptional(assignment.ExecutionInput.WorkspaceScope)) + if (assignment.Capability == protocol.RunCapabilityConfigWrite || assignment.Capability == protocol.RunCapabilityFilesList || assignment.Capability == protocol.RunCapabilityFilesRead || assignment.Capability == protocol.RunCapabilityFilesWrite) && assignment.ExecutionInput.WorkspaceScope != "" { + if executor.fileExecutor == nil { + log.Printf("RUN phase=lifecycle status=file_executor_unavailable job=%s", assignment.JobID) + return lifecycleFailure("file_executor_unavailable", "file executor is unavailable") + } + log.Printf("RUN phase=lifecycle status=file_executor_start job=%s capability=%s", assignment.JobID, assignment.Capability) + return executor.fileExecutor.Execute(ctx, assignment) + } + if !isSupportedLifecycleCapability(assignment.Capability) { + log.Printf("RUN phase=lifecycle status=unsupported_capability job=%s capability=%s", assignment.JobID, assignment.Capability) + return lifecycleFailure("unsupported_lifecycle_capability", "unsupported lifecycle capability") + } + if len(assignment.ExecutionInput.DLLExtensions) > 0 { + log.Printf("RUN phase=lifecycle.dll status=validating job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions)) + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + log.Printf("RUN phase=lifecycle.dll status=invalid job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("unsafe_dll_extension_plan", "DLL extension plan is invalid") + } + } + if assignment.ExecutionInput.ServerDeploymentPlan != nil { + log.Printf("RUN phase=lifecycle status=legacy_deployment_rejected job=%s", assignment.JobID) + return lifecycleFailure("unsupported_legacy_deployment_plan", "game-specific deployment plans must be implemented by plugin lifecycle actions") + } + if assignment.ExecutionInput.Deployment != nil && assignment.ExecutionInput.Deployment.Mode == "custom-command" { + log.Printf("RUN phase=lifecycle status=custom_deployment job=%s", assignment.JobID) + return executor.executeDeployment(ctx, assignment) + } + log.Printf("RUN phase=lifecycle.template status=loading job=%s target=%s", assignment.JobID, safeOptional(assignment.TargetKey)) + template, scope, err := executor.loadLifecycleTemplate(assignment) + if err != nil { + log.Printf("RUN phase=lifecycle.template status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("unsafe_lifecycle_command", err.Error()) + } + log.Printf("RUN phase=lifecycle.template status=loaded job=%s action=%s mode=%s scope=%s commandArgs=%d envKeys=%s", assignment.JobID, safeOptional(template.Action), safeOptional(template.Mode), scope, len(template.Command)+len(template.Arguments), envKeysSummary(template.Env, template.Environment)) + if template.Action != "" { + expectedAction := map[string]string{protocol.RunCapabilityProcessInstall: "install", protocol.RunCapabilityProcessStart: "start", protocol.RunCapabilityProcessStop: "stop", protocol.RunCapabilityProcessStatus: "status"}[assignment.Capability] + if template.Action != expectedAction { + log.Printf("RUN phase=lifecycle.template status=action_mismatch job=%s expected=%s actual=%s", assignment.JobID, expectedAction, template.Action) + return lifecycleFailure("unsafe_lifecycle_command", "typed lifecycle action does not match capability") + } + if assignment.Capability == protocol.RunCapabilityProcessStart && template.Mode != "supervised" { + log.Printf("RUN phase=lifecycle.template status=mode_mismatch job=%s expected=supervised actual=%s", assignment.JobID, template.Mode) + return lifecycleFailure("unsafe_lifecycle_command", "typed start action must be supervised") + } + if (assignment.Capability == protocol.RunCapabilityProcessStop || assignment.Capability == protocol.RunCapabilityProcessStatus) && template.Mode != "control" { + log.Printf("RUN phase=lifecycle.template status=mode_mismatch job=%s expected=control actual=%s", assignment.JobID, template.Mode) + return lifecycleFailure("unsafe_lifecycle_command", "typed control action must use control mode") + } + } + if assignment.Capability == protocol.RunCapabilityProcessStart && len(assignment.ExecutionInput.DLLExtensions) > 0 { + log.Printf("RUN phase=lifecycle.dll status=synchronizing job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions)) + if err := executor.synchronizeUE4SSDLLExtensions(ctx, assignment, template, scope); err != nil { + log.Printf("RUN phase=lifecycle.dll status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return dllExtensionLifecycleFailure(err) + } + log.Printf("RUN phase=lifecycle.dll status=complete job=%s", assignment.JobID) + } + if executor.managed != nil && template.Action != "" && (assignment.Capability == protocol.RunCapabilityProcessStart || assignment.Capability == protocol.RunCapabilityProcessStop || assignment.Capability == protocol.RunCapabilityProcessStatus) { + log.Printf("RUN phase=lifecycle.managed status=dispatch job=%s action=%s mode=%s", assignment.JobID, template.Action, template.Mode) + return executor.executeManaged(ctx, assignment, template, scope) + } + log.Printf("RUN phase=lifecycle.command status=building job=%s", assignment.JobID) + command, err := template.ToProcessCommand(scope, NewWorkspaceResolver(executor.workspaceRoot), assignment) + if err != nil { + log.Printf("RUN phase=lifecycle.command status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("unsafe_lifecycle_command", err.Error()) + } + log.Printf("RUN phase=lifecycle.command status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil)) + result, err := executor.supervisor.Run(ctx, command) + if err != nil && ctx.Err() != nil { + log.Printf("RUN phase=lifecycle.command status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error())) + return LifecycleExecutionResult{ + State: lifecycleResultStateCancelled, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action cancelled"}, + Message: "lifecycle action cancelled", + ErrorCode: "lifecycle_cancelled", + } + } + executor.writeProcessLogs(ctx, assignment, result) + if err != nil { + log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error())) + return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error())) + } + if result.ExitCode != 0 { + log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d", assignment.JobID, result.ExitCode) + return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode)) + } + log.Printf("RUN phase=lifecycle.command status=exited job=%s exitCode=%d stdoutBytes=%d stderrBytes=%d", assignment.JobID, result.ExitCode, len(result.Stdout), len(result.Stderr)) + artifactRef, err := executor.artifactHook.QueueLifecycleResult(ctx, assignment, result) + if err != nil { + log.Printf("RUN phase=lifecycle.artifact status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("lifecycle_artifact_hook_failed", err.Error()) + } + log.Printf("RUN phase=lifecycle status=succeeded job=%s resultRef=%s", assignment.JobID, safeOptional(artifactRef)) + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action completed"}, + ResultRef: artifactRef, + Message: fmt.Sprintf("%s completed", assignment.Capability), + } +} + +func (executor LifecycleExecutor) executeDeployment(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + definition := assignment.ExecutionInput.Deployment + log.Printf("RUN phase=deployment status=starting job=%s mode=%s revision=%d", assignment.JobID, definition.Mode, definition.Revision) + if definition.SchemaVersion != "1" || definition.Revision < 1 || definition.Mode == "" { + log.Printf("RUN phase=deployment status=invalid job=%s", assignment.JobID) + return lifecycleFailure("invalid_deployment_definition", "deployment definition is invalid") + } + action := map[string]string{protocol.RunCapabilityProcessInstall: "install", protocol.RunCapabilityProcessStart: "start", protocol.RunCapabilityProcessStop: "stop", protocol.RunCapabilityProcessStatus: "status"}[assignment.Capability] + if action == "" { + log.Printf("RUN phase=deployment status=invalid_action job=%s capability=%s", assignment.JobID, assignment.Capability) + return lifecycleFailure("invalid_deployment_action", "deployment action is invalid") + } + commandText := map[string]string{"start": definition.StartCommand, "stop": definition.StopCommand, "status": definition.StatusCommand}[action] + if action == "install" { + commandText = definition.StartCommand + } + if commandText == "" { + log.Printf("RUN phase=deployment status=missing_command job=%s action=%s", assignment.JobID, action) + return lifecycleFailure("deployment_command_missing", "deployment command is not configured") + } + workdir := definition.WorkingDirectory + if workdir == "" { + workdir = definition.ServerRoot + } + if workdir == "" { + log.Printf("RUN phase=deployment status=missing_workdir job=%s action=%s", assignment.JobID, action) + return lifecycleFailure("deployment_workdir_missing", "deployment working directory is not configured") + } + args := strings.Fields(commandText) + if len(args) == 0 { + log.Printf("RUN phase=deployment status=invalid_command job=%s", assignment.JobID) + return lifecycleFailure("deployment_command_invalid", "deployment command is invalid") + } + if definition.Shell != "" { + log.Printf("RUN phase=deployment status=unsupported_shell job=%s shell=%s", assignment.JobID, definition.Shell) + return lifecycleFailure("deployment_shell_unsupported", "deployment shell is not supported by this Run") + } + command := ProcessCommand{Args: args, WorkDir: workdir, JobID: assignment.JobID, Capability: assignment.Capability, Action: action} + log.Printf("RUN phase=deployment.command status=starting job=%s action=%s revision=%d root=%s workdir=%s command=%s", assignment.JobID, action, definition.Revision, safeOptional(definition.ServerRoot), safeOptional(workdir), redactedCommandLine(command.Args)) + result, err := executor.supervisor.Run(ctx, command) + if err != nil { + log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error())) + return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error())) + } + executor.writeProcessLogs(ctx, assignment, result) + if result.ExitCode != 0 { + log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d", assignment.JobID, result.ExitCode) + return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode)) + } + log.Printf("RUN phase=deployment.command status=exited job=%s exitCode=%d stdoutBytes=%d stderrBytes=%d", assignment.JobID, result.ExitCode, len(result.Stdout), len(result.Stderr)) + receipt := &protocol.ServerDeploymentExecutionReceipt{SchemaVersion: "1", Revision: definition.Revision, Action: action, Mode: definition.Mode, Shell: definition.Shell, UsedServerRoot: definition.ServerRoot != ""} + log.Printf("RUN phase=deployment status=succeeded job=%s action=%s revision=%d", assignment.JobID, action, definition.Revision) + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "deployment lifecycle action completed"}, Message: fmt.Sprintf("%s completed", assignment.Capability), ExecutionResult: protocol.RunJobExecutionResult{Kind: "deployment.lifecycle", Summary: "deployment revision confirmed", DeploymentReceipt: receipt}} +} + +func (executor LifecycleExecutor) ResolveCommand(assignment protocol.RunJobAssignment) (ProcessCommand, error) { + workdir, err := scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID) + if err != nil { + return ProcessCommand{}, err + } + if err := os.MkdirAll(workdir, 0o755); err != nil { + return ProcessCommand{}, fmt.Errorf("create scoped workspace: %w", err) + } + template := LifecycleActionTemplate{ + Command: []string{"true"}, + TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond), + } + if assignment.TargetKey != "" { + path, err := scopedPath(workdir, assignment.TargetKey) + if err != nil { + return ProcessCommand{}, err + } + file, err := os.Open(path) + if err != nil { + return ProcessCommand{}, fmt.Errorf("open lifecycle action template: %w", err) + } + decodeErr := json.NewDecoder(file).Decode(&template) + closeErr := file.Close() + if decodeErr != nil { + return ProcessCommand{}, fmt.Errorf("decode lifecycle action template: %w", decodeErr) + } + if closeErr != nil { + return ProcessCommand{}, fmt.Errorf("close lifecycle action template: %w", closeErr) + } + } + return template.ToProcessCommand(workdir, NewWorkspaceResolver(executor.workspaceRoot), assignment) +} + +func (executor LifecycleExecutor) writeProcessLogs(ctx context.Context, assignment protocol.RunJobAssignment, result ProcessResult) { + for _, item := range []struct { + stream string + body string + }{ + {stream: "stdout", body: result.Stdout}, + {stream: "stderr", body: result.Stderr}, + } { + for _, line := range splitBoundedLines(item.body) { + _ = executor.logSink.Append(ctx, assignment, item.stream, line) + } + } +} + +type LifecycleActionTemplate struct { + Version int `json:"version,omitempty"` + Action string `json:"action,omitempty"` + Mode string `json:"mode,omitempty"` + ExecutableKey string `json:"executableKey,omitempty"` + TargetExecutableKey string `json:"targetExecutableKey,omitempty"` + Arguments []string `json:"arguments,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + OutputMode string `json:"outputMode,omitempty"` + StopTimeoutMS int `json:"stopTimeoutMs,omitempty"` + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` + TimeoutMS int `json:"timeoutMs,omitempty"` +} + +func (executor LifecycleExecutor) loadLifecycleTemplate(assignment protocol.RunJobAssignment) (LifecycleActionTemplate, string, error) { + scope, err := executor.lifecycleScope(assignment) + if err != nil { + return LifecycleActionTemplate{}, "", err + } + template := LifecycleActionTemplate{Command: []string{"true"}, TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond)} + if assignment.TargetKey == "" { + if err := os.MkdirAll(scope, 0o755); err != nil { + return LifecycleActionTemplate{}, "", fmt.Errorf("create lifecycle workspace: %w", err) + } + return template, scope, nil + } + path, err := NewWorkspaceResolver(executor.workspaceRoot).ExistingTarget(scope, assignment.TargetKey) + if err != nil { + return LifecycleActionTemplate{}, "", fmt.Errorf("open lifecycle action template: %w", err) + } + file, err := os.Open(path) + if err != nil { + return LifecycleActionTemplate{}, "", fmt.Errorf("open lifecycle action template: %w", err) + } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, 16*1024)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&template); err != nil { + return LifecycleActionTemplate{}, "", fmt.Errorf("decode lifecycle action template: %w", err) + } + return template, scope, nil +} + +func (executor LifecycleExecutor) lifecycleScope(assignment protocol.RunJobAssignment) (string, error) { + if assignment.ExecutionInput.WorkspaceScope != "" { + return NewWorkspaceResolver(executor.workspaceRoot).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + } + return scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID) +} + +func (executor LifecycleExecutor) executeManaged(ctx context.Context, assignment protocol.RunJobAssignment, template LifecycleActionTemplate, scope string) LifecycleExecutionResult { + resolver := NewWorkspaceResolver(executor.workspaceRoot) + identity := ProcessIdentity{Scope: scope, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, JobID: assignment.JobID, Capability: assignment.Capability, ProfileKey: assignment.ExecutionInput.WorkspaceScope, Attempt: assignment.Attempt, StdoutStreamKey: processLogStreamKey(assignment, "process.stdout", "stdout"), StderrStreamKey: processLogStreamKey(assignment, "process.stderr", "stderr")} + if assignment.Capability == protocol.RunCapabilityProcessStart { + log.Printf("RUN phase=lifecycle.managed status=building_start_command job=%s scope=%s", assignment.JobID, scope) + command, err := template.ToManagedProcessCommand(resolver, scope, assignment) + if err != nil { + log.Printf("RUN phase=lifecycle.managed status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("unsafe_lifecycle_command", err.Error()) + } + log.Printf("RUN phase=lifecycle.managed status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil)) + item, err := executor.managed.Start(ctx, command, identity, executor.managedProcessOutput(ctx, assignment)) + if err != nil { + if ctx.Err() != nil { + log.Printf("RUN phase=lifecycle.managed status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error())) + return lifecycleExecutionFailure("lifecycle_cancelled", "lifecycle action cancelled", false) + } + log.Printf("RUN phase=lifecycle.managed status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error())) + } + log.Printf("RUN phase=lifecycle.managed status=started job=%s pid=%d state=%s stdoutRef=%s stderrRef=%s", assignment.JobID, item.PID, item.State, safeOptional(item.StdoutLogRef), safeOptional(item.StderrLogRef)) + return processExecutionResult(item, "process started") + } + if assignment.Capability == protocol.RunCapabilityProcessStop { + log.Printf("RUN phase=lifecycle.managed status=stopping job=%s scope=%s", assignment.JobID, scope) + item, err := executor.managed.Stop(ctx, identity) + if err != nil { + log.Printf("RUN phase=lifecycle.managed status=stop_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return lifecycleFailure("lifecycle_stop_failed", RedactText(err.Error())) + } + log.Printf("RUN phase=lifecycle.managed status=stopped job=%s pid=%d state=%s classification=%s", assignment.JobID, item.PID, item.State, safeOptional(item.ExitClassification)) + return processExecutionResult(item, "process stopped") + } + log.Printf("RUN phase=lifecycle.managed status=querying job=%s scope=%s", assignment.JobID, scope) + item := executor.managed.Status(identity) + log.Printf("RUN phase=lifecycle.managed status=queried job=%s pid=%d state=%s classification=%s", assignment.JobID, item.PID, item.State, safeOptional(item.ExitClassification)) + return processExecutionResult(item, "process status queried") +} + +func (executor LifecycleExecutor) ResumeManagedProcessLogs(ctx context.Context) { + if executor.managed == nil { + return + } + executor.managed.ResumeOutput(executor.managedProcessOutput(ctx, protocol.RunJobAssignment{})) +} + +func (executor LifecycleExecutor) managedProcessOutput(ctx context.Context, assignment protocol.RunJobAssignment) ManagedProcessOutput { + return ManagedProcessOutput{ + Stdout: func(identity ProcessIdentity, line ManagedProcessLine) error { + return executor.appendManagedProcessLog(ctx, assignment, identity, "stdout", line) + }, + Stderr: func(identity ProcessIdentity, line ManagedProcessLine) error { + return executor.appendManagedProcessLog(ctx, assignment, identity, "stderr", line) + }, + } +} + +func (executor LifecycleExecutor) appendManagedProcessLog(ctx context.Context, assignment protocol.RunJobAssignment, identity ProcessIdentity, stream string, line ManagedProcessLine) error { + if executor.logSink == nil { + return nil + } + if assignment.JobID == "" { + assignment = assignmentFromProcessIdentity(identity) + } + assignment.LogSessionID = identity.LogSessionID + assignment.SessionStartedAt = identity.StartedAt + if assignment.JobID == "" || assignment.ServerInstanceID == "" { + return nil + } + if sink, ok := executor.logSink.(ProcessLogCursorSink); ok { + return sink.AppendWithCursor(ctx, assignment, stream, line.Text, ProcessLogCursor{StartOffset: line.StartOffset, EndOffset: line.EndOffset}) + } + return executor.logSink.Append(ctx, assignment, stream, line.Text) +} + +func assignmentFromProcessIdentity(identity ProcessIdentity) protocol.RunJobAssignment { + assignment := protocol.RunJobAssignment{ + JobID: identity.JobID, + ServerInstanceID: identity.ServerInstanceID, + RunEndpointID: identity.RunEndpointID, + Capability: identity.Capability, + Attempt: identity.Attempt, + LogSessionID: identity.LogSessionID, + SessionStartedAt: identity.StartedAt, + } + if identity.StdoutStreamKey != "" { + assignment.ExecutionInput.LogSources = append(assignment.ExecutionInput.LogSources, protocol.RuntimeLogSourcePlan{Key: "process-stdout", Kind: "process.stdout", StreamKey: identity.StdoutStreamKey, CursorKind: "sequence"}) + } + if identity.StderrStreamKey != "" { + assignment.ExecutionInput.LogSources = append(assignment.ExecutionInput.LogSources, protocol.RuntimeLogSourcePlan{Key: "process-stderr", Kind: "process.stderr", StreamKey: identity.StderrStreamKey, CursorKind: "sequence"}) + } + return assignment +} + +func processLogStreamKey(assignment protocol.RunJobAssignment, kind string, fallback string) string { + for _, source := range assignment.ExecutionInput.LogSources { + if source.Kind == kind && strings.TrimSpace(source.StreamKey) != "" { + return source.StreamKey + } + } + return fallback +} + +func processExecutionResult(item ProcessIdentity, message string) LifecycleExecutionResult { + state := item.State + if state == "" { + state = "stopped" + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, Message: message, ExecutionResult: protocol.RunJobExecutionResult{Kind: "process", ProcessState: state, ExitClassification: item.ExitClassification, ExitCode: item.ExitCode, Summary: "private supervised process identity"}} +} + +func lifecycleExecutionFailure(code string, message string, retryable bool) LifecycleExecutionResult { + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)}, Message: RedactText(message), ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "file", Summary: code}} +} + +func (template LifecycleActionTemplate) ToProcessCommand(workdir string, resolver WorkspaceResolver, assignment protocol.RunJobAssignment) (ProcessCommand, error) { + if template.ExecutableKey != "" { + return template.ToExecutableProcessCommand(resolver, workdir, assignment) + } + args := append([]string(nil), template.Command...) + if len(args) == 0 { + return ProcessCommand{}, fmt.Errorf("command is required") + } + for i, part := range args { + if strings.TrimSpace(part) == "" { + return ProcessCommand{}, fmt.Errorf("command part is required") + } + if containsUnsafeRuntimeText(part) { + return ProcessCommand{}, fmt.Errorf("command contains unsafe content") + } + if i == 0 { + if !commandNamePattern.MatchString(part) || strings.Contains(part, "/") || filepath.IsAbs(part) { + return ProcessCommand{}, fmt.Errorf("command executable must be an allowlisted name") + } + if _, disallowed := disallowedExecutables[strings.ToLower(part)]; disallowed { + return ProcessCommand{}, fmt.Errorf("command executable must not be a shell") + } + continue + } + if strings.ContainsAny(part, "|;&`$<>") { + return ProcessCommand{}, fmt.Errorf("command arguments must not contain shell metacharacters") + } + } + env, err := template.executionEnvironment(assignment) + if err != nil { + return ProcessCommand{}, err + } + timeout := defaultLifecycleTimeout + if template.TimeoutMS > 0 { + timeout = time.Duration(template.TimeoutMS) * time.Millisecond + } + if timeout > maxLifecycleTimeout { + return ProcessCommand{}, fmt.Errorf("timeout is too large") + } + return ProcessCommand{WorkDir: workdir, Args: args, Env: env, Timeout: timeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: template.Action}, nil +} + +func (template LifecycleActionTemplate) ToExecutableProcessCommand(resolver WorkspaceResolver, scope string, assignment protocol.RunJobAssignment) (ProcessCommand, error) { + if template.ExecutableKey == "" { + return ProcessCommand{}, fmt.Errorf("typed executableKey is required") + } + executable, err := resolver.ExistingTarget(scope, template.ExecutableKey) + if err != nil { + return ProcessCommand{}, err + } + info, err := os.Stat(executable) + if err != nil || !info.Mode().IsRegular() || runtime.GOOS != "windows" && info.Mode().Perm()&0o111 == 0 { + return ProcessCommand{}, fmt.Errorf("typed executable is not executable") + } + args := append([]string{executable}, template.Arguments...) + for _, part := range args[1:] { + if strings.TrimSpace(part) == "" || containsUnsafeRuntimeText(part) || strings.ContainsAny(part, "|;&`$<>") { + return ProcessCommand{}, fmt.Errorf("typed argument is unsafe") + } + } + if template.OutputMode != "" && template.OutputMode != "pipes" && template.OutputMode != "console" { + return ProcessCommand{}, fmt.Errorf("typed output mode is unsupported") + } + env, err := template.executionEnvironment(assignment) + if err != nil { + return ProcessCommand{}, err + } + timeout := defaultLifecycleTimeout + if template.TimeoutMS > 0 { + timeout = time.Duration(template.TimeoutMS) * time.Millisecond + } + if timeout > maxLifecycleTimeout { + return ProcessCommand{}, fmt.Errorf("timeout is too large") + } + args = managedExecutableArgs(runtime.GOOS, args) + return ProcessCommand{WorkDir: scope, Args: args, Env: env, OutputMode: template.OutputMode, Timeout: timeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: template.Action}, nil +} + +func managedExecutableArgs(targetOS string, args []string) []string { + if targetOS != "windows" || len(args) == 0 { + return args + } + ext := strings.ToLower(filepath.Ext(args[0])) + if ext != ".cmd" && ext != ".bat" { + return args + } + // Keep the batch path as a normal /c argument. Passing a pre-quoted + // command string here makes ComposeCommandLine escape the inner quotes + // for CreateProcess; cmd.exe does not treat those backslashes as quote + // escapes and consequently fails before it can run the script. + return append([]string{"cmd.exe", "/d", "/c", "call"}, args...) +} + +func quoteWindowsCommandArg(value string) string { + if value != "" && !strings.ContainsAny(value, " \t\"") { + return value + } + var builder strings.Builder + builder.WriteByte('"') + backslashes := 0 + for _, char := range value { + if char == '\\' { + backslashes++ + continue + } + if char == '"' { + builder.WriteString(strings.Repeat("\\", backslashes*2+1)) + builder.WriteRune(char) + backslashes = 0 + continue + } + if backslashes > 0 { + builder.WriteString(strings.Repeat("\\", backslashes)) + backslashes = 0 + } + builder.WriteRune(char) + } + if backslashes > 0 { + builder.WriteString(strings.Repeat("\\", backslashes*2)) + } + builder.WriteByte('"') + return builder.String() +} + +func (template LifecycleActionTemplate) ToManagedProcessCommand(resolver WorkspaceResolver, scope string, assignment protocol.RunJobAssignment) (ProcessCommand, error) { + command, err := template.ToExecutableProcessCommand(resolver, scope, assignment) + if err != nil { + return ProcessCommand{}, err + } + return command, nil +} + +func (template LifecycleActionTemplate) executionEnvironment(assignment protocol.RunJobAssignment) (map[string]string, error) { + env := template.Environment + if env == nil { + env = template.Env + } + validated := make(map[string]string, len(env)+16) + for key, value := range env { + if !envNamePattern.MatchString(key) || (!strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_")) || containsUnsafeRuntimeText(value) { + return nil, fmt.Errorf("typed environment is unsafe") + } + validated[key] = value + } + add := func(key string, value string) error { + if strings.TrimSpace(value) == "" { + return nil + } + if !envNamePattern.MatchString(key) || (!strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_")) || containsUnsafeRuntimeText(value) { + return fmt.Errorf("typed environment is unsafe") + } + validated[key] = value + return nil + } + if err := add("SERVER_PLUGIN_ID", assignment.ExecutionInput.PluginID); err != nil { + return nil, err + } + if err := add("SERVER_LIFECYCLE_OPERATION", assignment.ExecutionInput.LifecycleOperation); err != nil { + return nil, err + } + if deployment := assignment.ExecutionInput.Deployment; deployment != nil { + if err := add("SERVER_DEPLOYMENT_MODE", deployment.Mode); err != nil { + return nil, err + } + if err := add("SERVER_PROFILE_KEY", deployment.ProfileKey); err != nil { + return nil, err + } + if err := add("SERVER_ROOT", deployment.ServerRoot); err != nil { + return nil, err + } + if err := add("SERVER_WORKING_DIRECTORY", deployment.WorkingDirectory); err != nil { + return nil, err + } + if deployment.Revision > 0 { + if err := add("SERVER_REVISION", fmt.Sprint(deployment.Revision)); err != nil { + return nil, err + } + } + for key, value := range deployment.CreateInputs { + suffix := envKeySuffix(key) + if suffix == "" { + return nil, fmt.Errorf("typed environment is unsafe") + } + if err := add("SERVER_CREATE_"+suffix, value); err != nil { + return nil, err + } + } + } + for key, value := range assignment.ExecutionInput.Inputs { + suffix := envKeySuffix(key) + if suffix == "" { + return nil, fmt.Errorf("typed environment is unsafe") + } + if err := add("SERVER_INPUT_"+suffix, value); err != nil { + return nil, err + } + } + return validated, nil +} + +func envKeySuffix(key string) string { + var builder strings.Builder + for _, char := range key { + switch { + case char >= 'a' && char <= 'z': + builder.WriteRune(char - 'a' + 'A') + case char >= 'A' && char <= 'Z': + builder.WriteRune(char) + case char >= '0' && char <= '9': + builder.WriteRune(char) + case char == '_' || char == '-' || char == '.' || char == '/': + builder.WriteByte('_') + default: + return "" + } + } + return strings.Trim(builder.String(), "_") +} + +type ProcessCommand struct { + WorkDir string + Args []string + Env map[string]string + OutputMode string + Timeout time.Duration + JobID string + Capability string + Action string +} + +type ProcessResult struct { + ExitCode int + Stdout string + Stderr string +} + +type ProcessSupervisor interface { + Run(context.Context, ProcessCommand) (ProcessResult, error) +} + +type OSProcessSupervisor struct{} + +func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) { + if len(command.Args) == 0 { + log.Printf("RUN phase=process.command status=missing_executable job=%s capability=%s action=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action)) + return ProcessResult{ExitCode: -1}, fmt.Errorf("command is required") + } + startedAt := time.Now() + log.Printf("RUN phase=process.command status=starting job=%s capability=%s action=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil)) + if command.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, command.Timeout) + defer cancel() + } + cmd := exec.CommandContext(ctx, command.Args[0], command.Args[1:]...) + cmd.Dir = command.WorkDir + cmd.Env = os.Environ() + for key, value := range command.Env { + cmd.Env = append(cmd.Env, key+"="+value) + } + var stdout bytes.Buffer + var stderr bytes.Buffer + stdoutWriter := newLifecycleOutputWriter(&stdout, maxLifecycleOutputBytes, command, "stdout") + stderrWriter := newLifecycleOutputWriter(&stderr, maxLifecycleOutputBytes, command, "stderr") + cmd.Stdout = stdoutWriter + cmd.Stderr = stderrWriter + if err := cmd.Start(); err != nil { + log.Printf("RUN phase=process.command status=start_failed job=%s capability=%s action=%s command=%s durationMs=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), redactedCommandLine(command.Args), time.Since(startedAt).Milliseconds(), RedactText(err.Error())) + return ProcessResult{ExitCode: -1}, err + } + pid := 0 + if cmd.Process != nil { + pid = cmd.Process.Pid + } + log.Printf("RUN phase=process.command status=started job=%s capability=%s action=%s pid=%d command=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args)) + err := cmd.Wait() + stdoutWriter.Flush() + stderrWriter.Flush() + result := ProcessResult{Stdout: RedactText(stdout.String()), Stderr: RedactText(stderr.String())} + if cmd.ProcessState != nil { + result.ExitCode = cmd.ProcessState.ExitCode() + } + if err != nil { + log.Printf("RUN phase=process.command status=failed job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr), RedactText(err.Error())) + return result, err + } + log.Printf("RUN phase=process.command status=exited job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr)) + return result, nil +} + +type lifecycleOutputWriter struct { + mu sync.Mutex + buffer *bytes.Buffer + limit int + command ProcessCommand + stream string + pending string +} + +func newLifecycleOutputWriter(buffer *bytes.Buffer, limit int, command ProcessCommand, stream string) *lifecycleOutputWriter { + return &lifecycleOutputWriter{buffer: buffer, limit: limit, command: command, stream: stream} +} + +func (writer *lifecycleOutputWriter) Write(p []byte) (int, error) { + writer.mu.Lock() + defer writer.mu.Unlock() + remaining := writer.limit - writer.buffer.Len() + if remaining > 0 { + if len(p) > remaining { + _, _ = writer.buffer.Write(p[:remaining]) + } else { + _, _ = writer.buffer.Write(p) + } + } + writer.pending += string(p) + for { + index := strings.IndexByte(writer.pending, '\n') + if index < 0 { + break + } + line := strings.TrimRight(writer.pending[:index], "\r") + writer.pending = writer.pending[index+1:] + writer.logLine(line) + } + return len(p), nil +} + +func (writer *lifecycleOutputWriter) Flush() { + writer.mu.Lock() + defer writer.mu.Unlock() + if strings.TrimSpace(writer.pending) == "" { + writer.pending = "" + return + } + writer.logLine(strings.TrimRight(writer.pending, "\r")) + writer.pending = "" +} + +func (writer *lifecycleOutputWriter) logLine(line string) { + if strings.TrimSpace(line) == "" { + return + } + log.Printf("RUN phase=process.command.output status=line job=%s capability=%s action=%s stream=%s line=%q", safeOptional(writer.command.JobID), safeOptional(writer.command.Capability), safeOptional(writer.command.Action), writer.stream, RedactText(line)) +} + +type ioLimitWriter struct { + Writer *bytes.Buffer + Limit int +} + +func (writer ioLimitWriter) Write(p []byte) (int, error) { + remaining := writer.Limit - writer.Writer.Len() + if remaining > 0 { + if len(p) > remaining { + _, _ = writer.Writer.Write(p[:remaining]) + } else { + _, _ = writer.Writer.Write(p) + } + } + return len(p), nil +} + +type ProcessLogSink interface { + Append(context.Context, protocol.RunJobAssignment, string, string) error +} + +type ProcessLogCursor struct { + StartOffset int64 + EndOffset int64 +} + +type ProcessLogCursorSink interface { + AppendWithCursor(context.Context, protocol.RunJobAssignment, string, string, ProcessLogCursor) error +} + +type NoopProcessLogSink struct{} + +func (NoopProcessLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error { + return nil +} + +type LifecycleArtifactHook interface { + QueueLifecycleResult(context.Context, protocol.RunJobAssignment, ProcessResult) (string, error) +} + +type StaticLifecycleArtifactHook struct{} + +func (StaticLifecycleArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) { + return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", url.PathEscape(assignment.JobID)), nil +} + +func LifecycleResultRequest(assignment protocol.RunJobAssignment, sessionToken string, result LifecycleExecutionResult) protocol.RunJobResultRequest { + return protocol.RunJobResultRequest{ + RunEndpointID: assignment.RunEndpointID, + SessionToken: sessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + State: result.State, + Progress: result.Progress, + ResultRef: result.ResultRef, + Message: result.Message, + ErrorCode: result.ErrorCode, + Retryable: result.Retryable, + ExecutionResult: protocol.RunJobExecutionResult{Kind: result.ExecutionResult.Kind, ProcessState: result.ExecutionResult.ProcessState, ExitClassification: result.ExecutionResult.ExitClassification, ExitCode: result.ExecutionResult.ExitCode, Version: result.ExecutionResult.Version, Checksum: result.ExecutionResult.Checksum, SizeBytes: result.ExecutionResult.SizeBytes, Summary: result.ExecutionResult.Summary, Content: result.ExecutionResult.Content, SQLiteSchemaProbe: result.ExecutionResult.SQLiteSchemaProbe, DeploymentReceipt: result.ExecutionResult.DeploymentReceipt, ServerDeploymentEvidence: result.ExecutionResult.ServerDeploymentEvidence}, + } +} + +func isSupportedLifecycleCapability(capability string) bool { + for _, supported := range SupportedLifecycleCapabilities() { + if capability == supported { + return true + } + } + return false +} + +func isSupportedRemoteCapability(capability string) bool { + for _, supported := range SupportedRemoteCapabilities() { + if capability == supported { + return true + } + } + return false +} + +func lifecycleFailure(code string, message string) LifecycleExecutionResult { + return LifecycleExecutionResult{ + State: lifecycleResultStateFailed, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)}, + Message: RedactText(message), + ErrorCode: code, + } +} + +func scopedServerWorkspace(root string, serverInstanceID string) (string, error) { + if strings.TrimSpace(serverInstanceID) == "" { + return "", fmt.Errorf("server instance id is required") + } + if containsUnsafeRuntimeText(serverInstanceID) || strings.ContainsAny(serverInstanceID, `/\`) || serverInstanceID == "." || serverInstanceID == ".." { + return "", fmt.Errorf("server instance id is unsafe") + } + return scopedPath(root, serverInstanceID) +} + +func scopedPath(root string, key string) (string, error) { + if strings.TrimSpace(root) == "" { + return "", fmt.Errorf("workspace root is required") + } + if strings.TrimSpace(key) == "" { + return "", fmt.Errorf("logical key is required") + } + if filepath.IsAbs(key) || strings.Contains(key, "..") || strings.Contains(key, `\`) || containsUnsafeRuntimeText(key) { + return "", fmt.Errorf("logical key is unsafe") + } + cleanRoot, err := filepath.Abs(root) + if err != nil { + return "", err + } + candidate := filepath.Clean(filepath.Join(cleanRoot, filepath.FromSlash(key))) + rel, err := filepath.Rel(cleanRoot, candidate) + if err != nil { + return "", err + } + if rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return "", fmt.Errorf("logical key escapes workspace") + } + return candidate, nil +} + +func containsUnsafeRuntimeText(value string) bool { + normalized := strings.ToLower(value) + for _, marker := range []string{"/users/", "/.ssh/", "password=", "apikey", "api_key", "secret=", "bearer ", "sk-", "unix://", "tcp://", "://"} { + if strings.Contains(normalized, marker) { + return true + } + } + return false +} + +func safeOptional(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "-" + } + return RedactText(value) +} + +func errorSummary(err error) string { + if err == nil { + return "-" + } + return RedactText(err.Error()) +} + +func redactedCommandLine(args []string) string { + if len(args) == 0 { + return "-" + } + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = strconv.Quote(RedactText(arg)) + } + return strings.Join(parts, " ") +} + +func envKeysSummary(first map[string]string, second map[string]string) string { + seen := map[string]struct{}{} + for key := range first { + seen[key] = struct{}{} + } + for key := range second { + seen[key] = struct{}{} + } + if len(seen) == 0 { + return "-" + } + keys := make([]string, 0, len(seen)) + for key := range seen { + keys = append(keys, key) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func RedactText(value string) string { + redacted := value + replacements := []string{"/Users/", "[host]/", "Bearer ", "Bearer [redacted] ", "sk-", "sk-[redacted]", "password=", "password=[redacted]", "api_key=", "api_key=[redacted]", "secret=", "secret=[redacted]", "unix://", "socket://"} + for i := 0; i+1 < len(replacements); i += 2 { + redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1]) + } + if len(redacted) > maxLifecycleOutputBytes { + return redacted[:maxLifecycleOutputBytes] + } + return redacted +} + +func splitBoundedLines(value string) []string { + value = RedactText(value) + lines := strings.Split(value, "\n") + out := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimRight(line, "\r") + if strings.TrimSpace(line) == "" { + continue + } + out = append(out, line) + } + return out +} + +func checksumForText(value string) string { + sum := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(sum[:]) +} diff --git a/runtime/lifecycle_test.go b/runtime/lifecycle_test.go new file mode 100644 index 0000000..af5acce --- /dev/null +++ b/runtime/lifecycle_test.go @@ -0,0 +1,583 @@ +package runtime + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "browser.local/run/config" + "browser.local/run/protocol" +) + +func TestLifecycleExecutorHandlesSupportedJobs(t *testing.T) { + executor := NewLifecycleExecutor() + for _, capability := range SupportedLifecycleCapabilities() { + assignment := lifecycleAssignment(capability) + result := executor.Execute(assignment) + if result.State != "succeeded" || result.Progress.Percent != 100 || result.ResultRef == "" { + t.Fatalf("expected successful bounded result for %s, got %+v", capability, result) + } + for _, forbidden := range []string{"host path", "/Users/", "run socket", "api_key", "sk-"} { + if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) { + t.Fatalf("lifecycle result exposed forbidden content %q: %+v", forbidden, result) + } + } + } +} + +func TestGeneratedRunOmitsBuildOnlyCapabilities(t *testing.T) { + generated := SupportedRunCapabilitiesForComponent("run") + if containsCapability(generated, protocol.RunCapabilityDistributionBuild) { + t.Fatalf("generated Run must not advertise %s: %v", protocol.RunCapabilityDistributionBuild, generated) + } + if !containsCapability(generated, protocol.RunCapabilityRunSelfUpdate) { + t.Fatalf("generated Run must advertise %s: %v", protocol.RunCapabilityRunSelfUpdate, generated) + } + for _, capability := range []string{"deployment.scum.v1"} { + if containsCapability(generated, capability) { + t.Fatalf("generated Run must not advertise %s: %v", capability, generated) + } + } + for _, capability := range []string{protocol.RunCapabilityProcessStart, protocol.RunCapabilityDependenciesInstall} { + if !containsCapability(generated, capability) { + t.Fatalf("generated Run must retain %s: %v", capability, generated) + } + } + if !containsCapability(SupportedRunCapabilitiesForComponent(""), protocol.RunCapabilityDistributionBuild) { + t.Fatal("generic build worker must retain distribution.build") + } + if containsCapability(SupportedRunCapabilitiesForComponent(""), "deployment.scum.v1") { + t.Fatal("generic run must not advertise game-specific SCUM deployment capability") + } +} + +func TestLifecycleExecutorRejectsLegacyGameSpecificDeploymentPlan(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityProcessInstall) + assignment.ExecutionInput.Deployment = &protocol.ServerDeploymentExecution{SchemaVersion: "1", Mode: "guided-install", ServerRoot: "C:/scumserver", Revision: 1} + assignment.ExecutionInput.ServerDeploymentPlan = &protocol.ServerDeploymentPlan{SchemaVersion: "1", PluginID: "game.scum", TemplateKey: "scum-steamcmd-windows"} + + result := NewLifecycleExecutor().Execute(assignment) + + if result.State != "failed" || result.ErrorCode != "unsupported_legacy_deployment_plan" { + t.Fatalf("expected legacy deployment plan rejection, got %+v", result) + } +} + +func TestLifecycleExecutorRejectsUnsupportedJobs(t *testing.T) { + result := NewLifecycleExecutor().Execute(lifecycleAssignment("files.write")) + if result.State != "failed" || result.ErrorCode != "unsupported_lifecycle_capability" || result.ResultRef != "" { + t.Fatalf("expected unsupported lifecycle failure, got %+v", result) + } +} + +func TestLifecycleResultRequestUsesAssignmentLease(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart) + execution := NewLifecycleExecutor().Execute(assignment) + request := LifecycleResultRequest(assignment, "session-token", execution) + + if request.RunEndpointID != assignment.RunEndpointID || request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken || request.Attempt != assignment.Attempt { + t.Fatalf("expected result request to use assignment lease, got %+v", request) + } + if request.SessionToken != "session-token" || request.State != "succeeded" { + t.Fatalf("unexpected result request: %+v", request) + } +} + +func TestLifecycleExecutorRunsScopedCommandTemplateAndHooks(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart) + serverRoot := filepath.Join(root, assignment.ServerInstanceID) + if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil { + t.Fatalf("create action dir: %v", err) + } + template := LifecycleActionTemplate{ + Command: []string{"echo", "server-ready"}, + Env: map[string]string{"GAME_MODE": "test"}, + } + body, err := json.Marshal(template) + if err != nil { + t.Fatalf("marshal template: %v", err) + } + if err := os.WriteFile(filepath.Join(serverRoot, "actions", "start.json"), body, 0o644); err != nil { + t.Fatalf("write template: %v", err) + } + assignment.TargetKey = "actions/start.json" + logSink := &recordingLogSink{} + artifactHook := &recordingArtifactHook{} + + result := NewLifecycleExecutor( + WithLifecycleWorkspaceRoot(root), + WithProcessLogSink(logSink), + WithLifecycleArtifactHook(artifactHook), + ).Execute(assignment) + + if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/lifecycle-result" { + t.Fatalf("expected scoped lifecycle success, got %+v", result) + } + if len(logSink.lines) != 1 || logSink.lines[0] != "stdout:server-ready" { + t.Fatalf("expected process stdout to be logged, got %+v", logSink.lines) + } + if !artifactHook.called { + t.Fatal("expected artifact hook to be called") + } +} + +func TestLifecycleExecutorRunsTypedInstallActionWithDeploymentInputs(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityProcessInstall) + assignment.TargetKey = "actions/install.json" + assignment.ExecutionInput.WorkspaceScope = "run-local" + assignment.ExecutionInput.PluginID = "game.example" + assignment.ExecutionInput.LifecycleOperation = "install" + assignment.ExecutionInput.Deployment = &protocol.ServerDeploymentExecution{ + SchemaVersion: "1", + Mode: "guided-install", + ProfileKey: "run-local", + ServerRoot: "D:/game-server", + CreateInputs: map[string]string{"gamePort": "27000", "maxPlayers": "128"}, + Revision: 3, + } + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("resolve scope: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o755); err != nil { + t.Fatalf("create action dir: %v", err) + } + if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o755); err != nil { + t.Fatalf("create bin dir: %v", err) + } + if err := os.WriteFile(filepath.Join(scope, "bin", "install-server"), []byte("plugin-owned helper"), 0o700); err != nil { + t.Fatalf("write helper: %v", err) + } + body, err := json.Marshal(LifecycleActionTemplate{Version: 1, Action: "install", Mode: "oneshot", ExecutableKey: "bin/install-server", Environment: map[string]string{"GAME_ID": "example"}, OutputMode: "console", TimeoutMS: int((90 * time.Minute) / time.Millisecond)}) + if err != nil { + t.Fatalf("marshal action: %v", err) + } + if err := os.WriteFile(filepath.Join(scope, "actions", "install.json"), body, 0o600); err != nil { + t.Fatalf("write action: %v", err) + } + supervisor := &recordingSupervisor{} + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessSupervisor(supervisor)).Execute(assignment) + + if result.State != "succeeded" || !strings.HasSuffix(supervisor.command.Args[0], filepath.Join("bin", "install-server")) { + t.Fatalf("expected typed install helper execution, result=%+v command=%+v", result, supervisor.command) + } + if supervisor.command.Env["SERVER_ROOT"] != "D:/game-server" || supervisor.command.Env["SERVER_CREATE_GAMEPORT"] != "27000" || supervisor.command.Env["SERVER_CREATE_MAXPLAYERS"] != "128" || supervisor.command.Env["SERVER_REVISION"] != "3" { + t.Fatalf("expected deployment inputs in typed action environment, got %+v", supervisor.command.Env) + } + if supervisor.command.Timeout != 90*time.Minute { + t.Fatalf("expected plugin-declared long lifecycle timeout, got %s", supervisor.command.Timeout) + } + if supervisor.command.OutputMode != "console" { + t.Fatalf("expected plugin-declared output mode, got %q", supervisor.command.OutputMode) + } +} + +func TestLifecycleExecutorRejectsUnsafeTemplates(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart) + serverRoot := filepath.Join(root, assignment.ServerInstanceID) + if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil { + t.Fatalf("create action dir: %v", err) + } + for name, template := range map[string]LifecycleActionTemplate{ + "absolute": {Command: []string{"/bin/echo", "nope"}}, + "shell": {Command: []string{"sh", "-c", "echo nope"}}, + "secret": {Command: []string{"echo", "sk-secret"}}, + "env": {Command: []string{"echo", "ok"}, Env: map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}}, + } { + body, err := json.Marshal(template) + if err != nil { + t.Fatalf("marshal %s: %v", name, err) + } + actionPath := filepath.Join(serverRoot, "actions", name+".json") + if err := os.WriteFile(actionPath, body, 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + unsafeAssignment := assignment + unsafeAssignment.TargetKey = "actions/" + name + ".json" + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(unsafeAssignment) + if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" { + t.Fatalf("expected unsafe command rejection for %s, got %+v", name, result) + } + } +} + +func TestLifecycleExecutorRejectsWorkspaceEscapes(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart) + assignment.TargetKey = "../outside.json" + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment) + + if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" { + t.Fatalf("expected workspace escape rejection, got %+v", result) + } +} + +func TestMaterializeWorkspaceSeedWritesPluginAssetsToProfileScope(t *testing.T) { + root := t.TempDir() + seed, err := json.Marshal([]workspaceSeedFile{ + {Path: "actions/install.json", Content: `{"version":1,"action":"install","mode":"oneshot"}`, Mode: 0o600}, + {Path: "bin/install-server", Content: "echo install\n", Mode: 0o700}, + {Path: "assets/map.bin", Content: base64.StdEncoding.EncodeToString([]byte{0xff, 0x00, 0x7f}), Encoding: "base64", Mode: 0o600}, + }) + if err != nil { + t.Fatalf("marshal seed: %v", err) + } + + err = MaterializeWorkspaceSeed(config.Config{ + WorkspaceRoot: root, + ServerInstanceID: "server-seeded", + ComponentKey: "run-local", + WorkspaceSeed: base64.StdEncoding.EncodeToString(seed), + ComponentKind: "run", + RegistrationToken: "unused", + }) + + if err != nil { + t.Fatalf("materialize workspace seed: %v", err) + } + scope, err := NewWorkspaceResolver(root).Scope("server-seeded", "run-local") + if err != nil { + t.Fatalf("resolve seeded scope: %v", err) + } + if body, err := os.ReadFile(filepath.Join(scope, "actions", "install.json")); err != nil || !strings.Contains(string(body), `"action":"install"`) { + t.Fatalf("expected seeded action file, body=%q err=%v", body, err) + } + info, err := os.Stat(filepath.Join(scope, "bin", "install-server")) + if err != nil || info.Mode().Perm()&0o111 == 0 { + t.Fatalf("expected executable seeded helper, info=%+v err=%v", info, err) + } + if body, err := os.ReadFile(filepath.Join(scope, "assets", "map.bin")); err != nil || string(body) != string([]byte{0xff, 0x00, 0x7f}) { + t.Fatalf("expected base64 seed file to materialize as binary bytes, body=%v err=%v", body, err) + } +} + +func TestLifecycleExecutorKeepsSiblingInstanceWorkspacesIsolated(t *testing.T) { + root := t.TempDir() + first := lifecycleAssignment(protocol.RunCapabilityProcessStart) + first.ServerInstanceID = "server-alpha" + second := lifecycleAssignment(protocol.RunCapabilityProcessStop) + second.JobID = "job-2" + second.ServerInstanceID = "server-beta" + for _, assignment := range []protocol.RunJobAssignment{first, second} { + serverRoot := filepath.Join(root, assignment.ServerInstanceID) + if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil { + t.Fatalf("create action dir for %s: %v", assignment.ServerInstanceID, err) + } + body, err := json.Marshal(LifecycleActionTemplate{Command: []string{"echo", assignment.ServerInstanceID}}) + if err != nil { + t.Fatalf("marshal template: %v", err) + } + if err := os.WriteFile(filepath.Join(serverRoot, "actions", "lifecycle.json"), body, 0o644); err != nil { + t.Fatalf("write template for %s: %v", assignment.ServerInstanceID, err) + } + } + first.TargetKey = "actions/lifecycle.json" + second.TargetKey = "actions/lifecycle.json" + logSink := &recordingLogSink{} + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink)) + + firstResult := executor.Execute(first) + secondResult := executor.Execute(second) + + if firstResult.State != "succeeded" || secondResult.State != "succeeded" { + t.Fatalf("expected both lifecycle jobs to succeed, got first=%+v second=%+v", firstResult, secondResult) + } + joined := strings.Join(logSink.lines, "\n") + if !strings.Contains(joined, "stdout:server-alpha") || !strings.Contains(joined, "stdout:server-beta") { + t.Fatalf("expected instance-specific output, got %q", joined) + } + if _, err := os.Stat(filepath.Join(root, "server-alpha", "actions", "lifecycle.json")); err != nil { + t.Fatalf("expected alpha template to remain scoped: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "server-beta", "actions", "lifecycle.json")); err != nil { + t.Fatalf("expected beta template to remain scoped: %v", err) + } +} + +func TestLifecycleExecutorCancelsRunningCommand(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + result := NewLifecycleExecutor(WithProcessSupervisor(blockingSupervisor{})).ExecuteContext(ctx, lifecycleAssignment(protocol.RunCapabilityProcessStart)) + + if result.State != "cancelled" || result.ErrorCode != "lifecycle_cancelled" { + t.Fatalf("expected cancelled lifecycle result, got %+v", result) + } +} + +func TestSmokeSummaryReportsLifecycleCapabilities(t *testing.T) { + summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"}) + for _, capability := range SupportedLifecycleCapabilities() { + if !containsCapability(summary.Capabilities, capability) { + t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities) + } + } +} + +func TestSmokeSummaryReportsLogReadCapability(t *testing.T) { + summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"}) + if !containsCapability(summary.Capabilities, protocol.RunCapabilityLogsRead) { + t.Fatalf("expected smoke capabilities to include %s, got %+v", protocol.RunCapabilityLogsRead, summary.Capabilities) + } +} + +func TestRemoteAccessExecutorCompletesBoundedJobs(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery) + assignment.TargetKey = "db/scum/query" + assignment.InputRef = "input://server-1/db/sqlite/query/1" + + result := ExecuteRemoteAccessJob(context.Background(), assignment) + + if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/remote-access-result" { + t.Fatalf("expected bounded remote result, got %+v", result) + } + for _, forbidden := range []string{"/Users/", "tcp://", "password=", "sk-"} { + if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) { + t.Fatalf("remote result exposed forbidden fragment %q: %+v", forbidden, result) + } + } +} + +func TestSmokeSummaryReportsRemoteCapabilities(t *testing.T) { + summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"}) + for _, capability := range []string{protocol.RunCapabilityRemoteRunRCONCommand, protocol.RunCapabilityRemoteRunDBMySQLQuery, protocol.RunCapabilityRemoteRunDBSQLiteQuery, protocol.RunCapabilityRemoteRunLogsTransfer} { + if !containsCapability(summary.Capabilities, capability) { + t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities) + } + } +} + +func TestSmokeSummaryReportsSQLiteSchemaProbeCapability(t *testing.T) { + summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"}) + if !containsCapability(summary.Capabilities, protocol.RunCapabilityRemoteRunDBSQLiteProbe) { + t.Fatalf("expected schema probe capability, got %+v", summary.Capabilities) + } +} + +func TestLifecycleResultRequestPreservesSQLiteSchemaProbeEnvelope(t *testing.T) { + assignment := sqliteSchemaProbeAssignment() + probe := &protocol.SQLiteSchemaProbeResult{RequestID: "probe-1", JobID: assignment.JobID, Binding: assignment.ExecutionInput.SQLiteSchemaProbe.Binding, Status: "succeeded", ResultDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Limits: assignment.ExecutionInput.SQLiteSchemaProbe.Limits} + request := LifecycleResultRequest(assignment, "session-token", LifecycleExecutionResult{State: lifecycleResultStateSucceeded, ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", SQLiteSchemaProbe: probe}}) + if request.ExecutionResult.SQLiteSchemaProbe == nil || request.ExecutionResult.SQLiteSchemaProbe.ResultDigest != probe.ResultDigest || request.ExecutionResult.SQLiteSchemaProbe.JobID != assignment.JobID { + t.Fatalf("expected SQLite probe terminal envelope to survive job result conversion: %+v", request.ExecutionResult) + } +} + +func TestSmokeSummaryReportsDistributionCapabilities(t *testing.T) { + summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"}) + for _, capability := range []string{protocol.RunCapabilityRunSelfUpdate, protocol.RunCapabilityDependenciesCheck, protocol.RunCapabilityDependenciesInstall, protocol.RunCapabilityLogsBackfill} { + if !containsCapability(summary.Capabilities, capability) { + t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities) + } + } +} + +func TestDistributionExecutorsReturnBoundedRefsAndRedactResults(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityLogsBackfill) + assignment.TargetKey = "logs/latest-log" + assignment.InputRef = "artifact://logs/checkpoint/1" + result := ExecuteDistributionJob(context.Background(), assignment) + if result.State != "succeeded" || result.Progress.Percent != 100 || !strings.HasPrefix(result.ResultRef, "artifact://jobs/") { + t.Fatalf("expected bounded log backfill success, got %+v", result) + } + for _, forbidden := range []string{"/Users/", "tcp://", "unix://", "password=", "sk-", "mysql://", "sqlite://"} { + if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) { + t.Fatalf("distribution result leaked forbidden fragment %q: %+v", forbidden, result) + } + } +} + +func TestDistributionExecutorsRejectUnsafeJobs(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityDependenciesInstall) + assignment.TargetKey = "dependencies/java-21" + + result := ExecuteDistributionJob(context.Background(), assignment) + + if result.State != "failed" || result.ErrorCode != "dependency_execution_requires_worker" { + t.Fatalf("expected dependency execution to require authenticated worker, got %+v", result) + } +} + +func TestResolveRuntimeProfilesSupportsDeclaredModesAndSafeMissingKeys(t *testing.T) { + profiles := RuntimeProfiles{ + Discovery: []RuntimeDiscoveryProbe{{Key: "steamcmd", Kind: "command.version", TargetKey: "steamcmd", Required: true}}, + LifecycleProfiles: []RuntimeLifecycleProfile{ + {Key: "run-local", Mode: RuntimeModeLocalProcess, Capabilities: []string{protocol.RunCapabilityProcessStart}, ActionRefs: map[string]string{"start": "actions/start.json"}, TransportKeys: []string{"server-files"}, Platforms: []string{"linux"}}, + {Key: "hosted-ftp", Mode: RuntimeModeHostedFTPRCON, Capabilities: []string{protocol.RunCapabilityRemoteFTPRead, protocol.RunCapabilityRemoteRunRCONCommand}, TransportKeys: []string{"ftp", "rcon"}}, + {Key: "ftp-only", Mode: RuntimeModeFTPOnly, Capabilities: []string{protocol.RunCapabilityRemoteFTPRead}, TransportKeys: []string{"ftp"}}, + {Key: "custom-client", Mode: RuntimeModeCustomClient, Capabilities: []string{protocol.RunCapabilityRemoteRunRCONCommand}, TransportKeys: []string{"rcon"}, ClientManagerRef: "scum-client-manager"}, + }, + LogSources: []RuntimeLogSource{{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest", StreamKey: "latest-log"}}, + TransportProfiles: []RuntimeTransportProfile{ + {Key: "server-files", Kind: "file", TargetKey: "server-root", Capabilities: []string{protocol.RunCapabilityRemoteRunFilesRead}}, + {Key: "ftp", Kind: "ftp", TargetKey: "ftp-root", Capabilities: []string{protocol.RunCapabilityRemoteFTPRead}}, + {Key: "rcon", Kind: "rcon", TargetKey: "rcon", Capabilities: []string{protocol.RunCapabilityRemoteRunRCONCommand}}, + }, + } + resolution, err := ResolveRuntimeProfile(profiles, "custom-client", "windows", RuntimeBindingSet{ + ProfileKey: "custom-client", + Mode: RuntimeModeCustomClient, + Bindings: map[string]string{ + "rcon": "binding://rcon/current", + "logs/latest": "binding://logs/latest", + "steamcmd": "binding://probe/steamcmd", + "scum-client-manager": "binding://client/current", + }, + }) + if err != nil { + t.Fatalf("resolve custom client profile: %v", err) + } + if !resolution.Available || resolution.Mode != RuntimeModeCustomClient || resolution.ClientManagerRef != "scum-client-manager" { + t.Fatalf("unexpected custom client resolution: %+v", resolution) + } + + missing, err := ResolveRuntimeProfile(profiles, "hosted-ftp", "linux", RuntimeBindingSet{ProfileKey: "hosted-ftp", Mode: RuntimeModeHostedFTPRCON, Bindings: map[string]string{"ftp-root": "binding://ftp/current"}}) + if err != nil { + t.Fatalf("resolve hosted profile: %v", err) + } + if missing.Available || strings.Join(missing.MissingKeys, ",") != "rcon,steamcmd" { + t.Fatalf("expected safe missing keys without raw binding values, got %+v", missing) + } +} + +func TestTailDeclaredFileLogSourceUsesCheckpointAndRedaction(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityLogsRead) + serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs") + if err := os.MkdirAll(serverRoot, 0o755); err != nil { + t.Fatalf("create logs dir: %v", err) + } + logPath := filepath.Join(serverRoot, "latest.log") + if err := os.WriteFile(logPath, []byte("first line\npassword=hidden\n"), 0o644); err != nil { + t.Fatalf("write log file: %v", err) + } + store := NewMemoryLogCheckpointStore() + sink := &recordingLogSink{} + source := RuntimeLogSource{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest.log", StreamKey: "latest-log", CursorKind: "offset"} + + result := TailDeclaredFileLogSource(context.Background(), root, assignment, source, sink, store) + + if result.State != "succeeded" || !strings.Contains(result.ResultRef, "live-log-checkpoint") { + t.Fatalf("expected file tail success, got %+v", result) + } + if len(sink.lines) != 2 || strings.Contains(strings.Join(sink.lines, "\n"), "password=hidden") { + t.Fatalf("expected redacted tailed lines, got %+v", sink.lines) + } + checkpoint := store.GetLogCheckpoint("latest-log") + if checkpoint.Offset == 0 || checkpoint.Sequence != 2 || strings.Contains(RedactedLogCheckpointSummary(checkpoint), "/Users/") { + t.Fatalf("expected durable safe checkpoint, got %+v", checkpoint) + } + + if err := os.WriteFile(logPath, []byte("first line\npassword=hidden\nsecond line\n"), 0o644); err != nil { + t.Fatalf("append log file: %v", err) + } + sink.lines = nil + result = TailDeclaredFileLogSource(context.Background(), root, assignment, source, sink, store) + if result.State != "succeeded" || len(sink.lines) != 1 || !strings.Contains(sink.lines[0], "second line") { + t.Fatalf("expected checkpointed incremental tail, result=%+v lines=%+v", result, sink.lines) + } +} + +func TestLifecycleExecutorExecutesDeclaredLogBackfillTail(t *testing.T) { + root := t.TempDir() + assignment := lifecycleAssignment(protocol.RunCapabilityLogsBackfill) + assignment.TargetKey = "logs/latest-log" + assignment.ExecutionInput.LogSource = &protocol.RuntimeLogSourcePlan{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest.log", StreamKey: "latest-log", CursorKind: "offset", RetentionDays: 30} + serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs") + if err := os.MkdirAll(serverRoot, 0o755); err != nil { + t.Fatalf("create logs dir: %v", err) + } + if err := os.WriteFile(filepath.Join(serverRoot, "latest.log"), []byte("scum latest line\n"), 0o644); err != nil { + t.Fatalf("write log file: %v", err) + } + sink := &recordingLogSink{} + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(sink)).ExecuteLogBackfill(context.Background(), assignment) + + if result.State != "succeeded" || !strings.Contains(result.Message, "tailed") { + t.Fatalf("expected tailed log backfill success, got %+v", result) + } + if len(sink.lines) != 1 || sink.lines[0] != "latest-log:scum latest line" { + t.Fatalf("expected tailed log line in sink, got %+v", sink.lines) + } +} + +type recordingSupervisor struct { + command ProcessCommand +} + +func (supervisor *recordingSupervisor) Run(_ context.Context, command ProcessCommand) (ProcessResult, error) { + supervisor.command = command + return ProcessResult{ExitCode: 0, Stdout: "recorded\n"}, nil +} + +type recordingLogSink struct { + mu sync.Mutex + lines []string +} + +func (sink *recordingLogSink) Append(_ context.Context, _ protocol.RunJobAssignment, stream string, line string) error { + sink.mu.Lock() + defer sink.mu.Unlock() + sink.lines = append(sink.lines, stream+":"+line) + return nil +} + +func (sink *recordingLogSink) snapshot() []string { + sink.mu.Lock() + defer sink.mu.Unlock() + return append([]string(nil), sink.lines...) +} + +type recordingArtifactHook struct { + called bool +} + +func (hook *recordingArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) { + hook.called = true + return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID), nil +} + +type blockingSupervisor struct{} + +func (blockingSupervisor) Run(ctx context.Context, _ ProcessCommand) (ProcessResult, error) { + <-ctx.Done() + return ProcessResult{ExitCode: -1}, ctx.Err() +} + +func lifecycleAssignment(capability string) protocol.RunJobAssignment { + now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC) + return protocol.RunJobAssignment{ + JobID: "job-1", + ServerInstanceID: "server-1", + RunEndpointID: "run-local", + Capability: capability, + IdempotencyKey: "idem-1", + State: "accepted", + LeaseToken: "lease-1", + Attempt: 1, + CreatedAt: now, + UpdatedAt: now, + } +} + +func containsCapability(capabilities []string, capability string) bool { + for _, item := range capabilities { + if item == capability { + return true + } + } + return false +} diff --git a/runtime/log_sources.go b/runtime/log_sources.go new file mode 100644 index 0000000..f686681 --- /dev/null +++ b/runtime/log_sources.go @@ -0,0 +1,118 @@ +package runtime + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + + "browser.local/run/protocol" +) + +type LogSourceCheckpoint struct { + SourceKey string + Offset int64 + Sequence uint64 + CursorRef string +} + +type LogCheckpointStore interface { + GetLogCheckpoint(sourceKey string) LogSourceCheckpoint + PutLogCheckpoint(checkpoint LogSourceCheckpoint) +} + +type MemoryLogCheckpointStore struct { + checkpoints map[string]LogSourceCheckpoint +} + +func NewMemoryLogCheckpointStore() *MemoryLogCheckpointStore { + return &MemoryLogCheckpointStore{checkpoints: map[string]LogSourceCheckpoint{}} +} + +func (store *MemoryLogCheckpointStore) GetLogCheckpoint(sourceKey string) LogSourceCheckpoint { + if store == nil || store.checkpoints == nil { + return LogSourceCheckpoint{SourceKey: sourceKey} + } + return store.checkpoints[sourceKey] +} + +func (store *MemoryLogCheckpointStore) PutLogCheckpoint(checkpoint LogSourceCheckpoint) { + if store == nil { + return + } + if store.checkpoints == nil { + store.checkpoints = map[string]LogSourceCheckpoint{} + } + store.checkpoints[checkpoint.SourceKey] = checkpoint +} + +func TailDeclaredFileLogSource(ctx context.Context, workspaceRoot string, assignment protocol.RunJobAssignment, source RuntimeLogSource, sink ProcessLogSink, store LogCheckpointStore) LifecycleExecutionResult { + if source.Kind != "file.tail" { + return lifecycleFailure("unsupported_log_source", "only file.tail sources are supported by the local tailer") + } + if !protocol.ValidLogicalFileKey(source.Key) || !protocol.ValidLogicalFileKey(source.TargetKey) || !protocol.ValidLogicalFileKey(source.StreamKey) { + return lifecycleFailure("unsafe_log_source", "log source is unsafe") + } + if sink == nil { + sink = NoopProcessLogSink{} + } + if store == nil { + store = NewMemoryLogCheckpointStore() + } + serverRoot, err := scopedServerWorkspace(workspaceRoot, assignment.ServerInstanceID) + if err != nil { + return lifecycleFailure("unsafe_log_workspace", err.Error()) + } + path, err := scopedPath(serverRoot, source.TargetKey) + if err != nil { + return lifecycleFailure("unsafe_log_source", err.Error()) + } + file, err := os.Open(path) + if err != nil { + return lifecycleFailure("log_source_open_failed", err.Error()) + } + defer file.Close() + + checkpoint := store.GetLogCheckpoint(source.Key) + if checkpoint.Offset > 0 { + if _, err := file.Seek(checkpoint.Offset, 0); err != nil { + return lifecycleFailure("log_source_seek_failed", err.Error()) + } + } + body := make([]byte, maxLifecycleOutputBytes) + n, err := file.Read(body) + if err != nil && n == 0 { + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint unchanged"}, + ResultRef: fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID)), + Message: "live log source had no new lines", + } + } + for _, line := range splitBoundedLines(string(body[:n])) { + checkpoint.Sequence++ + if err := sink.Append(ctx, assignment, source.StreamKey, line); err != nil { + return lifecycleFailure("log_source_sink_failed", err.Error()) + } + } + checkpoint.SourceKey = source.Key + checkpoint.Offset += int64(n) + checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID)) + store.PutLogCheckpoint(checkpoint) + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint updated"}, + ResultRef: checkpoint.CursorRef, + Message: "live log source tailed with durable offset checkpoint", + } +} + +func RedactedLogCheckpointSummary(checkpoint LogSourceCheckpoint) string { + return strings.Join([]string{ + "source=" + checkpoint.SourceKey, + fmt.Sprintf("offset=%d", checkpoint.Offset), + fmt.Sprintf("sequence=%d", checkpoint.Sequence), + "cursorRef=" + checkpoint.CursorRef, + }, " ") +} diff --git a/runtime/managed_command_test.go b/runtime/managed_command_test.go new file mode 100644 index 0000000..6dda8c4 --- /dev/null +++ b/runtime/managed_command_test.go @@ -0,0 +1,22 @@ +package runtime + +import ( + "reflect" + "testing" +) + +func TestManagedExecutableArgsWrapWindowsPluginScripts(t *testing.T) { + got := managedExecutableArgs("windows", []string{`C:\run workspace\bin\scum-start.cmd`, "--mode", "safe mode"}) + want := []string{"cmd.exe", "/d", "/c", "call", `C:\run workspace\bin\scum-start.cmd`, "--mode", "safe mode"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected Windows script command: got=%q want=%q", got, want) + } +} + +func TestManagedExecutableArgsKeepsDirectExecutables(t *testing.T) { + input := []string{"C:\\run\\bin\\server.exe", "--port", "7779"} + got := managedExecutableArgs("windows", input) + if !reflect.DeepEqual(got, input) { + t.Fatalf("direct executable was unexpectedly shell wrapped: got=%q want=%q", got, input) + } +} diff --git a/runtime/managed_process_default.go b/runtime/managed_process_default.go new file mode 100644 index 0000000..8b9ea0d --- /dev/null +++ b/runtime/managed_process_default.go @@ -0,0 +1,59 @@ +//go:build !windows + +package runtime + +import ( + "os" + "os/exec" +) + +type execManagedProcess struct { + cmd *exec.Cmd +} + +func startManagedProcess(command ProcessCommand, files managedProcessFiles, _ string) (managedProcess, error) { + cmd := exec.Command(command.Args[0], command.Args[1:]...) + cmd.Dir = command.WorkDir + cmd.Env = os.Environ() + for key, value := range command.Env { + cmd.Env = append(cmd.Env, key+"="+value) + } + cmd.Stdout = files.stdout + cmd.Stderr = files.stderr + if err := cmd.Start(); err != nil { + return nil, err + } + return &execManagedProcess{cmd: cmd}, nil +} + +func requestManagedProcessStop(identity ProcessIdentity) error { + process, err := os.FindProcess(identity.PID) + if err != nil { + return err + } + return process.Kill() +} + +func forceManagedProcessStop(identity ProcessIdentity) error { + return requestManagedProcessStop(identity) +} + +func (process *execManagedProcess) PID() int { + return process.cmd.Process.Pid +} + +func (process *execManagedProcess) TargetPID() int { + return process.PID() +} + +func (process *execManagedProcess) Wait() (int, error) { + err := process.cmd.Wait() + if process.cmd.ProcessState == nil { + return -1, err + } + return process.cmd.ProcessState.ExitCode(), err +} + +func (process *execManagedProcess) Kill() error { + return process.cmd.Process.Kill() +} diff --git a/runtime/metrics.go b/runtime/metrics.go new file mode 100644 index 0000000..073ab36 --- /dev/null +++ b/runtime/metrics.go @@ -0,0 +1,112 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "browser.local/run/config" + "browser.local/run/protocol" +) + +const metricReportTimeout = 5 * time.Second + +// MetricCollector provides only generic local utilization data. Game-specific +// observations belong to a plugin-declared bridge, never the Run runtime. +type MetricCollector interface { + Collect(context.Context, string) (MetricUtilization, error) +} + +type MetricUtilization struct { + CPUPercent *float64 + MemoryPercent *float64 + DiskPercent *float64 +} + +type defaultMetricCollector struct{} + +func (defaultMetricCollector) Collect(ctx context.Context, workspaceRoot string) (MetricUtilization, error) { + var utilization MetricUtilization + var collectionErrors []error + if cpuPercent, err := hostCPUPercent(ctx); err != nil { + collectionErrors = append(collectionErrors, fmt.Errorf("collect CPU utilization: %w", err)) + } else { + utilization.CPUPercent = &cpuPercent + } + if memoryPercent, err := hostMemoryPercent(); err != nil { + collectionErrors = append(collectionErrors, fmt.Errorf("collect memory utilization: %w", err)) + } else { + utilization.MemoryPercent = &memoryPercent + } + if err := ctx.Err(); err != nil { + collectionErrors = append(collectionErrors, err) + } else if diskPercent, err := workspaceDiskPercent(workspaceRoot); err != nil { + collectionErrors = append(collectionErrors, fmt.Errorf("collect disk utilization: %w", err)) + } else { + utilization.DiskPercent = &diskPercent + } + return utilization, errors.Join(collectionErrors...) +} + +// WithMetricCollector overrides the generic host collector for tests and +// platform-specific collectors. It has no access to plugin inputs or secrets. +func WithMetricCollector(collector MetricCollector) LifecycleExecutorOption { + return func(executor *LifecycleExecutor) { + executor.metricCollector = collector + } +} + +func (worker *Worker) reportMetricsDegraded(ctx context.Context, trigger string) { + if err := worker.ReportMetricsOnce(ctx); err != nil { + log.Printf("RUN phase=metrics status=degraded trigger=%s error=%s", safeOptional(trigger), RedactText(err.Error())) + } +} + +func (worker *Worker) ReportMetricsOnce(ctx context.Context) error { + if worker.cfg.ComponentKind != config.PackageComponentRun || worker.cfg.ServerInstanceID == "" { + return nil + } + state, err := worker.registeredState() + if err != nil { + return err + } + sample := protocol.MetricSample{ServerInstanceID: worker.cfg.ServerInstanceID, Online: worker.managedServerProcessOnline(state), Source: "run", CollectedAt: time.Now().UTC()} + if worker.metricCollector != nil { + utilization, collectErr := worker.metricCollector.Collect(ctx, worker.cfg.WorkspaceRoot) + sample.CPUPercent = utilization.CPUPercent + sample.MemoryPercent = utilization.MemoryPercent + sample.DiskPercent = utilization.DiskPercent + if collectErr != nil { + log.Printf("RUN phase=metrics status=utilization_unavailable error=%s", RedactText(collectErr.Error())) + } + } + reportCtx, cancel := context.WithTimeout(ctx, metricReportTimeout) + defer cancel() + response, err := worker.client.IngestMetricBatch(reportCtx, protocol.MetricBatchIngestRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, Samples: []protocol.MetricSample{sample}}) + if err != nil { + return err + } + if !response.Accepted || response.AcceptedCount != 1 { + return fmt.Errorf("metric batch was not accepted") + } + log.Printf("RUN phase=metrics status=accepted server=%s online=%t", sample.ServerInstanceID, sample.Online) + return nil +} + +func (worker *Worker) managedServerProcessOnline(state WorkerState) bool { + source, ok := worker.executor.managed.(ManagedProcessObservationSource) + if !ok { + return false + } + for _, identity := range source.ManagedProcessObservations() { + if identity.ServerInstanceID != worker.cfg.ServerInstanceID || identity.RunEndpointID != state.RunEndpointID { + continue + } + if worker.executor.managed.Status(identity).State == "running" { + return true + } + } + return false +} diff --git a/runtime/metrics_disk_unix.go b/runtime/metrics_disk_unix.go new file mode 100644 index 0000000..e5a35dd --- /dev/null +++ b/runtime/metrics_disk_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package runtime + +import "syscall" + +func workspaceDiskPercent(workspaceRoot string) (float64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(workspaceRoot, &stat); err != nil { + return 0, err + } + if stat.Blocks == 0 { + return 0, syscall.EINVAL + } + return 100 * float64(stat.Blocks-stat.Bavail) / float64(stat.Blocks), nil +} diff --git a/runtime/metrics_disk_windows.go b/runtime/metrics_disk_windows.go new file mode 100644 index 0000000..75806ae --- /dev/null +++ b/runtime/metrics_disk_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +package runtime + +import ( + "path/filepath" + "syscall" + "unsafe" +) + +var getDiskFreeSpaceEx = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW") + +func workspaceDiskPercent(workspaceRoot string) (float64, error) { + volume := filepath.VolumeName(workspaceRoot) + if volume == "" { + volume = workspaceRoot + } else { + volume += `\` + } + var available, total, free uint64 + success, _, callErr := getDiskFreeSpaceEx.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(volume))), uintptr(unsafe.Pointer(&available)), uintptr(unsafe.Pointer(&total)), uintptr(unsafe.Pointer(&free))) + if success == 0 { + return 0, callErr + } + if total == 0 { + return 0, syscall.EINVAL + } + return 100 * float64(total-free) / float64(total), nil +} diff --git a/runtime/metrics_host_linux.go b/runtime/metrics_host_linux.go new file mode 100644 index 0000000..d642471 --- /dev/null +++ b/runtime/metrics_host_linux.go @@ -0,0 +1,123 @@ +//go:build linux + +package runtime + +import ( + "bufio" + "context" + "fmt" + "os" + "strconv" + "strings" + "time" +) + +const cpuSampleInterval = 100 * time.Millisecond + +func hostCPUPercent(ctx context.Context) (float64, error) { + first, err := readLinuxCPUStat() + if err != nil { + return 0, err + } + timer := time.NewTimer(cpuSampleInterval) + defer timer.Stop() + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-timer.C: + } + second, err := readLinuxCPUStat() + if err != nil { + return 0, err + } + totalDelta := second.total - first.total + busyDelta := second.busy - first.busy + if totalDelta == 0 || busyDelta > totalDelta { + return 0, fmt.Errorf("CPU counters did not advance") + } + return clampMetricPercent(100 * float64(busyDelta) / float64(totalDelta)), nil +} + +type linuxCPUStat struct { + total uint64 + busy uint64 +} + +func readLinuxCPUStat() (linuxCPUStat, error) { + file, err := os.Open("/proc/stat") + if err != nil { + return linuxCPUStat{}, err + } + defer file.Close() + line, err := bufio.NewReader(file).ReadString('\n') + if err != nil { + return linuxCPUStat{}, err + } + fields := strings.Fields(line) + if len(fields) < 5 || fields[0] != "cpu" { + return linuxCPUStat{}, fmt.Errorf("/proc/stat CPU row is invalid") + } + values := make([]uint64, len(fields)-1) + for index, field := range fields[1:] { + value, parseErr := strconv.ParseUint(field, 10, 64) + if parseErr != nil { + return linuxCPUStat{}, fmt.Errorf("parse /proc/stat CPU counter: %w", parseErr) + } + values[index] = value + } + var total uint64 + for _, value := range values { + total += value + } + idle := values[3] + if len(values) > 4 { + idle += values[4] + } + if idle > total { + return linuxCPUStat{}, fmt.Errorf("/proc/stat idle counter exceeds total") + } + return linuxCPUStat{total: total, busy: total - idle}, nil +} + +func hostMemoryPercent() (float64, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer file.Close() + var total, available uint64 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 2 { + continue + } + value, parseErr := strconv.ParseUint(fields[1], 10, 64) + if parseErr != nil { + return 0, fmt.Errorf("parse /proc/meminfo: %w", parseErr) + } + switch fields[0] { + case "MemTotal:": + total = value + case "MemAvailable:": + available = value + } + } + if err := scanner.Err(); err != nil { + return 0, err + } + if total == 0 || available > total { + return 0, fmt.Errorf("/proc/meminfo memory counters are invalid") + } + return clampMetricPercent(100 * float64(total-available) / float64(total)), nil +} + +func clampMetricPercent(value float64) float64 { + if value < 0 { + return 0 + } + if value > 100 { + return 100 + } + return value +} diff --git a/runtime/metrics_host_other.go b/runtime/metrics_host_other.go new file mode 100644 index 0000000..cc2400e --- /dev/null +++ b/runtime/metrics_host_other.go @@ -0,0 +1,17 @@ +//go:build !linux && !windows + +package runtime + +import ( + "context" + "fmt" + "runtime" +) + +func hostCPUPercent(context.Context) (float64, error) { + return 0, fmt.Errorf("CPU utilization is not implemented for %s", runtime.GOOS) +} + +func hostMemoryPercent() (float64, error) { + return 0, fmt.Errorf("memory utilization is not implemented for %s", runtime.GOOS) +} diff --git a/runtime/metrics_host_windows.go b/runtime/metrics_host_windows.go new file mode 100644 index 0000000..8e8048e --- /dev/null +++ b/runtime/metrics_host_windows.go @@ -0,0 +1,101 @@ +//go:build windows + +package runtime + +import ( + "context" + "fmt" + "syscall" + "time" + "unsafe" +) + +const cpuSampleInterval = 100 * time.Millisecond + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + getSystemTimesProc = kernel32.NewProc("GetSystemTimes") + globalMemoryStatusExProc = kernel32.NewProc("GlobalMemoryStatusEx") +) + +type windowsFileTime struct { + lowDateTime uint32 + highDateTime uint32 +} + +type windowsMemoryStatusEx struct { + dwLength uint32 + dwMemoryLoad uint32 + ullTotalPhys uint64 + ullAvailPhys uint64 + ullTotalPageFile uint64 + ullAvailPageFile uint64 + ullTotalVirtual uint64 + ullAvailVirtual uint64 + ullAvailExtendedVirtual uint64 +} + +type windowsCPUStat struct { + idle uint64 + total uint64 +} + +func hostCPUPercent(ctx context.Context) (float64, error) { + first, err := readWindowsCPUStat() + if err != nil { + return 0, err + } + timer := time.NewTimer(cpuSampleInterval) + defer timer.Stop() + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-timer.C: + } + second, err := readWindowsCPUStat() + if err != nil { + return 0, err + } + totalDelta := second.total - first.total + idleDelta := second.idle - first.idle + if totalDelta == 0 || idleDelta > totalDelta { + return 0, fmt.Errorf("Windows CPU counters did not advance") + } + return clampMetricPercent(100 * float64(totalDelta-idleDelta) / float64(totalDelta)), nil +} + +func readWindowsCPUStat() (windowsCPUStat, error) { + var idle, kernel, user windowsFileTime + result, _, callErr := getSystemTimesProc.Call(uintptr(unsafe.Pointer(&idle)), uintptr(unsafe.Pointer(&kernel)), uintptr(unsafe.Pointer(&user))) + if result == 0 { + return windowsCPUStat{}, callErr + } + idleTicks := windowsFileTimeValue(idle) + return windowsCPUStat{idle: idleTicks, total: idleTicks + windowsFileTimeValue(kernel) + windowsFileTimeValue(user)}, nil +} + +func windowsFileTimeValue(value windowsFileTime) uint64 { + return uint64(value.highDateTime)<<32 | uint64(value.lowDateTime) +} + +func hostMemoryPercent() (float64, error) { + status := windowsMemoryStatusEx{dwLength: uint32(unsafe.Sizeof(windowsMemoryStatusEx{}))} + result, _, callErr := globalMemoryStatusExProc.Call(uintptr(unsafe.Pointer(&status))) + if result == 0 { + return 0, callErr + } + if status.ullTotalPhys == 0 || status.ullAvailPhys > status.ullTotalPhys { + return 0, fmt.Errorf("Windows memory counters are invalid") + } + return clampMetricPercent(100 * float64(status.ullTotalPhys-status.ullAvailPhys) / float64(status.ullTotalPhys)), nil +} + +func clampMetricPercent(value float64) float64 { + if value < 0 { + return 0 + } + if value > 100 { + return 100 + } + return value +} diff --git a/runtime/metrics_test.go b/runtime/metrics_test.go new file mode 100644 index 0000000..fe5ccea --- /dev/null +++ b/runtime/metrics_test.go @@ -0,0 +1,134 @@ +package runtime + +import ( + "context" + "errors" + "testing" + + "browser.local/run/config" +) + +func TestWorkerReportsMetricsWithCurrentSessionAndManagedProcessState(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.ComponentKind = config.PackageComponentRun + managed := &metricManagedSupervisor{identity: ProcessIdentity{ServerInstanceID: cfg.ServerInstanceID, RunEndpointID: cfg.RunEndpointID, Scope: "server", State: "running"}} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithMetricCollector(metricCollectorStub{disk: 42.5})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if err := worker.ReportMetricsOnce(context.Background()); err != nil { + t.Fatalf("report metrics: %v", err) + } + if len(client.metricRequests) != 1 { + t.Fatalf("expected one metric request, got %d", len(client.metricRequests)) + } + request := client.metricRequests[0] + if request.RunEndpointID != cfg.RunEndpointID || request.SessionToken != "session-token" || len(request.Samples) != 1 { + t.Fatalf("expected registered session metric request, got %+v", request) + } + sample := request.Samples[0] + if !sample.Online || sample.Source != "run" || sample.PlayerCount != nil || sample.TPS != nil || sample.LatencyMS != nil || sample.DiskPercent == nil || *sample.DiskPercent != 42.5 { + t.Fatalf("unexpected generic metric sample: %+v", sample) + } +} + +func TestWorkerReportsOfflineWhenManagedProcessExitedAndOmitsUnavailableUtilization(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.ComponentKind = config.PackageComponentRun + managed := &metricManagedSupervisor{identity: ProcessIdentity{ServerInstanceID: cfg.ServerInstanceID, RunEndpointID: cfg.RunEndpointID, Scope: "server", State: "exited"}} + worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithMetricCollector(metricCollectorError{})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if err := worker.ReportMetricsOnce(context.Background()); err != nil { + t.Fatalf("report metrics: %v", err) + } + sample := client.metricRequests[0].Samples[0] + if sample.Online || sample.CPUPercent != nil || sample.MemoryPercent != nil || sample.DiskPercent != nil { + t.Fatalf("expected offline sample without unavailable utilization, got %+v", sample) + } +} + +func TestWorkerReportsAvailableUtilizationWhenOneMetricFails(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.ComponentKind = config.PackageComponentRun + worker, err := NewWorker(cfg, client, WithMetricCollector(metricCollectorPartial{disk: 42.5})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if err := worker.ReportMetricsOnce(context.Background()); err != nil { + t.Fatalf("report metrics: %v", err) + } + sample := client.metricRequests[0].Samples[0] + if sample.DiskPercent == nil || *sample.DiskPercent != 42.5 { + t.Fatalf("expected available disk utilization to be reported, got %+v", sample) + } +} + +func TestMetricUploadFailureDoesNotFailReportingCycle(t *testing.T) { + client := newFakeWorkerClient() + client.metricErr = errors.New("platform unavailable") + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "server-worker" + cfg.ComponentKind = config.PackageComponentRun + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + worker.reportMetricsDegraded(context.Background(), "test") + if len(client.metricRequests) != 1 { + t.Fatalf("expected degraded metric attempt, got %d", len(client.metricRequests)) + } +} + +type metricManagedSupervisor struct{ identity ProcessIdentity } + +func (supervisor *metricManagedSupervisor) Start(context.Context, ProcessCommand, ProcessIdentity, ManagedProcessOutput) (ProcessIdentity, error) { + return supervisor.identity, nil +} +func (supervisor *metricManagedSupervisor) Stop(context.Context, ProcessIdentity) (ProcessIdentity, error) { + return supervisor.identity, nil +} +func (supervisor *metricManagedSupervisor) Status(ProcessIdentity) ProcessIdentity { + return supervisor.identity +} +func (supervisor *metricManagedSupervisor) ResumeOutput(ManagedProcessOutput) {} +func (supervisor *metricManagedSupervisor) ManagedProcessObservations() []ProcessIdentity { + return []ProcessIdentity{supervisor.identity} +} + +type metricCollectorStub struct{ disk float64 } + +func (collector metricCollectorStub) Collect(context.Context, string) (MetricUtilization, error) { + return MetricUtilization{DiskPercent: &collector.disk}, nil +} + +type metricCollectorError struct{} + +func (metricCollectorError) Collect(context.Context, string) (MetricUtilization, error) { + return MetricUtilization{}, errors.New("collector unavailable") +} + +type metricCollectorPartial struct{ disk float64 } + +func (collector metricCollectorPartial) Collect(context.Context, string) (MetricUtilization, error) { + return MetricUtilization{DiskPercent: &collector.disk}, errors.New("CPU unavailable") +} diff --git a/runtime/process_alive_default.go b/runtime/process_alive_default.go new file mode 100644 index 0000000..9354af9 --- /dev/null +++ b/runtime/process_alive_default.go @@ -0,0 +1,19 @@ +//go:build !windows + +package runtime + +import ( + "os" + "syscall" +) + +func processAlivePID(pid int) bool { + if pid <= 0 { + return false + } + process, err := os.FindProcess(pid) + if err != nil { + return false + } + return process.Signal(syscall.Signal(0)) == nil +} diff --git a/runtime/process_alive_windows.go b/runtime/process_alive_windows.go new file mode 100644 index 0000000..ffd2a88 --- /dev/null +++ b/runtime/process_alive_windows.go @@ -0,0 +1,55 @@ +//go:build windows + +package runtime + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + windowsProcessQueryLimitedInformation = 0x1000 + windowsStillActive = 259 +) + +func processAlivePID(pid int) bool { + if pid <= 0 { + return false + } + handle, err := windows.OpenProcess(windowsProcessQueryLimitedInformation, false, uint32(pid)) + if err != nil { + if processSnapshotContainsPID(pid) { + return true + } + return err == syscall.ERROR_ACCESS_DENIED + } + defer windows.CloseHandle(handle) + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err == nil { + return exitCode == windowsStillActive + } + return processSnapshotContainsPID(pid) +} + +func processSnapshotContainsPID(pid int) bool { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) + if err != nil { + return false + } + defer windows.CloseHandle(snapshot) + var entry windows.ProcessEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Process32First(snapshot, &entry); err != nil { + return false + } + for { + if entry.ProcessID == uint32(pid) { + return true + } + if err := windows.Process32Next(snapshot, &entry); err != nil { + return false + } + } +} diff --git a/runtime/process_control_test.go b/runtime/process_control_test.go new file mode 100644 index 0000000..db3ea8a --- /dev/null +++ b/runtime/process_control_test.go @@ -0,0 +1,49 @@ +package runtime + +import ( + "strings" + "testing" +) + +func TestManagedProcessStopEventNameIsStableAndScoped(t *testing.T) { + base := ProcessIdentity{Scope: `C:\workspace\instances\server-1\run-local`, RunEndpointID: "run-1", ServerInstanceID: "server-1", LogSessionID: "session-a"} + if got, want := managedProcessStopEventName(base), managedProcessStopEventName(base); got != want { + t.Fatalf("stop event name is not stable: got=%q want=%q", got, want) + } + if got := managedProcessStopEventName(base); !strings.HasPrefix(got, `Local\run-managed-stop-`) { + t.Fatalf("stop event name is not in the local namespace: %q", got) + } + if strings.Contains(managedProcessStopEventName(base), "workspace") { + t.Fatalf("stop event name leaked the process scope: %q", managedProcessStopEventName(base)) + } + for _, changed := range []ProcessIdentity{ + {Scope: base.Scope, RunEndpointID: "run-2", ServerInstanceID: base.ServerInstanceID, LogSessionID: base.LogSessionID}, + {Scope: base.Scope, RunEndpointID: base.RunEndpointID, ServerInstanceID: "server-2", LogSessionID: base.LogSessionID}, + {Scope: base.Scope, RunEndpointID: base.RunEndpointID, ServerInstanceID: base.ServerInstanceID, LogSessionID: "session-b"}, + } { + if got := managedProcessStopEventName(changed); got == managedProcessStopEventName(base) { + t.Fatalf("different process generation shared stop event name: base=%q changed=%q", managedProcessStopEventName(base), got) + } + } +} + +func TestManagedProcessGenerationRejectsOutputFileSwap(t *testing.T) { + current := ProcessIdentity{ + PID: 100, + LogSessionID: "session-a", + StdoutLogRef: "stdout-a.log", + StderrLogRef: "stderr-a.log", + } + + if !sameManagedProcessGeneration(current, current) { + t.Fatal("expected an identical managed process generation to match") + } + for _, expected := range []ProcessIdentity{ + {PID: 100, LogSessionID: "session-a", StdoutLogRef: "stdout-b.log", StderrLogRef: "stderr-a.log"}, + {PID: 100, LogSessionID: "session-a", StdoutLogRef: "stdout-a.log", StderrLogRef: "stderr-b.log"}, + } { + if sameManagedProcessGeneration(current, expected) { + t.Fatalf("output file swap was treated as the same generation: current=%+v expected=%+v", current, expected) + } + } +} diff --git a/runtime/process_state_isolation_test.go b/runtime/process_state_isolation_test.go new file mode 100644 index 0000000..66629f8 --- /dev/null +++ b/runtime/process_state_isolation_test.go @@ -0,0 +1,110 @@ +package runtime + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "browser.local/run/config" +) + +func TestManagedProcessStateRootIsolatedPerRunService(t *testing.T) { + workspace := t.TempDir() + base := config.Config{WorkspaceRoot: workspace, ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"} + first := base + first.RunEndpointID = "run-a" + second := base + second.RunEndpointID = "run-b" + third := base + third.RunEndpointID = "run-a" + third.ComponentKey = "run-secondary" + + firstRoot := managedProcessStateRoot(first) + secondRoot := managedProcessStateRoot(second) + thirdRoot := managedProcessStateRoot(third) + if firstRoot == secondRoot || firstRoot == thirdRoot || secondRoot == thirdRoot { + t.Fatalf("run services must not share managed process state: %q %q %q", firstRoot, secondRoot, thirdRoot) + } + wantPrefix := filepath.Join(workspace, "run-services") + string(filepath.Separator) + for _, root := range []string{firstRoot, secondRoot, thirdRoot} { + if !strings.HasPrefix(root, wantPrefix) { + t.Fatalf("managed process state escaped the isolated root: %q", root) + } + if len(filepath.Base(root)) != 64 { + t.Fatalf("managed process state namespace is not a sha256 directory: %q", root) + } + } +} + +func TestMigrateLegacyManagedProcessStateKeepsOnlyThisRunService(t *testing.T) { + workspace := t.TempDir() + cfg := config.Config{WorkspaceRoot: workspace, RunEndpointID: "run-a", ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"} + legacy := processJournal{Version: 1, Items: map[string]ProcessIdentity{ + "owned": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 101, State: "running"}, + "other-endpoint": {RunEndpointID: "run-b", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 102, State: "running"}, + "other-profile": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-other", PID: 103, State: "running"}, + }} + body, err := json.Marshal(legacy) + if err != nil { + t.Fatalf("marshal legacy journal: %v", err) + } + legacyPath := filepath.Join(workspace, "state", "processes.json") + if err := os.MkdirAll(filepath.Dir(legacyPath), 0o700); err != nil { + t.Fatalf("create legacy state directory: %v", err) + } + if err := os.WriteFile(legacyPath, body, 0o600); err != nil { + t.Fatalf("write legacy journal: %v", err) + } + + stateRoot := managedProcessStateRoot(cfg) + if err := migrateLegacyManagedProcessState(cfg, stateRoot); err != nil { + t.Fatalf("migrate legacy journal: %v", err) + } + migratedBody, err := os.ReadFile(filepath.Join(stateRoot, "state", "processes.json")) + if err != nil { + t.Fatalf("read isolated journal: %v", err) + } + var migrated processJournal + if err := json.Unmarshal(migratedBody, &migrated); err != nil { + t.Fatalf("decode isolated journal: %v", err) + } + if len(migrated.Items) != 1 || migrated.Items["owned"].PID != 101 { + t.Fatalf("isolated journal imported another Run service: %+v", migrated.Items) + } +} + +func TestMigrateManagedProcessStateImportsMatchingPriorNamespace(t *testing.T) { + workspace := t.TempDir() + cfg := config.Config{WorkspaceRoot: workspace, RunEndpointID: "run-a", ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"} + priorRoot := filepath.Join(workspace, "run-services", "prior", "state") + if err := os.MkdirAll(priorRoot, 0o700); err != nil { + t.Fatalf("create prior state directory: %v", err) + } + body, err := json.Marshal(processJournal{Version: 2, Items: map[string]ProcessIdentity{ + "owned": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 101, State: "exited"}, + "other": {RunEndpointID: "run-b", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 102, State: "running"}, + }}) + if err != nil { + t.Fatalf("marshal prior journal: %v", err) + } + if err := os.WriteFile(filepath.Join(priorRoot, "processes.json"), body, 0o600); err != nil { + t.Fatalf("write prior journal: %v", err) + } + stateRoot := managedProcessStateRoot(cfg) + if err := migrateManagedProcessState(cfg, stateRoot); err != nil { + t.Fatalf("migrate managed state: %v", err) + } + migratedBody, err := os.ReadFile(filepath.Join(stateRoot, "state", "processes.json")) + if err != nil { + t.Fatalf("read migrated journal: %v", err) + } + var migrated processJournal + if err := json.Unmarshal(migratedBody, &migrated); err != nil { + t.Fatalf("decode migrated journal: %v", err) + } + if len(migrated.Items) != 1 || migrated.Items["owned"].PID != 101 { + t.Fatalf("unexpected migrated matching state: %+v", migrated.Items) + } +} diff --git a/runtime/process_supervisor.go b/runtime/process_supervisor.go new file mode 100644 index 0000000..53aa2a7 --- /dev/null +++ b/runtime/process_supervisor.go @@ -0,0 +1,811 @@ +package runtime + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + managedProcessJournalVersion = 2 + managedProcessOutputPollInterval = 50 * time.Millisecond + managedProcessOutputDrainDelay = 750 * time.Millisecond + managedProcessOutputRetryDelay = 500 * time.Millisecond +) + +type ProcessIdentity struct { + Scope string `json:"scope"` + ServerInstanceID string `json:"serverInstanceId"` + RunEndpointID string `json:"runEndpointId,omitempty"` + JobID string `json:"jobId,omitempty"` + Capability string `json:"capability,omitempty"` + ProfileKey string `json:"profileKey"` + LogSessionID string `json:"logSessionId,omitempty"` + PID int `json:"pid"` + SupervisorPID int `json:"supervisorPid,omitempty"` + StartedAt time.Time `json:"startedAt"` + CommandFingerprint string `json:"commandFingerprint"` + State string `json:"state"` + ExitCode int `json:"exitCode,omitempty"` + ExitClassification string `json:"exitClassification,omitempty"` + ObservationSeq uint64 `json:"observationSeq,omitempty"` + Attempt int `json:"attempt"` + LeaseTokenHash string `json:"leaseTokenHash,omitempty"` + StdoutLogRef string `json:"stdoutLogRef,omitempty"` + StderrLogRef string `json:"stderrLogRef,omitempty"` + StdoutStreamKey string `json:"stdoutStreamKey,omitempty"` + StderrStreamKey string `json:"stderrStreamKey,omitempty"` + StopEventName string `json:"stopEventName,omitempty"` + StdoutOffset int64 `json:"stdoutOffset,omitempty"` + StderrOffset int64 `json:"stderrOffset,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type processJournal struct { + Version int `json:"version"` + Items map[string]ProcessIdentity `json:"items"` + Retired map[string]ProcessIdentity `json:"retired,omitempty"` +} + +type ManagedProcessSupervisor interface { + Start(context.Context, ProcessCommand, ProcessIdentity, ManagedProcessOutput) (ProcessIdentity, error) + Stop(context.Context, ProcessIdentity) (ProcessIdentity, error) + Status(ProcessIdentity) ProcessIdentity + ResumeOutput(ManagedProcessOutput) +} + +// ManagedProcessObservationSource exposes only generic supervised-process +// facts. Worker uses it to report persisted transitions after registration. +type ManagedProcessObservationSource interface { + ManagedProcessObservations() []ProcessIdentity +} + +type ManagedProcessLine struct { + Text string + StartOffset int64 + EndOffset int64 +} + +type ManagedProcessLineSink func(ProcessIdentity, ManagedProcessLine) error + +type ManagedProcessOutput struct { + Stdout ManagedProcessLineSink + Stderr ManagedProcessLineSink +} + +type managedProcessFiles struct { + stdout *os.File + stderr *os.File +} + +type managedProcess interface { + PID() int + TargetPID() int + Wait() (int, error) + Kill() error +} + +type managedProcessTailer struct { + cancel context.CancelFunc + drain chan struct{} +} + +type OSManagedProcessSupervisor struct { + root string + path string + outputRoot string + mu sync.Mutex + items map[string]ProcessIdentity + retired map[string]ProcessIdentity + tailers map[string]*managedProcessTailer +} + +func NewOSManagedProcessSupervisor(root string) (*OSManagedProcessSupervisor, error) { + return NewOSManagedProcessSupervisorWithOutputRoot(root, root) +} + +func NewOSManagedProcessSupervisorWithOutputRoot(root string, outputRoot string) (*OSManagedProcessSupervisor, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + outputRootAbs, err := filepath.Abs(outputRoot) + if err != nil { + return nil, err + } + stateDir := filepath.Join(rootAbs, "state") + if err := ensureDirectory(stateDir); err != nil { + return nil, err + } + supervisor := &OSManagedProcessSupervisor{root: rootAbs, path: filepath.Join(stateDir, "processes.json"), outputRoot: outputRootAbs, items: map[string]ProcessIdentity{}, retired: map[string]ProcessIdentity{}, tailers: map[string]*managedProcessTailer{}} + if err := supervisor.load(); err != nil { + return nil, err + } + if err := supervisor.migrateLegacySessions(); err != nil { + return nil, err + } + supervisor.Reconcile() + return supervisor, nil +} + +func (supervisor *OSManagedProcessSupervisor) Start(ctx context.Context, command ProcessCommand, identity ProcessIdentity, output ManagedProcessOutput) (ProcessIdentity, error) { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + key := identity.Scope + log.Printf("RUN phase=process.managed status=start_requested job=%s server=%s scope=%s command=%s workdir=%s", safeOptional(identity.JobID), identity.ServerInstanceID, safeOptional(identity.Scope), redactedCommandLine(command.Args), safeOptional(command.WorkDir)) + if existing, ok := supervisor.items[key]; ok && existing.State == "running" && supervisor.isAlive(existing) { + if existing.LogSessionID == "" { + logSessionID, err := newManagedProcessLogSessionID() + if err != nil { + return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err) + } + existing.LogSessionID = logSessionID + } + if existing.StopEventName == "" { + existing.StopEventName = managedProcessStopEventName(existing) + } + existing.State = "running" + existing.ObservationSeq++ + existing.UpdatedAt = time.Now().UTC() + supervisor.items[key] = existing + if err := supervisor.persistLocked(); err != nil { + return ProcessIdentity{}, fmt.Errorf("persist managed process session: %w", err) + } + supervisor.startTailersLocked(existing, output) + log.Printf("RUN phase=process.managed status=reusing_existing job=%s pid=%d state=%s", safeOptional(identity.JobID), existing.PID, existing.State) + return existing, nil + } + if err := ctx.Err(); err != nil { + log.Printf("RUN phase=process.managed status=context_done job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error())) + return ProcessIdentity{}, err + } + if len(command.Args) == 0 { + log.Printf("RUN phase=process.managed status=missing_executable job=%s", safeOptional(identity.JobID)) + return ProcessIdentity{}, fmt.Errorf("process executable is required") + } + startedAt := time.Now().UTC() + if identity.LogSessionID == "" { + logSessionID, err := newManagedProcessLogSessionID() + if err != nil { + return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err) + } + identity.LogSessionID = logSessionID + } + if identity.StopEventName == "" { + identity.StopEventName = managedProcessStopEventName(identity) + } + files, identity, err := supervisor.prepareOutputFilesLocked(identity, startedAt) + if err != nil { + log.Printf("RUN phase=process.managed status=prepare_output_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error())) + return ProcessIdentity{}, err + } + log.Printf("RUN phase=process.managed status=output_ready job=%s stdoutRef=%s stderrRef=%s", safeOptional(identity.JobID), safeOptional(identity.StdoutLogRef), safeOptional(identity.StderrLogRef)) + process, err := startManagedProcess(command, files, identity.StopEventName) + if err != nil { + files.close() + log.Printf("RUN phase=process.managed status=start_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error())) + return ProcessIdentity{}, err + } + identity.SupervisorPID = process.PID() + identity.PID = process.TargetPID() + identity.StartedAt = startedAt + identity.State = "running" + identity.ObservationSeq = 1 + identity.UpdatedAt = identity.StartedAt + identity.CommandFingerprint = fingerprintArgs(command.Args) + previous, hadPrevious := supervisor.items[key] + retiredKey := "" + if hadPrevious && supervisor.hasPendingOutput(previous) { + retiredKey = managedProcessGenerationKey(previous) + supervisor.retired[retiredKey] = previous + } + supervisor.items[key] = identity + if err := supervisor.persistLocked(); err != nil { + _ = forceManagedProcessStop(identity) + files.close() + if hadPrevious { + supervisor.items[key] = previous + } else { + delete(supervisor.items, key) + } + if retiredKey != "" { + delete(supervisor.retired, retiredKey) + } + log.Printf("RUN phase=process.managed status=persist_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error())) + return ProcessIdentity{}, err + } + supervisor.startTailersLocked(identity, output) + go supervisor.wait(key, process, identity.PID, files) + log.Printf("RUN phase=process.managed status=started job=%s pid=%d fingerprint=%s", safeOptional(identity.JobID), identity.PID, safeOptional(identity.CommandFingerprint)) + return identity, nil +} + +func (supervisor *OSManagedProcessSupervisor) Stop(ctx context.Context, identity ProcessIdentity) (ProcessIdentity, error) { + supervisor.mu.Lock() + current, ok := supervisor.items[identity.Scope] + if !ok || current.State != "running" || !supervisor.isAlive(current) { + if ok { + current.State = "stopped" + current.ExitClassification = "already-stopped" + current.ObservationSeq++ + current.UpdatedAt = time.Now().UTC() + supervisor.items[identity.Scope] = current + _ = supervisor.persistLocked() + } + supervisor.mu.Unlock() + log.Printf("RUN phase=process.managed status=already_stopped job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope)) + return current, nil + } + log.Printf("RUN phase=process.managed status=stop_requested job=%s pid=%d scope=%s", safeOptional(identity.JobID), current.PID, safeOptional(identity.Scope)) + if err := requestManagedProcessStop(current); err != nil { + log.Printf("RUN phase=process.managed status=stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error())) + } + supervisor.mu.Unlock() + deadline := time.NewTimer(2 * time.Second) + ticker := time.NewTicker(20 * time.Millisecond) + defer deadline.Stop() + defer ticker.Stop() + for { + if !supervisor.isAlive(current) { + current.State = "stopped" + current.ExitClassification = "requested-stop" + current.ObservationSeq++ + current.UpdatedAt = time.Now().UTC() + supervisor.mu.Lock() + supervisor.items[current.Scope] = current + _ = supervisor.persistLocked() + supervisor.mu.Unlock() + supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay) + log.Printf("RUN phase=process.managed status=stopped job=%s pid=%d classification=%s", safeOptional(identity.JobID), current.PID, current.ExitClassification) + return current, nil + } + select { + case <-ctx.Done(): + log.Printf("RUN phase=process.managed status=stop_context_done job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(ctx.Err().Error())) + return ProcessIdentity{}, ctx.Err() + case <-deadline.C: + if err := forceManagedProcessStop(current); err != nil { + log.Printf("RUN phase=process.managed status=forced_stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error())) + } + current.State = "stopped" + current.ExitClassification = "forced-stop" + current.ObservationSeq++ + current.UpdatedAt = time.Now().UTC() + supervisor.mu.Lock() + supervisor.items[current.Scope] = current + _ = supervisor.persistLocked() + supervisor.mu.Unlock() + supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay) + log.Printf("RUN phase=process.managed status=forced_stop job=%s pid=%d", safeOptional(identity.JobID), current.PID) + return current, nil + case <-ticker.C: + } + } +} + +func (supervisor *OSManagedProcessSupervisor) Status(identity ProcessIdentity) ProcessIdentity { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + current, ok := supervisor.items[identity.Scope] + if !ok { + log.Printf("RUN phase=process.managed status=not_started job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope)) + return ProcessIdentity{Scope: identity.Scope, State: "stopped", ExitClassification: "not-started"} + } + if current.State == "running" && !supervisor.isAlive(current) { + current.State = "exited" + if current.ExitClassification == "" { + current.ExitClassification = "unexpected-exit" + } + current.UpdatedAt = time.Now().UTC() + current.ObservationSeq++ + supervisor.items[current.Scope] = current + _ = supervisor.persistLocked() + supervisor.drainTailersLocked(current) + } + log.Printf("RUN phase=process.managed status=current job=%s pid=%d state=%s classification=%s", safeOptional(identity.JobID), current.PID, current.State, safeOptional(current.ExitClassification)) + return current +} + +func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcessOutput) { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + for _, item := range supervisor.items { + if item.State == "running" && supervisor.isAlive(item) { + supervisor.startTailersLocked(item, output) + log.Printf("RUN phase=process.managed status=resume_output pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope)) + } else if supervisor.hasPendingOutput(item) { + supervisor.startDrainTailersLocked(item, output) + log.Printf("RUN phase=process.managed status=resume_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope)) + } + } + changed := false + for key, item := range supervisor.retired { + if supervisor.hasPendingOutput(item) { + supervisor.startDrainTailersLocked(item, output) + log.Printf("RUN phase=process.managed status=resume_retired_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope)) + continue + } + delete(supervisor.retired, key) + changed = true + } + if changed { + _ = supervisor.persistLocked() + } +} + +func (supervisor *OSManagedProcessSupervisor) Reconcile() { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + changed := false + for key, item := range supervisor.items { + if item.State == "running" && !supervisor.isAlive(item) { + item.State = "exited" + item.ExitClassification = "unexpected-exit" + item.UpdatedAt = time.Now().UTC() + item.ObservationSeq++ + supervisor.items[key] = item + supervisor.drainTailersLocked(item) + changed = true + } + } + if changed { + _ = supervisor.persistLocked() + } +} + +func (supervisor *OSManagedProcessSupervisor) wait(key string, process managedProcess, pid int, files managedProcessFiles) { + exitCode, err := process.Wait() + files.close() + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + item, ok := supervisor.items[key] + if !ok || item.PID != pid { + return + } + // A durable helper may exit independently of the target process (legacy + // shell wrappers can detach their child). Preserve a live target so the + // next Run can continue monitoring it by PID. + if processAlivePID(item.PID) { + item.State = "running" + item.SupervisorPID = 0 + item.ExitClassification = "supervisor-exited-target-alive" + } else { + item.State = "exited" + } + item.UpdatedAt = time.Now().UTC() + item.ObservationSeq++ + item.ExitCode = exitCode + if item.State == "running" { + // Keep the classification assigned above. + } else if err == nil { + item.ExitClassification = "clean-exit" + } else { + item.ExitClassification = "unexpected-exit" + } + supervisor.items[key] = item + _ = supervisor.persistLocked() + if item.State != "running" { + supervisor.drainTailersAfter(item, managedProcessOutputDrainDelay) + } + if err != nil { + log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s error=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification, RedactText(err.Error())) + return + } + log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification) +} + +func processTerminalStatus(state string) string { + if state == "running" { + return "supervisor_exited_target_alive" + } + return "exited" +} + +func (supervisor *OSManagedProcessSupervisor) ManagedProcessObservations() []ProcessIdentity { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + items := make([]ProcessIdentity, 0, len(supervisor.items)) + for _, item := range supervisor.items { + items = append(items, item) + } + return items +} + +func (files managedProcessFiles) close() { + if files.stdout != nil { + _ = files.stdout.Close() + } + if files.stderr != nil { + _ = files.stderr.Close() + } +} + +func (supervisor *OSManagedProcessSupervisor) prepareOutputFilesLocked(identity ProcessIdentity, startedAt time.Time) (managedProcessFiles, ProcessIdentity, error) { + outputDir := filepath.Join(supervisor.outputRoot, "state", "process-output") + if err := ensureDirectory(outputDir); err != nil { + return managedProcessFiles{}, ProcessIdentity{}, err + } + base := processOutputBase(identity.Scope+"\x00"+identity.LogSessionID, startedAt) + identity.StdoutLogRef = base + ".stdout.log" + identity.StderrLogRef = base + ".stderr.log" + stdout, stdoutOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StdoutLogRef)) + if err != nil { + return managedProcessFiles{}, ProcessIdentity{}, err + } + stderr, stderrOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StderrLogRef)) + if err != nil { + _ = stdout.Close() + return managedProcessFiles{}, ProcessIdentity{}, err + } + identity.StdoutOffset = stdoutOffset + identity.StderrOffset = stderrOffset + return managedProcessFiles{stdout: stdout, stderr: stderr}, identity, nil +} + +func openManagedOutputFile(path string) (*os.File, int64, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, 0, err + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, 0, err + } + return file, info.Size(), nil +} + +func (supervisor *OSManagedProcessSupervisor) startTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) { + if output.Stdout != nil && identity.StdoutLogRef != "" { + supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, true) + } + if output.Stderr != nil && identity.StderrLogRef != "" { + supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, true) + } +} + +func (supervisor *OSManagedProcessSupervisor) startDrainTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) { + if output.Stdout != nil && identity.StdoutLogRef != "" { + supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, false) + } + if output.Stderr != nil && identity.StderrLogRef != "" { + supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, false) + } +} + +func (supervisor *OSManagedProcessSupervisor) startTailerLocked(identity ProcessIdentity, stream string, ref string, offset int64, sink ManagedProcessLineSink, follow bool) { + if sink == nil { + return + } + tailerID := managedProcessTailerID(identity, stream) + if tailer, exists := supervisor.tailers[tailerID]; exists { + if !follow { + beginManagedProcessTailerDrain(tailer) + } + log.Printf("RUN phase=process.managed.output status=tail_reuse job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset) + return + } + path := filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(ref)) + ctx, cancel := context.WithCancel(context.Background()) + tailer := &managedProcessTailer{cancel: cancel, drain: make(chan struct{})} + if !follow { + beginManagedProcessTailerDrain(tailer) + } + supervisor.tailers[tailerID] = tailer + log.Printf("RUN phase=process.managed.output status=tail_start job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset) + go supervisor.tailOutput(ctx, tailerID, tailer, identity, stream, path, offset, sink) +} + +func (supervisor *OSManagedProcessSupervisor) tailOutput(ctx context.Context, tailerID string, tailer *managedProcessTailer, identity ProcessIdentity, stream string, path string, offset int64, sink ManagedProcessLineSink) { + defer supervisor.removeTailer(tailerID, tailer, identity) + file, err := os.Open(path) + if err != nil { + log.Printf("RUN phase=process.managed.output status=tail_open_failed job=%s pid=%d stream=%s path=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), RedactText(err.Error())) + return + } + defer file.Close() + if offset > 0 { + if _, err := file.Seek(offset, io.SeekStart); err != nil { + log.Printf("RUN phase=process.managed.output status=tail_seek_failed job=%s pid=%d stream=%s path=%s offset=%d error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), offset, RedactText(err.Error())) + return + } + } + reader := bufio.NewReader(file) + defer func() { + log.Printf("RUN phase=process.managed.output status=tail_stop job=%s pid=%d stream=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, offset) + }() + for { + line, err := reader.ReadString('\n') + if len(line) > 0 { + startOffset := offset + endOffset := offset + int64(len(line)) + text := strings.TrimSpace(line) + if text != "" { + for { + if sinkErr := sink(identity, ManagedProcessLine{Text: text, StartOffset: startOffset, EndOffset: endOffset}); sinkErr == nil { + log.Printf("RUN phase=process.managed.output status=line job=%s pid=%d stream=%s line=%q", safeOptional(identity.JobID), identity.PID, stream, RedactText(text)) + break + } else { + log.Printf("RUN phase=process.managed.output status=spool_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(sinkErr.Error())) + } + select { + case <-ctx.Done(): + return + case <-time.After(managedProcessOutputRetryDelay): + } + } + } + for { + if offsetErr := supervisor.updateOutputOffset(identity, stream, endOffset); offsetErr == nil { + break + } else { + log.Printf("RUN phase=process.managed.output status=offset_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(offsetErr.Error())) + } + select { + case <-ctx.Done(): + return + case <-time.After(managedProcessOutputRetryDelay): + } + } + offset = endOffset + } + if err == nil { + continue + } + if err != io.EOF { + return + } + select { + case <-ctx.Done(): + return + case <-tailer.drain: + return + case <-time.After(managedProcessOutputPollInterval): + } + } +} + +func (supervisor *OSManagedProcessSupervisor) removeTailer(tailerID string, tailer *managedProcessTailer, identity ProcessIdentity) { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + if current, ok := supervisor.tailers[tailerID]; ok && current == tailer { + delete(supervisor.tailers, tailerID) + } + key := managedProcessGenerationKey(identity) + if retired, ok := supervisor.retired[key]; ok && !supervisor.hasPendingOutput(retired) { + delete(supervisor.retired, key) + if err := supervisor.persistLocked(); err != nil { + supervisor.retired[key] = retired + log.Printf("RUN phase=process.managed.output status=retired_prune_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error())) + } + } +} + +func (supervisor *OSManagedProcessSupervisor) updateOutputOffset(identity ProcessIdentity, stream string, offset int64) error { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + item, ok := supervisor.items[identity.Scope] + retiredKey := "" + if !ok || !sameManagedProcessGeneration(item, identity) { + retiredKey = managedProcessGenerationKey(identity) + item, ok = supervisor.retired[retiredKey] + if !ok || !sameManagedProcessGeneration(item, identity) { + return nil + } + } + if stream == "stdout" { + item.StdoutOffset = offset + } else { + item.StderrOffset = offset + } + item.UpdatedAt = time.Now().UTC() + if retiredKey == "" { + supervisor.items[identity.Scope] = item + } else { + supervisor.retired[retiredKey] = item + } + return supervisor.persistLocked() +} + +func sameManagedProcessGeneration(current ProcessIdentity, expected ProcessIdentity) bool { + if current.LogSessionID != "" || expected.LogSessionID != "" { + if current.LogSessionID == "" || current.LogSessionID != expected.LogSessionID { + return false + } + return sameManagedProcessOutputFiles(current, expected) + } + return current.PID == expected.PID && current.StdoutLogRef == expected.StdoutLogRef && current.StderrLogRef == expected.StderrLogRef +} + +func sameManagedProcessOutputFiles(current ProcessIdentity, expected ProcessIdentity) bool { + if current.StdoutLogRef != "" && expected.StdoutLogRef != "" && current.StdoutLogRef != expected.StdoutLogRef { + return false + } + if current.StderrLogRef != "" && expected.StderrLogRef != "" && current.StderrLogRef != expected.StderrLogRef { + return false + } + return true +} + +func (supervisor *OSManagedProcessSupervisor) drainTailersAfter(identity ProcessIdentity, delay time.Duration) { + go func() { + time.Sleep(delay) + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + supervisor.drainTailersLocked(identity) + }() +} + +func (supervisor *OSManagedProcessSupervisor) drainTailersLocked(identity ProcessIdentity) { + for _, stream := range []string{"stdout", "stderr"} { + if tailer, ok := supervisor.tailers[managedProcessTailerID(identity, stream)]; ok { + beginManagedProcessTailerDrain(tailer) + } + } +} + +func beginManagedProcessTailerDrain(tailer *managedProcessTailer) { + select { + case <-tailer.drain: + default: + close(tailer.drain) + } +} + +func (supervisor *OSManagedProcessSupervisor) stopTailersLocked(identity ProcessIdentity) { + for _, stream := range []string{"stdout", "stderr"} { + tailerID := managedProcessTailerID(identity, stream) + if tailer, ok := supervisor.tailers[tailerID]; ok { + tailer.cancel() + delete(supervisor.tailers, tailerID) + } + } +} + +func managedProcessTailerID(identity ProcessIdentity, stream string) string { + generation := identity.LogSessionID + if generation == "" { + generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef) + } else { + generation += "\x00" + identity.StdoutLogRef + "\x00" + identity.StderrLogRef + } + return identity.Scope + "\x00" + generation + "\x00" + stream +} + +func managedProcessGenerationKey(identity ProcessIdentity) string { + generation := identity.LogSessionID + if generation == "" { + generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef) + } + return identity.Scope + "\x00" + generation +} + +func (supervisor *OSManagedProcessSupervisor) hasPendingOutput(identity ProcessIdentity) bool { + for _, item := range []struct { + ref string + offset int64 + }{{identity.StdoutLogRef, identity.StdoutOffset}, {identity.StderrLogRef, identity.StderrOffset}} { + if item.ref == "" { + continue + } + info, err := os.Stat(filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(item.ref))) + if err == nil && info.Size() > item.offset { + return true + } + } + return false +} + +func (supervisor *OSManagedProcessSupervisor) isAlive(item ProcessIdentity) bool { + return processAlivePID(item.PID) +} + +func (supervisor *OSManagedProcessSupervisor) load() error { + body, err := os.ReadFile(supervisor.path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var file processJournal + if err := json.Unmarshal(body, &file); err != nil { + return err + } + for key, item := range file.Items { + supervisor.items[key] = item + } + for key, item := range file.Retired { + supervisor.retired[key] = item + } + return nil +} + +func (supervisor *OSManagedProcessSupervisor) migrateLegacySessions() error { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + changed := false + for key, item := range supervisor.items { + if item.State != "running" || !supervisor.isAlive(item) { + continue + } + if item.LogSessionID == "" { + logSessionID, err := newManagedProcessLogSessionID() + if err != nil { + return fmt.Errorf("generate legacy managed process log session: %w", err) + } + item.LogSessionID = logSessionID + changed = true + } + if item.StartedAt.IsZero() { + item.StartedAt = time.Now().UTC() + changed = true + } + item.UpdatedAt = time.Now().UTC() + supervisor.items[key] = item + } + if !changed { + return nil + } + if err := supervisor.persistLocked(); err != nil { + return fmt.Errorf("persist legacy managed process log session: %w", err) + } + return nil +} + +func (supervisor *OSManagedProcessSupervisor) persistLocked() error { + body, err := json.Marshal(processJournal{Version: managedProcessJournalVersion, Items: supervisor.items, Retired: supervisor.retired}) + if err != nil { + return err + } + temporary := supervisor.path + ".tmp" + if err := os.WriteFile(temporary, body, 0o600); err != nil { + return err + } + if err := os.Rename(temporary, supervisor.path); err != nil { + _ = os.Remove(temporary) + return err + } + return nil +} + +func processOutputBase(scope string, startedAt time.Time) string { + sum := sha256.Sum256([]byte(scope + "\x00" + startedAt.Format(time.RFC3339Nano))) + return hex.EncodeToString(sum[:]) +} + +func managedProcessStopEventName(identity ProcessIdentity) string { + seed := strings.Join([]string{ + "run-managed-stop-v1", + identity.RunEndpointID, + identity.ServerInstanceID, + identity.Scope, + identity.LogSessionID, + }, "\x00") + sum := sha256.Sum256([]byte(seed)) + return "Local\\run-managed-stop-" + hex.EncodeToString(sum[:]) +} + +func newManagedProcessLogSessionID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + return hex.EncodeToString(value[:]), nil +} + +func fingerprintArgs(args []string) string { + sum := sha256.Sum256([]byte(strings.Join(args, "\x00"))) + return "sha256:" + hex.EncodeToString(sum[:]) +} diff --git a/runtime/process_window_default.go b/runtime/process_window_default.go new file mode 100644 index 0000000..1a72f90 --- /dev/null +++ b/runtime/process_window_default.go @@ -0,0 +1,9 @@ +//go:build !windows + +package runtime + +import "os/exec" + +func configureManagedProcessCommand(_ *exec.Cmd) {} + +func RunManagedProcessHelper(_ []string) (bool, int) { return false, 0 } diff --git a/runtime/process_window_windows.go b/runtime/process_window_windows.go new file mode 100644 index 0000000..2919a7e --- /dev/null +++ b/runtime/process_window_windows.go @@ -0,0 +1,745 @@ +//go:build windows + +package runtime + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + "unicode/utf16" + "unsafe" + + "golang.org/x/sys/windows" +) + +const managedProcessHelperFlag = "--run-managed-process-helper" + +func configureManagedProcessCommand(cmd *exec.Cmd) { + // Run may itself be launched by a service or task scheduler job that + // terminates its process tree on shutdown. The helper is the durable + // owner of the game process, so ask Windows to keep it outside that job. + // The helper itself does not need a console: its stdout/stderr are already + // durable files. CREATE_NEW_CONSOLE creates a second hidden conhost in a + // non-interactive Task Scheduler session and prevents the child pseudo + // console from initializing on Windows Server. + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_NO_WINDOW | windows.CREATE_BREAKAWAY_FROM_JOB, HideWindow: true} +} + +// Windows console output is collected by a child helper that owns the +// pseudo-console. The helper inherits the durable stdout/stderr files, so it +// remains attached to the supervised process when Run itself is updated or +// restarted. The new Run instance resumes tailing those files by offset. +type windowsFileManagedProcess struct { + cmd *exec.Cmd + helperExecutable string + targetPID int +} + +type managedProcessHelperSpec struct { + Command ProcessCommand `json:"command"` + StopEventName string `json:"stopEventName,omitempty"` + StdoutPath string `json:"stdoutPath,omitempty"` + StderrPath string `json:"stderrPath,omitempty"` + PIDPath string `json:"pidPath,omitempty"` +} + +func startManagedProcess(command ProcessCommand, files managedProcessFiles, stopEventName string) (managedProcess, error) { + pidPath := files.stdout.Name() + ".pid" + _ = os.Remove(pidPath) + body, err := json.Marshal(managedProcessHelperSpec{Command: command, StopEventName: stopEventName, StdoutPath: files.stdout.Name(), StderrPath: files.stderr.Name(), PIDPath: pidPath}) + if err != nil { + return nil, fmt.Errorf("encode managed process helper spec: %w", err) + } + payload := base64.RawURLEncoding.EncodeToString(body) + executable, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve Run executable for managed process helper: %w", err) + } + helperExecutable, err := prepareManagedProcessHelperExecutable(executable, files.stdout.Name()) + if err != nil { + return nil, fmt.Errorf("prepare managed process helper executable: %w", err) + } + cmd := exec.Command(helperExecutable, managedProcessHelperFlag, payload) + cmd.Stdout = files.stdout + cmd.Stderr = files.stderr + configureManagedProcessCommand(cmd) + if err := cmd.Start(); err != nil { + _ = os.Remove(helperExecutable) + return nil, err + } + targetPID := cmd.Process.Pid + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if body, readErr := os.ReadFile(pidPath); readErr == nil { + if parsed, parseErr := strconv.Atoi(strings.TrimSpace(string(body))); parseErr == nil && parsed > 0 { + targetPID = parsed + break + } + } + time.Sleep(10 * time.Millisecond) + } + return &windowsFileManagedProcess{cmd: cmd, helperExecutable: helperExecutable, targetPID: targetPID}, nil +} + +func prepareManagedProcessHelperExecutable(executable, outputPath string) (string, error) { + directory := filepath.Dir(outputPath) + if err := os.MkdirAll(directory, 0o700); err != nil { + return "", err + } + temporary, err := os.CreateTemp(directory, ".run-managed-helper-*.exe") + if err != nil { + return "", err + } + helper := temporary.Name() + if err := temporary.Close(); err != nil { + _ = os.Remove(helper) + return "", err + } + if err := os.Remove(helper); err != nil { + return "", err + } + if err := copyExecutable(executable, helper); err != nil { + _ = os.Remove(helper) + return "", err + } + return helper, nil +} + +func (process *windowsFileManagedProcess) PID() int { + return process.cmd.Process.Pid +} + +func (process *windowsFileManagedProcess) TargetPID() int { + if process.targetPID > 0 { + return process.targetPID + } + return process.PID() +} + +func (process *windowsFileManagedProcess) Wait() (int, error) { + err := process.cmd.Wait() + if process.helperExecutable != "" { + _ = os.Remove(process.helperExecutable) + } + if process.cmd.ProcessState == nil { + return -1, err + } + return process.cmd.ProcessState.ExitCode(), err +} + +func (process *windowsFileManagedProcess) Kill() error { + return process.cmd.Process.Kill() +} + +// RunManagedProcessHelper is invoked by the same executable in a detached +// child process. It is deliberately not a worker mode and does not register +// with Platform. +func RunManagedProcessHelper(args []string) (bool, int) { + if len(args) < 3 || args[1] != managedProcessHelperFlag { + return false, 0 + } + body, err := base64.RawURLEncoding.DecodeString(args[2]) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid managed process helper payload: %v\n", err) + return true, 2 + } + var spec managedProcessHelperSpec + if err := json.Unmarshal(body, &spec); err != nil || len(spec.Command.Args) == 0 { + if err == nil { + err = fmt.Errorf("managed process command is empty") + } + fmt.Fprintf(os.Stderr, "invalid managed process helper spec: %v\n", err) + return true, 2 + } + code, err := runManagedProcessHelper(spec) + if err != nil { + fmt.Fprintf(os.Stderr, "managed process helper failed: %v\n", err) + if code == 0 { + code = 1 + } + } + return true, code +} + +func runManagedProcessHelper(spec managedProcessHelperSpec) (int, error) { + defer scheduleManagedProcessHelperCleanup() + stdout, closeStdout, err := openManagedProcessOutput(spec.StdoutPath, os.Stdout) + if err != nil { + return 1, fmt.Errorf("open managed process stdout: %w", err) + } + defer closeStdout() + stderr, closeStderr, err := openManagedProcessOutput(spec.StderrPath, os.Stderr) + if err != nil { + return 1, fmt.Errorf("open managed process stderr: %w", err) + } + defer closeStderr() + command := spec.Command + stopEventName := spec.StopEventName + // The plugin-declared pipes mode uses ordinary inherited handles. The + // durable output files are attached directly to the child process, so this + // path remains valid when Run itself is updated or restarted. Console mode + // is deliberately kept separate for applications that switch from + // redirected handles to a Windows console after startup. + if command.OutputMode != "console" { + return runManagedProcessWithPipes(command, stopEventName, spec.PIDPath, stdout, stderr) + } + return runManagedProcessWithPseudoConsole(command, stopEventName, spec.PIDPath, stdout, stderr) +} + +func scheduleManagedProcessHelperCleanup() { + executable, err := os.Executable() + if err != nil || strings.TrimSpace(executable) == "" { + return + } + commandLine := "ping 127.0.0.1 -n 2 >nul & del /f /q " + quoteWindowsCommandArg(executable) + cleanup := exec.Command("cmd.exe", "/d", "/c", commandLine) + cleanup.Stdout = io.Discard + cleanup.Stderr = io.Discard + configureManagedProcessCommand(cleanup) + _ = cleanup.Start() +} + +func openManagedProcessOutput(path string, fallback *os.File) (io.Writer, func(), error) { + if strings.TrimSpace(path) == "" { + return fallback, func() {}, nil + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, nil, err + } + return file, func() { _ = file.Close() }, nil +} + +func runManagedProcessWithPipes(command ProcessCommand, stopEventName string, pidPath string, stdout io.Writer, stderr io.Writer) (int, error) { + if len(command.Args) == 0 { + return 1, fmt.Errorf("managed process command is empty") + } + application := command.Args[0] + if strings.EqualFold(application, "cmd.exe") { + if comspec := os.Getenv("ComSpec"); comspec != "" { + application = comspec + } + } + cmd := exec.Command(application, command.Args[1:]...) + cmd.Dir = command.WorkDir + cmd.Env = os.Environ() + for key, value := range command.Env { + cmd.Env = append(cmd.Env, key+"="+value) + } + cmd.Stdout = stdout + cmd.Stderr = stderr + configureManagedProcessCommand(cmd) + if err := cmd.Start(); err != nil { + return 1, fmt.Errorf("start managed process with pipes: %w", err) + } + if err := writeManagedProcessPID(pidPath, cmd.Process.Pid); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return 1, fmt.Errorf("persist managed process pid: %w", err) + } + + processHandle, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(cmd.Process.Pid)) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return 1, fmt.Errorf("open managed process handle: %w", err) + } + job, err := createManagedProcessJob(processHandle) + _ = windows.CloseHandle(processHandle) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return 1, err + } + defer windows.CloseHandle(job) + stopCleanup, err := watchManagedProcessStop(job, stopEventName) + if err != nil { + _ = windows.TerminateJobObject(job, 1) + _ = cmd.Wait() + return 1, err + } + defer stopCleanup() + + waitErr := cmd.Wait() + if cmd.ProcessState == nil { + if waitErr != nil { + return 1, waitErr + } + return 1, fmt.Errorf("managed process has no exit state") + } + exitCode := cmd.ProcessState.ExitCode() + if waitErr != nil { + // exec.Cmd returns *exec.ExitError for a normal non-zero exit. Return + // the actual code without treating it as an internal supervisor error; + // the outer helper will propagate the code to the durable supervisor. + if _, ok := waitErr.(*exec.ExitError); !ok { + return 1, waitErr + } + } + if exitCode > 255 { + return 1, fmt.Errorf("managed process exited with code %d", exitCode) + } + return exitCode, nil +} + +func writeManagedProcessPID(path string, pid int) error { + if strings.TrimSpace(path) == "" || pid <= 0 { + return nil + } + temporary := path + ".tmp" + if err := os.WriteFile(temporary, []byte(strconv.Itoa(pid)+"\n"), 0o600); err != nil { + return err + } + if err := os.Rename(temporary, path); err != nil { + _ = os.Remove(temporary) + return err + } + return nil +} + +type synchronizedOutputWriter struct { + mu sync.Mutex + w io.Writer +} + +func (writer *synchronizedOutputWriter) Write(body []byte) (int, error) { + writer.mu.Lock() + defer writer.mu.Unlock() + return writer.w.Write(body) +} + +func runManagedProcessWithPseudoConsole(command ProcessCommand, stopEventName string, pidPath string, stdout io.Writer, stderr io.Writer) (int, error) { + // Direct console executables use a pseudo-console so programs that require + // a console handle still have a bounded, hidden console surface. This is + // also used for plugin-declared console capture around a Windows shell. + inputRead, inputWrite, err := createPseudoConsolePipe() + if err != nil { + return 1, fmt.Errorf("create pseudo-console input pipe: %w", err) + } + outputRead, outputWrite, err := createPseudoConsolePipe() + if err != nil { + closePseudoConsoleHandles(inputRead, inputWrite) + return 1, fmt.Errorf("create pseudo-console output pipe: %w", err) + } + + var console windows.Handle + if err := windows.CreatePseudoConsole(windows.Coord{X: 160, Y: 50}, inputRead, outputWrite, 0, &console); err != nil { + closePseudoConsoleHandles(inputRead, inputWrite, outputRead, outputWrite) + return 1, fmt.Errorf("create pseudo-console: %w", err) + } + _ = windows.CloseHandle(inputRead) + _ = windows.CloseHandle(outputWrite) + + attributes, err := windows.NewProcThreadAttributeList(1) + if err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("create pseudo-console process attributes: %w", err) + } + defer attributes.Delete() + // PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE expects the HPCON handle value as + // lpValue, not the address of the local variable that stores the handle. + if err := attributes.Update(windows.PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE, unsafe.Pointer(uintptr(console)), unsafe.Sizeof(console)); err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("configure pseudo-console process attributes: %w", err) + } + + applicationPath := command.Args[0] + commandIsCmd := strings.EqualFold(applicationPath, "cmd.exe") + if commandIsCmd { + applicationPath = os.Getenv("ComSpec") + if applicationPath == "" { + applicationPath = `C:\Windows\System32\cmd.exe` + } + } + // CreateProcess receives both an application name and a command line. + // Keep the command-line program name identical to the resolved application + // name; cmd.exe can fail with STATUS_DLL_INIT_FAILED when the former is + // left as the short name while the latter is an absolute path. + commandArgs := append([]string(nil), command.Args...) + commandArgs[0] = applicationPath + commandLine, err := windows.UTF16FromString(windows.ComposeCommandLine(commandArgs)) + if err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("encode managed process command: %w", err) + } + var applicationName *uint16 + if !commandIsCmd && strings.ContainsAny(applicationPath, `:\`) { + applicationName, err = windows.UTF16PtrFromString(applicationPath) + if err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("encode managed process executable: %w", err) + } + } + environment, err := managedProcessEnvironment(command.Env) + if err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("encode managed process environment: %w", err) + } + var workDir *uint16 + if command.WorkDir != "" { + workDir, err = windows.UTF16PtrFromString(command.WorkDir) + if err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("encode managed process working directory: %w", err) + } + } + + // A pseudo-console supplies the child's console surface itself; do not + // combine it with STARTF_USESHOWWINDOW, which makes cmd.exe fail during + // initialization on some non-interactive Windows Server sessions. + startup := &windows.StartupInfoEx{StartupInfo: windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfoEx{}))}, ProcThreadAttributeList: attributes.List()} + var processInfo windows.ProcessInformation + // The helper has already detached from any parent job. A second + // CREATE_BREAKAWAY_FROM_JOB on the pseudo-console client is rejected by + // some Windows Server builds during console initialization and surfaces as + // STATUS_DLL_INIT_FAILED from the otherwise valid child process. + if err := windows.CreateProcess(applicationName, &commandLine[0], nil, nil, false, windows.CREATE_UNICODE_ENVIRONMENT|windows.EXTENDED_STARTUPINFO_PRESENT, environment, workDir, &startup.StartupInfo, &processInfo); err != nil { + windows.ClosePseudoConsole(console) + closePseudoConsoleHandles(inputWrite, outputRead) + return 1, fmt.Errorf("start managed process in pseudo-console: %w", err) + } + _ = windows.CloseHandle(processInfo.Thread) + _ = windows.CloseHandle(inputWrite) + if err := writeManagedProcessPID(pidPath, int(processInfo.ProcessId)); err != nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + _ = windows.CloseHandle(processInfo.Process) + windows.ClosePseudoConsole(console) + _ = windows.CloseHandle(outputRead) + return 1, fmt.Errorf("persist managed process pid: %w", err) + } + + job, err := createManagedProcessJob(processInfo.Process) + if err != nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + _ = windows.CloseHandle(processInfo.Process) + windows.ClosePseudoConsole(console) + _ = windows.CloseHandle(outputRead) + return 1, err + } + defer windows.CloseHandle(job) + stopCleanup, err := watchManagedProcessStop(job, stopEventName) + if err != nil { + _ = windows.TerminateJobObject(job, 1) + windows.ClosePseudoConsole(console) + _ = windows.CloseHandle(processInfo.Process) + return 1, err + } + defer stopCleanup() + + outputFile := os.NewFile(uintptr(outputRead), "run-pseudo-console-output") + if outputFile == nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + _ = windows.CloseHandle(processInfo.Process) + windows.ClosePseudoConsole(console) + return 1, fmt.Errorf("open pseudo-console output") + } + outputDone := make(chan struct{}) + outputWriter := &synchronizedOutputWriter{w: stdout} + go func() { + _, _ = io.Copy(outputWriter, outputFile) + close(outputDone) + }() + + _, waitErr := windows.WaitForSingleObject(processInfo.Process, windows.INFINITE) + var exitCode uint32 + if waitErr == nil { + waitErr = windows.GetExitCodeProcess(processInfo.Process, &exitCode) + } + windows.ClosePseudoConsole(console) + <-outputDone + _ = outputFile.Close() + _ = windows.CloseHandle(processInfo.Process) + if waitErr != nil { + return 1, waitErr + } + if exitCode > 255 { + return 1, fmt.Errorf("managed process exited with code %d", exitCode) + } + return int(exitCode), nil +} + +func runManagedShellWithPipes(command ProcessCommand, stopEventName string, stdout io.Writer, stderr io.Writer) (int, error) { + inputRead, inputWrite, err := createPseudoConsolePipe() + if err != nil { + return 1, fmt.Errorf("create managed shell input pipe: %w", err) + } + stdoutRead, stdoutWrite, err := createPseudoConsolePipe() + if err != nil { + closePseudoConsoleHandles(inputRead, inputWrite) + return 1, fmt.Errorf("create managed shell stdout pipe: %w", err) + } + stderrRead, stderrWrite, err := createPseudoConsolePipe() + if err != nil { + closePseudoConsoleHandles(inputRead, inputWrite, stdoutRead, stdoutWrite) + return 1, fmt.Errorf("create managed shell stderr pipe: %w", err) + } + closeOnError := func() { + closePseudoConsoleHandles(inputRead, inputWrite, stdoutRead, stdoutWrite, stderrRead, stderrWrite) + } + if err := windows.SetHandleInformation(stdoutRead, windows.HANDLE_FLAG_INHERIT, 0); err != nil { + closeOnError() + return 1, fmt.Errorf("make managed shell stdout pipe private: %w", err) + } + if err := windows.SetHandleInformation(stderrRead, windows.HANDLE_FLAG_INHERIT, 0); err != nil { + closeOnError() + return 1, fmt.Errorf("make managed shell stderr pipe private: %w", err) + } + + applicationPath := os.Getenv("ComSpec") + if applicationPath == "" { + applicationPath = `C:\Windows\System32\cmd.exe` + } + commandArgs := append([]string(nil), command.Args...) + commandArgs[0] = applicationPath + commandLine, err := windows.UTF16FromString(windows.ComposeCommandLine(commandArgs)) + if err != nil { + closeOnError() + return 1, fmt.Errorf("encode managed shell command: %w", err) + } + applicationName, err := windows.UTF16PtrFromString(applicationPath) + if err != nil { + closeOnError() + return 1, fmt.Errorf("encode managed shell executable: %w", err) + } + environment, err := managedProcessEnvironment(command.Env) + if err != nil { + closeOnError() + return 1, fmt.Errorf("encode managed shell environment: %w", err) + } + var workDir *uint16 + if command.WorkDir != "" { + workDir, err = windows.UTF16PtrFromString(command.WorkDir) + if err != nil { + closeOnError() + return 1, fmt.Errorf("encode managed shell working directory: %w", err) + } + } + startup := &windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfo{})), Flags: windows.STARTF_USESTDHANDLES | windows.STARTF_USESHOWWINDOW, ShowWindow: windows.SW_HIDE, StdInput: inputRead, StdOutput: stdoutWrite, StdErr: stderrWrite} + var processInfo windows.ProcessInformation + if err := windows.CreateProcess(applicationName, &commandLine[0], nil, nil, true, windows.CREATE_BREAKAWAY_FROM_JOB|windows.CREATE_NO_WINDOW|windows.CREATE_UNICODE_ENVIRONMENT, environment, workDir, startup, &processInfo); err != nil { + closeOnError() + return 1, fmt.Errorf("start managed shell: %w", err) + } + _ = windows.CloseHandle(processInfo.Thread) + closePseudoConsoleHandles(inputRead, inputWrite, stdoutWrite, stderrWrite) + + job, err := createManagedProcessJob(processInfo.Process) + if err != nil { + _ = windows.TerminateProcess(processInfo.Process, 1) + _ = windows.CloseHandle(processInfo.Process) + closePseudoConsoleHandles(stdoutRead, stderrRead) + return 1, err + } + defer windows.CloseHandle(job) + stopCleanup, err := watchManagedProcessStop(job, stopEventName) + if err != nil { + _ = windows.TerminateJobObject(job, 1) + _ = windows.CloseHandle(processInfo.Process) + return 1, err + } + defer stopCleanup() + stdoutFile := os.NewFile(uintptr(stdoutRead), "run-managed-shell-stdout") + stderrFile := os.NewFile(uintptr(stderrRead), "run-managed-shell-stderr") + if stdoutFile == nil || stderrFile == nil { + if stdoutFile != nil { + _ = stdoutFile.Close() + } + if stderrFile != nil { + _ = stderrFile.Close() + } + _ = windows.TerminateProcess(processInfo.Process, 1) + _ = windows.CloseHandle(processInfo.Process) + return 1, fmt.Errorf("open managed shell output pipes") + } + outputDone := make(chan struct{}) + outputWriter := &synchronizedOutputWriter{w: stdout} + go func() { + _, _ = io.Copy(outputWriter, stdoutFile) + _ = stdoutFile.Close() + close(outputDone) + }() + errorDone := make(chan struct{}) + errorWriter := &synchronizedOutputWriter{w: stderr} + go func() { + _, _ = io.Copy(errorWriter, stderrFile) + _ = stderrFile.Close() + close(errorDone) + }() + + _, waitErr := windows.WaitForSingleObject(processInfo.Process, windows.INFINITE) + var exitCode uint32 + if waitErr == nil { + waitErr = windows.GetExitCodeProcess(processInfo.Process, &exitCode) + } + <-outputDone + <-errorDone + _ = windows.CloseHandle(processInfo.Process) + if waitErr != nil { + return 1, waitErr + } + if exitCode > 255 { + return 1, fmt.Errorf("managed shell exited with code %d", exitCode) + } + return int(exitCode), nil +} + +func createManagedProcessJob(process windows.Handle) (windows.Handle, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, fmt.Errorf("create managed process job: %w", err) + } + if err := windows.AssignProcessToJobObject(job, process); err != nil { + _ = windows.CloseHandle(job) + return 0, fmt.Errorf("assign managed process to job: %w", err) + } + return job, nil +} + +func watchManagedProcessStop(job windows.Handle, name string) (func(), error) { + if strings.TrimSpace(name) == "" { + return func() {}, nil + } + eventName, err := windows.UTF16PtrFromString(name) + if err != nil { + return nil, fmt.Errorf("encode managed process stop event: %w", err) + } + event, eventErr := windows.CreateEvent(nil, 1, 0, eventName) + if eventErr != nil && eventErr != windows.ERROR_ALREADY_EXISTS { + if event != 0 { + _ = windows.CloseHandle(event) + } + return nil, fmt.Errorf("create managed process stop event: %w", eventErr) + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + result, waitErr := windows.WaitForSingleObject(event, 100) + if waitErr != nil { + return + } + if result == windows.WAIT_OBJECT_0 { + _ = windows.TerminateJobObject(job, 1) + return + } + } + }() + return func() { + close(stop) + <-done + _ = windows.CloseHandle(event) + }, nil +} + +func requestManagedProcessStop(identity ProcessIdentity) error { + if strings.TrimSpace(identity.StopEventName) != "" { + name, err := windows.UTF16PtrFromString(identity.StopEventName) + if err != nil { + return err + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + event, openErr := windows.OpenEvent(windows.EVENT_MODIFY_STATE|windows.SYNCHRONIZE, false, name) + if openErr == nil { + setErr := windows.SetEvent(event) + _ = windows.CloseHandle(event) + return setErr + } + time.Sleep(25 * time.Millisecond) + } + } + return fmt.Errorf("managed process stop event is unavailable") +} + +func forceManagedProcessStop(identity ProcessIdentity) error { + if err := requestManagedProcessStop(identity); err == nil { + return nil + } + if identity.PID <= 0 { + return fmt.Errorf("managed process pid is invalid") + } + // A recovered process can outlive its helper. In that case no stop-event + // watcher remains. Target only the persisted process PID here: /T would + // recursively terminate descendants and can take down a game process that + // was deliberately kept alive while Run is being restarted or updated. + command := exec.Command("taskkill.exe", "/PID", strconv.Itoa(identity.PID), "/F") + command.Stdout = io.Discard + command.Stderr = io.Discard + return command.Run() +} + +func createPseudoConsolePipe() (windows.Handle, windows.Handle, error) { + var readHandle, writeHandle windows.Handle + // ConPTY owns these handles through its internal duplication. Keeping the + // pipe ends non-inheritable matches the Windows ConPTY contract and avoids + // leaking the pseudo-console handles into the client process. + if err := windows.CreatePipe(&readHandle, &writeHandle, nil, 0); err != nil { + return 0, 0, err + } + return readHandle, writeHandle, nil +} + +func closePseudoConsoleHandles(handles ...windows.Handle) { + for _, handle := range handles { + if handle != 0 && handle != windows.InvalidHandle { + _ = windows.CloseHandle(handle) + } + } +} + +func managedProcessEnvironment(values map[string]string) (*uint16, error) { + if len(values) == 0 { + // A nil environment tells CreateProcess to inherit the helper's + // environment. This preserves Windows' special drive-current-directory + // entries and avoids rebuilding a potentially incomplete environment + // block for the common case. + return nil, nil + } + environment := make(map[string]string, len(values)) + for _, entry := range os.Environ() { + keyEnd := strings.IndexByte(entry, '=') + if strings.HasPrefix(entry, "=") { + // Windows stores drive current directories as =C:=C:\\...; + // the first equals sign is part of that variable's name. + if next := strings.IndexByte(entry[1:], '='); next >= 0 { + keyEnd = next + 1 + } + } + if keyEnd > 0 { + key := entry[:keyEnd] + environment[strings.ToUpper(key)] = entry + } + } + for key, value := range values { + environment[strings.ToUpper(key)] = key + "=" + value + } + entries := make([]string, 0, len(environment)) + for _, entry := range environment { + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { return strings.ToUpper(entries[i]) < strings.ToUpper(entries[j]) }) + encoded := utf16.Encode([]rune(strings.Join(entries, "\x00") + "\x00\x00")) + return &encoded[0], nil +} diff --git a/runtime/process_window_windows_test.go b/runtime/process_window_windows_test.go new file mode 100644 index 0000000..b173f8a --- /dev/null +++ b/runtime/process_window_windows_test.go @@ -0,0 +1,69 @@ +//go:build windows + +package runtime + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "golang.org/x/sys/windows" +) + +func TestConfigureManagedProcessCommandPreservesInheritedConsole(t *testing.T) { + cmd := exec.Command("cmd.exe") + configureManagedProcessCommand(cmd) + if cmd.SysProcAttr == nil { + t.Fatal("expected Windows process attributes") + } + if !cmd.SysProcAttr.HideWindow { + t.Fatal("expected managed console window to stay hidden") + } + expected := uint32(windows.CREATE_NO_WINDOW | windows.CREATE_BREAKAWAY_FROM_JOB) + if cmd.SysProcAttr.CreationFlags != expected { + t.Fatalf("expected a hidden breakaway helper without a console, got %#x", cmd.SysProcAttr.CreationFlags) + } +} + +func TestWriteManagedProcessPIDUsesAtomicSidecar(t *testing.T) { + path := filepath.Join(t.TempDir(), "process.stdout.log.pid") + if err := writeManagedProcessPID(path, 321); err != nil { + t.Fatalf("write managed process pid: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read managed process pid: %v", err) + } + if got, err := strconv.Atoi(string(body[:len(body)-1])); err != nil || got != 321 { + t.Fatalf("expected pid sidecar to contain 321, got %q err=%v", body, err) + } + if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) { + t.Fatalf("pid sidecar temp file should not remain, err=%v", err) + } +} + +func TestPrepareManagedProcessHelperExecutableCopiesRunOutsideCurrentPath(t *testing.T) { + root := t.TempDir() + current := filepath.Join(root, "run.exe") + output := filepath.Join(root, "state", "process-output", "stdout.log") + if err := os.WriteFile(current, []byte("run-binary"), 0o700); err != nil { + t.Fatalf("write current executable: %v", err) + } + helper, err := prepareManagedProcessHelperExecutable(current, output) + if err != nil { + t.Fatalf("prepare helper executable: %v", err) + } + defer os.Remove(helper) + if helper == current || filepath.Dir(helper) != filepath.Dir(output) { + t.Fatalf("expected helper beside durable process output, helper=%q current=%q", helper, current) + } + body, err := os.ReadFile(helper) + if err != nil { + t.Fatalf("read helper executable: %v", err) + } + if string(body) != "run-binary" { + t.Fatalf("expected helper to copy current executable, got %q", body) + } +} diff --git a/runtime/protected_request.go b/runtime/protected_request.go new file mode 100644 index 0000000..3c37efc --- /dev/null +++ b/runtime/protected_request.go @@ -0,0 +1,210 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "browser.local/run/protocol" +) + +const ( + ProtectedRequestStatusSucceeded = "succeeded" + ProtectedRequestStatusFailed = "failed" + ProtectedRequestStatusUnknown = "unknown" +) + +// ErrProtectedRequestUnknown lets a Run-owned transport report that a request +// is syntactically safe but not one of its declared operations. It is terminal +// and affects only this request. +var ErrProtectedRequestUnknown = errors.New("protected request outcome unknown") + +// ProtectedRequest contains only Platform-authorized text and logical binding. +// Handlers resolve their own private transport configuration locally; they must +// not return it in an outcome, error, log line, or result. +type ProtectedRequest struct { + JobID string + ServerInstanceID string + FencingToken uint64 + Kind string + TransportKey string + TargetKey string + RequestText string +} + +type ProtectedRequestOutcome struct { + Status string + Stdout string + Stderr string +} + +type ProtectedRequestHandler interface { + ExecuteProtectedRequest(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error) +} + +type ProtectedRequestHandlerFunc func(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error) + +func (fn ProtectedRequestHandlerFunc) ExecuteProtectedRequest(ctx context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) { + return fn(ctx, request) +} + +// ProtectedRequestRegistry is configured by the local Run package owner. It +// uses only logical lookup keys, so plugin and Platform payloads cannot select +// an arbitrary program, DSN, socket, or host path. +type ProtectedRequestRegistry struct { + mu sync.RWMutex + handlers map[string]ProtectedRequestHandler +} + +func NewProtectedRequestRegistry() *ProtectedRequestRegistry { + return &ProtectedRequestRegistry{handlers: map[string]ProtectedRequestHandler{}} +} + +func (registry *ProtectedRequestRegistry) Register(kind string, transportKey string, handler ProtectedRequestHandler) error { + if !validProtectedRequestKind(kind) || !protocol.ValidLogicalFileKey(transportKey) || handler == nil { + return fmt.Errorf("protected request kind, transport key, and handler are required") + } + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.handlers == nil { + registry.handlers = map[string]ProtectedRequestHandler{} + } + registry.handlers[protectedRequestRegistryKey(kind, transportKey)] = handler + return nil +} + +func (registry *ProtectedRequestRegistry) handler(kind string, transportKey string) (ProtectedRequestHandler, bool) { + if registry == nil { + return nil, false + } + registry.mu.RLock() + defer registry.mu.RUnlock() + handler, exists := registry.handlers[protectedRequestRegistryKey(kind, transportKey)] + return handler, exists +} + +func protectedRequestRegistryKey(kind string, transportKey string) string { + return kind + "\x00" + transportKey +} + +func (executor LifecycleExecutor) ExecuteProtectedRequest(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.ProtectedRequestExecutionInputResponse) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil || !protectedRequestInputMatchesExecutor(input, assignment) { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_binding_invalid") + } + handler, exists := executor.protectedRequests.handler(input.Kind, input.TransportKey) + if !exists { + if input.Kind == "rcon" && assignment.ExecutionInput.SourceRCON != nil { + return executor.executeProtectedSourceRCON(ctx, assignment, input) + } + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusUnknown, "protected_request_transport_unknown") + } + executionCtx, cancel := context.WithTimeout(ctx, time.Duration(assignment.ExecutionInput.TimeoutSeconds)*time.Second) + defer cancel() + outcome, err := handler.ExecuteProtectedRequest(executionCtx, ProtectedRequest{JobID: input.JobID, ServerInstanceID: input.ServerInstanceID, FencingToken: input.FencingToken, Kind: input.Kind, TransportKey: input.TransportKey, TargetKey: input.TargetKey, RequestText: input.RequestText}) + if input.Kind == "program" { + executor.writeProtectedProgramLogs(ctx, assignment, outcome) + } + if errors.Is(executionCtx.Err(), context.Canceled) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request cancelled"}, Message: "protected request cancelled", ErrorCode: "protected_request_cancelled"} + } + if errors.Is(executionCtx.Err(), context.DeadlineExceeded) { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_timeout") + } + if errors.Is(err, ErrProtectedRequestUnknown) || outcome.Status == ProtectedRequestStatusUnknown { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusUnknown, "protected_request_unknown") + } + if err != nil || outcome.Status == ProtectedRequestStatusFailed || outcome.Status != "" && outcome.Status != ProtectedRequestStatusSucceeded { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_failed") + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request completed"}, Message: "protected request completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + input.Kind, Summary: "approved protected request executed through logical Run transport"}} +} + +func (executor LifecycleExecutor) executeProtectedSourceRCON(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.ProtectedRequestExecutionInputResponse) LifecycleExecutionResult { + result := executor.ExecuteSourceRCON(ctx, assignment, input.RequestText) + if result.State != lifecycleResultStateSucceeded { + return result + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected RCON request delivered"}, Message: "protected RCON request delivered", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected.rcon", Summary: "approved protected RCON request delivered through Source RCON"}} +} + +func protectedRequestInputMatchesExecutor(input protocol.ProtectedRequestExecutionInputResponse, assignment protocol.RunJobAssignment) bool { + return protocol.ValidProtectedRequestExecutionInput(input) && input.JobID == assignment.JobID && input.ServerInstanceID == assignment.ServerInstanceID && input.FencingToken == assignment.FencingToken && input.TargetKey == assignment.TargetKey && input.TransportKey == assignment.ExecutionInput.RemoteAdapterKey && input.Kind == protectedRequestKindForCapability(assignment.Capability) +} + +func protectedRequestKindForCapability(capability string) string { + switch capability { + case protocol.RunCapabilityRemoteRunProtectedSQL: + return "sql" + case protocol.RunCapabilityRemoteRunProtectedRCON: + return "rcon" + case protocol.RunCapabilityRemoteRunProgram: + return "program" + default: + return "" + } +} + +func validProtectedRequestKind(kind string) bool { + return kind == "sql" || kind == "rcon" || kind == "program" +} + +func (executor LifecycleExecutor) writeProtectedProgramLogs(ctx context.Context, assignment protocol.RunJobAssignment, outcome ProtectedRequestOutcome) { + for _, item := range []struct{ stream, body string }{{"management-program.stdout", outcome.Stdout}, {"management-program.stderr", outcome.Stderr}} { + for _, line := range splitProtectedProgramLines(item.body) { + _ = executor.logSink.Append(ctx, assignment, item.stream, sanitizeProtectedProgramLogLine(line)) + } + } +} + +func splitProtectedProgramLines(value string) []string { + value = strings.ToValidUTF8(strings.TrimSpace(value), "") + if len(value) > maxLifecycleOutputBytes { + value = strings.ToValidUTF8(value[:maxLifecycleOutputBytes], "") + } + lines := strings.Split(value, "\n") + bounded := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimRight(line, "\r") + if strings.TrimSpace(line) != "" { + bounded = append(bounded, line) + } + } + return bounded +} + +func sanitizeProtectedProgramLogLine(line string) string { + if containsProtectedProgramPrivateText(line) { + return "[redacted protected management-program output]" + } + return RedactText(line) +} + +func containsProtectedProgramPrivateText(value string) bool { + lower := strings.ToLower(value) + for _, marker := range []string{"password=", "password:", "secret=", "secret:", "token=", "token:", "credential", "bearer ", "api_key", "apikey", "dsn=", "path=", "file=", "database=", "://", "mysql:", "postgres:", "sqlite:", "file:", "socket", "named pipe"} { + if strings.Contains(lower, marker) { + return true + } + } + for _, field := range strings.Fields(value) { + field = strings.Trim(field, "\"'()[]{}<>,;") + if strings.HasPrefix(field, "/") || strings.HasPrefix(field, "./") || strings.HasPrefix(field, "../") || strings.HasPrefix(field, `\\`) || len(field) >= 3 && ((field[0] >= 'a' && field[0] <= 'z') || (field[0] >= 'A' && field[0] <= 'Z')) && field[1] == ':' && (field[2] == '/' || field[2] == '\\') { + return true + } + } + return false +} + +func protectedRequestFailure(capability string, status string, code string) LifecycleExecutionResult { + kind := protectedRequestKindForCapability(capability) + if kind == "" { + kind = "unknown" + } + if status == ProtectedRequestStatusUnknown { + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request outcome is unknown"}, Message: "protected request outcome is unknown", ErrorCode: "protected_request_unknown", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + kind + ".unknown", Summary: "protected request outcome is unknown"}} + } + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request failed"}, Message: "protected request failed", ErrorCode: code, ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + kind + ".failed", Summary: "protected request failed safely"}} +} diff --git a/runtime/protected_request_test.go b/runtime/protected_request_test.go new file mode 100644 index 0000000..5567123 --- /dev/null +++ b/runtime/protected_request_test.go @@ -0,0 +1,186 @@ +package runtime + +import ( + "context" + "encoding/json" + "net" + "strings" + "testing" + "time" + + "browser.local/run/protocol" + "browser.local/run/spool" +) + +func TestWorkerExecutesFencedProtectedProgramAndSpoolsDedicatedLogs(t *testing.T) { + registry := NewProtectedRequestRegistry() + requestText := `{"operation":"status"}` + handlerCalled := false + if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) { + handlerCalled = true + if request.ServerInstanceID != "server-worker" || request.FencingToken != 12 || request.TargetKey != "scum-program" || request.RequestText != requestText { + t.Fatalf("unexpected protected request: %+v", request) + } + return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded, Stdout: "SCUM ready\npassword=hidden\nconfig /Users/private/scum.ini\ndsn mysql://private", Stderr: "bounded warning"}, nil + })); err != nil { + t.Fatalf("register protected handler: %v", err) + } + assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 12) + client := newFakeWorkerClient() + client.claimJob = assignment + client.protectedInput = protectedWorkerInput(assignment, "program", requestText) + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("log spool: %v", err) + } + worker, err := NewWorker(workerTestConfig(t), client, WithProtectedRequestRegistry(registry), WithProcessLogSink(&SpoolLogSink{Spool: logSpool})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register worker: %v", err) + } + if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled { + t.Fatalf("claim protected request handled=%v err=%v", handled, err) + } + if !handlerCalled || len(client.protectedRequests) != 1 || client.protectedRequests[0].FencingToken != assignment.FencingToken { + t.Fatalf("expected one fenced protected input read: %+v", client.protectedRequests) + } + if len(client.resultRequests) != 1 || client.resultRequests[0].State != lifecycleResultStateSucceeded || client.resultRequests[0].ExecutionResult.Kind != "protected.program" { + t.Fatalf("unexpected protected result: %+v", client.resultRequests) + } + batches, err := logSpool.Pending() + entryCount := 0 + for _, batch := range batches { + entryCount += len(batch.Entries) + } + if err != nil || entryCount != 5 { + t.Fatalf("expected five program log lines: batches=%+v err=%v", batches, err) + } + redactedEntries := 0 + for _, batch := range batches { + if batch.Source != "management-program" || batch.Source == "file" || batch.Source == "process" { + t.Fatalf("program output used wrong log source: %+v", batch) + } + for _, entry := range batch.Entries { + if entry.Redacted { + redactedEntries++ + } + if strings.Contains(entry.Line, "password=hidden") || strings.Contains(entry.Line, "/Users/") || strings.Contains(entry.Line, "://") { + t.Fatalf("program log leaked protected output: %+v", entry) + } + } + } + if redactedEntries != 3 { + t.Fatalf("expected three explicitly redacted private lines, got %d", redactedEntries) + } + serialized, err := json.Marshal([]any{client.resultRequests, client.protectedRequests, worker.journal.ActiveJobs()}) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{requestText, "password=hidden", "/Users/private/scum.ini", "mysql://private", "bounded warning"} { + if strings.Contains(string(serialized), private) { + t.Fatalf("protected text or output leaked into control projection %q: %s", private, serialized) + } + } +} + +func TestProtectedRequestUnknownAndBindingFailureAreIsolated(t *testing.T) { + registry := NewProtectedRequestRegistry() + called := 0 + if err := registry.Register("rcon", "scum-management", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) { + called++ + if request.RequestText == "unknown.command" { + return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown}, ErrProtectedRequestUnknown + } + return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded}, nil + })); err != nil { + t.Fatal(err) + } + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry)) + assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 21) + unknown := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "unknown.command")) + if unknown.State != lifecycleResultStateFailed || unknown.ErrorCode != "protected_request_unknown" || unknown.Retryable { + t.Fatalf("unexpected unknown outcome: %+v", unknown) + } + succeeded := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "status")) + if succeeded.State != lifecycleResultStateSucceeded || called != 2 { + t.Fatalf("unknown request affected later request: result=%+v called=%d", succeeded, called) + } + mismatched := protectedWorkerInput(assignment, "rcon", "status") + mismatched.FencingToken++ + failed := executor.ExecuteProtectedRequest(context.Background(), assignment, mismatched) + if failed.ErrorCode != "protected_request_binding_invalid" || called != 2 { + t.Fatalf("binding failure reached handler: result=%+v called=%d", failed, called) + } + encoded, _ := json.Marshal([]LifecycleExecutionResult{unknown, failed}) + if strings.Contains(string(encoded), "unknown.command") || strings.Contains(string(encoded), "status") { + t.Fatalf("safe failure leaked request text: %s", encoded) + } +} + +func TestProtectedRCONFallsBackToSourceRCONPlan(t *testing.T) { + listener, port := newSourceRCONListener(t) + defer listener.Close() + password := strings.Repeat("f", 64) + assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 41) + assignment.ExecutionInput.WorkspaceScope = "run-local" + assignment.ExecutionInput.TimeoutSeconds = 5 + assignment.ExecutionInput.SourceRCON = &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port} + root := t.TempDir() + writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password) + commands := make(chan string, 1) + serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, packet sourceRCONPacket) error { + commands <- packet.body + return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue}) + }) + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "#ListPlayers")) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "protected.rcon" { + t.Fatalf("expected protected RCON delivery through Source RCON, got %+v", result) + } + if got := <-commands; got != "#ListPlayers" { + t.Fatalf("expected SCUM command delivery, got %q", got) + } + awaitSourceRCONServer(t, serverDone) + serialized, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(serialized), "#ListPlayers") || strings.Contains(string(serialized), password) { + t.Fatalf("protected Source RCON result leaked private input: %s", serialized) + } +} + +func TestUnknownProtectedProgramKeepsSafeDiagnosticInProgramLogOnly(t *testing.T) { + registry := NewProtectedRequestRegistry() + if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error) { + return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown, Stderr: "unknown field database=/private/scum.db"}, ErrProtectedRequestUnknown + })); err != nil { + t.Fatal(err) + } + sink := &recordingLogSink{} + executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry), WithProcessLogSink(sink)) + assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 31) + result := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "program", `{"unexpected":true}`)) + if result.ErrorCode != "protected_request_unknown" || result.State != lifecycleResultStateFailed { + t.Fatalf("unexpected unknown program result: %+v", result) + } + if len(sink.lines) != 1 || !strings.HasPrefix(sink.lines[0], "management-program.stderr:") || strings.Contains(sink.lines[0], "/private/") || !strings.Contains(sink.lines[0], "[redacted protected") { + t.Fatalf("unknown program diagnostic was not safely channelized: %+v", sink.lines) + } +} + +func protectedWorkerAssignment(capability string, kind string, key string, fence uint64) protocol.RunJobAssignment { + assignment := workerJobAssignment(capability) + assignment.TargetKey = key + assignment.InputRef = "input://protected-request/" + assignment.JobID + assignment.FencingToken = fence + assignment.MaxAttempts = 1 + assignment.ExecutionInput = protocol.RunJobExecutionInput{RemoteAdapterKey: key, RemoteAdapterKind: "protected-" + kind, TimeoutSeconds: 5} + return assignment +} + +func protectedWorkerInput(assignment protocol.RunJobAssignment, kind string, text string) protocol.ProtectedRequestExecutionInputResponse { + return protocol.ProtectedRequestExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, FencingToken: assignment.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: time.Now().UTC().Add(time.Minute), Kind: kind, TransportKey: assignment.ExecutionInput.RemoteAdapterKey, TargetKey: assignment.TargetKey, RequestText: text} +} diff --git a/runtime/remote_access.go b/runtime/remote_access.go new file mode 100644 index 0000000..a82e798 --- /dev/null +++ b/runtime/remote_access.go @@ -0,0 +1,188 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "sync" + "time" + + "browser.local/run/protocol" +) + +type RemoteAdapterRequest struct { + JobID string + ServerInstanceID string + AdapterKey string + AdapterKind string + TargetKey string + Capability string + InputRef string +} + +type RemoteAdapterOutcome struct { + Message string + ResultRef string + Retryable bool +} + +type RemoteAdapter interface { + Execute(context.Context, RemoteAdapterRequest) (RemoteAdapterOutcome, error) +} + +type RemoteAdapterFunc func(context.Context, RemoteAdapterRequest) (RemoteAdapterOutcome, error) + +func (fn RemoteAdapterFunc) Execute(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) { + return fn(ctx, request) +} + +type RemoteAdapterRegistry struct { + mu sync.RWMutex + adapters map[string]RemoteAdapter +} + +func NewRemoteAdapterRegistry() *RemoteAdapterRegistry { + registry := &RemoteAdapterRegistry{adapters: map[string]RemoteAdapter{}} + for _, kind := range []string{"ftp", "rsync", "run-file", "run-process", "database", "log-transfer"} { + registry.adapters[kind] = declaredRemoteAdapter{kind: kind} + } + return registry +} + +func (registry *RemoteAdapterRegistry) Register(kind string, adapter RemoteAdapter) error { + kind = strings.TrimSpace(kind) + if kind == "" || adapter == nil { + return fmt.Errorf("remote adapter kind and implementation are required") + } + registry.mu.Lock() + defer registry.mu.Unlock() + registry.adapters[kind] = adapter + return nil +} + +func (registry *RemoteAdapterRegistry) adapter(kind string) (RemoteAdapter, bool) { + registry.mu.RLock() + defer registry.mu.RUnlock() + adapter, exists := registry.adapters[kind] + return adapter, exists +} + +func ExecuteRemoteAccessJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + return ExecuteRemoteAccessJobWithRegistry(ctx, assignment, NewRemoteAdapterRegistry()) +} + +func ExecuteRemoteAccessJobWithRegistry(ctx context.Context, assignment protocol.RunJobAssignment, registry *RemoteAdapterRegistry) LifecycleExecutionResult { + if assignment.ExecutionInput.SourceRCON != nil { + return lifecycleFailure("source_rcon_requires_worker_transport", "Source RCON commands require the one-time worker transport") + } + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + if strings.Contains(err.Error(), "remoteAdapterKey") { + return lifecycleFailure("unsafe_remote_adapter_target", "remote adapter key must be an approved logical key") + } + return lifecycleFailure("unsafe_remote_access_job", err.Error()) + } + if !isSupportedRemoteCapability(assignment.Capability) { + return lifecycleFailure("unsupported_remote_access_capability", "unsupported remote access capability") + } + if registry == nil { + return lifecycleFailure("remote_adapter_unavailable", "remote adapter registry is unavailable") + } + adapterKind := strings.TrimSpace(assignment.ExecutionInput.RemoteAdapterKind) + if adapterKind == "" { + adapterKind = adapterKindForCapability(assignment.Capability) + } + adapterKey := strings.TrimSpace(assignment.ExecutionInput.RemoteAdapterKey) + if adapterKey == "" { + adapterKey = assignment.TargetKey + } + if !protocol.ValidLogicalFileKey(adapterKey) || !protocol.ValidLogicalFileKey(assignment.TargetKey) { + return lifecycleFailure("unsafe_remote_adapter_target", "remote adapter and target must use approved logical keys") + } + if !adapterKindAllowsCapability(adapterKind, assignment.Capability) { + return lifecycleFailure("remote_adapter_capability_mismatch", "remote adapter kind does not allow requested capability") + } + adapter, exists := registry.adapter(adapterKind) + if !exists { + return lifecycleFailure("remote_adapter_unavailable", "declared remote adapter is unavailable") + } + + executionCtx := ctx + cancel := func() {} + if timeout := assignment.ExecutionInput.TimeoutSeconds; timeout > 0 { + if timeout > 300 { + return lifecycleFailure("unsafe_remote_adapter_timeout", "remote adapter timeout exceeds bound") + } + executionCtx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + } + defer cancel() + + request := RemoteAdapterRequest{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, AdapterKey: adapterKey, AdapterKind: adapterKind, TargetKey: assignment.TargetKey, Capability: assignment.Capability, InputRef: assignment.InputRef} + outcome, err := adapter.Execute(executionCtx, request) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter timed out"}, Message: "remote adapter timed out", ErrorCode: "remote_adapter_timeout", Retryable: true} + } + if errors.Is(err, context.Canceled) || errors.Is(executionCtx.Err(), context.Canceled) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter cancelled"}, Message: "remote adapter cancelled", ErrorCode: "remote_adapter_cancelled"} + } + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter failed"}, Message: "remote adapter failed", ErrorCode: "remote_adapter_failed", Retryable: outcome.Retryable} + } + if err := executionCtx.Err(); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter timed out"}, Message: "remote adapter timed out", ErrorCode: "remote_adapter_timeout", Retryable: true} + } + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter cancelled"}, Message: "remote adapter cancelled", ErrorCode: "remote_adapter_cancelled"} + } + resultRef := outcome.ResultRef + if resultRef == "" { + resultRef = fmt.Sprintf("artifact://jobs/%s/remote-access-result", url.PathEscape(assignment.JobID)) + } + message := strings.TrimSpace(outcome.Message) + if message == "" { + message = fmt.Sprintf("%s completed through declared %s adapter", assignment.Capability, adapterKind) + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter completed"}, ResultRef: resultRef, Message: message} +} + +type declaredRemoteAdapter struct { + kind string +} + +func (adapter declaredRemoteAdapter) Execute(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) { + select { + case <-ctx.Done(): + return RemoteAdapterOutcome{}, ctx.Err() + default: + } + if request.AdapterKind != adapter.kind || request.ServerInstanceID == "" || request.JobID == "" { + return RemoteAdapterOutcome{}, fmt.Errorf("remote adapter request identity mismatch") + } + return RemoteAdapterOutcome{Message: fmt.Sprintf("%s completed through bounded remote access envelope", request.Capability), ResultRef: fmt.Sprintf("artifact://jobs/%s/remote-access-result", url.PathEscape(request.JobID))}, nil +} + +func adapterKindForCapability(capability string) string { + switch capability { + case protocol.RunCapabilityRemoteFTPRead, protocol.RunCapabilityRemoteFTPWrite: + return "ftp" + case protocol.RunCapabilityRemoteRsyncRead, protocol.RunCapabilityRemoteRsyncWrite: + return "rsync" + case protocol.RunCapabilityRemoteRunFilesRead, protocol.RunCapabilityRemoteRunFilesWrite: + return "run-file" + case protocol.RunCapabilityRemoteRunProcessStart, protocol.RunCapabilityRemoteRunProcessStop: + return "run-process" + case protocol.RunCapabilityRemoteRunDBMySQLQuery, protocol.RunCapabilityRemoteRunDBSQLiteQuery: + return "database" + case protocol.RunCapabilityRemoteRunRCONCommand: + return "rcon" + case protocol.RunCapabilityRemoteRunLogsTransfer: + return "log-transfer" + default: + return "" + } +} + +func adapterKindAllowsCapability(kind string, capability string) bool { + return kind != "" && kind == adapterKindForCapability(capability) +} diff --git a/runtime/remote_access_test.go b/runtime/remote_access_test.go new file mode 100644 index 0000000..6ef9d96 --- /dev/null +++ b/runtime/remote_access_test.go @@ -0,0 +1,55 @@ +package runtime + +import ( + "context" + "strings" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestRemoteAdapterRegistryHonorsTimeoutAndCancellation(t *testing.T) { + registry := NewRemoteAdapterRegistry() + if err := registry.Register("database", RemoteAdapterFunc(func(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) { + <-ctx.Done() + return RemoteAdapterOutcome{}, ctx.Err() + })); err != nil { + t.Fatalf("register blocking adapter: %v", err) + } + assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery) + assignment.TargetKey = "db/sqlite/query" + assignment.InputRef = "input://server-1/db/sqlite/query/1" + assignment.ExecutionInput.RemoteAdapterKind = "database" + assignment.ExecutionInput.RemoteAdapterKey = "db-sqlite" + assignment.ExecutionInput.TimeoutSeconds = 1 + started := time.Now() + result := ExecuteRemoteAccessJobWithRegistry(context.Background(), assignment, registry) + if result.ErrorCode != "remote_adapter_timeout" || !result.Retryable || time.Since(started) > 3*time.Second { + t.Fatalf("expected bounded remote timeout, got %+v", result) + } + + cancelCtx, cancel := context.WithCancel(context.Background()) + cancel() + result = ExecuteRemoteAccessJobWithRegistry(cancelCtx, assignment, NewRemoteAdapterRegistry()) + if result.ErrorCode != "remote_adapter_cancelled" || result.State != "cancelled" { + t.Fatalf("expected remote cancellation, got %+v", result) + } +} + +func TestRemoteAdapterRejectsKindMismatchAndUnsafeProjection(t *testing.T) { + assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery) + assignment.TargetKey = "db/sqlite/query" + assignment.InputRef = "input://server-1/db/sqlite/query/1" + assignment.ExecutionInput.RemoteAdapterKind = "rcon" + result := ExecuteRemoteAccessJob(context.Background(), assignment) + if result.ErrorCode != "remote_adapter_capability_mismatch" { + t.Fatalf("expected kind mismatch rejection, got %+v", result) + } + assignment.ExecutionInput.RemoteAdapterKind = "database" + assignment.ExecutionInput.RemoteAdapterKey = "tcp://unapproved" + result = ExecuteRemoteAccessJob(context.Background(), assignment) + if result.ErrorCode != "unsafe_remote_adapter_target" || strings.Contains(result.Message, "tcp://") { + t.Fatalf("expected unsafe adapter target rejection, got %+v", result) + } +} diff --git a/runtime/runtime_profiles.go b/runtime/runtime_profiles.go new file mode 100644 index 0000000..4b8f876 --- /dev/null +++ b/runtime/runtime_profiles.go @@ -0,0 +1,297 @@ +package runtime + +import ( + "fmt" + "sort" + "strings" + + "browser.local/run/protocol" +) + +const ( + RuntimeModeLocalProcess = "local-process" + RuntimeModeHostedFTPRCON = "hosted-ftp-rcon" + RuntimeModeFTPOnly = "ftp-only" + RuntimeModeCustomClient = "custom-client" +) + +type RuntimeProfiles struct { + Discovery []RuntimeDiscoveryProbe `json:"discovery,omitempty"` + LifecycleProfiles []RuntimeLifecycleProfile `json:"lifecycleProfiles,omitempty"` + DependencyProbes []RuntimeDependencyProbe `json:"dependencyProbes,omitempty"` + InstallPlans []RuntimeInstallPlan `json:"installPlans,omitempty"` + LogSources []RuntimeLogSource `json:"logSources,omitempty"` + TransportProfiles []RuntimeTransportProfile `json:"transportProfiles,omitempty"` + ClientManagers []RuntimeClientManagerSpec `json:"clientManagers,omitempty"` +} + +type RuntimeDiscoveryProbe struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey"` + Required bool `json:"required,omitempty"` + Platforms []string `json:"platforms,omitempty"` +} + +type RuntimeLifecycleProfile struct { + Key string `json:"key"` + Mode string `json:"mode"` + Capabilities []string `json:"capabilities"` + ActionRefs map[string]string `json:"actionRefs,omitempty"` + TransportKeys []string `json:"transportKeys,omitempty"` + ClientManagerRef string `json:"clientManagerRef,omitempty"` + Platforms []string `json:"platforms,omitempty"` +} + +type RuntimeDependencyProbe struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey"` + Required bool `json:"required,omitempty"` + Platforms []string `json:"platforms,omitempty"` +} + +type RuntimeInstallPlan struct { + Key string `json:"key"` + Title string `json:"title"` + Platforms []string `json:"platforms,omitempty"` + Steps []RuntimeInstallStep `json:"steps"` +} + +type RuntimeInstallStep struct { + Type string `json:"type"` + TargetKey string `json:"targetKey"` + PackageManager string `json:"packageManager,omitempty"` + PackageName string `json:"packageName,omitempty"` + Version string `json:"version,omitempty"` + DownloadRef string `json:"downloadRef,omitempty"` + Checksum string `json:"checksum,omitempty"` +} + +type RuntimeLogSource struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey,omitempty"` + StreamKey string `json:"streamKey"` + CursorKind string `json:"cursorKind,omitempty"` + RetentionDays int `json:"retentionDays,omitempty"` +} + +type RuntimeTransportProfile struct { + Key string `json:"key"` + Kind string `json:"kind"` + TargetKey string `json:"targetKey,omitempty"` + Capabilities []string `json:"capabilities"` +} + +type RuntimeClientManagerSpec struct { + Key string `json:"key"` +} + +type RuntimeBindingSet struct { + ProfileKey string `json:"profileKey"` + Mode string `json:"mode"` + Bindings map[string]string `json:"bindings,omitempty"` + MissingKeys []string `json:"missingKeys,omitempty"` +} + +type RuntimeResolution struct { + ProfileKey string `json:"profileKey"` + Mode string `json:"mode"` + Capabilities []string `json:"capabilities"` + ActionRefs map[string]string `json:"actionRefs,omitempty"` + TransportKeys []string `json:"transportKeys,omitempty"` + Transports []RuntimeTransportProfile `json:"transports,omitempty"` + LogSources []RuntimeLogSource `json:"logSources,omitempty"` + Discovery []RuntimeDiscoveryProbe `json:"discovery,omitempty"` + ClientManagerRef string `json:"clientManagerRef,omitempty"` + MissingKeys []string `json:"missingKeys,omitempty"` + Available bool `json:"available"` +} + +func ResolveRuntimeProfile(profiles RuntimeProfiles, profileKey string, targetOS string, binding RuntimeBindingSet) (RuntimeResolution, error) { + profile, ok := findLifecycleProfile(profiles.LifecycleProfiles, profileKey) + if !ok { + return RuntimeResolution{}, fmt.Errorf("runtime profile is not declared") + } + if !supportedRuntimeMode(profile.Mode) { + return RuntimeResolution{}, fmt.Errorf("runtime mode is unsupported") + } + if targetOS != "" && !supportsPlatform(profile.Platforms, targetOS) { + return RuntimeResolution{}, fmt.Errorf("runtime profile does not support target platform") + } + if binding.ProfileKey != "" && binding.ProfileKey != profile.Key { + return RuntimeResolution{}, fmt.Errorf("runtime binding profile does not match") + } + if binding.Mode != "" && binding.Mode != profile.Mode { + return RuntimeResolution{}, fmt.Errorf("runtime binding mode does not match") + } + if err := validateRuntimeProfile(profile); err != nil { + return RuntimeResolution{}, err + } + + transports, err := resolveTransports(profile.TransportKeys, profiles.TransportProfiles) + if err != nil { + return RuntimeResolution{}, err + } + missing := missingRuntimeBindingKeys(profile, transports, profiles.Discovery, profiles.LogSources, binding) + return RuntimeResolution{ + ProfileKey: profile.Key, + Mode: profile.Mode, + Capabilities: append([]string(nil), profile.Capabilities...), + ActionRefs: copyStringMap(profile.ActionRefs), + TransportKeys: append([]string(nil), profile.TransportKeys...), + Transports: transports, + LogSources: safeLogSources(profiles.LogSources, targetOS), + Discovery: safeDiscovery(profiles.Discovery, targetOS), + ClientManagerRef: profile.ClientManagerRef, + MissingKeys: missing, + Available: len(missing) == 0, + }, nil +} + +func findLifecycleProfile(profiles []RuntimeLifecycleProfile, key string) (RuntimeLifecycleProfile, bool) { + for _, profile := range profiles { + if profile.Key == key { + return profile, true + } + } + return RuntimeLifecycleProfile{}, false +} + +func validateRuntimeProfile(profile RuntimeLifecycleProfile) error { + if !protocol.ValidLogicalFileKey(profile.Key) { + return fmt.Errorf("runtime profile key is unsafe") + } + for _, capability := range profile.Capabilities { + if strings.TrimSpace(capability) == "" || containsUnsafeRuntimeText(capability) { + return fmt.Errorf("runtime capability is unsafe") + } + } + for action, ref := range profile.ActionRefs { + if !protocol.ValidLogicalFileKey(action) || !protocol.ValidLogicalFileKey(ref) { + return fmt.Errorf("runtime action ref is unsafe") + } + } + if profile.ClientManagerRef != "" && !protocol.ValidLogicalFileKey(profile.ClientManagerRef) { + return fmt.Errorf("client manager ref is unsafe") + } + return nil +} + +func resolveTransports(keys []string, profiles []RuntimeTransportProfile) ([]RuntimeTransportProfile, error) { + out := make([]RuntimeTransportProfile, 0, len(keys)) + for _, key := range keys { + if !protocol.ValidLogicalFileKey(key) { + return nil, fmt.Errorf("transport key is unsafe") + } + found := false + for _, profile := range profiles { + if profile.Key != key { + continue + } + if !protocol.ValidLogicalFileKey(profile.Key) || (profile.TargetKey != "" && !protocol.ValidLogicalFileKey(profile.TargetKey)) { + return nil, fmt.Errorf("transport profile is unsafe") + } + out = append(out, profile) + found = true + break + } + if !found { + return nil, fmt.Errorf("transport profile %q is not declared", key) + } + } + return out, nil +} + +func missingRuntimeBindingKeys(profile RuntimeLifecycleProfile, transports []RuntimeTransportProfile, discovery []RuntimeDiscoveryProbe, logs []RuntimeLogSource, binding RuntimeBindingSet) []string { + required := map[string]struct{}{} + for _, transport := range transports { + if transport.TargetKey != "" { + required[transport.TargetKey] = struct{}{} + } + } + for _, probe := range discovery { + if probe.Required && probe.TargetKey != "" { + required[probe.TargetKey] = struct{}{} + } + } + logTargets := map[string]struct{}{} + for _, source := range logs { + if source.TargetKey != "" { + logTargets[source.TargetKey] = struct{}{} + } + } + if profile.ClientManagerRef != "" { + required[profile.ClientManagerRef] = struct{}{} + } + for _, key := range binding.MissingKeys { + if _, logTarget := logTargets[key]; logTarget { + continue + } + if protocol.ValidLogicalFileKey(key) { + required[key] = struct{}{} + } + } + missing := make([]string, 0, len(required)) + for key := range required { + if _, ok := binding.Bindings[key]; !ok { + missing = append(missing, key) + } + } + sort.Strings(missing) + return missing +} + +func safeDiscovery(probes []RuntimeDiscoveryProbe, targetOS string) []RuntimeDiscoveryProbe { + out := []RuntimeDiscoveryProbe{} + for _, probe := range probes { + if supportsPlatform(probe.Platforms, targetOS) && protocol.ValidLogicalFileKey(probe.Key) && protocol.ValidLogicalFileKey(probe.TargetKey) { + out = append(out, probe) + } + } + return out +} + +func safeLogSources(sources []RuntimeLogSource, targetOS string) []RuntimeLogSource { + _ = targetOS + out := []RuntimeLogSource{} + for _, source := range sources { + if protocol.ValidLogicalFileKey(source.Key) && protocol.ValidLogicalFileKey(source.StreamKey) && (source.TargetKey == "" || protocol.ValidLogicalFileKey(source.TargetKey)) { + out = append(out, source) + } + } + return out +} + +func supportedRuntimeMode(mode string) bool { + switch mode { + case RuntimeModeLocalProcess, RuntimeModeHostedFTPRCON, RuntimeModeFTPOnly, RuntimeModeCustomClient: + return true + default: + return false + } +} + +func supportsPlatform(platforms []string, targetOS string) bool { + if targetOS == "" || len(platforms) == 0 { + return true + } + for _, platform := range platforms { + if platform == targetOS { + return true + } + } + return false +} + +func copyStringMap(values map[string]string) map[string]string { + if len(values) == 0 { + return nil + } + out := make(map[string]string, len(values)) + for key, value := range values { + out[key] = value + } + return out +} diff --git a/runtime/self_update.go b/runtime/self_update.go new file mode 100644 index 0000000..9cd5a54 --- /dev/null +++ b/runtime/self_update.go @@ -0,0 +1,611 @@ +package runtime + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "browser.local/run/protocol" +) + +const ( + selfUpdateManifestVersion = 1 + maxSelfUpdateBytes = int64(512 * 1024 * 1024) + maxSelfUpdateEntries = 8 + defaultUpdateHealthWait = 30 * time.Second +) + +var ErrSelfUpdateRestartRequested = errors.New("Run self-update restart requested") + +type SelfUpdateManifest struct { + Version int `json:"version"` + JobID string `json:"jobId"` + Attempt int `json:"attempt"` + LeaseToken string `json:"leaseToken"` + ArtifactID string `json:"artifactId"` + ArtifactChecksum string `json:"artifactChecksum"` + ArtifactSizeBytes int64 `json:"artifactSizeBytes"` + TargetOS string `json:"targetOs"` + TargetArch string `json:"targetArch"` + TargetRelease string `json:"targetRelease"` + CurrentExecutable string `json:"currentExecutable"` + StagedExecutable string `json:"stagedExecutable"` + BackupExecutable string `json:"backupExecutable"` + HealthFile string `json:"healthFile"` + WorkingDirectory string `json:"workingDirectory"` + Phase string `json:"phase"` + DownloadedBytes int64 `json:"downloadedBytes"` + BinaryChecksum string `json:"binaryChecksum,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type SelfUpdateActivator interface { + Activate(string) error +} + +type ProcessSelfUpdateActivator struct{} + +func (ProcessSelfUpdateActivator) Activate(manifestPath string) error { + manifest, err := loadSelfUpdateManifest(manifestPath) + if err != nil { + return err + } + command := exec.Command(manifest.StagedExecutable) + command.Dir = manifest.WorkingDirectory + command.Env = append(cleanUpdateEnvironment(os.Environ()), "RUN_MODE=self-update-helper", "RUN_UPDATE_MANIFEST="+manifestPath) + command.Stdout = io.Discard + command.Stderr = io.Discard + return command.Start() +} + +func (worker *Worker) executeRunSelfUpdate(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_self_update_job", err.Error()) + } + state, err := worker.registeredState() + if err != nil { + return lifecycleFailure("self_update_unregistered", "Run worker is not registered") + } + input, err := worker.client.GetRunUpdateInput(ctx, protocol.RunUpdateInputRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt}) + if err != nil { + return lifecycleFailure("self_update_input_failed", "could not load fenced Run update input") + } + if err := validateRunUpdateInput(assignment, input); err != nil { + return lifecycleFailure("unsafe_self_update_input", err.Error()) + } + transactionRoot := filepath.Join(worker.cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID)) + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + return lifecycleFailure("self_update_workspace_failed", "could not create update transaction workspace") + } + manifestPath := filepath.Join(transactionRoot, "manifest.json") + archivePath := filepath.Join(transactionRoot, "update.archive") + manifest, err := prepareSelfUpdateManifest(manifestPath, assignment, input, transactionRoot) + if err != nil { + return lifecycleFailure("self_update_manifest_failed", err.Error()) + } + if manifest.Phase != "staged" { + manifest.Phase = "downloading" + if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { + return lifecycleFailure("self_update_manifest_failed", err.Error()) + } + if err := worker.downloadRunUpdate(ctx, assignment, input, archivePath, manifestPath, &manifest); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update download cancelled"}, Message: "Run update download cancelled", ErrorCode: "run_self_update_cancelled"} + } + return lifecycleFailure("self_update_download_failed", err.Error()) + } + stagedPath := filepath.Join(transactionRoot, input.ExecutableName+".staged") + binaryChecksum, err := stageRunUpdateBinary(archivePath, input.PackageFormat, input.ExecutableName, stagedPath) + if err != nil { + return lifecycleFailure("self_update_extract_failed", err.Error()) + } + manifest.StagedExecutable = stagedPath + manifest.BinaryChecksum = binaryChecksum + manifest.Phase = "staged" + manifest.UpdatedAt = time.Now().UTC() + if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { + return lifecycleFailure("self_update_manifest_failed", err.Error()) + } + } + evidence, _ := json.Marshal(protocol.RunUpdateExecutionEvidence{TargetRelease: input.TargetRelease, Phase: "staged"}) + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", safeWorkspaceName(assignment.JobID)), Message: "Run update verified and staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: input.Checksum, SizeBytes: input.SizeBytes, Summary: "verified update staged", Content: string(evidence)}, ActivationManifest: manifestPath} +} + +func validateRunUpdateInput(assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse) error { + if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID || assignment.InputRef != "artifact://"+input.ArtifactID { + return fmt.Errorf("Run update input scope does not match job") + } + if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH { + return fmt.Errorf("Run update target does not match this executable") + } + if input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" && input.PackageFormat != "raw-executable" { + return fmt.Errorf("Run update package format is unsupported") + } + if input.SizeBytes <= 0 || input.SizeBytes > maxSelfUpdateBytes || input.ChunkSizeBytes <= 0 || input.ChunkSizeBytes > 1024*1024 || !validSHA256(input.Checksum) { + return fmt.Errorf("Run update artifact bounds are invalid") + } + expectedName := "run" + if runtime.GOOS == "windows" { + expectedName = "run.exe" + } + if input.ExecutableName != expectedName || !protocol.ValidLogicalFileKey(input.TargetRelease) { + return fmt.Errorf("Run update executable or release identity is unsafe") + } + return nil +} + +func prepareSelfUpdateManifest(path string, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, root string) (SelfUpdateManifest, error) { + if existing, err := loadSelfUpdateManifest(path); err == nil { + if existing.JobID != assignment.JobID || existing.ArtifactID != input.ArtifactID || existing.ArtifactChecksum != input.Checksum || existing.TargetRelease != input.TargetRelease || existing.Attempt > assignment.Attempt { + return SelfUpdateManifest{}, fmt.Errorf("existing update transaction does not match active attempt") + } + if existing.Phase == "staged" { + if existing.StagedExecutable == "" || !pathWithinRoot(root, existing.StagedExecutable) || existing.BinaryChecksum == "" { + return SelfUpdateManifest{}, fmt.Errorf("staged update manifest is outside the transaction workspace") + } + checksum, _, checksumErr := checksumFile(existing.StagedExecutable) + if checksumErr != nil || checksum != existing.BinaryChecksum { + return SelfUpdateManifest{}, fmt.Errorf("staged Run binary checksum changed") + } + } + existing.Attempt = assignment.Attempt + existing.LeaseToken = assignment.LeaseToken + return existing, nil + } else if !errors.Is(err, os.ErrNotExist) { + return SelfUpdateManifest{}, err + } + current, err := os.Executable() + if err != nil { + return SelfUpdateManifest{}, err + } + current, err = filepath.Abs(current) + if err != nil { + return SelfUpdateManifest{}, err + } + info, err := os.Lstat(current) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return SelfUpdateManifest{}, fmt.Errorf("current Run executable is not a regular file") + } + workingDirectory, err := os.Getwd() + if err != nil { + return SelfUpdateManifest{}, err + } + now := time.Now().UTC() + manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, CurrentExecutable: current, BackupExecutable: filepath.Join(root, "previous-run.backup"), HealthFile: filepath.Join(root, "healthy"), WorkingDirectory: workingDirectory, Phase: "downloading", CreatedAt: now, UpdatedAt: now} + return manifest, nil +} + +func pathWithinRoot(root, path string) bool { + rootAbs, rootErr := filepath.Abs(root) + pathAbs, pathErr := filepath.Abs(path) + if rootErr != nil || pathErr != nil { + return false + } + relative, err := filepath.Rel(rootAbs, pathAbs) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && relative != "." +} + +func (worker *Worker) downloadRunUpdate(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, archivePath, manifestPath string, manifest *SelfUpdateManifest) error { + file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + offset := info.Size() + if offset < 0 || offset > input.SizeBytes { + return fmt.Errorf("partial update artifact has invalid size") + } + if _, err := file.Seek(offset, io.SeekStart); err != nil { + return err + } + for offset < input.SizeBytes { + if ctx.Err() != nil { + return ctx.Err() + } + state, err := worker.registeredState() + if err != nil { + return err + } + length := input.ChunkSizeBytes + if remaining := input.SizeBytes - offset; int64(length) > remaining { + length = int(remaining) + } + chunk, err := worker.client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Offset: offset, Length: length}) + if err != nil { + return err + } + if chunk.JobID != assignment.JobID || chunk.ArtifactID != input.ArtifactID || chunk.Offset != offset || chunk.TotalBytes != input.SizeBytes || chunk.Checksum != input.Checksum || len(chunk.Payload) == 0 || len(chunk.Payload) > length { + return fmt.Errorf("Run update chunk acknowledgement does not match request") + } + if _, err := file.Write(chunk.Payload); err != nil { + return err + } + if err := file.Sync(); err != nil { + return err + } + offset += int64(len(chunk.Payload)) + manifest.DownloadedBytes = offset + manifest.UpdatedAt = time.Now().UTC() + if err := persistSelfUpdateManifest(manifestPath, *manifest); err != nil { + return err + } + } + if err := file.Close(); err != nil { + return err + } + checksum, size, err := checksumFile(archivePath) + if err != nil { + return err + } + if size != input.SizeBytes || checksum != input.Checksum { + _ = os.Remove(archivePath) + return fmt.Errorf("Run update artifact checksum mismatch") + } + return nil +} + +func stageRunUpdateBinary(artifactPath, format, executableName, destination string) (string, error) { + if format != "raw-executable" { + return extractRunUpdateBinary(artifactPath, format, executableName, destination) + } + info, err := os.Stat(artifactPath) + if err != nil { + return "", err + } + if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxSelfUpdateBytes { + return "", fmt.Errorf("Run update executable exceeds bounds") + } + input, err := os.Open(artifactPath) + if err != nil { + return "", err + } + defer input.Close() + temporary := destination + ".tmp" + output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) + if err != nil { + return "", err + } + written, copyErr := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)) + if copyErr == nil && written != info.Size() { + copyErr = fmt.Errorf("Run update executable size does not match artifact") + } + if syncErr := output.Sync(); copyErr == nil { + copyErr = syncErr + } + if closeErr := output.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + _ = os.Remove(temporary) + return "", copyErr + } + if err := os.Rename(temporary, destination); err != nil { + _ = os.Remove(temporary) + return "", err + } + if err := os.Chmod(destination, 0o700); err != nil { + return "", err + } + checksum, _, err := checksumFile(destination) + return checksum, err +} + +func extractRunUpdateBinary(archivePath, format, executableName, destination string) (string, error) { + found := false + entries := 0 + writeEntry := func(name string, mode os.FileMode, reader io.Reader, size int64) error { + entries++ + if entries > maxSelfUpdateEntries || size < 0 || size > maxSelfUpdateBytes { + return fmt.Errorf("Run update archive exceeds bounds") + } + clean := filepath.ToSlash(filepath.Clean(name)) + if clean != name || strings.Contains(clean, "../") || strings.HasPrefix(clean, "/") || strings.Contains(clean, `\`) { + return fmt.Errorf("Run update archive entry is unsafe") + } + if clean == "config.json" { + _, err := io.Copy(io.Discard, io.LimitReader(reader, size+1)) + return err + } + if clean != executableName || found || mode&os.ModeSymlink != 0 { + return fmt.Errorf("Run update archive contains unexpected entry") + } + found = true + temporary := destination + ".tmp" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) + if err != nil { + return err + } + written, copyErr := io.Copy(file, io.LimitReader(reader, maxSelfUpdateBytes+1)) + if copyErr == nil && written != size { + copyErr = fmt.Errorf("Run update binary size does not match archive") + } + if syncErr := file.Sync(); copyErr == nil { + copyErr = syncErr + } + if closeErr := file.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + _ = os.Remove(temporary) + return copyErr + } + if err := os.Rename(temporary, destination); err != nil { + _ = os.Remove(temporary) + return err + } + return os.Chmod(destination, 0o700) + } + + if format == "zip" { + info, err := os.Stat(archivePath) + if err != nil { + return "", err + } + reader, err := zip.OpenReader(archivePath) + if err != nil { + return "", err + } + defer reader.Close() + if info.Size() > maxSelfUpdateBytes { + return "", fmt.Errorf("Run update archive exceeds size limit") + } + for _, entry := range reader.File { + if entry.FileInfo().IsDir() || entry.Mode()&os.ModeType != 0 { + return "", fmt.Errorf("Run update archive contains non-regular entry") + } + stream, err := entry.Open() + if err != nil { + return "", err + } + err = writeEntry(entry.Name, entry.Mode(), stream, int64(entry.UncompressedSize64)) + _ = stream.Close() + if err != nil { + return "", err + } + } + } else { + file, err := os.Open(archivePath) + if err != nil { + return "", err + } + defer file.Close() + gzipReader, err := gzip.NewReader(file) + if err != nil { + return "", err + } + defer gzipReader.Close() + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", err + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { + return "", fmt.Errorf("Run update archive contains non-regular entry") + } + if err := writeEntry(header.Name, os.FileMode(header.Mode), tarReader, header.Size); err != nil { + return "", err + } + } + } + if !found { + return "", fmt.Errorf("Run update archive does not contain expected executable") + } + checksum, _, err := checksumFile(destination) + return checksum, err +} + +func ApplySelfUpdateManifest(manifestPath string) error { + manifest, err := loadSelfUpdateManifest(manifestPath) + if err != nil { + return err + } + helper, err := os.Executable() + if err != nil { + return err + } + helper, _ = filepath.Abs(helper) + staged, _ := filepath.Abs(manifest.StagedExecutable) + if helper != staged || manifest.TargetOS != runtime.GOOS || manifest.TargetArch != runtime.GOARCH || manifest.Phase != "staged" { + return fmt.Errorf("self-update helper scope does not match staged transaction") + } + manifest.Phase = "activating" + manifest.UpdatedAt = time.Now().UTC() + if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { + return err + } + if err := replaceRunExecutable(manifest); err != nil { + manifest.Phase = "rolled-back" + manifest.UpdatedAt = time.Now().UTC() + _ = persistSelfUpdateManifest(manifestPath, manifest) + _, _ = startRunAfterUpdate(manifest, "rolled-back") + return err + } + _ = os.Remove(manifest.HealthFile) + command, err := startRunAfterUpdate(manifest, "succeeded") + if err != nil { + _ = rollbackRunExecutable(manifest) + _, _ = startRunAfterUpdate(manifest, "rolled-back") + return err + } + wait := defaultUpdateHealthWait + if value, parseErr := strconv.Atoi(os.Getenv("RUN_UPDATE_HEALTH_TIMEOUT_MS")); parseErr == nil && value > 0 && value <= 300000 { + wait = time.Duration(value) * time.Millisecond + } + deadline := time.Now().Add(wait) + for time.Now().Before(deadline) { + if _, err := os.Stat(manifest.HealthFile); err == nil { + manifest.Phase = "succeeded" + manifest.UpdatedAt = time.Now().UTC() + return persistSelfUpdateManifest(manifestPath, manifest) + } + time.Sleep(100 * time.Millisecond) + } + _ = command.Process.Kill() + if err := rollbackRunExecutable(manifest); err != nil { + return fmt.Errorf("updated Run health timed out and rollback failed: %w", err) + } + manifest.Phase = "rolled-back" + manifest.UpdatedAt = time.Now().UTC() + _ = persistSelfUpdateManifest(manifestPath, manifest) + _, _ = startRunAfterUpdate(manifest, "rolled-back") + return fmt.Errorf("updated Run did not become healthy before timeout") +} + +func replaceRunExecutable(manifest SelfUpdateManifest) error { + if checksum, _, err := checksumFile(manifest.StagedExecutable); err != nil || checksum != manifest.BinaryChecksum { + return fmt.Errorf("staged Run binary checksum changed") + } + _ = os.Remove(manifest.BackupExecutable) + var lastErr error + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if err := os.Rename(manifest.CurrentExecutable, manifest.BackupExecutable); err != nil { + lastErr = err + continue + } + if err := copyExecutable(manifest.StagedExecutable, manifest.CurrentExecutable); err != nil { + _ = os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable) + return err + } + return nil + } + return fmt.Errorf("could not back up current Run executable: %w", lastErr) +} + +func rollbackRunExecutable(manifest SelfUpdateManifest) error { + if _, err := os.Stat(manifest.BackupExecutable); err != nil { + return err + } + _ = os.Remove(manifest.CurrentExecutable) + return os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable) +} + +func startRunAfterUpdate(manifest SelfUpdateManifest, outcome string) (*exec.Cmd, error) { + command := exec.Command(manifest.CurrentExecutable) + command.Dir = manifest.WorkingDirectory + environment := cleanUpdateEnvironment(os.Environ()) + environment = append(environment, "RUN_MODE=worker", "RUN_UPDATE_JOB_ID="+manifest.JobID, "RUN_UPDATE_OUTCOME="+outcome, "RUN_UPDATE_ATTEMPT="+strconv.Itoa(manifest.Attempt), "RUN_UPDATE_LEASE_TOKEN="+manifest.LeaseToken) + if outcome == "succeeded" { + environment = append(environment, "RUN_VERSION="+manifest.TargetRelease, "RUN_UPDATE_HEALTH_FILE="+manifest.HealthFile) + } + command.Env = environment + command.Stdout = io.Discard + command.Stderr = io.Discard + if err := command.Start(); err != nil { + return command, err + } + return command, nil +} + +func MarkSelfUpdateHealthy(path string) error { + if strings.TrimSpace(path) == "" { + return nil + } + return writeRuntimeAtomicFile(path, []byte("healthy\n"), 0o600) +} + +func loadSelfUpdateManifest(path string) (SelfUpdateManifest, error) { + body, err := os.ReadFile(path) + if err != nil { + return SelfUpdateManifest{}, err + } + var manifest SelfUpdateManifest + if err := json.Unmarshal(body, &manifest); err != nil { + return SelfUpdateManifest{}, fmt.Errorf("decode self-update manifest: %w", err) + } + if manifest.Version != selfUpdateManifestVersion || manifest.JobID == "" || manifest.Attempt <= 0 || manifest.LeaseToken == "" || manifest.ArtifactID == "" || !validSHA256(manifest.ArtifactChecksum) || manifest.ArtifactSizeBytes <= 0 || manifest.ArtifactSizeBytes > maxSelfUpdateBytes || !protocol.ValidLogicalFileKey(manifest.TargetRelease) { + return SelfUpdateManifest{}, fmt.Errorf("self-update manifest is invalid") + } + return manifest, nil +} + +func persistSelfUpdateManifest(path string, manifest SelfUpdateManifest) error { + body, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + return writeRuntimeAtomicFile(path, body, 0o600) +} + +func checksumFile(path string) (string, int64, error) { + file, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer file.Close() + hash := sha256.New() + size, err := io.Copy(hash, io.LimitReader(file, maxSelfUpdateBytes+1)) + if err != nil { + return "", 0, err + } + if size > maxSelfUpdateBytes { + return "", size, fmt.Errorf("file exceeds self-update size limit") + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil +} + +func copyExecutable(source, destination string) error { + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + temporary := destination + ".update-tmp" + output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) + if err != nil { + return err + } + if _, err := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)); err != nil { + _ = output.Close() + _ = os.Remove(temporary) + return err + } + if err := output.Sync(); err != nil { + _ = output.Close() + _ = os.Remove(temporary) + return err + } + if err := output.Close(); err != nil { + _ = os.Remove(temporary) + return err + } + if err := os.Rename(temporary, destination); err != nil { + _ = os.Remove(temporary) + return err + } + return os.Chmod(destination, 0o700) +} + +func cleanUpdateEnvironment(environment []string) []string { + blocked := map[string]bool{"RUN_UPDATE_MANIFEST": true, "RUN_UPDATE_HEALTH_FILE": true, "RUN_UPDATE_HEALTH_TIMEOUT_MS": true, "RUN_UPDATE_JOB_ID": true, "RUN_UPDATE_OUTCOME": true, "RUN_UPDATE_ATTEMPT": true, "RUN_UPDATE_LEASE_TOKEN": true, "RUN_MODE": true, "RUN_VERSION": true} + out := make([]string, 0, len(environment)) + for _, entry := range environment { + key, _, _ := strings.Cut(entry, "=") + if !blocked[key] { + out = append(out, entry) + } + } + return out +} diff --git a/runtime/self_update_test.go b/runtime/self_update_test.go new file mode 100644 index 0000000..685adf9 --- /dev/null +++ b/runtime/self_update_test.go @@ -0,0 +1,189 @@ +package runtime + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestRunSelfUpdateResumesPartialDownloadAndRejectsChecksum(t *testing.T) { + assignment, input, payload := selfUpdateTestFixture(t) + client := newFakeWorkerClient() + client.updateInput = input + client.updatePayload = payload + cfg := workerTestConfig(t) + transactionRoot := filepath.Join(cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID)) + if err := os.MkdirAll(transactionRoot, 0o700); err != nil { + t.Fatalf("create transaction root: %v", err) + } + partial := len(payload) / 3 + if err := os.WriteFile(filepath.Join(transactionRoot, "update.archive"), payload[:partial], 0o600); err != nil { + t.Fatalf("write partial update: %v", err) + } + worker, err := NewWorker(cfg, client, WithSelfUpdateActivator(&recordingSelfUpdateActivator{})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + result := worker.executeRunSelfUpdate(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "run.update.staged" || result.ActivationManifest == "" { + t.Fatalf("expected staged self-update, got %+v", result) + } + if len(client.updateChunkOffsets) == 0 || client.updateChunkOffsets[0] != int64(partial) { + t.Fatalf("expected resumable range from %d, got %+v", partial, client.updateChunkOffsets) + } + + badClient := newFakeWorkerClient() + badInput := input + badInput.Checksum = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + badClient.updateInput = badInput + badClient.updatePayload = payload + badWorker, err := NewWorker(workerTestConfig(t), badClient, WithSelfUpdateActivator(&recordingSelfUpdateActivator{})) + if err != nil { + t.Fatalf("new bad checksum worker: %v", err) + } + badWorker.state.SessionToken = "session-token" + bad := badWorker.executeRunSelfUpdate(context.Background(), assignment) + if bad.State != lifecycleResultStateFailed || bad.ErrorCode != "self_update_download_failed" { + t.Fatalf("expected final checksum rejection, got %+v", bad) + } +} + +func TestSelfUpdateActivationWaitsForAcceptedTerminalResult(t *testing.T) { + assignment, input, payload := selfUpdateTestFixture(t) + client := newFakeWorkerClient() + client.claimJob = assignment + client.updateInput = input + client.updatePayload = payload + client.resultErr = errors.New("stale lease") + activator := &recordingSelfUpdateActivator{} + worker, err := NewWorker(workerTestConfig(t), client, WithSelfUpdateActivator(activator)) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + handled, err := worker.ClaimAndRunOnce(context.Background()) + if !handled || err == nil { + t.Fatalf("expected rejected result error, handled=%v err=%v", handled, err) + } + if activator.manifestPath != "" { + t.Fatalf("stale terminal result must not activate update: %s", activator.manifestPath) + } + if worker.journal.ActiveCount() != 1 { + t.Fatal("staged result must remain recoverable until Platform accepts it") + } +} + +func TestSelfUpdateReplacementPreservesRollbackAndRejectsTraversal(t *testing.T) { + root := t.TempDir() + current := filepath.Join(root, "run") + staged := filepath.Join(root, "staged-run") + backup := filepath.Join(root, "previous-run") + if err := os.WriteFile(current, []byte("old-run"), 0o700); err != nil { + t.Fatalf("write current: %v", err) + } + if err := os.WriteFile(staged, []byte("new-run"), 0o700); err != nil { + t.Fatalf("write staged: %v", err) + } + checksum, _, err := checksumFile(staged) + if err != nil { + t.Fatalf("checksum staged: %v", err) + } + manifest := SelfUpdateManifest{StagedExecutable: staged, CurrentExecutable: current, BackupExecutable: backup, BinaryChecksum: checksum} + if err := replaceRunExecutable(manifest); err != nil { + t.Fatalf("replace executable: %v", err) + } + if body, _ := os.ReadFile(current); string(body) != "new-run" { + t.Fatalf("expected new executable, got %q", body) + } + if err := rollbackRunExecutable(manifest); err != nil { + t.Fatalf("rollback executable: %v", err) + } + if body, _ := os.ReadFile(current); string(body) != "old-run" { + t.Fatalf("expected previous executable after rollback, got %q", body) + } + + archivePath := filepath.Join(root, "unsafe.tar.gz") + var archive bytes.Buffer + gzipWriter := gzip.NewWriter(&archive) + tarWriter := tar.NewWriter(gzipWriter) + body := []byte("escape") + if err := tarWriter.WriteHeader(&tar.Header{Name: "../run", Mode: 0o700, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatalf("write unsafe header: %v", err) + } + _, _ = tarWriter.Write(body) + _ = tarWriter.Close() + _ = gzipWriter.Close() + if err := os.WriteFile(archivePath, archive.Bytes(), 0o600); err != nil { + t.Fatalf("write unsafe archive: %v", err) + } + if _, err := extractRunUpdateBinary(archivePath, "tar.gz", "run", filepath.Join(root, "escaped")); err == nil || !strings.Contains(err.Error(), "unsafe") { + t.Fatalf("expected traversal rejection, got %v", err) + } +} + +func TestPrepareSelfUpdateManifestRevalidatesStagedBinaryAfterRestart(t *testing.T) { + root := t.TempDir() + assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate) + assignment.TargetKey = "run/update" + assignment.InputRef = "artifact://artifact-run-staged" + input := protocol.RunUpdateInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, ArtifactID: "artifact-run-staged", Checksum: "sha256:" + strings.Repeat("a", 64), SizeBytes: 16, TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, PackageFormat: "tar.gz", ExecutableName: "run", TargetRelease: "release-staged", ChunkSizeBytes: 8} + manifestPath := filepath.Join(root, "manifest.json") + stagedPath := filepath.Join(root, "run.staged") + if err := os.WriteFile(stagedPath, []byte("staged-binary"), 0o700); err != nil { + t.Fatalf("write staged binary: %v", err) + } + checksum, _, err := checksumFile(stagedPath) + if err != nil { + t.Fatalf("checksum staged binary: %v", err) + } + manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, StagedExecutable: stagedPath, BinaryChecksum: checksum, Phase: "staged", CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC()} + if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { + t.Fatalf("persist staged manifest: %v", err) + } + if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err != nil { + t.Fatalf("revalidate staged manifest: %v", err) + } + if err := os.WriteFile(stagedPath, []byte("tampered-binary"), 0o700); err != nil { + t.Fatalf("tamper staged binary: %v", err) + } + if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err == nil || !strings.Contains(err.Error(), "checksum changed") { + t.Fatalf("expected staged checksum rejection, got %v", err) + } + manifest.StagedExecutable = filepath.Join(root, "..", "outside") + if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { + t.Fatalf("persist unsafe staged manifest: %v", err) + } + if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err == nil || !strings.Contains(err.Error(), "outside") { + t.Fatalf("expected staged path rejection, got %v", err) + } +} + +func selfUpdateTestFixture(t *testing.T) (protocol.RunJobAssignment, protocol.RunUpdateInputResponse, []byte) { + t.Helper() + assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate) + assignment.TargetKey = "run/update" + assignment.InputRef = "artifact://artifact-run-latest" + assignment.State = "running" + assignment.LeaseToken = "lease-update" + assignment.Attempt = 1 + executableName := "run" + if runtime.GOOS == "windows" { + executableName = "run.exe" + } + payload := []byte("self-update-test-binary") + input := protocol.RunUpdateInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, ArtifactID: "artifact-run-latest", Checksum: bytesChecksum(payload), SizeBytes: int64(len(payload)), TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, PackageFormat: "raw-executable", ExecutableName: executableName, TargetRelease: "run-release-test", ChunkSizeBytes: 64} + return assignment, input, payload +} diff --git a/runtime/smoke.go b/runtime/smoke.go new file mode 100644 index 0000000..fcf454b --- /dev/null +++ b/runtime/smoke.go @@ -0,0 +1,19 @@ +package runtime + +import ( + "browser.local/run/config" + "browser.local/run/domain" +) + +func SmokeSummary(cfg config.Config) domain.ExecutorStatus { + return domain.ExecutorStatus{ + Mode: cfg.Mode, + PlatformURL: cfg.PlatformURL, + Status: "ok", + ExposedHostPath: false, + Capabilities: append([]string{ + "control.hello", + "control.heartbeat", + }, SupportedRunCapabilitiesForComponent(cfg.ComponentKind)...), + } +} diff --git a/runtime/smoke_test.go b/runtime/smoke_test.go new file mode 100644 index 0000000..927159b --- /dev/null +++ b/runtime/smoke_test.go @@ -0,0 +1,24 @@ +package runtime + +import ( + "testing" + + "browser.local/run/config" +) + +func TestSmokeSummaryDoesNotExposeHostPaths(t *testing.T) { + summary := SmokeSummary(config.Config{ + Mode: "smoke", + PlatformURL: "http://platform.test", + }) + + if summary.Status != "ok" { + t.Fatalf("expected ok status, got %q", summary.Status) + } + if summary.ExposedHostPath { + t.Fatal("smoke summary must not expose host paths") + } + if len(summary.Capabilities) == 0 { + t.Fatal("expected baseline capabilities") + } +} diff --git a/runtime/source_rcon.go b/runtime/source_rcon.go new file mode 100644 index 0000000..5a66720 --- /dev/null +++ b/runtime/source_rcon.go @@ -0,0 +1,305 @@ +package runtime + +import ( + "context" + "encoding/binary" + "encoding/hex" + "errors" + "io" + "net" + "strconv" + "strings" + "time" + "unicode/utf8" + + "browser.local/run/protocol" +) + +const ( + sourceRCONAuthRequestID int32 = 1 + sourceRCONCommandRequestID int32 = 2 + sourceRCONResponseValue int32 = 0 + sourceRCONAuthResponse int32 = 2 + sourceRCONExecuteCommand int32 = 2 + sourceRCONAuthenticate int32 = 3 + sourceRCONMaxPacketSize = 4096 + sourceRCONMaxCommandBytes = 4000 + sourceRCONMaxResponsePackets = 32 + sourceRCONMaxResponseBytes = 64 * 1024 + sourceRCONIOTimeout = 10 * time.Second + sourceRCONMaxExecutionTimeout = 60 * time.Second +) + +type sourceRCONPacket struct { + id int32 + typeCode int32 + body string +} + +type sourceRCONConfig struct { + password string +} + +type sourceRCONError struct { + code string +} + +func (err sourceRCONError) Error() string { return err.code } + +// ExecuteSourceRCON connects only to the local UE4SS listener described by a +// frozen plan. The command, config, password, and response body are transient. +func (executor LifecycleExecutor) ExecuteSourceRCON(ctx context.Context, assignment protocol.RunJobAssignment, command string) LifecycleExecutionResult { + if assignment.ExecutionInput.SourceRCON == nil || protocol.ValidateRunJobAssignment(assignment) != nil { + return lifecycleFailure("unsafe_source_rcon_plan", "Source RCON plan is invalid") + } + if executor.runtimeTargetOS != "windows" || executor.runtimeTargetArch != "amd64" { + return lifecycleFailure("unsupported_extension_platform", "Source RCON requires Windows amd64") + } + if !validSourceRCONCommand(command) { + return lifecycleFailure("source_rcon_input_invalid", "Source RCON command input is invalid") + } + + executionCtx, cancel := sourceRCONExecutionContext(ctx, assignment.ExecutionInput.TimeoutSeconds) + defer cancel() + resolver := NewWorkspaceResolver(executor.workspaceRoot) + scope, err := resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + return lifecycleFailure("source_rcon_workspace_unavailable", "Source RCON workspace is unavailable") + } + configPath, err := sourceRCONConfigPath(resolver, scope, *assignment.ExecutionInput.SourceRCON) + if err != nil { + return lifecycleFailure("source_rcon_config_unavailable", "Source RCON configuration is unavailable") + } + config, err := loadSourceRCONConfig(configPath, assignment.ExecutionInput.SourceRCON.Port) + if err != nil { + return lifecycleFailure("source_rcon_config_invalid", "Source RCON configuration is invalid") + } + if err := executeSourceRCONWire(executionCtx, assignment.ExecutionInput.SourceRCON.Port, config.password, command); err != nil { + if errors.Is(executionCtx.Err(), context.Canceled) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Source RCON command cancelled"}, Message: "Source RCON command cancelled", ErrorCode: "source_rcon_cancelled"} + } + if errors.Is(executionCtx.Err(), context.DeadlineExceeded) { + return lifecycleFailure("source_rcon_timeout", "Source RCON command timed out") + } + var sourceErr sourceRCONError + if errors.As(err, &sourceErr) { + return lifecycleFailure(sourceErr.code, sourceRCONSafeMessage(sourceErr.code)) + } + return lifecycleFailure("source_rcon_execution_failed", "Source RCON command failed") + } + return LifecycleExecutionResult{ + State: lifecycleResultStateSucceeded, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Source RCON command delivered"}, + Message: "Source RCON command delivered", + ExecutionResult: protocol.RunJobExecutionResult{ + Kind: "source-rcon", + Summary: "one-time loopback Source RCON command delivered", + }, + } +} + +func sourceRCONExecutionContext(ctx context.Context, timeoutSeconds int) (context.Context, context.CancelFunc) { + timeout := time.Duration(timeoutSeconds) * time.Second + if timeout <= 0 || timeout > sourceRCONMaxExecutionTimeout { + timeout = sourceRCONMaxExecutionTimeout + } + return context.WithTimeout(ctx, timeout) +} + +func validSourceRCONCommand(command string) bool { + return strings.TrimSpace(command) != "" && utf8.ValidString(command) && len([]byte(command)) <= sourceRCONMaxCommandBytes && !strings.ContainsAny(command, "\x00\r\n") +} + +func loadSourceRCONConfig(path string, expectedPort int) (sourceRCONConfig, error) { + body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes) + if err != nil || !found || !utf8.Valid(body) { + return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"} + } + content := strings.ReplaceAll(string(body), "\r\n", "\n") + if !strings.Contains(content, managedRCONConfigMarker) { + return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"} + } + values := map[string]string{} + inRCON := false + for _, rawLine := range strings.Split(content, "\n") { + line := strings.TrimSpace(rawLine) + if line == "[rcon]" { + inRCON = true + continue + } + if strings.HasPrefix(line, "[") { + inRCON = false + continue + } + if !inRCON || line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") { + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + if key != "bind_address" && key != "port" && key != "password" { + continue + } + if _, duplicate := values[key]; duplicate { + return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"} + } + values[key] = strings.TrimSpace(value) + } + configuredPort, err := strconv.Atoi(values["port"]) + if err != nil || values["bind_address"] != "127.0.0.1" || configuredPort != expectedPort || len(values["password"]) != 64 { + return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"} + } + if _, err := hex.DecodeString(values["password"]); err != nil { + return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"} + } + return sourceRCONConfig{password: values["password"]}, nil +} + +func sourceRCONConfigPath(resolver WorkspaceResolver, scope string, plan protocol.RuntimeSourceRCONPlan) (string, error) { + markerPath, err := resolver.ExistingTarget(scope, plan.DeploymentStateRef) + if err != nil { + return "", sourceRCONError{code: "source_rcon_config_unavailable"} + } + marker, found, err := loadManagedDLLExtensionMarker(markerPath) + if err != nil || !found || !sourceRCONMarkerMatchesPlan(marker, plan) { + return "", sourceRCONError{code: "source_rcon_config_unavailable"} + } + configPath, err := resolver.ExistingTarget(scope, marker.ConfigRef) + if err != nil { + return "", sourceRCONError{code: "source_rcon_config_unavailable"} + } + return configPath, nil +} + +func sourceRCONMarkerMatchesPlan(marker managedDLLExtensionMarker, plan protocol.RuntimeSourceRCONPlan) bool { + if marker.ExtensionKey != plan.ExtensionKey || marker.ModKey != plan.ModKey || marker.RCONPort != plan.Port || !managedRCONConfigRefForMod(marker.ConfigRef, plan.ModKey) { + return false + } + return marker.ConfigRef == plan.ConfigRef || strings.HasSuffix(marker.ConfigRef, "/"+plan.ConfigRef) +} + +func executeSourceRCONWire(ctx context.Context, port int, password string, command string) error { + dialer := net.Dialer{Timeout: sourceRCONIOTimeout} + connection, err := dialer.DialContext(ctx, "tcp4", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + return sourceRCONError{code: "source_rcon_connection_failed"} + } + defer connection.Close() + if err := writeSourceRCONPacket(ctx, connection, sourceRCONPacket{id: sourceRCONAuthRequestID, typeCode: sourceRCONAuthenticate, body: password}); err != nil { + return sourceRCONError{code: "source_rcon_connection_failed"} + } + auth, err := readSourceRCONPacket(ctx, connection) + if err != nil { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + if auth.typeCode != sourceRCONAuthResponse || auth.id == -1 { + return sourceRCONError{code: "source_rcon_authentication_failed"} + } + if auth.id != sourceRCONAuthRequestID || auth.body != "" { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + if err := writeSourceRCONPacket(ctx, connection, sourceRCONPacket{id: sourceRCONCommandRequestID, typeCode: sourceRCONExecuteCommand, body: command}); err != nil { + return sourceRCONError{code: "source_rcon_connection_failed"} + } + responseBytes := 0 + sourceError := false + responsePrefix := make([]byte, 0, len("error:")) + for packetIndex := 0; packetIndex < sourceRCONMaxResponsePackets; packetIndex++ { + response, err := readSourceRCONPacket(ctx, connection) + if err != nil { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + if response.id != sourceRCONCommandRequestID || response.typeCode != sourceRCONResponseValue { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + responseBytes += len([]byte(response.body)) + if responseBytes > sourceRCONMaxResponseBytes { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + if len(responsePrefix) < cap(responsePrefix) { + remaining := cap(responsePrefix) - len(responsePrefix) + chunk := []byte(response.body) + if len(chunk) > remaining { + chunk = chunk[:remaining] + } + responsePrefix = append(responsePrefix, chunk...) + } + if len(responsePrefix) == len("error:") && strings.EqualFold(string(responsePrefix), "error:") { + sourceError = true + } + if response.body == "" { + if sourceError { + return sourceRCONError{code: "source_rcon_command_failed"} + } + return nil + } + } + return sourceRCONError{code: "source_rcon_protocol_failed"} +} + +func writeSourceRCONPacket(ctx context.Context, connection net.Conn, packet sourceRCONPacket) error { + if !utf8.ValidString(packet.body) || len([]byte(packet.body)) > sourceRCONMaxCommandBytes { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + size := 8 + len(packet.body) + 2 + if size < 10 || size > sourceRCONMaxPacketSize { + return sourceRCONError{code: "source_rcon_protocol_failed"} + } + buffer := make([]byte, 4+size) + binary.LittleEndian.PutUint32(buffer[0:4], uint32(size)) + binary.LittleEndian.PutUint32(buffer[4:8], uint32(packet.id)) + binary.LittleEndian.PutUint32(buffer[8:12], uint32(packet.typeCode)) + copy(buffer[12:], packet.body) + if err := setSourceRCONDeadline(ctx, connection); err != nil { + return err + } + _, err := connection.Write(buffer) + return err +} + +func readSourceRCONPacket(ctx context.Context, connection net.Conn) (sourceRCONPacket, error) { + if err := setSourceRCONDeadline(ctx, connection); err != nil { + return sourceRCONPacket{}, err + } + var sizeBuffer [4]byte + if _, err := io.ReadFull(connection, sizeBuffer[:]); err != nil { + return sourceRCONPacket{}, err + } + size := int(int32(binary.LittleEndian.Uint32(sizeBuffer[:]))) + if size < 10 || size > sourceRCONMaxPacketSize { + return sourceRCONPacket{}, sourceRCONError{code: "source_rcon_protocol_failed"} + } + payload := make([]byte, size) + if _, err := io.ReadFull(connection, payload); err != nil { + return sourceRCONPacket{}, err + } + if payload[size-2] != 0 || payload[size-1] != 0 || !utf8.Valid(payload[8:size-2]) { + return sourceRCONPacket{}, sourceRCONError{code: "source_rcon_protocol_failed"} + } + return sourceRCONPacket{id: int32(binary.LittleEndian.Uint32(payload[0:4])), typeCode: int32(binary.LittleEndian.Uint32(payload[4:8])), body: string(payload[8 : size-2])}, nil +} + +func setSourceRCONDeadline(ctx context.Context, connection net.Conn) error { + deadline := time.Now().Add(sourceRCONIOTimeout) + if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { + deadline = contextDeadline + } + return connection.SetDeadline(deadline) +} + +func sourceRCONSafeMessage(code string) string { + switch code { + case "source_rcon_connection_failed": + return "Source RCON listener is unavailable" + case "source_rcon_authentication_failed": + return "Source RCON authentication failed" + case "source_rcon_command_failed": + return "Source RCON command was rejected" + case "source_rcon_protocol_failed": + return "Source RCON protocol exchange failed" + default: + return "Source RCON command failed" + } +} diff --git a/runtime/source_rcon_test.go b/runtime/source_rcon_test.go new file mode 100644 index 0000000..3132731 --- /dev/null +++ b/runtime/source_rcon_test.go @@ -0,0 +1,349 @@ +package runtime + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestExecuteSourceRCONAuthenticatesRunsAndRedacts(t *testing.T) { + listener, port := newSourceRCONListener(t) + defer listener.Close() + password := strings.Repeat("a", 64) + assignment := sourceRCONAssignment(port) + root := t.TempDir() + writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password) + commands := make(chan string, 1) + serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, command sourceRCONPacket) error { + commands <- command.body + if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue, body: "queued"}); err != nil { + return err + } + return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue}) + }) + + command := "SetTime 12" + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, command) + if result.State != lifecycleResultStateSucceeded || result.ErrorCode != "" || result.ExecutionResult.Kind != "source-rcon" { + t.Fatalf("expected successful Source RCON delivery, got %+v", result) + } + if got := <-commands; got != command { + t.Fatalf("expected transient command delivery, got %q", got) + } + awaitSourceRCONServer(t, serverDone) + serialized, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal result: %v", err) + } + for _, private := range []string{command, password, "queued"} { + if strings.Contains(string(serialized), private) { + t.Fatalf("Source RCON result exposed private wire data %q: %s", private, serialized) + } + } +} + +func TestExecuteSourceRCONReadsConfigFromManagedNestedDeployment(t *testing.T) { + listener, port := newSourceRCONListener(t) + defer listener.Close() + password := strings.Repeat("e", 64) + assignment := sourceRCONAssignment(port) + root := t.TempDir() + writeSourceRCONConfigAt(t, root, assignment, "bin/"+assignment.ExecutionInput.SourceRCON.ConfigRef, "127.0.0.1", password) + serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, command sourceRCONPacket) error { + if command.body != "rcon.status" { + return fmt.Errorf("unexpected nested deployment command %q", command.body) + } + return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue}) + }) + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status") + if result.State != lifecycleResultStateSucceeded { + t.Fatalf("expected nested managed config delivery, got %+v", result) + } + awaitSourceRCONServer(t, serverDone) +} + +func TestExecuteSourceRCONRedactsSourceErrorsAndMalformedPackets(t *testing.T) { + password := strings.Repeat("b", 64) + command := "SpawnItem secret-item" + for _, testCase := range []struct { + name string + respond func(net.Conn, sourceRCONPacket) error + wantCode string + private string + }{ + { + name: "source error response", + respond: func(connection net.Conn, packet sourceRCONPacket) error { + if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "err"}); err != nil { + return err + } + if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "or: denied secret-item"}); err != nil { + return err + } + return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue}) + }, + wantCode: "source_rcon_command_failed", + private: "denied secret-item", + }, + { + name: "malformed response packet", + respond: func(connection net.Conn, _ sourceRCONPacket) error { + var size [4]byte + binary.LittleEndian.PutUint32(size[:], sourceRCONMaxPacketSize+1) + _, err := connection.Write(size[:]) + return err + }, + wantCode: "source_rcon_protocol_failed", + private: "source response body", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + listener, port := newSourceRCONListener(t) + defer listener.Close() + assignment := sourceRCONAssignment(port) + root := t.TempDir() + writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password) + serverDone := serveSourceRCONSession(listener, password, testCase.respond) + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, command) + if result.State != lifecycleResultStateFailed || result.ErrorCode != testCase.wantCode || result.Retryable { + t.Fatalf("expected safe non-retryable %s failure, got %+v", testCase.wantCode, result) + } + awaitSourceRCONServer(t, serverDone) + serialized, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal result: %v", err) + } + for _, private := range []string{command, password, testCase.private} { + if strings.Contains(string(serialized), private) { + t.Fatalf("Source RCON failure exposed private wire data %q: %s", private, serialized) + } + } + }) + } +} + +func TestExecuteSourceRCONRejectsUnsafeConfigAndNonWindowsBeforeDial(t *testing.T) { + listener, port := newSourceRCONListener(t) + acceptResult := make(chan error, 1) + go func() { + connection, err := listener.Accept() + if err == nil { + _ = connection.Close() + } + acceptResult <- err + }() + password := strings.Repeat("c", 64) + assignment := sourceRCONAssignment(port) + root := t.TempDir() + writeSourceRCONConfig(t, root, assignment, "0.0.0.0", password) + + result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status") + if result.ErrorCode != "source_rcon_config_invalid" { + t.Fatalf("expected unsafe local config rejection, got %+v", result) + } + acceptReturned := false + acceptedConnection := false + select { + case err := <-acceptResult: + acceptReturned = true + if err == nil { + acceptedConnection = true + } + case <-time.After(150 * time.Millisecond): + // No connection is expected before the listener is closed below. + } + if acceptedConnection { + t.Fatal("unsafe config opened a socket") + } + if err := listener.Close(); err != nil { + t.Fatalf("close listener: %v", err) + } + if !acceptReturned { + if err := <-acceptResult; err == nil { + t.Fatal("unsafe config opened a socket") + } + } + + linuxResult := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithDLLExtensionRuntimeTarget("linux", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status") + if linuxResult.ErrorCode != "unsupported_extension_platform" { + t.Fatalf("expected non-Windows rejection, got %+v", linuxResult) + } +} + +func TestWorkerSourceRCONConsumesInputOnceWithoutJournalOrResultLeakage(t *testing.T) { + listener, port := newSourceRCONListener(t) + defer listener.Close() + password := strings.Repeat("d", 64) + command := "SendChat 4 \"maintenance complete\"" + client := newFakeWorkerClient() + assignment := workerJobAssignment(protocol.RunCapabilityRemoteRunRCONCommand) + assignment.TargetKey = "rcon.password" + assignment.InputRef = "input://source-rcon/job-worker" + assignment.MaxAttempts = 1 + assignment.ExecutionInput = protocol.RunJobExecutionInput{ + WorkspaceScope: "run-local", + RemoteAdapterKey: "rcon", + RemoteAdapterKind: "rcon", + TimeoutSeconds: 5, + SourceRCON: &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port}, + } + client.claimJob = assignment + client.sourceRCONInput = protocol.SourceRCONExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, Command: command} + serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, packet sourceRCONPacket) error { + if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "accepted"}); err != nil { + return err + } + return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue}) + }) + config := workerTestConfig(t) + writeSourceRCONConfig(t, config.WorkspaceRoot, assignment, "127.0.0.1", password) + worker, err := NewWorker(config, client, WithDLLExtensionRuntimeTarget("windows", "amd64")) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register worker: %v", err) + } + if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled { + t.Fatalf("claim Source RCON job handled=%v err=%v", handled, err) + } + awaitSourceRCONServer(t, serverDone) + if len(client.sourceRCONRequests) != 1 || client.sourceRCONRequests[0].JobID != assignment.JobID { + t.Fatalf("expected one active-lease Source RCON input read, got %+v", client.sourceRCONRequests) + } + if len(client.resultRequests) != 1 || client.resultRequests[0].Retryable || client.resultRequests[0].State != lifecycleResultStateSucceeded { + t.Fatalf("expected one non-retryable safe result, got %+v", client.resultRequests) + } + for _, projection := range []any{worker.journal.ActiveJobs(), client.resultRequests, client.sourceRCONRequests} { + body, marshalErr := json.Marshal(projection) + if marshalErr != nil { + t.Fatalf("marshal safe projection: %v", marshalErr) + } + for _, private := range []string{command, password, "accepted"} { + if strings.Contains(string(body), private) { + t.Fatalf("journal or result projection exposed %q: %s", private, body) + } + } + } +} + +func newSourceRCONListener(t *testing.T) (net.Listener, int) { + t.Helper() + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen Source RCON fixture: %v", err) + } + address, ok := listener.Addr().(*net.TCPAddr) + if !ok || address.Port < 1024 { + _ = listener.Close() + t.Fatal("invalid Source RCON fixture address") + } + return listener, address.Port +} + +func sourceRCONAssignment(port int) protocol.RunJobAssignment { + assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunRCONCommand) + assignment.TargetKey = "rcon.password" + assignment.InputRef = "input://source-rcon/job-1" + assignment.MaxAttempts = 1 + assignment.ExecutionInput = protocol.RunJobExecutionInput{ + WorkspaceScope: "run-local", + RemoteAdapterKey: "rcon", + RemoteAdapterKind: "rcon", + TimeoutSeconds: 5, + SourceRCON: &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port}, + } + return assignment +} + +func writeSourceRCONConfig(t *testing.T, root string, assignment protocol.RunJobAssignment, bindAddress string, password string) { + writeSourceRCONConfigAt(t, root, assignment, assignment.ExecutionInput.SourceRCON.ConfigRef, bindAddress, password) +} + +func writeSourceRCONConfigAt(t *testing.T, root string, assignment protocol.RunJobAssignment, configRef string, bindAddress string, password string) { + t.Helper() + resolver := NewWorkspaceResolver(root) + scope, err := resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("create Source RCON scope: %v", err) + } + path, _, err := resolver.WritableTarget(scope, configRef) + if err != nil { + t.Fatalf("resolve Source RCON config: %v", err) + } + body := fmt.Sprintf("%s\n[rcon]\nbind_address=%s\nport=%d\npassword=%s\n", managedRCONConfigMarker, bindAddress, assignment.ExecutionInput.SourceRCON.Port, password) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write Source RCON config: %v", err) + } + markerPath, _, err := resolver.WritableTarget(scope, assignment.ExecutionInput.SourceRCON.DeploymentStateRef) + if err != nil { + t.Fatalf("resolve Source RCON deployment marker: %v", err) + } + markerBody, err := json.Marshal(managedDLLExtensionMarker{Version: ue4ssExtensionMarkerVersion, ReleaseVersion: "1.0.0", Checksum: "sha256:" + strings.Repeat("a", 64), SizeBytes: 1, ExtensionKey: assignment.ExecutionInput.SourceRCON.ExtensionKey, ModKey: assignment.ExecutionInput.SourceRCON.ModKey, ConfigRef: configRef, RCONPort: assignment.ExecutionInput.SourceRCON.Port}) + if err != nil { + t.Fatalf("marshal Source RCON deployment marker: %v", err) + } + if err := os.WriteFile(markerPath, markerBody, 0o600); err != nil { + t.Fatalf("write Source RCON deployment marker: %v", err) + } +} + +func serveSourceRCONSession(listener net.Listener, password string, respond func(net.Conn, sourceRCONPacket) error) <-chan error { + done := make(chan error, 1) + go func() { + connection, err := listener.Accept() + if err != nil { + done <- err + return + } + defer connection.Close() + context, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + auth, err := readSourceRCONPacket(context, connection) + if err != nil { + done <- err + return + } + if auth.id != sourceRCONAuthRequestID || auth.typeCode != sourceRCONAuthenticate || auth.body != password { + done <- fmt.Errorf("unexpected auth packet") + return + } + if err := writeSourceRCONPacket(context, connection, sourceRCONPacket{id: auth.id, typeCode: sourceRCONAuthResponse}); err != nil { + done <- err + return + } + command, err := readSourceRCONPacket(context, connection) + if err != nil { + done <- err + return + } + if command.id != sourceRCONCommandRequestID || command.typeCode != sourceRCONExecuteCommand { + done <- fmt.Errorf("unexpected command packet") + return + } + done <- respond(connection, command) + }() + return done +} + +func awaitSourceRCONServer(t *testing.T, done <-chan error) { + t.Helper() + select { + case err := <-done: + if err != nil { + t.Fatalf("Source RCON fixture: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Source RCON fixture did not finish") + } +} diff --git a/runtime/sqlite_schema_probe.go b/runtime/sqlite_schema_probe.go new file mode 100644 index 0000000..e855932 --- /dev/null +++ b/runtime/sqlite_schema_probe.go @@ -0,0 +1,312 @@ +package runtime + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "browser.local/run/protocol" + _ "modernc.org/sqlite" +) + +const sqliteSchemaProbeDriver = "sqlite" + +// SQLiteSchemaProbeExecutor performs only fixed SQLite introspection queries. +// The assignment carries no SQL and the only local target is a package-scoped, +// logical database key. +type SQLiteSchemaProbeExecutor struct{ resolver WorkspaceResolver } + +func NewSQLiteSchemaProbeExecutor(workspaceRoot string) *SQLiteSchemaProbeExecutor { + return &SQLiteSchemaProbeExecutor{resolver: NewWorkspaceResolver(workspaceRoot)} +} + +func (executor *SQLiteSchemaProbeExecutor) Execute(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return sqliteSchemaProbeFailure(assignment, "invalid_request", false) + } + request := *assignment.ExecutionInput.SQLiteSchemaProbe + probe := protocol.SQLiteSchemaProbeResult{RequestID: request.RequestID, JobID: assignment.JobID, Binding: request.Binding, Status: "failed", Limits: request.Limits} + scope, err := executor.resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + return sqliteSchemaProbeTerminalFailure(probe, "target_unavailable", false) + } + path, err := executor.resolver.ExistingTarget(scope, assignment.TargetKey) + if err != nil { + return sqliteSchemaProbeTerminalFailure(probe, "target_unavailable", false) + } + sourceFingerprint, err := fingerprintSQLiteSource(path) + if err != nil { + return sqliteSchemaProbeTerminalFailure(probe, "source_unavailable", true) + } + probe.SourceFingerprint = sourceFingerprint + + probeCtx, cancel := context.WithTimeout(ctx, time.Duration(request.Limits.TimeoutMS)*time.Millisecond) + defer cancel() + database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+path+"?mode=ro") + if err != nil { + return sqliteSchemaProbeTerminalFailure(probe, "sqlite_open_failed", true) + } + defer database.Close() + database.SetMaxOpenConns(1) + database.SetConnMaxLifetime(time.Minute) + if _, err := database.ExecContext(probeCtx, "PRAGMA query_only = ON"); err != nil { + return sqliteSchemaProbeTerminalFailure(probe, sqliteProbeErrorCode(probeCtx, err), true) + } + objects, err := inspectSQLiteSchema(probeCtx, database, request.Limits) + if err != nil { + return sqliteSchemaProbeTerminalFailure(probe, sqliteProbeErrorCode(probeCtx, err), true) + } + probe.Objects = objects + if sourceFingerprintAfter, err := fingerprintSQLiteSource(path); err != nil || sourceFingerprintAfter != probe.SourceFingerprint { + return sqliteSchemaProbeTerminalFailure(probe, "source_changed", true) + } + probe.SchemaFingerprint = digestValue(schemaFingerprintInput(objects)) + probe.ObservedAt = time.Now().UTC() + probe.Status = "succeeded" + if !finalizeSQLiteSchemaProbe(&probe) || sqliteSchemaProbeSize(probe) > request.Limits.MaxResultBytes { + return sqliteSchemaProbeTerminalFailure(probe, "result_limit_exceeded", false) + } + return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "SQLite schema probe completed"}, Message: "SQLite schema probe completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", Checksum: probe.ResultDigest, SizeBytes: int64(sqliteSchemaProbeSize(probe)), Summary: "bounded query-only SQLite schema metadata", SQLiteSchemaProbe: &probe}} +} + +func sqliteSchemaProbeFailure(assignment protocol.RunJobAssignment, code string, retryable bool) LifecycleExecutionResult { + probe := protocol.SQLiteSchemaProbeResult{JobID: assignment.JobID, Status: "failed", SafeError: protocol.SQLiteSchemaProbeSafeError{Code: code, Retryable: retryable}} + if assignment.ExecutionInput.SQLiteSchemaProbe != nil { + probe.RequestID, probe.Binding, probe.Limits = assignment.ExecutionInput.SQLiteSchemaProbe.RequestID, assignment.ExecutionInput.SQLiteSchemaProbe.Binding, assignment.ExecutionInput.SQLiteSchemaProbe.Limits + } + return sqliteSchemaProbeTerminalFailure(probe, code, retryable) +} + +func sqliteSchemaProbeTerminalFailure(probe protocol.SQLiteSchemaProbeResult, code string, retryable bool) LifecycleExecutionResult { + probe.Status, probe.ObservedAt, probe.SafeError = "failed", time.Now().UTC(), protocol.SQLiteSchemaProbeSafeError{Code: code, Retryable: retryable} + _ = finalizeSQLiteSchemaProbe(&probe) + return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "SQLite schema probe failed"}, Message: "SQLite schema probe failed", ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", Checksum: probe.ResultDigest, SizeBytes: int64(sqliteSchemaProbeSize(probe)), Summary: "bounded query-only SQLite schema probe failed", SQLiteSchemaProbe: &probe}} +} + +func inspectSQLiteSchema(ctx context.Context, database *sql.DB, limits protocol.SQLiteSchemaProbeLimits) ([]protocol.SQLiteSchemaProbeObject, error) { + rows, err := database.QueryContext(ctx, "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY type, name LIMIT ?", limits.MaxObjects) + if err != nil { + return nil, err + } + type sqliteObjectIdentity struct{ name, kind string } + identities := make([]sqliteObjectIdentity, 0, limits.MaxObjects) + for rows.Next() { + var name, kind string + if err := rows.Scan(&name, &kind); err != nil { + rows.Close() + return nil, err + } + identities = append(identities, sqliteObjectIdentity{name: name, kind: kind}) + } + if err := rows.Close(); err != nil { + return nil, err + } + objects := make([]protocol.SQLiteSchemaProbeObject, 0, len(identities)) + for _, identity := range identities { + object, err := inspectSQLiteObject(ctx, database, identity.name, identity.kind, limits, len(objects) < limits.MaxCardinalityReads) + if err != nil { + return nil, err + } + objects = append(objects, object) + } + return objects, nil +} + +func inspectSQLiteObject(ctx context.Context, database *sql.DB, name, kind string, limits protocol.SQLiteSchemaProbeLimits, includeCardinality bool) (protocol.SQLiteSchemaProbeObject, error) { + object := protocol.SQLiteSchemaProbeObject{ObjectHash: digestValue(kind + "\x00" + name), Kind: kind, NameFingerprint: digestValue(name)} + columns, err := database.QueryContext(ctx, "SELECT cid, name, type, [notnull], pk FROM pragma_table_info(?) ORDER BY cid LIMIT ?", name, limits.MaxColumnsPerObject) + if err != nil { + return object, err + } + for columns.Next() { + var ordinal, notNull, primaryKey int + var columnName, declaredType string + if err := columns.Scan(&ordinal, &columnName, &declaredType, ¬Null, &primaryKey); err != nil { + columns.Close() + return object, err + } + nullable := notNull == 0 + object.DeclaredColumns = append(object.DeclaredColumns, protocol.SQLiteSchemaProbeColumn{NameFingerprint: digestValue(columnName), DeclaredType: safeSQLiteDeclaredType(declaredType), Nullable: &nullable, PrimaryKey: primaryKey != 0, Ordinal: ordinal}) + } + if err := columns.Close(); err != nil { + return object, err + } + indexes, err := database.QueryContext(ctx, "SELECT name, [unique] FROM pragma_index_list(?) ORDER BY seq LIMIT ?", name, limits.MaxIndexesPerObject) + if err != nil { + return object, err + } + type sqliteIndexIdentity struct { + name string + unique bool + } + indexIdentities := make([]sqliteIndexIdentity, 0, limits.MaxIndexesPerObject) + for indexes.Next() { + var indexName string + var unique int + if err := indexes.Scan(&indexName, &unique); err != nil { + indexes.Close() + return object, err + } + indexIdentities = append(indexIdentities, sqliteIndexIdentity{name: indexName, unique: unique != 0}) + } + if err := indexes.Close(); err != nil { + return object, err + } + for _, identity := range indexIdentities { + item, err := inspectSQLiteIndex(ctx, database, identity.name, identity.unique, limits.MaxColumnsPerObject) + if err != nil { + return object, err + } + object.Indexes = append(object.Indexes, item) + } + foreignKeys, err := database.QueryContext(ctx, "SELECT [table], [from], [to] FROM pragma_foreign_key_list(?) ORDER BY id, seq LIMIT ?", name, limits.MaxForeignKeys) + if err != nil { + return object, err + } + for foreignKeys.Next() { + var destination, from, to string + if err := foreignKeys.Scan(&destination, &from, &to); err != nil { + foreignKeys.Close() + return object, err + } + object.ForeignKeys = append(object.ForeignKeys, protocol.SQLiteSchemaProbeForeignKey{FromColumnHash: digestValue(from), ToObjectHash: digestValue("table\x00" + destination), ToColumnHash: digestValue(to)}) + } + if err := foreignKeys.Close(); err != nil { + return object, err + } + if includeCardinality { + var count int64 + if err := database.QueryRowContext(ctx, "SELECT count(*) FROM "+quoteSQLiteIdentifier(name)).Scan(&count); err != nil { + return object, err + } + object.ApproximateRows = &count + } + if limits.MaxSampleRows > 0 { + rows, err := database.QueryContext(ctx, "SELECT * FROM "+quoteSQLiteIdentifier(name)+" LIMIT ?", limits.MaxSampleRows) + if err != nil { + return object, err + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + return object, err + } + for rows.Next() { + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + rows.Close() + return object, err + } + object.SampleFingerprints = append(object.SampleFingerprints, digestValue(canonicalSQLiteRow(columns, values))) + } + if err := rows.Close(); err != nil { + return object, err + } + } + return object, nil +} + +func inspectSQLiteIndex(ctx context.Context, database *sql.DB, name string, unique bool, maxColumns int) (protocol.SQLiteSchemaProbeIndex, error) { + index := protocol.SQLiteSchemaProbeIndex{NameFingerprint: digestValue(name), Unique: unique} + rows, err := database.QueryContext(ctx, "SELECT name FROM pragma_index_info(?) ORDER BY seqno LIMIT ?", name, maxColumns) + if err != nil { + return index, err + } + defer rows.Close() + for rows.Next() { + var column string + if err := rows.Scan(&column); err != nil { + return index, err + } + index.ColumnHashes = append(index.ColumnHashes, digestValue(column)) + } + return index, rows.Err() +} + +func quoteSQLiteIdentifier(value string) string { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` +} +func digestBytes(value []byte) string { + sum := sha256.Sum256(value) + return "sha256:" + hex.EncodeToString(sum[:]) +} +func digestValue(value string) string { return digestBytes([]byte(value)) } +func fingerprintSQLiteSource(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil +} +func schemaFingerprintInput(objects []protocol.SQLiteSchemaProbeObject) string { + body, _ := json.Marshal(objects) + return string(body) +} +func finalizeSQLiteSchemaProbe(probe *protocol.SQLiteSchemaProbeResult) bool { + probe.ResultDigest = "" + body, err := json.Marshal(probe) + if err != nil { + return false + } + probe.ResultDigest = digestBytes(body) + return true +} +func sqliteSchemaProbeSize(probe protocol.SQLiteSchemaProbeResult) int { + body, _ := json.Marshal(probe) + return len(body) +} +func canonicalSQLiteRow(columns []string, values []any) string { + body, _ := json.Marshal(struct { + Columns []string `json:"columns"` + Values []any `json:"values"` + }{columns, values}) + return string(body) +} + +func safeSQLiteDeclaredType(value string) string { + value = strings.ToUpper(strings.TrimSpace(value)) + if value == "" { + return "" + } + if len(value) > 80 { + return "OTHER" + } + for _, char := range value { + if (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || strings.ContainsRune("_(), ", char) { + continue + } + return "OTHER" + } + return value +} + +func sqliteProbeErrorCode(ctx context.Context, err error) string { + if errors.Is(ctx.Err(), context.Canceled) { + return "cancelled" + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "timeout" + } + lower := strings.ToLower(fmt.Sprint(err)) + if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") { + return "database_busy" + } + return "sqlite_read_failed" +} diff --git a/runtime/sqlite_schema_probe_test.go b/runtime/sqlite_schema_probe_test.go new file mode 100644 index 0000000..1175216 --- /dev/null +++ b/runtime/sqlite_schema_probe_test.go @@ -0,0 +1,115 @@ +package runtime + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "browser.local/run/protocol" +) + +func TestSQLiteSchemaProbeExecutesOnlyAgainstScopedLogicalTarget(t *testing.T) { + root := t.TempDir() + assignment := sqliteSchemaProbeAssignment() + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("create workspace scope: %v", err) + } + databasePath := filepath.Join(scope, "databases", "current.db") + createSQLiteProbeFixture(t, databasePath) + database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+databasePath+"?mode=ro") + if err != nil { + t.Fatal(err) + } + if _, err := inspectSQLiteSchema(context.Background(), database, assignment.ExecutionInput.SQLiteSchemaProbe.Limits); err != nil { + t.Fatalf("inspect fixture directly: %v", err) + } + database.Close() + + result := NewSQLiteSchemaProbeExecutor(root).Execute(context.Background(), assignment) + if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil { + t.Fatalf("expected successful probe envelope, got %+v", result) + } + probe := result.ExecutionResult.SQLiteSchemaProbe + if probe.JobID != assignment.JobID || probe.Binding != assignment.ExecutionInput.SQLiteSchemaProbe.Binding || probe.Status != "succeeded" || probe.ObservedAt.IsZero() || probe.ResultDigest == "" || probe.SourceFingerprint == "" || probe.SchemaFingerprint == "" { + t.Fatalf("unexpected probe identity envelope: %+v", probe) + } + if len(probe.Objects) != 2 || probe.Objects[0].ApproximateRows == nil || len(probe.Objects[1].SampleFingerprints) == 0 { + t.Fatalf("expected bounded table evidence, got %+v", probe.Objects) + } + serialized := mustJSON(t, probe) + for _, value := range []string{"current.db", "members", "alpha", root, "SELECT", "sqlite:"} { + if strings.Contains(serialized, value) { + t.Fatalf("probe leaked protected source material %q: %s", value, serialized) + } + } + if result.ExecutionResult.Content != "" { + t.Fatalf("probe must not return content: %+v", result.ExecutionResult) + } +} + +func TestSQLiteSchemaProbeRejectsUnscopedOrUnsafeRequests(t *testing.T) { + assignment := sqliteSchemaProbeAssignment() + assignment.TargetKey = "/tmp/current.db" + result := NewSQLiteSchemaProbeExecutor(t.TempDir()).Execute(context.Background(), assignment) + if result.ErrorCode != "invalid_request" || result.ExecutionResult.SQLiteSchemaProbe == nil { + t.Fatalf("expected safe invalid probe failure, got %+v", result) + } + assignment = sqliteSchemaProbeAssignment() + assignment.ExecutionInput.SQLiteSchemaProbe.Binding.DatabaseIdentity = "C:/host/path" + result = NewSQLiteSchemaProbeExecutor(t.TempDir()).Execute(context.Background(), assignment) + if result.ErrorCode != "invalid_request" || strings.Contains(result.Message, "C:/") { + t.Fatalf("expected redacted binding rejection, got %+v", result) + } +} + +func TestSQLiteSchemaProbeEnforcesResultLimit(t *testing.T) { + root := t.TempDir() + assignment := sqliteSchemaProbeAssignment() + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatal(err) + } + createSQLiteProbeFixture(t, filepath.Join(scope, "databases", "current.db")) + assignment.ExecutionInput.SQLiteSchemaProbe.Limits.MaxResultBytes = 1 + result := NewSQLiteSchemaProbeExecutor(root).Execute(context.Background(), assignment) + if result.ErrorCode != "result_limit_exceeded" || result.State != lifecycleResultStateFailed { + t.Fatalf("expected result bound failure, got %+v", result) + } +} + +func sqliteSchemaProbeAssignment() protocol.RunJobAssignment { + assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteProbe) + assignment.TargetKey, assignment.MaxAttempts, assignment.FencingToken = "databases/current.db", 1, 7 + assignment.ExecutionInput.WorkspaceScope = "profile-default" + assignment.ExecutionInput.SQLiteSchemaProbe = &protocol.SQLiteSchemaProbeRequest{RequestID: "probe-1", Binding: protocol.SQLiteSchemaProbeBinding{ServerInstanceID: assignment.ServerInstanceID, RunBindingID: "binding-1", RunEndpointID: assignment.RunEndpointID, PluginID: "game.example", PluginVersion: "1.0.0", AdapterVersion: "adapter-1", GameVersion: "1.0", DatabaseIdentity: "database-1"}, Limits: protocol.SQLiteSchemaProbeLimits{MaxObjects: 8, MaxColumnsPerObject: 8, MaxIndexesPerObject: 8, MaxForeignKeys: 8, MaxCardinalityReads: 8, MaxSampleRows: 2, TimeoutMS: 1000, MaxResultBytes: 128 * 1024}} + return assignment +} + +func createSQLiteProbeFixture(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + database, err := sql.Open(sqliteSchemaProbeDriver, path) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := database.Exec("CREATE TABLE members (id INTEGER PRIMARY KEY, name TEXT NOT NULL); CREATE TABLE groups (id INTEGER PRIMARY KEY, member_id INTEGER REFERENCES members(id)); CREATE UNIQUE INDEX members_name ON members(name); INSERT INTO members(name) VALUES ('alpha'), ('beta');"); err != nil { + t.Fatal(err) + } +} + +func mustJSON(t *testing.T, value any) string { + t.Helper() + body, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(body) +} diff --git a/runtime/ue4ss_dll_extension.go b/runtime/ue4ss_dll_extension.go new file mode 100644 index 0000000..a82d704 --- /dev/null +++ b/runtime/ue4ss_dll_extension.go @@ -0,0 +1,556 @@ +package runtime + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + pathpkg "path" + "path/filepath" + "strconv" + "strings" + "time" + + "browser.local/run/protocol" +) + +const ( + ue4ssExtensionStateRoot = "runtime/ue4ss-dll" + ue4ssExtensionMarkerVersion = 2 + maxUE4SSMetadataBytes int64 = 16 * 1024 + maxUE4SSDLLBytes int64 = 128 * 1024 * 1024 + maxSCUMExecutableBytes int64 = 2 * 1024 * 1024 * 1024 + managedRCONConfigMarker = "; managed by Run UE4SS DLL extension" +) + +type dllExtensionError struct { + code string + message string +} + +func (err dllExtensionError) Error() string { return err.message } + +type managedDLLExtensionMarker struct { + Version int `json:"version"` + ReleaseVersion string `json:"releaseVersion"` + Checksum string `json:"checksum"` + SizeBytes int64 `json:"sizeBytes"` + ExtensionKey string `json:"extensionKey"` + ModKey string `json:"modKey"` + ConfigRef string `json:"configRef"` + RCONPort int `json:"rconPort"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (executor LifecycleExecutor) synchronizeUE4SSDLLExtensions(ctx context.Context, assignment protocol.RunJobAssignment, template LifecycleActionTemplate, scope string) error { + if executor.runtimeTargetOS != "windows" || executor.runtimeTargetArch != "amd64" { + return dllExtensionError{code: "unsupported_extension_platform", message: "UE4SS DLL extensions require Windows amd64"} + } + if err := ctx.Err(); err != nil { + return err + } + executableKey := template.TargetExecutableKey + // Older generated plugin packages do not have targetExecutableKey yet, + // but SCUM's existing start action already carries the same logical path + // in SERVER_EXECUTABLE_REF. Keep those packages forward-compatible. + if executableKey == "" && template.Environment != nil { + executableKey = template.Environment["SERVER_EXECUTABLE_REF"] + } + if executableKey == "" && template.Env != nil { + executableKey = template.Env["SERVER_EXECUTABLE_REF"] + } + if executableKey == "" && strings.HasSuffix(strings.ToLower(template.ExecutableKey), ".exe") { + executableKey = template.ExecutableKey + } + if executableKey == "" || !strings.HasSuffix(strings.ToLower(executableKey), ".exe") { + return dllExtensionError{code: "extension_scum_executable_invalid", message: "UE4SS DLL extensions require a declared SCUM executable"} + } + + resolver := NewWorkspaceResolver(executor.workspaceRoot) + targetResolver, targetScope := resolver, scope + if deployment := assignment.ExecutionInput.Deployment; deployment != nil && deployment.ServerRoot != "" { + root := filepath.Clean(deployment.ServerRoot) + if root == "." || !filepath.IsAbs(root) { + return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared executable root is unsafe"} + } + targetResolver = NewWorkspaceResolver(filepath.Dir(root)) + targetScope = root + } + executable, err := declaredLifecycleExecutable(targetResolver, targetScope, executableKey) + if err != nil { + return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable is unavailable"} + } + executableChecksum, _, err := checksumRegularFile(executable, maxSCUMExecutableBytes) + if err != nil { + return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable cannot be verified"} + } + for _, plan := range assignment.ExecutionInput.DLLExtensions { + if !strings.EqualFold(executableChecksum, plan.SCUMExecutableChecksum) { + return dllExtensionError{code: "extension_scum_checksum_mismatch", message: "declared SCUM executable does not match the extension release"} + } + } + + gameRootKey, err := gameRootKeyForExecutable(template.ExecutableKey) + if err != nil { + return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable location is unsafe"} + } + if err := verifyUE4SSBootstrap(targetResolver, targetScope, gameRootKey); err != nil { + return err + } + for _, plan := range assignment.ExecutionInput.DLLExtensions { + if err := executor.synchronizeUE4SSDLLExtension(ctx, targetResolver, targetScope, gameRootKey, plan); err != nil { + return err + } + } + return nil +} + +func declaredLifecycleExecutable(resolver WorkspaceResolver, scope string, executableKey string) (string, error) { + if !protocol.ValidLogicalFileKey(filepath.ToSlash(executableKey)) { + return "", fmt.Errorf("declared executable key is unsafe") + } + return resolver.ExistingTarget(scope, executableKey) +} + +func (executor LifecycleExecutor) synchronizeUE4SSDLLExtension(ctx context.Context, resolver WorkspaceResolver, scope string, gameRootKey string, plan protocol.RuntimeDLLExtensionPlan) error { + activeKey := gameRelativeKey(gameRootKey, plan.DLLRef) + configRef := managedRCONConfigRef(gameRootKey, plan.ModKey) + activePath, _, err := resolver.WritableTarget(scope, activeKey) + if err != nil { + return dllExtensionError{code: "dll_extension_workspace_failed", message: "extension deployment workspace is unavailable"} + } + markerPath, stagePath, previousPath, err := extensionStatePaths(resolver, scope, plan) + if err != nil { + return dllExtensionError{code: "dll_extension_workspace_failed", message: "extension state workspace is unavailable"} + } + marker, markerFound, err := loadManagedDLLExtensionMarker(markerPath) + if err != nil { + return dllExtensionError{code: "dll_extension_state_failed", message: "extension release state cannot be read"} + } + unchanged := markerFound && markerMatchesDeployment(marker, plan, configRef) && managedDLLMatchesPlan(activePath, plan) + if !unchanged { + _ = os.Remove(stagePath) + defer os.Remove(stagePath) + downloadedSize, downloadedChecksum, downloadErr := executor.dependencyDownloader.Download(ctx, plan.ReleaseURL, stagePath, plan.SizeBytes) + if downloadErr != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return dllExtensionError{code: "dll_extension_download_failed", message: "declared DLL download failed"} + } + verifiedChecksum, verifiedSize, verifyErr := checksumRegularFile(stagePath, plan.SizeBytes) + if verifyErr != nil || downloadedSize != plan.SizeBytes || verifiedSize != plan.SizeBytes || !strings.EqualFold(downloadedChecksum, plan.Checksum) || !strings.EqualFold(verifiedChecksum, plan.Checksum) { + return dllExtensionError{code: "dll_extension_verify_failed", message: "declared DLL did not match its fixed release checksum"} + } + } + if err := executor.ensureLoopbackRCONConfig(resolver, scope, gameRootKey, plan); err != nil { + return err + } + if err := executor.ensureUE4SSModsIndex(resolver, scope, gameRootKey, plan.ModKey); err != nil { + return err + } + if unchanged { + return nil + } + if err := executor.activateManagedDLLExtension(activePath, stagePath, previousPath, markerPath, plan, configRef); err != nil { + return err + } + return nil +} + +func verifyUE4SSBootstrap(resolver WorkspaceResolver, scope string, gameRootKey string) error { + for _, filename := range []string{"dwmapi.dll", "UE4SS.dll"} { + if _, err := resolver.ExistingTarget(scope, gameRelativeKey(gameRootKey, filename)); err != nil { + return dllExtensionError{code: "ue4ss_bootstrap_missing", message: "required UE4SS bootstrap files are not installed"} + } + } + if err := existingRuntimeDirectory(scope, gameRelativeKey(gameRootKey, "ue4ss")); err != nil { + return dllExtensionError{code: "ue4ss_bootstrap_missing", message: "required UE4SS bootstrap files are not installed"} + } + return nil +} + +func gameRootKeyForExecutable(executableKey string) (string, error) { + normalized := filepath.ToSlash(executableKey) + if !protocol.ValidLogicalFileKey(normalized) || strings.HasPrefix(normalized, "/") || strings.Contains(normalized, `\`) { + return "", fmt.Errorf("executable key is unsafe") + } + parent := pathpkg.Dir(normalized) + if parent == "." { + return "", nil + } + return parent, nil +} + +func gameRelativeKey(gameRootKey string, relativeKey string) string { + if gameRootKey == "" { + return relativeKey + } + return gameRootKey + "/" + relativeKey +} + +func extensionStatePaths(resolver WorkspaceResolver, scope string, plan protocol.RuntimeDLLExtensionPlan) (string, string, string, error) { + baseKey := ue4ssExtensionStateRoot + "/" + plan.TargetKey + markerPath, _, err := resolver.WritableTarget(scope, baseKey+"/release.json") + if err != nil { + return "", "", "", err + } + stagePath, _, err := resolver.WritableTarget(scope, baseKey+"/download.staged") + if err != nil { + return "", "", "", err + } + previousPath, _, err := resolver.WritableTarget(scope, baseKey+"/previous.dll") + if err != nil { + return "", "", "", err + } + return markerPath, stagePath, previousPath, nil +} + +func loadManagedDLLExtensionMarker(path string) (managedDLLExtensionMarker, bool, error) { + body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes) + if err != nil || !found { + return managedDLLExtensionMarker{}, found, err + } + var marker managedDLLExtensionMarker + if err := json.Unmarshal(body, &marker); err != nil { + return managedDLLExtensionMarker{}, false, nil + } + if marker.Version != ue4ssExtensionMarkerVersion || !protocol.ValidLogicalFileKey(marker.ExtensionKey) || !protocol.ValidLogicalFileKey(marker.ModKey) || !managedRCONConfigRefForMod(marker.ConfigRef, marker.ModKey) || !protocolValidSHA256(marker.Checksum) || marker.SizeBytes < 1 || marker.ReleaseVersion == "" || marker.RCONPort < 1024 || marker.RCONPort > 65535 { + return managedDLLExtensionMarker{}, false, nil + } + return marker, true, nil +} + +func markerMatchesPlan(marker managedDLLExtensionMarker, plan protocol.RuntimeDLLExtensionPlan) bool { + return marker.Version == ue4ssExtensionMarkerVersion && marker.ReleaseVersion == plan.Version && strings.EqualFold(marker.Checksum, plan.Checksum) && marker.SizeBytes == plan.SizeBytes && marker.ExtensionKey == plan.Key && marker.ModKey == plan.ModKey && marker.RCONPort == plan.RCONPort +} + +func markerMatchesDeployment(marker managedDLLExtensionMarker, plan protocol.RuntimeDLLExtensionPlan, configRef string) bool { + return markerMatchesPlan(marker, plan) && marker.ConfigRef == configRef +} + +func managedDLLMatchesPlan(path string, plan protocol.RuntimeDLLExtensionPlan) bool { + checksum, size, err := checksumRegularFile(path, plan.SizeBytes) + return err == nil && size == plan.SizeBytes && strings.EqualFold(checksum, plan.Checksum) +} + +func (executor LifecycleExecutor) activateManagedDLLExtension(activePath string, stagePath string, previousPath string, markerPath string, plan protocol.RuntimeDLLExtensionPlan, configRef string) error { + if _, _, err := checksumRegularFile(stagePath, plan.SizeBytes); err != nil { + return dllExtensionError{code: "dll_extension_verify_failed", message: "staged DLL cannot be verified"} + } + previousMarker, previousMarkerFound, markerErr := readBoundedRegularFile(markerPath, maxUE4SSMetadataBytes) + if markerErr != nil { + return dllExtensionError{code: "dll_extension_state_failed", message: "extension release state cannot be read"} + } + activeExists := false + if _, _, err := checksumRegularFile(activePath, maxUE4SSDLLBytes); err == nil { + activeExists = true + if err := copyRegularFileAtomic(activePath, previousPath, maxUE4SSDLLBytes, 0o600); err != nil { + return dllExtensionError{code: "dll_extension_activation_failed", message: "previous DLL could not be retained"} + } + } else if !errors.Is(err, os.ErrNotExist) { + return dllExtensionError{code: "dll_extension_activation_failed", message: "current DLL cannot be safely replaced"} + } + if err := os.Rename(stagePath, activePath); err != nil { + return dllExtensionError{code: "dll_extension_activation_failed", message: "verified DLL could not be activated"} + } + if err := os.Chmod(activePath, 0o600); err != nil { + rollbackManagedDLLExtension(activePath, previousPath, activeExists) + return dllExtensionError{code: "dll_extension_activation_failed", message: "activated DLL permissions could not be secured"} + } + marker := managedDLLExtensionMarker{Version: ue4ssExtensionMarkerVersion, ReleaseVersion: plan.Version, Checksum: strings.ToLower(plan.Checksum), SizeBytes: plan.SizeBytes, ExtensionKey: plan.Key, ModKey: plan.ModKey, ConfigRef: configRef, RCONPort: plan.RCONPort, UpdatedAt: time.Now().UTC()} + body, err := json.Marshal(marker) + if err != nil || executor.writeRuntimeFile(markerPath, body, 0o600) != nil { + rollbackManagedDLLExtension(activePath, previousPath, activeExists) + if previousMarkerFound { + _ = executor.writeRuntimeFile(markerPath, previousMarker, 0o600) + } else { + _ = os.Remove(markerPath) + } + return dllExtensionError{code: "dll_extension_activation_failed", message: "extension release state could not be activated"} + } + return nil +} + +func rollbackManagedDLLExtension(activePath string, previousPath string, activeExists bool) { + if activeExists { + _ = copyRegularFileAtomic(previousPath, activePath, maxUE4SSDLLBytes, 0o600) + return + } + _ = os.Remove(activePath) +} + +func (executor LifecycleExecutor) ensureLoopbackRCONConfig(resolver WorkspaceResolver, scope string, gameRootKey string, plan protocol.RuntimeDLLExtensionPlan) error { + configKey := managedRCONConfigRef(gameRootKey, plan.ModKey) + configPath, _, err := resolver.WritableTarget(scope, configKey) + if err != nil { + return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be prepared"} + } + if managedLoopbackRCONConfigMatches(configPath, plan.RCONPort) { + return nil + } + password, err := randomRCONPassword() + if err != nil { + return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be secured"} + } + body := fmt.Sprintf("%s\n[rcon]\nbind_address=127.0.0.1\nport=%d\npassword=%s\n", managedRCONConfigMarker, plan.RCONPort, password) + if err := executor.writeRuntimeFile(configPath, []byte(body), 0o600); err != nil { + return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be written"} + } + return nil +} + +func managedRCONConfigRef(gameRootKey string, modKey string) string { + return gameRelativeKey(gameRootKey, "ue4ss/Mods/"+modKey+"/config.ini") +} + +func managedRCONConfigRefForMod(configRef string, modKey string) bool { + baseRef := "ue4ss/Mods/" + modKey + "/config.ini" + return protocol.ValidLogicalFileKey(configRef) && (configRef == baseRef || strings.HasSuffix(configRef, "/"+baseRef)) +} + +func managedLoopbackRCONConfigMatches(path string, port int) bool { + body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes) + if err != nil || !found { + return false + } + content := strings.ReplaceAll(string(body), "\r\n", "\n") + if !strings.Contains(content, managedRCONConfigMarker) { + return false + } + values := map[string]string{} + inRCON := false + for _, rawLine := range strings.Split(content, "\n") { + line := strings.TrimSpace(rawLine) + if line == "[rcon]" { + inRCON = true + continue + } + if strings.HasPrefix(line, "[") { + inRCON = false + continue + } + if !inRCON || line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") { + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + if key != "bind_address" && key != "port" && key != "password" { + continue + } + if _, duplicate := values[key]; duplicate { + return false + } + values[key] = strings.TrimSpace(value) + } + configuredPort, err := strconv.Atoi(values["port"]) + if err != nil || values["bind_address"] != "127.0.0.1" || configuredPort != port || len(values["password"]) != 64 { + return false + } + _, err = hex.DecodeString(values["password"]) + return err == nil +} + +func randomRCONPassword() (string, error) { + bytes := make([]byte, 32) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes), nil +} + +func (executor LifecycleExecutor) ensureUE4SSModsIndex(resolver WorkspaceResolver, scope string, gameRootKey string, modKey string) error { + modsKey := gameRelativeKey(gameRootKey, "ue4ss/Mods/mods.txt") + modsPath, _, err := resolver.WritableTarget(scope, modsKey) + if err != nil { + return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be prepared"} + } + body, found, err := readBoundedRegularFile(modsPath, maxUE4SSMetadataBytes) + if err != nil { + return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be read"} + } + content := "" + if found { + content = strings.ReplaceAll(string(body), "\r\n", "\n") + } + lines := strings.Split(content, "\n") + if content == "" { + lines = nil + } + updated := make([]string, 0, len(lines)+1) + declared := false + for _, line := range lines { + if modsIndexLineKey(line) == modKey { + if !declared { + updated = append(updated, modKey+" : 1") + declared = true + } + continue + } + updated = append(updated, line) + } + if !declared { + updated = append(updated, modKey+" : 1") + } + next := strings.Join(updated, "\n") + if !strings.HasSuffix(next, "\n") { + next += "\n" + } + if content == next { + return nil + } + if err := executor.writeRuntimeFile(modsPath, []byte(next), 0o600); err != nil { + return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be updated"} + } + return nil +} + +func modsIndexLineKey(line string) string { + withoutComment := strings.SplitN(line, "#", 2)[0] + parts := strings.SplitN(strings.TrimSpace(withoutComment), ":", 2) + if len(parts) != 2 { + return "" + } + return strings.TrimSpace(parts[0]) +} + +func existingRuntimeDirectory(scope string, key string) error { + if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) { + return fmt.Errorf("directory key is unsafe") + } + cleanScope, err := filepath.Abs(scope) + if err != nil { + return err + } + current := cleanScope + for _, part := range strings.Split(filepath.ToSlash(key), "/") { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("required directory is unavailable") + } + } + return nil +} + +func checksumRegularFile(path string, maxBytes int64) (string, int64, error) { + info, err := os.Lstat(path) + if err != nil { + return "", 0, err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return "", 0, fmt.Errorf("file is not regular") + } + file, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer file.Close() + hash := sha256.New() + size, err := io.Copy(hash, io.LimitReader(file, maxBytes+1)) + if err != nil { + return "", 0, err + } + if size > maxBytes { + return "", size, fmt.Errorf("file exceeds maximum size") + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil +} + +func copyRegularFileAtomic(source string, destination string, maxBytes int64, mode os.FileMode) error { + checksum, size, err := checksumRegularFile(source, maxBytes) + if err != nil || checksum == "" || size < 1 { + if err != nil { + return err + } + return fmt.Errorf("source file is empty") + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + temporary := destination + ".copying" + output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + remove := true + defer func() { + _ = output.Close() + if remove { + _ = os.Remove(temporary) + } + }() + written, err := io.Copy(output, io.LimitReader(input, maxBytes+1)) + if err != nil || written != size || written > maxBytes { + if err != nil { + return err + } + return fmt.Errorf("source file changed during copy") + } + if err := output.Sync(); err != nil { + return err + } + if err := output.Close(); err != nil { + return err + } + if err := os.Rename(temporary, destination); err != nil { + return err + } + remove = false + return os.Chmod(destination, mode) +} + +func readBoundedRegularFile(path string, maxBytes int64) ([]byte, bool, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Size() > maxBytes { + return nil, false, fmt.Errorf("file is not a bounded regular file") + } + file, err := os.Open(path) + if err != nil { + return nil, false, err + } + defer file.Close() + body, err := io.ReadAll(io.LimitReader(file, maxBytes+1)) + if err != nil || int64(len(body)) > maxBytes { + if err != nil { + return nil, false, err + } + return nil, false, fmt.Errorf("file exceeds maximum size") + } + return body, true, nil +} + +func protocolValidSHA256(value string) bool { + if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func dllExtensionLifecycleFailure(err error) LifecycleExecutionResult { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "DLL extension synchronization cancelled"}, Message: "DLL extension synchronization cancelled", ErrorCode: "dll_extension_cancelled"} + } + var extensionErr dllExtensionError + if errors.As(err, &extensionErr) { + return lifecycleFailure(extensionErr.code, extensionErr.message) + } + return lifecycleFailure("dll_extension_sync_failed", "DLL extension synchronization failed") +} diff --git a/runtime/ue4ss_dll_extension_test.go b/runtime/ue4ss_dll_extension_test.go new file mode 100644 index 0000000..fc10c75 --- /dev/null +++ b/runtime/ue4ss_dll_extension_test.go @@ -0,0 +1,448 @@ +package runtime + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "browser.local/run/protocol" +) + +type ue4ssTestDownloader struct { + mu sync.Mutex + payload []byte + calls int + lastURL string + started chan struct{} + unblock <-chan struct{} + startOnce sync.Once +} + +func (downloader *ue4ssTestDownloader) Download(ctx context.Context, sourceURL string, destination string, _ int64) (int64, string, error) { + downloader.mu.Lock() + downloader.calls++ + downloader.lastURL = sourceURL + payload := append([]byte(nil), downloader.payload...) + started := downloader.started + unblock := downloader.unblock + downloader.mu.Unlock() + if started != nil { + downloader.startOnce.Do(func() { close(started) }) + } + if unblock != nil { + select { + case <-unblock: + case <-ctx.Done(): + return 0, "", ctx.Err() + } + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return 0, "", err + } + if err := os.WriteFile(destination, payload, 0o600); err != nil { + return 0, "", err + } + return int64(len(payload)), bytesChecksum(payload), nil +} + +func (downloader *ue4ssTestDownloader) SetPayload(payload []byte) { + downloader.mu.Lock() + defer downloader.mu.Unlock() + downloader.payload = append([]byte(nil), payload...) +} + +func (downloader *ue4ssTestDownloader) Count() int { + downloader.mu.Lock() + defer downloader.mu.Unlock() + return downloader.calls +} + +func (downloader *ue4ssTestDownloader) LastURL() string { + downloader.mu.Lock() + defer downloader.mu.Unlock() + return downloader.lastURL +} + +type ue4ssManagedProcessSupervisor struct { + mu sync.Mutex + starts int + command ProcessCommand +} + +func (supervisor *ue4ssManagedProcessSupervisor) Start(_ context.Context, command ProcessCommand, identity ProcessIdentity, output ManagedProcessOutput) (ProcessIdentity, error) { + supervisor.mu.Lock() + supervisor.starts++ + supervisor.command = ProcessCommand{WorkDir: command.WorkDir, Args: append([]string(nil), command.Args...), Env: command.Env, Timeout: command.Timeout} + supervisor.mu.Unlock() + if output.Stdout != nil { + _ = output.Stdout(identity, ManagedProcessLine{Text: "ue4ss managed process started", EndOffset: int64(len("ue4ss managed process started"))}) + } + identity.State = "running" + return identity, nil +} + +func (supervisor *ue4ssManagedProcessSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) { + identity.State = "stopped" + return identity, nil +} + +func (supervisor *ue4ssManagedProcessSupervisor) Status(identity ProcessIdentity) ProcessIdentity { + if identity.State == "" { + identity.State = "stopped" + } + return identity +} + +func (supervisor *ue4ssManagedProcessSupervisor) ResumeOutput(ManagedProcessOutput) {} + +func (supervisor *ue4ssManagedProcessSupervisor) Starts() int { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + return supervisor.starts +} + +func (supervisor *ue4ssManagedProcessSupervisor) Command() ProcessCommand { + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + return ProcessCommand{WorkDir: supervisor.command.WorkDir, Args: append([]string(nil), supervisor.command.Args...), Env: supervisor.command.Env, Timeout: supervisor.command.Timeout} +} + +type ue4ssExtensionFixture struct { + assignment protocol.RunJobAssignment + plan protocol.RuntimeDLLExtensionPlan + activePath string + markerPath string + previous string + stagePath string + configPath string + modsPath string +} + +func TestUE4SSDLLExtensionSynchronizesNoOpsUpdatesAndKeepsPriorRelease(t *testing.T) { + root := t.TempDir() + fixture := newUE4SSExtensionFixture(t, root, true) + payloadV1 := []byte("scum-simple-rcon DLL release one") + downloader := &ue4ssTestDownloader{payload: payloadV1} + supervisor := &ue4ssManagedProcessSupervisor{} + executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64") + + first := executor.Execute(fixture.assignment) + if first.State != lifecycleResultStateSucceeded { + t.Fatalf("expected first DLL sync and start to succeed, got %+v", first) + } + if downloader.Count() != 1 || downloader.LastURL() != fixture.plan.ReleaseURL { + t.Fatalf("expected one frozen release download, count=%d url=%q", downloader.Count(), downloader.LastURL()) + } + assertUE4SSFileEquals(t, fixture.activePath, payloadV1) + marker, found, err := loadManagedDLLExtensionMarker(fixture.markerPath) + if err != nil || !found || !markerMatchesPlan(marker, fixture.plan) || marker.ConfigRef != "bin/ue4ss/Mods/scum_simple_rcon/config.ini" { + t.Fatalf("expected managed release marker for first DLL, marker=%+v found=%v err=%v", marker, found, err) + } + config := readUE4SSFile(t, fixture.configPath) + if !managedLoopbackRCONConfigMatches(fixture.configPath, fixture.plan.RCONPort) { + t.Fatal("expected protected loopback RCON configuration") + } + password := rconPasswordFromConfig(string(config)) + if len(password) != 64 { + t.Fatal("expected a 32-byte generated RCON password") + } + if _, err := hex.DecodeString(password); err != nil { + t.Fatalf("expected hexadecimal generated RCON password: %v", err) + } + if info, err := os.Stat(fixture.configPath); err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("expected protected RCON configuration permissions, info=%v err=%v", info, err) + } + mods := string(readUE4SSFile(t, fixture.modsPath)) + if strings.Count(mods, "scum_simple_rcon : 1\n") != 1 || strings.Contains(mods, "scum_simple_rcon : 0") { + t.Fatalf("expected exactly one enabled managed mod entry, mods=%q", mods) + } + command := supervisor.Command() + if len(command.Args) != 1 || filepath.Base(command.Args[0]) != "SCUMServer.exe" || strings.HasSuffix(strings.ToLower(command.Args[0]), ".dll") { + t.Fatalf("expected normal SCUM executable start without a DLL loader, command=%+v", command) + } + + second := executor.Execute(fixture.assignment) + if second.State != lifecycleResultStateSucceeded || downloader.Count() != 1 { + t.Fatalf("expected unchanged release to skip download, result=%+v downloads=%d", second, downloader.Count()) + } + if nextConfig := readUE4SSFile(t, fixture.configPath); !bytes.Equal(config, nextConfig) { + t.Fatal("expected unchanged release to retain its protected RCON configuration") + } + + payloadV2 := []byte("scum-simple-rcon DLL release two") + planV2 := fixture.plan + planV2.Version = "1.1.0" + planV2.Checksum = bytesChecksum(payloadV2) + planV2.SizeBytes = int64(len(payloadV2)) + downloader.SetPayload(payloadV2) + updatedAssignment := fixture.assignment + updatedAssignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{planV2} + updated := executor.Execute(updatedAssignment) + if updated.State != lifecycleResultStateSucceeded || downloader.Count() != 2 { + t.Fatalf("expected changed release to update before start, result=%+v downloads=%d", updated, downloader.Count()) + } + assertUE4SSFileEquals(t, fixture.activePath, payloadV2) + assertUE4SSFileEquals(t, fixture.previous, payloadV1) + marker, found, err = loadManagedDLLExtensionMarker(fixture.markerPath) + if err != nil || !found || !markerMatchesPlan(marker, planV2) { + t.Fatalf("expected managed release marker for updated DLL, marker=%+v found=%v err=%v", marker, found, err) + } + if mods := string(readUE4SSFile(t, fixture.modsPath)); strings.Count(mods, "scum_simple_rcon : 1\n") != 1 { + t.Fatalf("expected deterministic mod index after update, mods=%q", mods) + } + + badPlan := planV2 + badPlan.Version = "1.2.0" + badPlan.Checksum = bytesChecksum([]byte("expected-but-not-delivered")) + badPlan.SizeBytes = int64(len([]byte("expected-but-not-delivered"))) + downloader.SetPayload([]byte("tampered release payload")) + badAssignment := fixture.assignment + badAssignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{badPlan} + failed := executor.Execute(badAssignment) + if failed.State != lifecycleResultStateFailed || failed.ErrorCode != "dll_extension_verify_failed" { + t.Fatalf("expected checksum mismatch to fail before process start, got %+v", failed) + } + if supervisor.Starts() != 3 { + t.Fatalf("expected failed update not to start SCUM, starts=%d", supervisor.Starts()) + } + assertUE4SSFileEquals(t, fixture.activePath, payloadV2) + assertUE4SSFileEquals(t, fixture.previous, payloadV1) + marker, found, err = loadManagedDLLExtensionMarker(fixture.markerPath) + if err != nil || !found || !markerMatchesPlan(marker, planV2) { + t.Fatalf("expected failed update to retain prior managed release, marker=%+v found=%v err=%v", marker, found, err) + } + if _, err := os.Stat(fixture.stagePath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected failed staged DLL to be removed, err=%v", err) + } +} + +func TestUE4SSDLLExtensionRollsBackWhenReleaseMarkerCannotActivate(t *testing.T) { + root := t.TempDir() + fixture := newUE4SSExtensionFixture(t, root, true) + payloadV1 := []byte("scum-simple-rcon DLL release one") + downloader := &ue4ssTestDownloader{payload: payloadV1} + supervisor := &ue4ssManagedProcessSupervisor{} + executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64") + if result := executor.Execute(fixture.assignment); result.State != lifecycleResultStateSucceeded { + t.Fatalf("install initial release: %+v", result) + } + + payloadV2 := []byte("scum-simple-rcon DLL release two") + planV2 := fixture.plan + planV2.Version = "1.1.0" + planV2.Checksum = bytesChecksum(payloadV2) + planV2.SizeBytes = int64(len(payloadV2)) + downloader.SetPayload(payloadV2) + failMarkerWrite := true + executor.runtimeFileWriter = func(path string, body []byte, mode os.FileMode) error { + if path == fixture.markerPath && failMarkerWrite { + failMarkerWrite = false + return errors.New("injected marker write failure") + } + return writeRuntimeAtomicFile(path, body, mode) + } + assignment := fixture.assignment + assignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{planV2} + result := executor.Execute(assignment) + if result.State != lifecycleResultStateFailed || result.ErrorCode != "dll_extension_activation_failed" { + t.Fatalf("expected failed marker activation to fail closed, got %+v", result) + } + if supervisor.Starts() != 1 { + t.Fatalf("expected marker activation failure not to start SCUM, starts=%d", supervisor.Starts()) + } + assertUE4SSFileEquals(t, fixture.activePath, payloadV1) + assertUE4SSFileEquals(t, fixture.previous, payloadV1) + marker, found, err := loadManagedDLLExtensionMarker(fixture.markerPath) + if err != nil || !found || !markerMatchesPlan(marker, fixture.plan) { + t.Fatalf("expected previous release marker after rollback, marker=%+v found=%v err=%v", marker, found, err) + } +} + +func TestUE4SSDLLExtensionRejectsLinuxBeforeDownloadOrStart(t *testing.T) { + root := t.TempDir() + fixture := newUE4SSExtensionFixture(t, root, false) + downloader := &ue4ssTestDownloader{payload: []byte("must not download")} + supervisor := &ue4ssManagedProcessSupervisor{} + executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "linux", "amd64") + + result := executor.Execute(fixture.assignment) + if result.State != lifecycleResultStateFailed || result.ErrorCode != "unsupported_extension_platform" { + t.Fatalf("expected Linux DLL extension rejection, got %+v", result) + } + if downloader.Count() != 0 || supervisor.Starts() != 0 { + t.Fatalf("expected Linux rejection before download or start, downloads=%d starts=%d", downloader.Count(), supervisor.Starts()) + } +} + +func TestUE4SSDLLExtensionRequiresInstalledBootstrap(t *testing.T) { + root := t.TempDir() + fixture := newUE4SSExtensionFixture(t, root, false) + downloader := &ue4ssTestDownloader{payload: []byte("must not download")} + supervisor := &ue4ssManagedProcessSupervisor{} + executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64") + + result := executor.Execute(fixture.assignment) + if result.State != lifecycleResultStateFailed || result.ErrorCode != "ue4ss_bootstrap_missing" { + t.Fatalf("expected bootstrap precondition failure, got %+v", result) + } + if downloader.Count() != 0 || supervisor.Starts() != 0 { + t.Fatalf("expected missing bootstrap to prevent download and start, downloads=%d starts=%d", downloader.Count(), supervisor.Starts()) + } +} + +func TestUE4SSDLLExtensionSlowDownloadPreservesHeartbeat(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + fixture := newUE4SSExtensionFixture(t, cfg.WorkspaceRoot, true) + releaseDownload := make(chan struct{}) + downloader := &ue4ssTestDownloader{payload: []byte("scum-simple-rcon DLL release one"), started: make(chan struct{}), unblock: releaseDownload} + supervisor := &ue4ssManagedProcessSupervisor{} + worker, err := NewWorker(cfg, client, + WithDependencyDownloader(downloader), + WithManagedProcessSupervisor(supervisor), + WithDLLExtensionRuntimeTarget("windows", "amd64"), + ) + if err != nil { + t.Fatalf("new worker: %v", err) + } + worker.state.SessionToken = "session-token" + + executionDone := make(chan LifecycleExecutionResult, 1) + go func() { executionDone <- worker.executeAssignment(context.Background(), fixture.assignment) }() + select { + case <-downloader.started: + case <-time.After(2 * time.Second): + close(releaseDownload) + t.Fatal("expected lifecycle job to begin its bounded DLL download") + } + + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- worker.HeartbeatOnce(context.Background()) }() + select { + case err := <-heartbeatDone: + if err != nil { + close(releaseDownload) + t.Fatalf("heartbeat during DLL download: %v", err) + } + case <-time.After(500 * time.Millisecond): + close(releaseDownload) + t.Fatal("slow DLL download blocked the control heartbeat") + } + if len(client.heartbeatRequests) != 1 { + close(releaseDownload) + t.Fatalf("expected heartbeat request during DLL download, got %d", len(client.heartbeatRequests)) + } + close(releaseDownload) + select { + case result := <-executionDone: + if result.State != lifecycleResultStateSucceeded { + t.Fatalf("expected lifecycle job to finish after download release, got %+v", result) + } + case <-time.After(2 * time.Second): + t.Fatal("lifecycle job did not finish after slow download was released") + } +} + +func newUE4SSExtensionExecutor(root string, downloader DependencyDownloader, supervisor ManagedProcessSupervisor, targetOS string, targetArch string) LifecycleExecutor { + return NewLifecycleExecutor( + WithLifecycleWorkspaceRoot(root), + WithDependencyDownloader(downloader), + WithManagedProcessSupervisor(supervisor), + WithDLLExtensionRuntimeTarget(targetOS, targetArch), + ) +} + +func newUE4SSExtensionFixture(t *testing.T, root string, bootstrap bool) ue4ssExtensionFixture { + t.Helper() + assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart) + assignment.TargetKey = "actions/start.json" + assignment.ExecutionInput.WorkspaceScope = "run-local" + scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope) + if err != nil { + t.Fatalf("create workspace scope: %v", err) + } + writeUE4SSFixtureFile(t, filepath.Join(scope, "actions", "start.json"), []byte(`{"version":1,"action":"start","mode":"supervised","executableKey":"bin/SCUMServer.exe"}`), 0o600) + executable := []byte("SCUM server executable fixture") + writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "SCUMServer.exe"), executable, 0o700) + if bootstrap { + writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "dwmapi.dll"), []byte("UE4SS proxy fixture"), 0o600) + writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "UE4SS.dll"), []byte("UE4SS loader fixture"), 0o600) + writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "ue4ss", "Mods", "mods.txt"), []byte("OtherMod : 1\nscum_simple_rcon : 0\nscum_simple_rcon : 1\n"), 0o600) + } + plan := protocol.RuntimeDLLExtensionPlan{ + Key: "scum-simple-rcon", + Version: "1.0.0", + ReleaseURL: "https://cdn.npc0.com/scum_simple_rcon_ue4s.dll", + Checksum: bytesChecksum([]byte("scum-simple-rcon DLL release one")), + SizeBytes: int64(len([]byte("scum-simple-rcon DLL release one"))), + TargetKey: "ue4ss/scum-simple-rcon", + ModKey: "scum_simple_rcon", + DLLRef: "ue4ss/Mods/scum_simple_rcon/dlls/main.dll", + SCUMExecutableChecksum: bytesChecksum(executable), + UE4SSABI: "ue4ss-3.0", + RCONPort: 27015, + } + assignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{plan} + return ue4ssExtensionFixture{ + assignment: assignment, + plan: plan, + activePath: filepath.Join(scope, "bin", "ue4ss", "Mods", plan.ModKey, "dlls", "main.dll"), + markerPath: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "release.json"), + previous: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "previous.dll"), + stagePath: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "download.staged"), + configPath: filepath.Join(scope, "bin", "ue4ss", "Mods", plan.ModKey, "config.ini"), + modsPath: filepath.Join(scope, "bin", "ue4ss", "Mods", "mods.txt"), + } +} + +func writeUE4SSFixtureFile(t *testing.T, path string, body []byte, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("create fixture directory: %v", err) + } + if err := os.WriteFile(path, body, mode); err != nil { + t.Fatalf("write fixture file: %v", err) + } +} + +func assertUE4SSFileEquals(t *testing.T, path string, expected []byte) { + t.Helper() + if actual := readUE4SSFile(t, path); !bytes.Equal(actual, expected) { + t.Fatalf("unexpected managed file contents at %s", filepath.Base(path)) + } +} + +func readUE4SSFile(t *testing.T, path string) []byte { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture file %s: %v", filepath.Base(path), err) + } + return body +} + +func rconPasswordFromConfig(content string) string { + inRCON := false + for _, rawLine := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") { + line := strings.TrimSpace(rawLine) + if line == "[rcon]" { + inRCON = true + continue + } + if strings.HasPrefix(line, "[") { + inRCON = false + continue + } + if inRCON && strings.HasPrefix(line, "password=") { + return strings.TrimPrefix(line, "password=") + } + } + return "" +} diff --git a/runtime/worker.go b/runtime/worker.go new file mode 100644 index 0000000..83fafe0 --- /dev/null +++ b/runtime/worker.go @@ -0,0 +1,1364 @@ +package runtime + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "browser.local/run/config" + "browser.local/run/protocol" + "browser.local/run/spool" +) + +type WorkerClient interface { + Hello(context.Context, protocol.RunHelloRequest) (protocol.RunHelloResponse, error) + Heartbeat(context.Context, protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) + ReportLifecycle(context.Context, protocol.RunLifecycleReportRequest) (protocol.RunLifecycleReportResponse, error) + GetRunLogStreamProgress(context.Context, protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) + ClaimJob(context.Context, protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) + AckJob(context.Context, protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) + UpdateJobProgress(context.Context, protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) + CompleteJob(context.Context, protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) + GetDistributionBuildInput(context.Context, protocol.DistributionBuildInputRequest) (protocol.DistributionBuildInputResponse, error) + GetDependencyExecutionInput(context.Context, protocol.DependencyExecutionInputRequest) (protocol.DependencyExecutionInputResponse, error) + GetSourceRCONExecutionInput(context.Context, protocol.SourceRCONExecutionInputRequest) (protocol.SourceRCONExecutionInputResponse, error) + GetProtectedRequestExecutionInput(context.Context, protocol.ProtectedRequestExecutionInputRequest) (protocol.ProtectedRequestExecutionInputResponse, error) + GetRunUpdateInput(context.Context, protocol.RunUpdateInputRequest) (protocol.RunUpdateInputResponse, error) + ReadRunUpdateChunk(context.Context, protocol.RunUpdateChunkRequest) (protocol.RunUpdateChunkResponse, error) + ReportRunUpdateHealth(context.Context, protocol.RunUpdateHealthRequest) (protocol.RunUpdateHealthResponse, error) + IngestMetricBatch(context.Context, protocol.MetricBatchIngestRequest) (protocol.MetricBatchIngestResponse, error) + OpenArtifactTransfer(context.Context, protocol.ArtifactTransferOpenRequest) (protocol.ArtifactTransferOpenResponse, error) + UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) + CompleteArtifactTransfer(context.Context, protocol.ArtifactTransferCompleteRequest) (protocol.ArtifactTransferCompleteResponse, error) + PollJobCancel(context.Context, protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) + ReconcileJobs(context.Context, protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) +} + +const ( + jobActivePollInterval = 10 * time.Second + durableUploaderFlushTimeout = 5 * time.Second +) + +type Worker struct { + cfg config.Config + client WorkerClient + executor LifecycleExecutor + state WorkerState + journal *JobJournal + stateMu sync.RWMutex + sessionRefreshMu sync.Mutex + sequenceMu sync.Mutex + restartMu sync.Mutex + restartRequested bool + observationMu sync.Mutex + reportedObservations map[string]uint64 + metricCollector MetricCollector +} + +type WorkerState struct { + RunEndpointID string + SessionToken string + SessionExpiresAt time.Time + Capabilities []string + Capacity protocol.RunCapacityReport + LastHeartbeat time.Time + Sequence uint64 +} + +func NewWorker(cfg config.Config, client WorkerClient, options ...LifecycleExecutorOption) (*Worker, error) { + if client == nil { + return nil, fmt.Errorf("worker client is required") + } + if cfg.RunEndpointID == "" { + cfg.RunEndpointID = config.DefaultEndpointID + } + if cfg.DisplayName == "" { + cfg.DisplayName = config.DefaultDisplayName + } + if cfg.Version == "" { + cfg.Version = config.DefaultVersion + } + if cfg.MaxJobs <= 0 { + cfg.MaxJobs = 1 + } + executorOptions := append([]LifecycleExecutorOption{ + WithLifecycleWorkspaceRoot(cfg.WorkspaceRoot), + WithManagedProcessStateRoot(managedProcessStateRoot(cfg)), + WithManagedProcessOutputRoot(cfg.WorkspaceRoot), + WithLocalStartupDiagnostics(cfg.LocalStartupDiagnostics), + }, options...) + if err := migrateManagedProcessState(cfg, managedProcessStateRoot(cfg)); err != nil { + return nil, err + } + journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot) + if err != nil { + return nil, err + } + worker := &Worker{ + cfg: cfg, + client: client, + executor: NewLifecycleExecutor(executorOptions...), + state: WorkerState{ + RunEndpointID: cfg.RunEndpointID, + Capabilities: SupportedRunCapabilitiesForComponent(cfg.ComponentKind), + Capacity: protocol.RunCapacityReport{MaxJobs: cfg.MaxJobs}, + }, + journal: journal, + reportedObservations: map[string]uint64{}, + metricCollector: defaultMetricCollector{}, + } + if worker.executor.metricCollector != nil { + worker.metricCollector = worker.executor.metricCollector + } + return worker, nil +} + +func migrateLegacyManagedProcessState(cfg config.Config, stateRoot string) error { + workspaceRoot := cfg.WorkspaceRoot + if strings.TrimSpace(workspaceRoot) == "" { + workspaceRoot = filepath.Join(".", ".run-workspace") + } + legacyPath := filepath.Join(workspaceRoot, "state", "processes.json") + newPath := filepath.Join(stateRoot, "state", "processes.json") + if filepath.Clean(legacyPath) == filepath.Clean(newPath) { + return nil + } + if _, err := os.Stat(newPath); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + body, err := os.ReadFile(legacyPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("read legacy managed process state: %w", err) + } + var legacy processJournal + if err := json.Unmarshal(body, &legacy); err != nil { + return fmt.Errorf("decode legacy managed process state: %w", err) + } + filtered := processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{}, Retired: map[string]ProcessIdentity{}} + for key, item := range legacy.Items { + if managedProcessBelongsToRun(item, cfg) { + filtered.Items[key] = item + } + } + for key, item := range legacy.Retired { + if managedProcessBelongsToRun(item, cfg) { + filtered.Retired[key] = item + } + } + if len(filtered.Items) == 0 && len(filtered.Retired) == 0 { + return nil + } + if err := ensureDirectory(filepath.Dir(newPath)); err != nil { + return fmt.Errorf("create isolated managed process state: %w", err) + } + encoded, err := json.Marshal(filtered) + if err != nil { + return fmt.Errorf("encode isolated managed process state: %w", err) + } + if err := os.WriteFile(newPath, encoded, 0o600); err != nil { + return fmt.Errorf("write isolated managed process state: %w", err) + } + log.Printf("RUN phase=process.managed status=legacy_state_migrated items=%d retired=%d", len(filtered.Items), len(filtered.Retired)) + return nil +} + +// migrateManagedProcessState also imports matching journals from prior Run +// state namespaces. A generated package can change its component metadata +// while retaining the same endpoint/server/profile; that must not create a +// second autonomous process for the same server during a Run update. +func migrateManagedProcessState(cfg config.Config, stateRoot string) error { + if err := migrateLegacyManagedProcessState(cfg, stateRoot); err != nil { + return err + } + workspaceRoot := cfg.WorkspaceRoot + if strings.TrimSpace(workspaceRoot) == "" { + workspaceRoot = filepath.Join(".", ".run-workspace") + } + targetPath := filepath.Join(stateRoot, "state", "processes.json") + target := processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{}, Retired: map[string]ProcessIdentity{}} + if body, err := os.ReadFile(targetPath); err == nil { + if err := json.Unmarshal(body, &target); err != nil { + return fmt.Errorf("decode managed process state: %w", err) + } + if target.Items == nil { + target.Items = map[string]ProcessIdentity{} + } + if target.Retired == nil { + target.Retired = map[string]ProcessIdentity{} + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("read managed process state: %w", err) + } + changed := false + err := filepath.WalkDir(workspaceRoot, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || entry.Name() != "processes.json" { + return nil + } + if filepath.Clean(path) == filepath.Clean(targetPath) { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + var source processJournal + if err := json.Unmarshal(body, &source); err != nil { + return fmt.Errorf("decode managed process state candidate: %w", err) + } + for key, item := range source.Items { + if !managedProcessBelongsToRun(item, cfg) { + continue + } + existing, exists := target.Items[key] + if !exists || (!processAlivePID(existing.PID) && processAlivePID(item.PID)) { + target.Items[key] = item + changed = true + } + } + for key, item := range source.Retired { + if managedProcessBelongsToRun(item, cfg) { + if _, exists := target.Retired[key]; !exists { + target.Retired[key] = item + changed = true + } + } + } + return nil + }) + if err != nil { + return fmt.Errorf("scan managed process state: %w", err) + } + if !changed { + return nil + } + if err := ensureDirectory(filepath.Dir(targetPath)); err != nil { + return fmt.Errorf("create managed process state directory: %w", err) + } + body, err := json.Marshal(target) + if err != nil { + return fmt.Errorf("encode managed process state: %w", err) + } + temporary := targetPath + ".tmp" + if err := os.WriteFile(temporary, body, 0o600); err != nil { + return fmt.Errorf("write managed process state: %w", err) + } + if err := os.Rename(temporary, targetPath); err != nil { + _ = os.Remove(temporary) + return fmt.Errorf("replace managed process state: %w", err) + } + log.Printf("RUN phase=process.managed status=state_namespaces_merged items=%d retired=%d", len(target.Items), len(target.Retired)) + return nil +} + +func managedProcessBelongsToRun(item ProcessIdentity, cfg config.Config) bool { + if item.RunEndpointID == "" || item.ServerInstanceID == "" || item.RunEndpointID != cfg.RunEndpointID || item.ServerInstanceID != cfg.ServerInstanceID { + return false + } + return cfg.ComponentKey == "" || item.ProfileKey == "" || item.ProfileKey == cfg.ComponentKey +} + +func managedProcessStateRoot(cfg config.Config) string { + workspaceRoot := cfg.WorkspaceRoot + if strings.TrimSpace(workspaceRoot) == "" { + workspaceRoot = filepath.Join(".", ".run-workspace") + } + identity := strings.Join([]string{ + "run-process-state-v1", + cfg.RunEndpointID, + cfg.ServerInstanceID, + cfg.PluginID, + cfg.ComponentKind, + cfg.ComponentKey, + }, "\x00") + digest := sha256.Sum256([]byte(identity)) + return filepath.Join(workspaceRoot, "run-services", hex.EncodeToString(digest[:])) +} + +func (worker *Worker) Register(ctx context.Context) error { + worker.sessionRefreshMu.Lock() + defer worker.sessionRefreshMu.Unlock() + return worker.registerUnlocked(ctx) +} + +func (worker *Worker) registerUnlocked(ctx context.Context) error { + state := worker.State() + log.Printf("RUN phase=register status=starting endpoint=%s version=%s server=%s plugin=%s component=%s componentKey=%s capabilities=%d maxJobs=%d", worker.cfg.RunEndpointID, worker.cfg.Version, worker.cfg.ServerInstanceID, worker.cfg.PluginID, worker.cfg.ComponentKind, safeOptional(worker.cfg.ComponentKey), len(state.Capabilities), worker.cfg.MaxJobs) + response, err := worker.client.Hello(ctx, protocol.RunHelloRequest{ + RegistrationToken: worker.cfg.RegistrationToken, + RunEndpointID: worker.cfg.RunEndpointID, + ServerInstanceID: worker.cfg.ServerInstanceID, + PluginID: worker.cfg.PluginID, + ComponentKind: worker.cfg.ComponentKind, + ComponentKey: worker.cfg.ComponentKey, + KeyGeneration: worker.cfg.KeyGeneration, + DisplayName: worker.cfg.DisplayName, + Version: worker.cfg.Version, + Status: "online", + Platform: runtime.GOOS, + Architecture: runtime.GOARCH, + CapabilityReport: protocol.RunCapabilityReport{ + Capabilities: state.Capabilities, + Fingerprint: capabilityFingerprint(state.Capabilities), + }, + Capacity: worker.capacityReportFor(state), + }) + if err != nil { + log.Printf("RUN phase=register status=failed endpoint=%s error=%s", worker.cfg.RunEndpointID, RedactText(err.Error())) + return err + } + if !response.Accepted || response.SessionToken == "" { + log.Printf("RUN phase=register status=rejected endpoint=%s accepted=%t sessionTokenPresent=%t", worker.cfg.RunEndpointID, response.Accepted, response.SessionToken != "") + return fmt.Errorf("run hello was not accepted") + } + worker.stateMu.Lock() + worker.state.SessionToken = response.SessionToken + worker.state.SessionExpiresAt = response.SessionExpiresAt + state = worker.state + state.Capabilities = append([]string(nil), state.Capabilities...) + worker.stateMu.Unlock() + if sink, ok := worker.executor.logSink.(*SpoolLogSink); ok { + sink.mu.Lock() + sink.RunEndpointID = state.RunEndpointID + sink.SessionToken = state.SessionToken + sink.Progress = func(ctx context.Context, serverInstanceID string, streamID string) (uint64, error) { + current, err := worker.registeredState() + if err != nil { + return 0, err + } + response, err := worker.client.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{RunEndpointID: current.RunEndpointID, SessionToken: current.SessionToken, ServerInstanceID: serverInstanceID, LogStreamID: streamID}) + if err != nil { + return 0, err + } + if !response.Accepted || response.LogStreamID != streamID { + return 0, fmt.Errorf("run log stream progress was not accepted") + } + return response.LatestSeq, nil + } + sink.mu.Unlock() + } + if hook, ok := worker.executor.artifactHook.(*QueueArtifactHook); ok { + hook.RunEndpointID = state.RunEndpointID + hook.SessionToken = state.SessionToken + } + worker.executor.ResumeManagedProcessLogs(ctx) + log.Printf("RUN phase=register status=accepted endpoint=%s sessionExpiresAt=%s heartbeatSeconds=%d", state.RunEndpointID, response.SessionExpiresAt.Format(time.RFC3339), response.HeartbeatIntervalSeconds) + return nil +} + +func (worker *Worker) registeredState() (WorkerState, error) { + state := worker.State() + if state.SessionToken == "" { + return state, fmt.Errorf("worker is not registered") + } + return state, nil +} + +func (worker *Worker) HeartbeatOnce(ctx context.Context) error { + state, err := worker.registeredState() + if err != nil { + return err + } + if !state.SessionExpiresAt.IsZero() && !time.Now().UTC().Add(time.Minute).Before(state.SessionExpiresAt) { + log.Printf("RUN phase=heartbeat status=session_expiring endpoint=%s sessionExpiresAt=%s", state.RunEndpointID, state.SessionExpiresAt.Format(time.RFC3339)) + return worker.reregisterAndReconcile(ctx, "heartbeat_session_expiring", state.SessionToken) + } + log.Printf("RUN phase=heartbeat status=starting endpoint=%s activeJobs=%d", state.RunEndpointID, worker.journal.ActiveCount()) + response, err := worker.client.Heartbeat(ctx, protocol.RunHeartbeatRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + Version: worker.cfg.Version, + Status: "online", + CapabilityFingerprint: capabilityFingerprint(state.Capabilities), + Capacity: worker.capacityReportFor(state), + }) + if err != nil { + if sessionInvalidError(err) { + log.Printf("RUN phase=heartbeat status=session_invalid endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) + return worker.reregisterAndReconcile(ctx, "heartbeat_session_invalid", state.SessionToken) + } + log.Printf("RUN phase=heartbeat status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) + return err + } + if !response.Accepted { + log.Printf("RUN phase=heartbeat status=rejected endpoint=%s", state.RunEndpointID) + return fmt.Errorf("heartbeat was not accepted") + } + worker.stateMu.Lock() + if worker.state.SessionToken == state.SessionToken { + worker.state.LastHeartbeat = response.ServerTime + } + worker.stateMu.Unlock() + log.Printf("RUN phase=heartbeat status=accepted endpoint=%s serverTime=%s nextSeconds=%d", state.RunEndpointID, response.ServerTime.Format(time.RFC3339), response.NextHeartbeatSeconds) + return nil +} + +func (worker *Worker) reregisterAndReconcile(ctx context.Context, reason string, observedToken string) error { + worker.sessionRefreshMu.Lock() + defer worker.sessionRefreshMu.Unlock() + state := worker.State() + if observedToken != "" && state.SessionToken != "" && state.SessionToken != observedToken { + log.Printf("RUN phase=register status=already_refreshed reason=%s endpoint=%s", safeOptional(reason), state.RunEndpointID) + return worker.ReconcileOnce(ctx) + } + log.Printf("RUN phase=register status=refreshing reason=%s endpoint=%s", safeOptional(reason), state.RunEndpointID) + if err := worker.registerUnlocked(ctx); err != nil { + return err + } + return worker.ReconcileOnce(ctx) +} + +func (worker *Worker) ClaimAndRunOnce(ctx context.Context) (bool, error) { + state, err := worker.registeredState() + if err != nil { + return false, err + } + log.Printf("RUN phase=claim status=polling endpoint=%s activeJobs=%d", state.RunEndpointID, worker.journal.ActiveCount()) + claim, err := worker.client.ClaimJob(ctx, protocol.RunJobClaimRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + Capabilities: state.Capabilities, + Capacity: worker.capacityReportFor(state), + }) + if err != nil { + if sessionInvalidError(err) { + log.Printf("RUN phase=claim status=session_invalid endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) + return false, worker.reregisterAndReconcile(ctx, "claim_session_invalid", state.SessionToken) + } + log.Printf("RUN phase=claim status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) + return false, err + } + if !claim.Accepted || !claim.HasJob || claim.Job == nil { + log.Printf("RUN phase=claim status=idle endpoint=%s accepted=%t hasJob=%t", state.RunEndpointID, claim.Accepted, claim.HasJob) + return false, nil + } + assignment := *claim.Job + log.Printf("RUN phase=claim status=assigned job=%s capability=%s target=%s attempt=%d server=%s workspaceScope=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), assignment.Attempt, assignment.ServerInstanceID, safeOptional(assignment.ExecutionInput.WorkspaceScope)) + return true, worker.runAssignment(ctx, assignment) +} + +func sessionInvalidError(err error) bool { + var sessionError interface{ SessionInvalid() bool } + return errors.As(err, &sessionError) && sessionError.SessionInvalid() +} + +func (worker *Worker) runAssignment(ctx context.Context, assignment protocol.RunJobAssignment) error { + state, err := worker.registeredState() + if err != nil { + return err + } + if assignment.RunEndpointID != state.RunEndpointID { + return fmt.Errorf("job assignment endpoint does not match registered Run endpoint") + } + log.Printf("RUN phase=job status=journal_store job=%s capability=%s attempt=%d", assignment.JobID, assignment.Capability, assignment.Attempt) + if err := worker.journal.Store(assignment); err != nil { + log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + log.Printf("RUN phase=job status=ack_start job=%s capability=%s", assignment.JobID, assignment.Capability) + ack, err := worker.client.AckJob(ctx, protocol.RunJobAckRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + Message: "job accepted by run worker", + }) + if err != nil { + log.Printf("RUN phase=job status=ack_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + assignment = ack.Job + if !ack.Accepted { + log.Printf("RUN phase=job status=ack_rejected job=%s", assignment.JobID) + return fmt.Errorf("job acknowledgement was not accepted") + } + log.Printf("RUN phase=job status=ack_accepted job=%s attempt=%d", assignment.JobID, assignment.Attempt) + if err := worker.journal.Store(assignment); err != nil { + log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + progressSequence := worker.nextProgressSequence(assignment.ProgressSequence) + state, err = worker.registeredState() + if err != nil { + return err + } + log.Printf("RUN phase=job status=progress_start job=%s sequence=%d", assignment.JobID, progressSequence) + progress, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + Progress: protocol.RunJobProgressReport{Percent: 10, Message: "lifecycle execution started"}, + Sequence: progressSequence, + }) + if err != nil { + log.Printf("RUN phase=job status=progress_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + if !progress.Accepted { + log.Printf("RUN phase=job status=progress_rejected job=%s", assignment.JobID) + return fmt.Errorf("job progress was not accepted") + } + assignment = progress.Job + log.Printf("RUN phase=job status=progress_accepted job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) + if err := worker.journal.Store(assignment); err != nil { + log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + log.Printf("RUN phase=job status=execute_start job=%s capability=%s", assignment.JobID, assignment.Capability) + execution, assignment, cancelledByPlatform, err := worker.executeWithJobPolling(ctx, assignment) + if err != nil { + log.Printf("RUN phase=job status=execute_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + log.Printf("RUN phase=job status=execute_done job=%s state=%s errorCode=%s message=%s cancelledByPlatform=%t", assignment.JobID, execution.State, safeOptional(execution.ErrorCode), safeOptional(execution.Message), cancelledByPlatform) + if cancelledByPlatform && execution.State == lifecycleResultStateSucceeded { + execution = LifecycleExecutionResult{ + State: lifecycleResultStateCancelled, + Progress: protocol.RunJobProgressReport{Percent: 100, Message: "cancelled by platform"}, + Message: "cancelled by platform", + ErrorCode: "lifecycle_cancelled", + } + } + state, err = worker.registeredState() + if err != nil { + return err + } + resultRequest := LifecycleResultRequest(assignment, state.SessionToken, execution) + log.Printf("RUN phase=job status=result_store job=%s state=%s", assignment.JobID, resultRequest.State) + if err := worker.journal.StorePendingResult(resultRequest, execution.ActivationManifest); err != nil { + log.Printf("RUN phase=job status=result_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + log.Printf("RUN phase=job status=result_submit job=%s state=%s", assignment.JobID, resultRequest.State) + result, err := worker.client.CompleteJob(ctx, resultRequest) + if err != nil { + log.Printf("RUN phase=job status=result_submit_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + if !result.Accepted { + log.Printf("RUN phase=job status=result_rejected job=%s", assignment.JobID) + return fmt.Errorf("job result was not accepted") + } + log.Printf("RUN phase=job status=result_accepted job=%s state=%s", assignment.JobID, resultRequest.State) + if err := worker.journal.Delete(assignment.JobID); err != nil { + log.Printf("RUN phase=job status=journal_delete_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + log.Printf("RUN phase=job status=complete job=%s state=%s", assignment.JobID, resultRequest.State) + if execution.ActivationManifest != "" { + log.Printf("RUN phase=self_update status=activate_start job=%s", assignment.JobID) + if err := worker.executor.selfUpdateActivator.Activate(execution.ActivationManifest); err != nil { + log.Printf("RUN phase=self_update status=activate_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return fmt.Errorf("launch self-update helper: %w", err) + } + worker.restartMu.Lock() + worker.restartRequested = true + worker.restartMu.Unlock() + } + return nil +} + +func (worker *Worker) executeWithJobPolling(ctx context.Context, assignment protocol.RunJobAssignment) (LifecycleExecutionResult, protocol.RunJobAssignment, bool, error) { + jobCtx, cancel := context.WithCancel(ctx) + defer cancel() + cancelledByPlatform := false + pollCancel := func() { + state, err := worker.registeredState() + if err != nil { + log.Printf("RUN phase=job.cancel_poll status=skipped_unregistered job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return + } + log.Printf("RUN phase=job.cancel_poll status=starting job=%s", assignment.JobID) + response, err := worker.client.PollJobCancel(ctx, protocol.RunJobCancelPollRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + }) + if err != nil { + log.Printf("RUN phase=job.cancel_poll status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return + } + if err == nil && response.HasCancel { + log.Printf("RUN phase=job.cancel_poll status=cancel_requested job=%s", assignment.JobID) + cancelledByPlatform = true + cancel() + return + } + log.Printf("RUN phase=job.cancel_poll status=clear job=%s", assignment.JobID) + } + pollCancel() + + executionCh := make(chan LifecycleExecutionResult, 1) + executionAssignment := assignment + go func() { + executionCh <- worker.executeAssignment(jobCtx, executionAssignment) + }() + log.Printf("RUN phase=job.execute status=worker_started job=%s pollSeconds=%d", assignment.JobID, int(jobActivePollInterval/time.Second)) + ticker := time.NewTicker(jobActivePollInterval) + defer ticker.Stop() + for { + select { + case execution := <-executionCh: + log.Printf("RUN phase=job.execute status=worker_finished job=%s state=%s", assignment.JobID, execution.State) + return execution, assignment, cancelledByPlatform, nil + case <-ctx.Done(): + log.Printf("RUN phase=job.execute status=context_done job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error())) + cancel() + execution := <-executionCh + return execution, assignment, cancelledByPlatform, ctx.Err() + case <-ticker.C: + log.Printf("RUN phase=job.execute status=active job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) + if !cancelledByPlatform { + pollCancel() + } + if cancelledByPlatform { + continue + } + state, err := worker.registeredState() + if err != nil { + cancel() + log.Printf("RUN phase=job.execute status=lease_renew_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err + } + progress, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + Progress: protocol.RunJobProgressReport{Percent: assignment.Progress.Percent, Message: "lifecycle execution active"}, + Sequence: worker.nextProgressSequence(assignment.ProgressSequence), + }) + if err != nil || !progress.Accepted { + cancel() + if err == nil { + err = fmt.Errorf("job lease renewal was not accepted") + } + log.Printf("RUN phase=job.execute status=lease_renew_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err + } + assignment = progress.Job + log.Printf("RUN phase=job.execute status=lease_renewed job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) + if err := worker.journal.Store(assignment); err != nil { + cancel() + log.Printf("RUN phase=job.execute status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err + } + } + } +} + +func (worker *Worker) nextProgressSequence(minimum uint64) uint64 { + worker.sequenceMu.Lock() + defer worker.sequenceMu.Unlock() + worker.stateMu.Lock() + defer worker.stateMu.Unlock() + if worker.state.Sequence < minimum { + worker.state.Sequence = minimum + } + worker.state.Sequence++ + return worker.state.Sequence +} + +func (worker *Worker) executeAssignment(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + log.Printf("RUN phase=job.dispatch status=select job=%s capability=%s inputRef=%s target=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.InputRef), safeOptional(assignment.TargetKey)) + if protocol.IsProtectedRequestCapability(assignment.Capability) { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=protected_request", assignment.JobID) + return worker.executeProtectedRequestJob(ctx, assignment) + } + if assignment.ExecutionInput.SourceRCON != nil { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=source_rcon", assignment.JobID) + return worker.executeSourceRCONJob(ctx, assignment) + } + if assignment.Capability == protocol.RunCapabilityDistributionBuild { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=distribution_build", assignment.JobID) + return worker.executeDistributionBuild(ctx, assignment) + } + if assignment.Capability == protocol.RunCapabilityDependenciesCheck || assignment.Capability == protocol.RunCapabilityDependenciesInstall { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=dependencies", assignment.JobID) + return worker.executeDependencyJob(ctx, assignment) + } + if assignment.Capability == protocol.RunCapabilityRunSelfUpdate { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=self_update", assignment.JobID) + return worker.executeRunSelfUpdate(ctx, assignment) + } + if assignment.Capability == protocol.RunCapabilityLogsBackfill { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=logs_backfill", assignment.JobID) + return worker.executor.ExecuteLogBackfill(ctx, assignment) + } + if assignment.Capability == protocol.RunCapabilityRemoteRunDBSQLiteProbe { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=sqlite_schema_probe", assignment.JobID) + if worker.executor.sqliteSchemaProbe == nil { + return lifecycleFailure("sqlite_probe_unavailable", "SQLite schema probe executor is unavailable") + } + if assignment.ExecutionInput.SQLiteSchemaProbe != nil { + targetKey, err := worker.materializeSQLiteProbeDataTarget(ctx, assignment) + if err != nil { + return sqliteProbeFailureForDataTarget(assignment, err) + } + assignment.TargetKey = targetKey + } + return worker.executor.sqliteSchemaProbe.Execute(ctx, assignment) + } + if isSupportedLifecycleCapability(assignment.Capability) { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=lifecycle", assignment.JobID) + return worker.executor.ExecuteContext(ctx, assignment) + } + if supportedCapability(SupportedFileCapabilities(), assignment.Capability) { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=file", assignment.JobID) + return worker.executor.ExecuteContext(ctx, assignment) + } + if isSupportedDistributionCapability(assignment.Capability) { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=distribution", assignment.JobID) + return ExecuteDistributionJob(ctx, assignment) + } + if isSupportedRemoteCapability(assignment.Capability) { + log.Printf("RUN phase=job.dispatch status=selected job=%s executor=remote", assignment.JobID) + return ExecuteRemoteAccessJob(ctx, assignment) + } + log.Printf("RUN phase=job.dispatch status=unsupported job=%s capability=%s", assignment.JobID, assignment.Capability) + return lifecycleFailure("unsupported_run_capability", "unsupported run capability") +} + +func (worker *Worker) executeProtectedRequestJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_assignment_invalid") + } + state, err := worker.registeredState() + if err != nil { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_unregistered") + } + input, err := worker.client.GetProtectedRequestExecutionInput(ctx, protocol.ProtectedRequestExecutionInputRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + FencingToken: assignment.FencingToken, + }) + if err != nil || !protectedRequestInputMatchesAssignment(input, assignment, state.RunEndpointID) { + return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_input_unavailable") + } + return worker.executor.ExecuteProtectedRequest(ctx, assignment, input) +} + +func protectedRequestInputMatchesAssignment(input protocol.ProtectedRequestExecutionInputResponse, assignment protocol.RunJobAssignment, endpointID string) bool { + return protocol.ValidProtectedRequestExecutionInput(input) && input.JobID == assignment.JobID && input.ServerInstanceID == assignment.ServerInstanceID && input.RunEndpointID == endpointID && input.FencingToken == assignment.FencingToken && input.TargetKey == assignment.TargetKey && input.TransportKey == assignment.ExecutionInput.RemoteAdapterKey && input.Kind == protectedRequestKindForCapability(assignment.Capability) +} + +func (worker *Worker) executeSourceRCONJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { + if err := protocol.ValidateRunJobAssignment(assignment); err != nil { + return lifecycleFailure("unsafe_source_rcon_plan", "Source RCON plan is invalid") + } + state, err := worker.registeredState() + if err != nil { + return lifecycleFailure("source_rcon_unregistered", "Run worker is not registered") + } + input, err := worker.client.GetSourceRCONExecutionInput(ctx, protocol.SourceRCONExecutionInputRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: assignment.JobID, + LeaseToken: assignment.LeaseToken, + Attempt: assignment.Attempt, + }) + if err != nil || input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != state.RunEndpointID { + return lifecycleFailure("source_rcon_input_unavailable", "Source RCON command input is unavailable") + } + return worker.executor.ExecuteSourceRCON(ctx, assignment, input.Command) +} + +func supportedCapability(capabilities []string, target string) bool { + for _, capability := range capabilities { + if capability == target { + return true + } + } + return false +} + +func (worker *Worker) ReconcileOnce(ctx context.Context) error { + state, err := worker.registeredState() + if err != nil { + return err + } + activeBefore := worker.journal.ActiveCount() + log.Printf("RUN phase=reconcile status=starting endpoint=%s activeJobs=%d", state.RunEndpointID, activeBefore) + response, err := worker.client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + ActiveJobs: worker.journal.ReconcileEntries(), + }) + if err != nil { + log.Printf("RUN phase=reconcile status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) + return err + } + if !response.Accepted { + log.Printf("RUN phase=reconcile status=rejected endpoint=%s", state.RunEndpointID) + return fmt.Errorf("job reconciliation was not accepted") + } + confirmed := map[string]struct{}{} + for _, job := range response.ConfirmedJobs { + if job.RunEndpointID != state.RunEndpointID { + return fmt.Errorf("reconciled job endpoint does not match registered Run endpoint") + } + if err := worker.journal.Store(job); err != nil { + return err + } + confirmed[job.JobID] = struct{}{} + } + for _, jobID := range response.DiscardJobIDs { + if err := worker.journal.Delete(jobID); err != nil { + return err + } + confirmed[jobID] = struct{}{} + } + for _, job := range worker.journal.ActiveJobs() { + if _, accounted := confirmed[job.JobID]; !accounted { + if err := worker.journal.Delete(job.JobID); err != nil { + return err + } + } + } + log.Printf("RUN phase=reconcile status=accepted endpoint=%s confirmed=%d discarded=%d activeBefore=%d activeAfter=%d", state.RunEndpointID, len(response.ConfirmedJobs), len(response.DiscardJobIDs), activeBefore, worker.journal.ActiveCount()) + return nil +} + +func (worker *Worker) RecoverActiveJobs(ctx context.Context) error { + activeJobs := worker.journal.ActiveJobs() + log.Printf("RUN phase=recover status=starting activeJobs=%d", len(activeJobs)) + for _, assignment := range activeJobs { + if pending, ok := worker.journal.PendingResult(assignment.JobID); ok { + log.Printf("RUN phase=recover status=pending_result job=%s", assignment.JobID) + activationManifest := worker.journal.PendingActivation(assignment.JobID) + state, err := worker.registeredState() + if err != nil { + return err + } + pending.SessionToken = state.SessionToken + result, err := worker.client.CompleteJob(ctx, pending) + if err != nil { + log.Printf("RUN phase=recover status=result_submit_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + if !result.Accepted { + log.Printf("RUN phase=recover status=result_rejected job=%s", assignment.JobID) + return fmt.Errorf("recovered job result was not accepted") + } + if err := worker.journal.Delete(assignment.JobID); err != nil { + return err + } + if activationManifest != "" { + if err := worker.executor.selfUpdateActivator.Activate(activationManifest); err != nil { + return fmt.Errorf("launch recovered self-update helper: %w", err) + } + worker.restartMu.Lock() + worker.restartRequested = true + worker.restartMu.Unlock() + } + continue + } + log.Printf("RUN phase=recover status=rerun_active_job job=%s capability=%s", assignment.JobID, assignment.Capability) + if err := worker.runAssignment(ctx, assignment); err != nil { + log.Printf("RUN phase=recover status=rerun_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) + return err + } + } + log.Printf("RUN phase=recover status=complete activeJobs=%d", worker.journal.ActiveCount()) + return nil +} + +func (worker *Worker) Run(ctx context.Context) error { + log.Printf("RUN phase=run status=starting endpoint=%s", worker.cfg.RunEndpointID) + if err := worker.Register(ctx); err != nil { + return err + } + if err := worker.ReconcileOnce(ctx); err != nil { + return err + } + if err := worker.RecoverActiveJobs(ctx); err != nil { + return err + } + if err := worker.RunAutonomousLifecycleOnce(ctx); err != nil { + return err + } + if err := worker.reportAutonomousProcessObservations(ctx); err != nil { + log.Printf("RUN phase=autonomous_lifecycle.observation status=degraded error=%s", RedactText(err.Error())) + } + worker.reportMetricsDegraded(ctx, "startup") + if err := MarkSelfUpdateHealthy(worker.cfg.UpdateHealthFile); err != nil { + log.Printf("RUN phase=self_update_health status=mark_failed error=%s", RedactText(err.Error())) + return err + } + if err := worker.reportRunUpdateHealth(ctx); err != nil { + return err + } + heartbeatInterval := durationOrDefault(worker.cfg.HeartbeatInterval, 15*time.Second) + jobInterval := durationOrDefault(worker.cfg.PollInterval, 2*time.Second) + state := worker.State() + log.Printf("RUN phase=run status=ready endpoint=%s heartbeatSeconds=%d jobPollSeconds=%d", state.RunEndpointID, int(heartbeatInterval/time.Second), int(jobInterval/time.Second)) + heartbeatTicker := time.NewTicker(heartbeatInterval) + defer heartbeatTicker.Stop() + workerCtx, cancelWorker := context.WithCancel(ctx) + uploaderDone := make(chan struct{}) + go worker.runDurableUploaders(workerCtx, uploaderDone) + jobDone := make(chan error, 1) + go func() { jobDone <- worker.runJobLoop(workerCtx, jobInterval) }() + defer func() { + cancelWorker() + <-uploaderDone + }() + for { + select { + case <-ctx.Done(): + log.Printf("RUN phase=run status=context_done error=%s", RedactText(ctx.Err().Error())) + return ctx.Err() + case err := <-jobDone: + log.Printf("RUN phase=run status=job_loop_done error=%s", errorSummary(err)) + return err + case <-heartbeatTicker.C: + if err := worker.HeartbeatOnce(ctx); err != nil { + log.Printf("RUN phase=heartbeat status=retry_scheduled backoffMs=%d", boundedRetryBackoff(worker.cfg.RetryBackoff).Milliseconds()) + heartbeatTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff)) + continue + } + if err := worker.reportAutonomousProcessObservations(ctx); err != nil { + log.Printf("RUN phase=autonomous_lifecycle.observation status=degraded error=%s", RedactText(err.Error())) + } + worker.reportMetricsDegraded(ctx, "heartbeat") + heartbeatTicker.Reset(heartbeatInterval) + } + } +} + +func (worker *Worker) reportRunUpdateHealth(ctx context.Context) error { + if worker.cfg.UpdateJobID == "" && worker.cfg.UpdateOutcome == "" && worker.cfg.UpdateAttempt == 0 && worker.cfg.UpdateLeaseToken == "" { + log.Printf("RUN phase=self_update_health status=skipped") + return nil + } + if worker.cfg.UpdateJobID == "" || (worker.cfg.UpdateOutcome != "succeeded" && worker.cfg.UpdateOutcome != "rolled-back") || worker.cfg.UpdateAttempt <= 0 || worker.cfg.UpdateLeaseToken == "" { + log.Printf("RUN phase=self_update_health status=invalid_config") + return fmt.Errorf("self-update health report configuration is incomplete") + } + log.Printf("RUN phase=self_update_health status=reporting job=%s outcome=%s attempt=%d", worker.cfg.UpdateJobID, worker.cfg.UpdateOutcome, worker.cfg.UpdateAttempt) + state, err := worker.registeredState() + if err != nil { + return err + } + response, err := worker.client.ReportRunUpdateHealth(ctx, protocol.RunUpdateHealthRequest{ + RunEndpointID: state.RunEndpointID, + SessionToken: state.SessionToken, + JobID: worker.cfg.UpdateJobID, + LeaseToken: worker.cfg.UpdateLeaseToken, + Attempt: worker.cfg.UpdateAttempt, + Outcome: worker.cfg.UpdateOutcome, + Version: worker.cfg.Version, + }) + if err != nil { + log.Printf("RUN phase=self_update_health status=failed job=%s error=%s", worker.cfg.UpdateJobID, RedactText(err.Error())) + return err + } + if !response.Accepted || response.JobID != worker.cfg.UpdateJobID { + log.Printf("RUN phase=self_update_health status=rejected job=%s", worker.cfg.UpdateJobID) + return fmt.Errorf("self-update health report was not accepted") + } + log.Printf("RUN phase=self_update_health status=accepted job=%s", worker.cfg.UpdateJobID) + return nil +} + +func (worker *Worker) runJobLoop(ctx context.Context, interval time.Duration) error { + log.Printf("RUN phase=job_loop status=starting pollMs=%d", interval.Milliseconds()) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Printf("RUN phase=job_loop status=context_done error=%s", RedactText(ctx.Err().Error())) + return ctx.Err() + case <-ticker.C: + if worker.journal.ActiveCount() > 0 { + log.Printf("RUN phase=job_loop status=active_jobs activeJobs=%d", worker.journal.ActiveCount()) + if err := worker.ReconcileOnce(ctx); err != nil { + log.Printf("RUN phase=job_loop status=reconcile_failed error=%s", RedactText(err.Error())) + ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff)) + continue + } + if err := worker.RecoverActiveJobs(ctx); err != nil { + log.Printf("RUN phase=job_loop status=recover_failed error=%s", RedactText(err.Error())) + ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff)) + continue + } + } + if _, err := worker.ClaimAndRunOnce(ctx); err != nil { + log.Printf("RUN phase=job_loop status=claim_failed error=%s", RedactText(err.Error())) + ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff)) + continue + } + worker.restartMu.Lock() + restartRequested := worker.restartRequested + worker.restartMu.Unlock() + if restartRequested { + log.Printf("RUN phase=job_loop status=restart_requested") + return ErrSelfUpdateRestartRequested + } + ticker.Reset(interval) + } + } +} + +type durableLogClient interface { + IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) +} + +type durableArtifactClient interface { + UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) +} + +type sessionLogBatchClient struct { + client durableLogClient + runEndpointID string + sessionToken string +} + +type sessionLogStreamProgressClient struct { + client interface { + GetRunLogStreamProgress(context.Context, protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) + } + runEndpointID string + sessionToken string + serverID string +} + +func (client sessionLogStreamProgressClient) GetRunLogStreamProgress(ctx context.Context, streamID string) (uint64, error) { + response, err := client.client.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{ + RunEndpointID: client.runEndpointID, + SessionToken: client.sessionToken, + ServerInstanceID: client.serverID, + LogStreamID: streamID, + }) + if err != nil { + return 0, err + } + if !response.Accepted || response.LogStreamID != streamID { + return 0, fmt.Errorf("platform log stream progress response is invalid") + } + return response.LatestSeq, nil +} + +func (client sessionLogBatchClient) IngestLogBatch(ctx context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + batch.RunEndpointID = client.runEndpointID + batch.SessionToken = client.sessionToken + if checksum, err := checksumForLogEntries(batch.Entries); err == nil { + batch.Checksum = checksum + } + response, err := client.client.IngestLogBatch(ctx, batch) + if err != nil && (logBatchSequenceGapError(err) || logBatchAcknowledgedRangeConflict(err)) { + reason := "platform_sequence_gap" + if logBatchAcknowledgedRangeConflict(err) { + reason = "platform_acknowledged_range_conflict" + } + log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=%s error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, reason, RedactText(err.Error())) + return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection(reason, err) + } + if err != nil && logBatchLegacySessionMetadataError(err) && strings.TrimSpace(batch.LogSessionID) == "" && !batch.SessionStartedAt.IsZero() { + log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=legacy_session_metadata error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, RedactText(err.Error())) + return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection("legacy_session_metadata", err) + } + if err != nil && logBatchSessionMetadataMismatchError(err) { + log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=session_metadata_mismatch error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, RedactText(err.Error())) + return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection("session_metadata_mismatch", err) + } + return response, err +} + +func logBatchSequenceGapError(err error) bool { + var gapError interface{ LogBatchSequenceGap() bool } + return errors.As(err, &gapError) && gapError.LogBatchSequenceGap() +} + +func logBatchAcknowledgedRangeConflict(err error) bool { + var conflict interface{ LogBatchAcknowledgedRangeConflict() bool } + return errors.As(err, &conflict) && conflict.LogBatchAcknowledgedRangeConflict() +} + +func logBatchLegacySessionMetadataError(err error) bool { + var legacyMetadata interface{ LogBatchLegacySessionMetadata() bool } + return errors.As(err, &legacyMetadata) && legacyMetadata.LogBatchLegacySessionMetadata() +} + +func logBatchSessionMetadataMismatchError(err error) bool { + var mismatch interface{ LogBatchSessionMetadataMismatch() bool } + return errors.As(err, &mismatch) && mismatch.LogBatchSessionMetadataMismatch() +} + +type sessionArtifactChunkClient struct { + client durableArtifactClient + runEndpointID string + sessionToken string +} + +func (client sessionArtifactChunkClient) UploadArtifactChunk(ctx context.Context, chunk protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) { + chunk.RunEndpointID = client.runEndpointID + chunk.SessionToken = client.sessionToken + return client.client.UploadArtifactChunk(ctx, chunk) +} + +func (worker *Worker) runDurableUploaders(ctx context.Context, done chan<- struct{}) { + defer close(done) + logSink, hasLogSink := worker.executor.logSink.(*SpoolLogSink) + artifactHook, hasArtifactHook := worker.executor.artifactHook.(*QueueArtifactHook) + logClient, hasLogClient := worker.client.(durableLogClient) + artifactClient, hasArtifactClient := worker.client.(durableArtifactClient) + if (!hasLogSink || !hasLogClient) && (!hasArtifactHook || !hasArtifactClient) { + log.Printf("RUN phase=durable_uploaders status=disabled logs=%t artifacts=%t", hasLogSink && hasLogClient, hasArtifactHook && hasArtifactClient) + return + } + log.Printf("RUN phase=durable_uploaders status=starting logs=%t artifacts=%t", hasLogSink && hasLogClient, hasArtifactHook && hasArtifactClient) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Printf("RUN phase=durable_uploaders status=stopping") + return + case <-ticker.C: + state, err := worker.registeredState() + if err != nil { + log.Printf("RUN phase=durable_uploaders status=skipped_unregistered error=%s", RedactText(err.Error())) + continue + } + if hasLogSink && hasLogClient { + startedAt := time.Now() + log.Printf("RUN phase=durable_uploaders.logs status=flush_start timeoutMs=%d", durableUploaderFlushTimeout.Milliseconds()) + flushCtx, cancel := context.WithTimeout(ctx, durableUploaderFlushTimeout) + client := sessionLogBatchClient{client: logClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken} + flushed, err := logSink.Spool.Flush(flushCtx, client) + if err != nil { + log.Printf("RUN phase=durable_uploaders.logs status=flush_failed flushed=%d durationMs=%d error=%s", flushed, time.Since(startedAt).Milliseconds(), RedactText(err.Error())) + } else if flushed > 0 { + log.Printf("RUN phase=durable_uploaders.logs status=flushed count=%d durationMs=%d", flushed, time.Since(startedAt).Milliseconds()) + } + cancel() + } + if hasArtifactHook && hasArtifactClient { + startedAt := time.Now() + log.Printf("RUN phase=durable_uploaders.artifacts status=flush_start timeoutMs=%d", durableUploaderFlushTimeout.Milliseconds()) + flushCtx, cancel := context.WithTimeout(ctx, durableUploaderFlushTimeout) + client := sessionArtifactChunkClient{client: artifactClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken} + flushed, err := artifactHook.Queue.Flush(flushCtx, client) + if err != nil { + log.Printf("RUN phase=durable_uploaders.artifacts status=flush_failed flushed=%d durationMs=%d error=%s", flushed, time.Since(startedAt).Milliseconds(), RedactText(err.Error())) + } else if flushed > 0 { + log.Printf("RUN phase=durable_uploaders.artifacts status=flushed count=%d durationMs=%d", flushed, time.Since(startedAt).Milliseconds()) + } + cancel() + } + } + } +} + +func checksumForLogEntries(entries []protocol.LogEntry) (string, error) { + stable := make([]logEntryChecksumBody, len(entries)) + for i, entry := range entries { + stable[i] = logEntryChecksumBody{ + Seq: entry.Seq, + Timestamp: entry.Timestamp.UTC().Format("2006-01-02T15:04:05.000000000Z07:00"), + Level: entry.Level, + Line: entry.Line, + Fields: entry.Fields, + Redacted: entry.Redacted, + } + } + encoded, err := json.Marshal(stable) + if err != nil { + return "", err + } + sum := sha256.Sum256(encoded) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +type logEntryChecksumBody struct { + Seq uint64 `json:"seq"` + Timestamp string `json:"timestamp"` + Level string `json:"level,omitempty"` + Line string `json:"line"` + Fields map[string]string `json:"fields,omitempty"` + Redacted bool `json:"redacted"` +} + +func (worker *Worker) capacityReport() protocol.RunCapacityReport { + return worker.capacityReportFor(worker.State()) +} + +func (worker *Worker) capacityReportFor(state WorkerState) protocol.RunCapacityReport { + return protocol.RunCapacityReport{ + MaxJobs: state.Capacity.MaxJobs, + RunningJobs: worker.journal.ActiveCount(), + QueuedJobs: 0, + Summary: "worker control active; job capacity reported separately", + } +} + +func (worker *Worker) State() WorkerState { + worker.stateMu.RLock() + defer worker.stateMu.RUnlock() + state := worker.state + state.Capabilities = append([]string(nil), state.Capabilities...) + return state +} + +type SpoolLogSink struct { + RunEndpointID string + SessionToken string + Spool spool.LogSpool + mu sync.Mutex + Progress func(context.Context, string, string) (uint64, error) +} + +func (sink *SpoolLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error { + return sink.append(ctx, assignment, stream, line, nil) +} + +func (sink *SpoolLogSink) AppendWithCursor(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, cursor ProcessLogCursor) error { + return sink.append(ctx, assignment, stream, line, &spool.LogSourceCursor{StartOffset: cursor.StartOffset, EndOffset: cursor.EndOffset}) +} + +func (sink *SpoolLogSink) append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, cursor *spool.LogSourceCursor) error { + sink.mu.Lock() + defer sink.mu.Unlock() + redactedLine := RedactText(line) + redacted := line != redactedLine || strings.HasPrefix(redactedLine, "[redacted protected ") + streamKey := declaredProcessStreamKey(assignment, stream) + logStreamID := logStreamIDForAssignment(assignment, streamKey) + entry := protocol.LogEntry{Timestamp: time.Now().UTC(), Level: "info", Line: redactedLine, Redacted: redacted} + source := "process" + if strings.HasPrefix(stream, "management-program.") { + source = "management-program" + } else if assignment.Capability == protocol.RunCapabilityLogsBackfill { + source = "file" + } + var recoverProgress func(context.Context, string) (uint64, error) + runStreamPrefix := "run." + assignment.RunEndpointID + "." + assignment.ServerInstanceID + "." + if sink.Progress != nil && strings.HasPrefix(logStreamID, runStreamPrefix) { + recoverProgress = func(ctx context.Context, streamID string) (uint64, error) { + return sink.Progress(ctx, assignment.ServerInstanceID, streamID) + } + } + if strings.TrimSpace(assignment.LogSessionID) != "" { + // A generation-scoped stream is globally fresh, so its first durable + // append must not depend on platform availability or the worker ctx. + recoverProgress = nil + } + _, _, err := sink.Spool.EnqueueNextAggregated(ctx, protocol.LogBatchIngestRequest{ + RunEndpointID: sink.RunEndpointID, + SessionToken: sink.SessionToken, + LogStreamID: logStreamID, + ServerInstanceID: assignment.ServerInstanceID, + StreamKey: streamKey, + Source: source, + LogSessionID: assignment.LogSessionID, + SessionStartedAt: assignment.SessionStartedAt, + Entries: []protocol.LogEntry{entry}, + }, cursor, recoverProgress, checksumForLogEntries) + return err +} + +func logStreamIDForAssignment(assignment protocol.RunJobAssignment, streamKey string) string { + if strings.TrimSpace(assignment.LogSessionID) != "" { + return fmt.Sprintf("run.%s.%s.%s.%s", assignment.RunEndpointID, assignment.ServerInstanceID, assignment.LogSessionID, streamKey) + } + if autonomousLifecycleLogAssignment(assignment) { + return fmt.Sprintf("run.%s.%s.%s", assignment.RunEndpointID, assignment.ServerInstanceID, streamKey) + } + return fmt.Sprintf("job.%s.%s", assignment.JobID, streamKey) +} + +func autonomousLifecycleLogAssignment(assignment protocol.RunJobAssignment) bool { + return strings.HasPrefix(assignment.IdempotencyKey, "autonomous:") || strings.HasPrefix(assignment.LeaseToken, "local-autonomous-") || strings.HasPrefix(assignment.JobID, "autonomous-") +} + +func declaredProcessStreamKey(assignment protocol.RunJobAssignment, stream string) string { + kind := "" + if stream == "stdout" { + kind = "process.stdout" + } else if stream == "stderr" { + kind = "process.stderr" + } + if kind != "" { + for _, source := range assignment.ExecutionInput.LogSources { + if source.Kind == kind && strings.TrimSpace(source.StreamKey) != "" { + return source.StreamKey + } + } + } + return stream +} + +type QueueArtifactHook struct { + RunEndpointID string + SessionToken string + Queue spool.ArtifactQueue +} + +func (hook QueueArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, result ProcessResult) (string, error) { + ref := fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID) + payload := []byte(RedactText(result.Stdout + result.Stderr)) + if len(payload) == 0 { + payload = []byte("lifecycle result metadata") + } + artifactID := "artifact-" + assignment.JobID + "-lifecycle" + if err := hook.Queue.Enqueue(protocol.ArtifactChunkUploadRequest{ + RunEndpointID: hook.RunEndpointID, + SessionToken: hook.SessionToken, + TransferID: "transfer-" + assignment.JobID, + ArtifactID: artifactID, + ChunkIndex: 0, + Offset: 0, + SizeBytes: len(payload), + Checksum: checksumForText(string(payload)), + Payload: payload, + }); err != nil { + return "", err + } + return ref, nil +} + +func capabilityFingerprint(capabilities []string) string { + return checksumForText(strings.Join(capabilities, ",")) +} + +func durationOrDefault(value time.Duration, fallback time.Duration) time.Duration { + if value <= 0 { + return fallback + } + return value +} + +func boundedRetryBackoff(value time.Duration) time.Duration { + value = durationOrDefault(value, time.Second) + if value > 30*time.Second { + return 30 * time.Second + } + return value +} diff --git a/runtime/worker_test.go b/runtime/worker_test.go new file mode 100644 index 0000000..d97c7d1 --- /dev/null +++ b/runtime/worker_test.go @@ -0,0 +1,1063 @@ +package runtime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "runtime" + "strings" + "sync" + "testing" + "time" + + "browser.local/run/api" + "browser.local/run/config" + "browser.local/run/protocol" + "browser.local/run/spool" +) + +func TestWorkerRegistersHeartbeatsAndStoresSession(t *testing.T) { + client := newFakeWorkerClient() + worker, err := NewWorker(workerTestConfig(t), client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if worker.State().SessionToken != "session-token" { + t.Fatalf("expected session token stored, got %+v", worker.State()) + } + if len(client.helloRequests) != 1 || client.helloRequests[0].RegistrationToken != "registration-token" || len(client.helloRequests[0].CapabilityReport.Capabilities) == 0 { + t.Fatalf("unexpected hello request: %+v", client.helloRequests) + } + + if err := worker.HeartbeatOnce(context.Background()); err != nil { + t.Fatalf("heartbeat: %v", err) + } + if len(client.heartbeatRequests) != 1 { + t.Fatalf("expected heartbeat request") + } + heartbeat := client.heartbeatRequests[0] + if heartbeat.SessionToken != "session-token" || heartbeat.Capacity.MaxJobs != 2 || heartbeat.Capacity.RunningJobs != 0 { + t.Fatalf("unexpected heartbeat request: %+v", heartbeat) + } + for _, forbidden := range []string{"/Users/", "unix://", "Bearer ", "sk-", "password=", "artifact", "log"} { + if containsText(heartbeat.Capacity.Summary, forbidden) { + t.Fatalf("heartbeat summary exposed forbidden fragment %q: %+v", forbidden, heartbeat) + } + } +} + +func TestManagedProcessLogStreamIsScopedToItsGeneration(t *testing.T) { + assignment := protocol.RunJobAssignment{RunEndpointID: "run-test", ServerInstanceID: "server-test", JobID: "job-test", LogSessionID: "generation-2"} + if got, want := logStreamIDForAssignment(assignment, "game.console.stdout"), "run.run-test.server-test.generation-2.game.console.stdout"; got != want { + t.Fatalf("expected session-scoped supervised stream, got %q", got) + } + assignment.LogSessionID = "" + if got, want := logStreamIDForAssignment(assignment, "game.console.stdout"), "job.job-test.game.console.stdout"; got != want { + t.Fatalf("expected non-session job stream to retain legacy ID, got %q", got) + } +} + +func TestWorkerRenewsExpiredOrUnauthorizedSession(t *testing.T) { + client := newFakeWorkerClient() + worker, err := NewWorker(workerTestConfig(t), client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + worker.state.SessionExpiresAt = time.Now().UTC().Add(30 * time.Second) + if err := worker.HeartbeatOnce(context.Background()); err != nil { + t.Fatalf("renew expiring session: %v", err) + } + if len(client.helloRequests) != 2 || len(client.heartbeatRequests) != 0 { + t.Fatalf("expected proactive hello renewal, hello=%d heartbeat=%d", len(client.helloRequests), len(client.heartbeatRequests)) + } + + client.heartbeatErr = api.PlatformRequestError{Status: http.StatusUnauthorized} + worker.state.SessionExpiresAt = time.Time{} + if err := worker.HeartbeatOnce(context.Background()); err != nil { + t.Fatalf("recover unauthorized session: %v", err) + } + if len(client.helloRequests) != 3 || worker.State().SessionToken == "" { + t.Fatalf("expected unauthorized heartbeat to re-register: state=%+v hello=%d", worker.State(), len(client.helloRequests)) + } + + client.heartbeatErr = api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"sessionToken is invalid"}} + if err := worker.HeartbeatOnce(context.Background()); err != nil { + t.Fatalf("recover legacy invalid session: %v", err) + } + if len(client.helloRequests) != 4 || worker.State().SessionToken == "" { + t.Fatalf("expected legacy invalid session to re-register: state=%+v hello=%d", worker.State(), len(client.helloRequests)) + } +} + +func TestWorkerReRegistersWhenClaimReportsInvalidSession(t *testing.T) { + client := newFakeWorkerClient() + client.claimErr = api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"sessionToken is invalid"}} + worker, err := NewWorker(workerTestConfig(t), client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || handled { + t.Fatalf("claim session renewal handled=%v err=%v", handled, err) + } + if len(client.helloRequests) != 2 || len(client.reconcileRequests) != 1 { + t.Fatalf("expected claim session renewal, hello=%d reconcile=%d", len(client.helloRequests), len(client.reconcileRequests)) + } +} + +func TestWorkerSerializesConcurrentSessionRefresh(t *testing.T) { + client := newRotatingSessionClient() + worker, err := NewWorker(workerTestConfig(t), client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + observedToken := worker.State().SessionToken + + start := make(chan struct{}) + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + go func() { + <-start + errs <- worker.reregisterAndReconcile(context.Background(), "test_concurrent_refresh", observedToken) + }() + } + close(start) + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("refresh %d: %v", i, err) + } + } + if client.HelloCount() != 2 { + t.Fatalf("expected initial registration plus one refresh, got %d hello calls", client.HelloCount()) + } + if got := worker.State().SessionToken; got != "session-token-2" { + t.Fatalf("expected refreshed session token, got %q", got) + } + if len(client.reconcileRequests) != 2 { + t.Fatalf("expected both refresh callers to reconcile, got %d", len(client.reconcileRequests)) + } +} + +func TestWorkerClaimsAcksProgressAndCompletesJob(t *testing.T) { + client := newFakeWorkerClient() + client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart) + client.claimJob.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Key: "game-console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}} + worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(staticSupervisor{stdout: "server ready\n"})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + handled, err := worker.ClaimAndRunOnce(context.Background()) + if err != nil || !handled { + t.Fatalf("claim/run handled=%v err=%v", handled, err) + } + if len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || len(client.resultRequests) != 1 || len(client.cancelPollRequests) != 1 { + t.Fatalf("expected ack/progress/result/cancel calls, got ack=%d progress=%d result=%d cancel=%d", len(client.ackRequests), len(client.progressRequests), len(client.resultRequests), len(client.cancelPollRequests)) + } + if client.progressRequests[0].Progress.Percent != 10 || client.resultRequests[0].State != "succeeded" || client.resultRequests[0].ResultRef == "" { + t.Fatalf("unexpected job channel payloads: progress=%+v result=%+v", client.progressRequests[0], client.resultRequests[0]) + } + if worker.journal.ActiveCount() != 0 { + t.Fatalf("expected terminal job removed from journal") + } +} + +func TestWorkerDispatchesSelfUpdateJob(t *testing.T) { + client := newFakeWorkerClient() + assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate) + assignment.TargetKey = "run/update" + assignment.InputRef = "artifact://artifact-run-latest" + client.claimJob = assignment + executableName := "run" + if runtime.GOOS == "windows" { + executableName = "run.exe" + } + client.updatePayload = []byte("staged run binary") + client.updateInput = protocol.RunUpdateInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, ArtifactID: "artifact-run-latest", Checksum: bytesChecksum(client.updatePayload), SizeBytes: int64(len(client.updatePayload)), TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, PackageFormat: "raw-executable", ExecutableName: executableName, TargetRelease: "run-release-test", ChunkSizeBytes: 64} + activator := &recordingSelfUpdateActivator{} + worker, err := NewWorker(workerTestConfig(t), client, WithSelfUpdateActivator(activator)) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + handled, err := worker.ClaimAndRunOnce(context.Background()) + if err != nil || !handled { + t.Fatalf("claim/run handled=%v err=%v", handled, err) + } + if len(client.resultRequests) != 1 || client.resultRequests[0].State != "succeeded" || !strings.Contains(client.resultRequests[0].ResultRef, "run-update-staged") { + t.Fatalf("expected self-update result, got %+v", client.resultRequests) + } + if activator.manifestPath == "" { + t.Fatal("expected activation only after accepted terminal result") + } +} + +func TestWorkerRegistersPackageIdentity(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.RegistrationToken = "current-run-key" + cfg.ServerInstanceID = "server-worker" + cfg.PluginID = "game.minecraft" + cfg.ComponentKind = "run" + cfg.KeyGeneration = 7 + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + hello := client.helloRequests[0] + if hello.RegistrationToken != "current-run-key" || hello.ServerInstanceID != "server-worker" || hello.ComponentKind != "run" || hello.KeyGeneration != 7 { + t.Fatalf("expected package identity in hello request, got %+v", hello) + } +} + +func TestWorkerReportsSelfUpdateHealthOnlyAfterRegistrationAndReconciliation(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.Version = "run-release-2" + cfg.UpdateJobID = "job-update-health" + cfg.UpdateOutcome = "succeeded" + cfg.UpdateAttempt = 2 + cfg.UpdateLeaseToken = "lease-update-health" + worker, err := NewWorker(cfg, client) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if len(client.helloRequests) != 1 || client.helloRequests[0].UpdateOutcome != "" || len(client.updateHealthReports) != 0 { + t.Fatalf("hello must not report update success before reconciliation: hello=%+v reports=%+v", client.helloRequests, client.updateHealthReports) + } + if err := worker.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if err := worker.reportRunUpdateHealth(context.Background()); err != nil { + t.Fatalf("report reconciled update health: %v", err) + } + if len(client.updateHealthReports) != 1 { + t.Fatalf("expected one update health report, got %+v", client.updateHealthReports) + } + report := client.updateHealthReports[0] + if report.SessionToken != "session-token" || report.JobID != cfg.UpdateJobID || report.Attempt != cfg.UpdateAttempt || report.LeaseToken != cfg.UpdateLeaseToken || report.Version != cfg.Version || report.Outcome != "succeeded" { + t.Fatalf("unexpected update health report: %+v", report) + } +} + +func TestWorkerHandlesCancellationAndReconcile(t *testing.T) { + client := newFakeWorkerClient() + client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart) + client.cancelResponse = protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", HasCancel: true, JobID: "job-worker", Reason: "operator requested", ServerTime: workerTestTime()} + worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(blockingSupervisor{})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + + handled, err := worker.ClaimAndRunOnce(context.Background()) + if err != nil || !handled { + t.Fatalf("claim/run handled=%v err=%v", handled, err) + } + if len(client.resultRequests) != 1 || client.resultRequests[0].State != "cancelled" || client.resultRequests[0].ErrorCode != "lifecycle_cancelled" { + t.Fatalf("expected cancelled terminal result, got %+v", client.resultRequests) + } + + worker.journal.MarkActive(workerJobAssignment(protocol.RunCapabilityProcessStart)) + client.reconcileResponse = protocol.RunJobReconcileResponse{ + Accepted: true, + RunEndpointID: "run-test", + ConfirmedJobs: []protocol.RunJobAssignment{workerJobAssignment(protocol.RunCapabilityProcessStop)}, + DiscardJobIDs: []string{"job-worker"}, + ServerTime: workerTestTime(), + } + if err := worker.ReconcileOnce(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if ids := worker.journal.ActiveJobIDs(); !reflect.DeepEqual(ids, []string{"job-worker-stop"}) { + t.Fatalf("expected reconcile to replace active job ids, got %+v", ids) + } +} + +func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) { + client := newFakeWorkerClient() + client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart) + client.claimJob.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Key: "game-console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}} + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("log spool: %v", err) + } + artifactQueue, err := spool.NewArtifactQueue(t.TempDir()) + if err != nil { + t.Fatalf("artifact queue: %v", err) + } + worker, err := NewWorker( + workerTestConfig(t), + client, + WithProcessSupervisor(staticSupervisor{stdout: "started password=hidden\n"}), + WithProcessLogSink(&SpoolLogSink{Spool: logSpool}), + WithLifecycleArtifactHook(&QueueArtifactHook{Queue: artifactQueue}), + ) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if _, err := worker.ClaimAndRunOnce(context.Background()); err != nil { + t.Fatalf("claim/run: %v", err) + } + logs, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || logs[0].StreamKey != "game.console.stdout" || containsText(logs[0].Entries[0].Line, "password=hidden") { + t.Fatalf("unexpected spooled logs: %+v", logs) + } + chunks, err := artifactQueue.Pending() + if err != nil { + t.Fatalf("pending artifact chunks: %v", err) + } + if len(chunks) != 1 || chunks[0].RunEndpointID != "run-test" || chunks[0].SessionToken != "session-token" { + t.Fatalf("unexpected artifact chunks: %+v", chunks) + } +} + +func TestSpoolLogSinkUsesRunScopedStreamForAutonomousLifecycle(t *testing.T) { + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("log spool: %v", err) + } + sink := &SpoolLogSink{Spool: logSpool} + assignment := protocol.RunJobAssignment{ + JobID: "autonomous-bootstrap-start", + RunEndpointID: "run-test", + ServerInstanceID: "server-worker", + Capability: protocol.RunCapabilityProcessStart, + IdempotencyKey: "autonomous:release:start", + LeaseToken: "local-autonomous-bootstrap", + ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{ + {Key: "game-console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}, + }}, + } + + if err := sink.Append(context.Background(), assignment, "stdout", "autonomous output"); err != nil { + t.Fatalf("append autonomous log: %v", err) + } + logs, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(logs) != 1 || logs[0].LogStreamID != "run.run-test.server-worker.game.console.stdout" || logs[0].StreamKey != "game.console.stdout" { + t.Fatalf("expected run-scoped autonomous stream, got %+v", logs) + } +} + +func TestSpoolLogSinkDurablyAppendsFreshSessionWithCanceledWorkerContext(t *testing.T) { + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("log spool: %v", err) + } + progressCalls := 0 + sink := &SpoolLogSink{RunEndpointID: "run-test", SessionToken: "session-token", Spool: logSpool, Progress: func(context.Context, string, string) (uint64, error) { + progressCalls++ + return 0, context.Canceled + }} + assignment := protocol.RunJobAssignment{ + JobID: "managed-start", + RunEndpointID: "run-test", + ServerInstanceID: "server-worker", + Capability: protocol.RunCapabilityProcessStart, + LogSessionID: "generation-current", + SessionStartedAt: workerTestTime(), + ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{ + {Kind: "process.stdout", StreamKey: "game.console.stdout"}, + }}, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := sink.Append(ctx, assignment, "stdout", "output during shutdown"); err != nil { + t.Fatalf("append fresh session with canceled worker context: %v", err) + } + if progressCalls != 0 { + t.Fatalf("fresh generation unexpectedly depended on remote progress: calls=%d", progressCalls) + } + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(pending) != 1 || pending[0].FirstSeq != 1 || pending[0].LogSessionID != assignment.LogSessionID || pending[0].Entries[0].Line != "output during shutdown" { + t.Fatalf("expected durable session batch despite canceled context, got %+v", pending) + } +} + +func TestGenericWorkerJobLogDoesNotUseRunStreamProgress(t *testing.T) { + client := newFakeWorkerClient() + cfg := workerTestConfig(t) + cfg.ServerInstanceID = "" + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + sink := &SpoolLogSink{Spool: logSpool} + worker, err := NewWorker(cfg, client, WithProcessLogSink(sink)) + if err != nil { + t.Fatalf("new generic worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register generic worker: %v", err) + } + assignment := protocol.RunJobAssignment{JobID: "job-generic", RunEndpointID: cfg.RunEndpointID, ServerInstanceID: "server-from-job", Capability: protocol.RunCapabilityProcessStart, ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "game.console.stdout"}}}} + if err := sink.Append(context.Background(), assignment, "stdout", "generic job output"); err != nil { + t.Fatalf("append generic job output: %v", err) + } + if len(client.logProgressRequests) != 0 { + t.Fatalf("job-scoped stream must not use run-scoped progress recovery: %+v", client.logProgressRequests) + } + pending, err := logSpool.Pending() + if err != nil || len(pending) != 1 || pending[0].FirstSeq != 1 || pending[0].ServerInstanceID != assignment.ServerInstanceID { + t.Fatalf("generic job output did not enter the local spool: pending=%+v err=%v", pending, err) + } +} + +func TestManagedProcessObservationIDBindsLogSession(t *testing.T) { + identity := ProcessIdentity{LogSessionID: "generation-current", RunEndpointID: "run-test", ServerInstanceID: "server-worker"} + if got := managedProcessObservationID(identity); got != "log-session:generation-current" { + t.Fatalf("managed process observation did not bind the log session: %q", got) + } +} + +func TestSpoolLogSinkReconcilesStableStreamBeforeAllocating(t *testing.T) { + logSpool, err := spool.NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("log spool: %v", err) + } + client := newFakeWorkerClient() + client.progressResponse.LatestSeq = 3816 + sink := &SpoolLogSink{Spool: logSpool, RunEndpointID: "run-test", SessionToken: "session-token", Progress: func(ctx context.Context, serverInstanceID string, streamID string) (uint64, error) { + response, err := client.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{RunEndpointID: "run-test", SessionToken: "session-token", ServerInstanceID: serverInstanceID, LogStreamID: streamID}) + return response.LatestSeq, err + }} + assignment := protocol.RunJobAssignment{ + JobID: "autonomous-bootstrap-start", + RunEndpointID: "run-test", + ServerInstanceID: "server-worker", + Capability: protocol.RunCapabilityProcessStart, + IdempotencyKey: "autonomous:release:start", + LeaseToken: "local-autonomous-bootstrap", + ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "scum.console.stdout"}}}, + } + + if err := sink.Append(context.Background(), assignment, "stdout", "current output"); err != nil { + t.Fatalf("append: %v", err) + } + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(pending) != 1 || pending[0].FirstSeq != 3817 || pending[0].LastSeq != 3817 { + t.Fatalf("expected reconciled sequence 3817, got %+v", pending) + } + if len(client.logProgressRequests) != 1 || client.logProgressRequests[0].LogStreamID != "run.run-test.server-worker.scum.console.stdout" || client.logProgressRequests[0].ServerInstanceID != assignment.ServerInstanceID { + t.Fatalf("expected one scoped progress query, got %+v", client.logProgressRequests) + } +} + +func TestSpoolLogSinkKeepsSequencesIndependentAndDurable(t *testing.T) { + root := t.TempDir() + logSpool, err := spool.NewLogSpool(root) + if err != nil { + t.Fatalf("log spool: %v", err) + } + sink := &SpoolLogSink{Spool: logSpool} + assignment := protocol.RunJobAssignment{JobID: "autonomous-bootstrap-start", RunEndpointID: "run-test", ServerInstanceID: "server-worker", Capability: protocol.RunCapabilityProcessStart, IdempotencyKey: "autonomous:release:start", LeaseToken: "local-autonomous-bootstrap", ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "stdout"}, {Kind: "process.stderr", StreamKey: "stderr"}}}} + for _, item := range []struct { + stream string + line string + }{{"stdout", "out-1"}, {"stderr", "err-1"}, {"stdout", "out-2"}} { + if err := sink.Append(context.Background(), assignment, item.stream, item.line); err != nil { + t.Fatalf("append %s: %v", item.stream, err) + } + } + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(pending) != 2 || pending[0].StreamKey != "stderr" || pending[0].FirstSeq != 1 || pending[0].LastSeq != 1 || pending[1].StreamKey != "stdout" || pending[1].FirstSeq != 1 || pending[1].LastSeq != 2 || len(pending[1].Entries) != 2 { + t.Fatalf("expected independent per-stream sequences, got %+v", pending) + } + client := &recordingDurableLogClient{} + if _, err := logSpool.Flush(context.Background(), client); err != nil { + t.Fatalf("flush logs: %v", err) + } + restarted, err := spool.NewLogSpool(root) + if err != nil { + t.Fatalf("reopen log spool: %v", err) + } + restartedSink := &SpoolLogSink{Spool: restarted} + if err := restartedSink.Append(context.Background(), assignment, "stdout", "out-3"); err != nil { + t.Fatalf("append after restart: %v", err) + } + pending, err = restarted.Pending() + if err != nil { + t.Fatalf("pending restarted logs: %v", err) + } + if len(pending) != 1 || pending[0].FirstSeq != 3 { + t.Fatalf("expected durable stdout sequence 3 after restart, got %+v", pending) + } +} + +func TestSessionLogBatchClientOverridesSpooledIdentityAndChecksum(t *testing.T) { + recorder := &recordingDurableLogClient{} + entry := protocol.LogEntry{Seq: 7, Timestamp: workerTestTime(), Level: "info", Line: "server ready", Redacted: false} + batch := protocol.LogBatchIngestRequest{ + RunEndpointID: "old-endpoint", + SessionToken: "old-token", + LogStreamID: "job.job-1.stdout", + ServerInstanceID: "server-worker", + StreamKey: "stdout", + Source: "process", + FirstSeq: 7, + LastSeq: 7, + Checksum: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + Entries: []protocol.LogEntry{entry}, + } + client := sessionLogBatchClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"} + response, err := client.IngestLogBatch(context.Background(), batch) + if err != nil || !response.Accepted { + t.Fatalf("ingest through session client accepted=%t err=%v", response.Accepted, err) + } + if recorder.batch.RunEndpointID != "run-current" || recorder.batch.SessionToken != "token-current" { + t.Fatalf("expected current identity, got %+v", recorder.batch) + } + expectedChecksum, err := checksumForLogEntries([]protocol.LogEntry{entry}) + if err != nil { + t.Fatalf("checksum: %v", err) + } + if recorder.batch.Checksum != expectedChecksum { + t.Fatalf("expected recomputed checksum %s, got %s", expectedChecksum, recorder.batch.Checksum) + } +} + +func TestSessionLogBatchClientQuarantinesSequenceGap(t *testing.T) { + recorder := &recordingDurableLogClient{err: api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"log batch firstSeq must follow latest acknowledged sequence"}}} + client := sessionLogBatchClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"} + _, err := client.IngestLogBatch(context.Background(), protocol.LogBatchIngestRequest{LogStreamID: "job.example.stdout", FirstSeq: 2256, LastSeq: 2256, Entries: []protocol.LogEntry{{Seq: 2256, Timestamp: workerTestTime(), Line: "line"}}}) + var permanent spool.PermanentLogBatchError + if !errors.As(err, &permanent) || permanent.Reason != "platform_sequence_gap" { + t.Fatalf("expected permanent platform_sequence_gap rejection, got %#v", err) + } +} + +func TestSessionLogBatchClientQuarantinesSequenceConflict(t *testing.T) { + recorder := &recordingDurableLogClient{err: api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"log batch conflicts with acknowledged range"}}} + client := sessionLogBatchClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"} + _, err := client.IngestLogBatch(context.Background(), protocol.LogBatchIngestRequest{LogStreamID: "run.run-current.server-worker.scum.console.stdout", FirstSeq: 1, LastSeq: 1, Entries: []protocol.LogEntry{{Seq: 1, Timestamp: workerTestTime(), Line: "line"}}}) + var permanent spool.PermanentLogBatchError + if !errors.As(err, &permanent) || permanent.Reason != "platform_acknowledged_range_conflict" { + t.Fatalf("expected permanent platform_acknowledged_range_conflict rejection, got %#v", err) + } +} + +func TestSessionLogBatchClientQuarantinesSessionMetadataMismatch(t *testing.T) { + recorder := &recordingDurableLogClient{err: api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"log session metadata must match stream"}}} + client := sessionLogBatchClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"} + _, err := client.IngestLogBatch(context.Background(), protocol.LogBatchIngestRequest{LogStreamID: "run.run-current.server-worker.session.stdout", LogSessionID: "session-old", SessionStartedAt: workerTestTime(), FirstSeq: 1, LastSeq: 1, Entries: []protocol.LogEntry{{Seq: 1, Timestamp: workerTestTime(), Line: "line"}}}) + var permanent spool.PermanentLogBatchError + if !errors.As(err, &permanent) || permanent.Reason != "session_metadata_mismatch" { + t.Fatalf("expected permanent session_metadata_mismatch rejection, got %#v", err) + } +} + +func TestSessionArtifactChunkClientOverridesSpooledIdentity(t *testing.T) { + recorder := &recordingDurableArtifactClient{} + chunk := protocol.ArtifactChunkUploadRequest{RunEndpointID: "old-endpoint", SessionToken: "old-token", TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 2} + client := sessionArtifactChunkClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"} + response, err := client.UploadArtifactChunk(context.Background(), chunk) + if err != nil || !response.Accepted { + t.Fatalf("upload through session client accepted=%t err=%v", response.Accepted, err) + } + if recorder.chunk.RunEndpointID != "run-current" || recorder.chunk.SessionToken != "token-current" { + t.Fatalf("expected current identity, got %+v", recorder.chunk) + } +} + +func TestWorkerRetryBackoffIsBounded(t *testing.T) { + if got := boundedRetryBackoff(75 * time.Millisecond); got != 75*time.Millisecond { + t.Fatalf("expected configured backoff, got %s", got) + } + if got := boundedRetryBackoff(time.Minute); got != 30*time.Second { + t.Fatalf("expected capped backoff, got %s", got) + } +} + +func TestWorkerIntegrationWithPlatformLikeServer(t *testing.T) { + assignment := workerJobAssignment(protocol.RunCapabilityProcessStart) + seen := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.URL.Path) + switch r.URL.Path { + case "/api/v1/run/control/hello": + var request protocol.RunHelloRequest + decodeWorkerTestJSON(t, r, &request) + if request.RunEndpointID != "run-test" || request.Capacity.MaxJobs != 2 { + t.Fatalf("unexpected hello: %+v", request) + } + writeWorkerTestJSON(t, w, protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", HeartbeatIntervalSeconds: 15, ServerTime: workerTestTime()}) + case "/api/v1/run/control/heartbeat": + var request protocol.RunHeartbeatRequest + decodeWorkerTestJSON(t, r, &request) + if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 0 { + t.Fatalf("unexpected heartbeat: %+v", request) + } + writeWorkerTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()}) + case "/api/v1/run/jobs/claim": + var request protocol.RunJobClaimRequest + decodeWorkerTestJSON(t, r, &request) + if request.SessionToken != "session-token" || len(request.Capabilities) == 0 { + t.Fatalf("unexpected claim: %+v", request) + } + writeWorkerTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: workerTestTime()}) + case "/api/v1/run/jobs/ack": + var request protocol.RunJobAckRequest + decodeWorkerTestJSON(t, r, &request) + if request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken { + t.Fatalf("unexpected ack: %+v", request) + } + assignment.State = "running" + writeWorkerTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()}) + case "/api/v1/run/jobs/progress": + var request protocol.RunJobProgressRequest + decodeWorkerTestJSON(t, r, &request) + if request.Progress.Percent != 10 { + t.Fatalf("unexpected progress: %+v", request) + } + assignment.Progress = request.Progress + writeWorkerTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()}) + case "/api/v1/run/jobs/cancel": + var request protocol.RunJobCancelPollRequest + decodeWorkerTestJSON(t, r, &request) + if request.JobID != assignment.JobID { + t.Fatalf("unexpected cancel poll: %+v", request) + } + writeWorkerTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ServerTime: workerTestTime()}) + case "/api/v1/run/jobs/result": + var request protocol.RunJobResultRequest + decodeWorkerTestJSON(t, r, &request) + if request.State != "succeeded" || request.ResultRef == "" { + t.Fatalf("unexpected result: %+v", request) + } + assignment.State = request.State + assignment.ResultRef = request.ResultRef + writeWorkerTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := api.NewPlatformClient(server.URL) + if err != nil { + t.Fatalf("platform client: %v", err) + } + cfg := workerTestConfig(t) + cfg.PlatformURL = server.URL + worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "integration ok\n"})) + if err != nil { + t.Fatalf("new worker: %v", err) + } + if err := worker.Register(context.Background()); err != nil { + t.Fatalf("register: %v", err) + } + if err := worker.HeartbeatOnce(context.Background()); err != nil { + t.Fatalf("heartbeat: %v", err) + } + if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled { + t.Fatalf("claim/run handled=%v err=%v", handled, err) + } + expected := []string{ + "/api/v1/run/control/hello", + "/api/v1/run/control/heartbeat", + "/api/v1/run/jobs/claim", + "/api/v1/run/jobs/ack", + "/api/v1/run/jobs/progress", + "/api/v1/run/jobs/cancel", + "/api/v1/run/jobs/result", + } + if !reflect.DeepEqual(seen, expected) { + t.Fatalf("unexpected platform flow: %+v", seen) + } +} + +type fakeWorkerClient struct { + helloRequests []protocol.RunHelloRequest + heartbeatRequests []protocol.RunHeartbeatRequest + lifecycleReports []protocol.RunLifecycleReportRequest + claimRequests []protocol.RunJobClaimRequest + ackRequests []protocol.RunJobAckRequest + progressRequests []protocol.RunJobProgressRequest + resultRequests []protocol.RunJobResultRequest + cancelPollRequests []protocol.RunJobCancelPollRequest + reconcileRequests []protocol.RunJobReconcileRequest + logProgressRequests []protocol.RunLogStreamProgressRequest + metricRequests []protocol.MetricBatchIngestRequest + claimJob protocol.RunJobAssignment + claimErr error + cancelResponse protocol.RunJobCancelPollResponse + reconcileResponse protocol.RunJobReconcileResponse + progressResponse protocol.RunLogStreamProgressResponse + buildInput protocol.DistributionBuildInputResponse + dependencyInput protocol.DependencyExecutionInputResponse + sourceRCONInput protocol.SourceRCONExecutionInputResponse + sourceRCONRequests []protocol.SourceRCONExecutionInputRequest + sourceRCONInputErr error + protectedInput protocol.ProtectedRequestExecutionInputResponse + protectedRequests []protocol.ProtectedRequestExecutionInputRequest + protectedInputErr error + updateInput protocol.RunUpdateInputResponse + updatePayload []byte + updateChunkOffsets []int64 + updateHealthReports []protocol.RunUpdateHealthRequest + artifactPayload []byte + artifactOpenRequests []protocol.ArtifactTransferOpenRequest + artifactTransfer string + heartbeatErr error + lifecycleReportErr error + logStreamProgress uint64 + resultErr error + metricErr error +} + +type recordingSelfUpdateActivator struct { + manifestPath string + err error +} + +func (activator *recordingSelfUpdateActivator) Activate(path string) error { + activator.manifestPath = path + return activator.err +} + +func newFakeWorkerClient() *fakeWorkerClient { + return &fakeWorkerClient{ + cancelResponse: protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()}, + reconcileResponse: protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()}, + progressResponse: protocol.RunLogStreamProgressResponse{Accepted: true, RunEndpointID: "run-test", ServerInstanceID: "server-worker", ServerTime: workerTestTime()}, + } +} + +type rotatingSessionClient struct { + *fakeWorkerClient + mu sync.Mutex + helloCount int +} + +func newRotatingSessionClient() *rotatingSessionClient { + return &rotatingSessionClient{fakeWorkerClient: newFakeWorkerClient()} +} + +func (client *rotatingSessionClient) Hello(_ context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) { + client.mu.Lock() + defer client.mu.Unlock() + client.helloCount++ + client.helloRequests = append(client.helloRequests, request) + return protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: fmt.Sprintf("session-token-%d", client.helloCount), ServerTime: workerTestTime(), HeartbeatIntervalSeconds: 15}, nil +} + +func (client *rotatingSessionClient) HelloCount() int { + client.mu.Lock() + defer client.mu.Unlock() + return client.helloCount +} + +type recordingDurableLogClient struct { + batch protocol.LogBatchIngestRequest + err error +} + +func (client *recordingDurableLogClient) IngestLogBatch(_ context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + client.batch = batch + if client.err != nil { + return protocol.LogBatchIngestResponse{}, client.err + } + return protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: batch.LogStreamID, AcceptedFrom: batch.FirstSeq, AcceptedTo: batch.LastSeq}, nil +} + +type recordingDurableArtifactClient struct { + chunk protocol.ArtifactChunkUploadRequest +} + +func (client *recordingDurableArtifactClient) UploadArtifactChunk(_ context.Context, chunk protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) { + client.chunk = chunk + return protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: chunk.TransferID, ArtifactID: chunk.ArtifactID, ChunkIndex: chunk.ChunkIndex}, nil +} + +func (client *fakeWorkerClient) Hello(_ context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) { + client.helloRequests = append(client.helloRequests, request) + return protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", ServerTime: workerTestTime(), HeartbeatIntervalSeconds: 15}, nil +} + +func (client *fakeWorkerClient) Heartbeat(_ context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) { + client.heartbeatRequests = append(client.heartbeatRequests, request) + if client.heartbeatErr != nil { + err := client.heartbeatErr + client.heartbeatErr = nil + return protocol.RunHeartbeatResponse{}, err + } + return protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) ReportLifecycle(_ context.Context, request protocol.RunLifecycleReportRequest) (protocol.RunLifecycleReportResponse, error) { + client.lifecycleReports = append(client.lifecycleReports, request) + if client.lifecycleReportErr != nil { + return protocol.RunLifecycleReportResponse{}, client.lifecycleReportErr + } + return protocol.RunLifecycleReportResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ServerInstanceID: request.ServerInstanceID, ProjectedState: request.ExecutionResult.ProcessState, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) ClaimJob(_ context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) { + client.claimRequests = append(client.claimRequests, request) + if client.claimErr != nil { + return protocol.RunJobClaimResponse{}, client.claimErr + } + if client.claimJob.JobID == "" { + return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: false, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil + } + job := client.claimJob + return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) AckJob(_ context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) { + client.ackRequests = append(client.ackRequests, request) + job := client.claimJob + job.State = "running" + return protocol.RunJobAckResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) UpdateJobProgress(_ context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) { + client.progressRequests = append(client.progressRequests, request) + job := client.claimJob + job.Progress = request.Progress + job.ProgressSequence = request.Sequence + return protocol.RunJobProgressResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) CompleteJob(_ context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) { + client.resultRequests = append(client.resultRequests, request) + if client.resultErr != nil { + return protocol.RunJobResultResponse{}, client.resultErr + } + job := client.claimJob + job.State = request.State + job.Progress = request.Progress + job.ResultRef = request.ResultRef + return protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) GetDistributionBuildInput(_ context.Context, request protocol.DistributionBuildInputRequest) (protocol.DistributionBuildInputResponse, error) { + if client.buildInput.JobID == "" { + return protocol.DistributionBuildInputResponse{}, fmt.Errorf("distribution build input is not configured") + } + if request.JobID != client.buildInput.JobID { + return protocol.DistributionBuildInputResponse{}, fmt.Errorf("unexpected build job") + } + return client.buildInput, nil +} + +func (client *fakeWorkerClient) GetDependencyExecutionInput(_ context.Context, request protocol.DependencyExecutionInputRequest) (protocol.DependencyExecutionInputResponse, error) { + if client.dependencyInput.JobID == "" || request.JobID != client.dependencyInput.JobID { + return protocol.DependencyExecutionInputResponse{}, fmt.Errorf("dependency input is not configured") + } + return client.dependencyInput, nil +} + +func (client *fakeWorkerClient) GetSourceRCONExecutionInput(_ context.Context, request protocol.SourceRCONExecutionInputRequest) (protocol.SourceRCONExecutionInputResponse, error) { + client.sourceRCONRequests = append(client.sourceRCONRequests, request) + if client.sourceRCONInputErr != nil { + return protocol.SourceRCONExecutionInputResponse{}, client.sourceRCONInputErr + } + if client.sourceRCONInput.JobID == "" || request.JobID != client.sourceRCONInput.JobID { + return protocol.SourceRCONExecutionInputResponse{}, fmt.Errorf("Source RCON input is not configured") + } + return client.sourceRCONInput, nil +} + +func (client *fakeWorkerClient) GetProtectedRequestExecutionInput(_ context.Context, request protocol.ProtectedRequestExecutionInputRequest) (protocol.ProtectedRequestExecutionInputResponse, error) { + client.protectedRequests = append(client.protectedRequests, request) + if client.protectedInputErr != nil { + return protocol.ProtectedRequestExecutionInputResponse{}, client.protectedInputErr + } + if client.protectedInput.JobID == "" || request.JobID != client.protectedInput.JobID { + return protocol.ProtectedRequestExecutionInputResponse{}, fmt.Errorf("protected request input is not configured") + } + return client.protectedInput, nil +} + +func (client *fakeWorkerClient) GetRunUpdateInput(_ context.Context, request protocol.RunUpdateInputRequest) (protocol.RunUpdateInputResponse, error) { + if client.updateInput.JobID == "" || request.JobID != client.updateInput.JobID { + return protocol.RunUpdateInputResponse{}, fmt.Errorf("Run update input is not configured") + } + return client.updateInput, nil +} + +func (client *fakeWorkerClient) ReadRunUpdateChunk(_ context.Context, request protocol.RunUpdateChunkRequest) (protocol.RunUpdateChunkResponse, error) { + client.updateChunkOffsets = append(client.updateChunkOffsets, request.Offset) + if client.updateInput.JobID == "" || request.JobID != client.updateInput.JobID || request.Offset < 0 || request.Offset >= int64(len(client.updatePayload)) { + return protocol.RunUpdateChunkResponse{}, fmt.Errorf("Run update chunk is not configured") + } + end := request.Offset + int64(request.Length) + if end > int64(len(client.updatePayload)) { + end = int64(len(client.updatePayload)) + } + return protocol.RunUpdateChunkResponse{JobID: request.JobID, ArtifactID: client.updateInput.ArtifactID, Offset: request.Offset, TotalBytes: int64(len(client.updatePayload)), Checksum: client.updateInput.Checksum, Payload: append([]byte(nil), client.updatePayload[request.Offset:end]...), Complete: end == int64(len(client.updatePayload))}, nil +} + +func (client *fakeWorkerClient) ReportRunUpdateHealth(_ context.Context, request protocol.RunUpdateHealthRequest) (protocol.RunUpdateHealthResponse, error) { + client.updateHealthReports = append(client.updateHealthReports, request) + return protocol.RunUpdateHealthResponse{Accepted: true, JobID: request.JobID, Phase: request.Outcome, ServerTime: workerTestTime()}, nil +} + +func (client *fakeWorkerClient) OpenArtifactTransfer(_ context.Context, request protocol.ArtifactTransferOpenRequest) (protocol.ArtifactTransferOpenResponse, error) { + if client.buildInput.JobID == "" { + return protocol.ArtifactTransferOpenResponse{}, fmt.Errorf("artifact transfer is not configured") + } + client.artifactOpenRequests = append(client.artifactOpenRequests, request) + client.artifactTransfer = "transfer-build" + return protocol.ArtifactTransferOpenResponse{Accepted: true, TransferID: client.artifactTransfer, Artifact: protocol.ArtifactMetadata{ID: request.ArtifactID, OwnerKind: request.OwnerKind, OwnerID: request.OwnerID, State: "uploading"}, ChunkSizeBytes: request.ChunkSizeBytes}, nil +} + +func (client *fakeWorkerClient) UploadArtifactChunk(_ context.Context, request protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) { + if request.TransferID != client.artifactTransfer { + return protocol.ArtifactChunkUploadResponse{}, fmt.Errorf("unexpected artifact transfer") + } + client.artifactPayload = append(client.artifactPayload, request.Payload...) + return protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: request.TransferID, ArtifactID: request.ArtifactID, ChunkIndex: request.ChunkIndex}, nil +} + +func (client *fakeWorkerClient) CompleteArtifactTransfer(_ context.Context, request protocol.ArtifactTransferCompleteRequest) (protocol.ArtifactTransferCompleteResponse, error) { + if request.TransferID != client.artifactTransfer || request.SizeBytes != int64(len(client.artifactPayload)) { + return protocol.ArtifactTransferCompleteResponse{}, fmt.Errorf("unexpected artifact completion") + } + return protocol.ArtifactTransferCompleteResponse{Accepted: true, TransferID: request.TransferID, Artifact: protocol.ArtifactMetadata{ID: request.ArtifactID, OwnerKind: "job", OwnerID: client.buildInput.JobID, SizeBytes: request.SizeBytes, Checksum: request.Checksum, State: "available"}, Completed: true}, nil +} + +func (client *fakeWorkerClient) PollJobCancel(_ context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) { + client.cancelPollRequests = append(client.cancelPollRequests, request) + return client.cancelResponse, nil +} + +func (client *fakeWorkerClient) ReconcileJobs(_ context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) { + client.reconcileRequests = append(client.reconcileRequests, request) + return client.reconcileResponse, nil +} + +func (client *fakeWorkerClient) GetRunLogStreamProgress(_ context.Context, request protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) { + client.logProgressRequests = append(client.logProgressRequests, request) + response := client.progressResponse + response.RunEndpointID = request.RunEndpointID + response.ServerInstanceID = request.ServerInstanceID + response.LogStreamID = request.LogStreamID + if response.LatestSeq == 0 { + response.LatestSeq = client.logStreamProgress + } + return response, nil +} + +func (client *fakeWorkerClient) IngestMetricBatch(_ context.Context, request protocol.MetricBatchIngestRequest) (protocol.MetricBatchIngestResponse, error) { + client.metricRequests = append(client.metricRequests, request) + if client.metricErr != nil { + return protocol.MetricBatchIngestResponse{}, client.metricErr + } + return protocol.MetricBatchIngestResponse{Accepted: true, AcceptedCount: len(request.Samples), ServerTime: workerTestTime()}, nil +} + +type staticSupervisor struct { + stdout string + stderr string + err error +} + +func (supervisor staticSupervisor) Run(context.Context, ProcessCommand) (ProcessResult, error) { + return ProcessResult{ExitCode: 0, Stdout: supervisor.stdout, Stderr: supervisor.stderr}, supervisor.err +} + +func workerTestConfig(t *testing.T) config.Config { + t.Helper() + return config.Config{ + Mode: "worker", + PlatformURL: "http://platform.test", + RunEndpointID: "run-test", + DisplayName: "Run Test", + Version: "0.1.0-test", + RegistrationToken: "registration-token", + WorkspaceRoot: t.TempDir(), + SpoolRoot: t.TempDir(), + MaxJobs: 2, + HeartbeatInterval: time.Second, + PollInterval: time.Second, + RetryBackoff: time.Millisecond, + } +} + +func workerJobAssignment(capability string) protocol.RunJobAssignment { + job := lifecycleAssignment(capability) + job.JobID = "job-worker" + if capability == protocol.RunCapabilityProcessStop { + job.JobID = "job-worker-stop" + } + job.RunEndpointID = "run-test" + job.ServerInstanceID = "server-worker" + return job +} + +func workerTestTime() time.Time { + return time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC) +} + +func containsText(value string, needle string) bool { + return strings.Contains(value, needle) +} + +func decodeWorkerTestJSON(t *testing.T, r *http.Request, target any) { + t.Helper() + if r.Method != http.MethodPost { + t.Fatalf("expected POST, got %s", r.Method) + } + if err := json.NewDecoder(r.Body).Decode(target); err != nil { + t.Fatalf("decode request: %v", err) + } +} + +func writeWorkerTestJSON(t *testing.T, w http.ResponseWriter, value any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(value); err != nil { + t.Fatalf("encode response: %v", err) + } +} diff --git a/runtime/workspace.go b/runtime/workspace.go new file mode 100644 index 0000000..d0cd374 --- /dev/null +++ b/runtime/workspace.go @@ -0,0 +1,197 @@ +package runtime + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "browser.local/run/protocol" +) + +const maxExecutionContentBytes = 64 * 1024 + +type WorkspaceResolver struct { + root string +} + +func NewWorkspaceResolver(root string) WorkspaceResolver { + if strings.TrimSpace(root) == "" { + root = filepath.Join(".", ".run-workspace") + } + return WorkspaceResolver{root: root} +} + +func (resolver WorkspaceResolver) Scope(serverInstanceID string, profileKey string) (string, error) { + if err := validateWorkspaceComponent(serverInstanceID, "serverInstanceId"); err != nil { + return "", err + } + if err := validateWorkspaceComponent(profileKey, "profileKey"); err != nil { + return "", err + } + root, err := filepath.Abs(resolver.root) + if err != nil { + return "", fmt.Errorf("resolve workspace root: %w", err) + } + if err := ensureDirectory(root); err != nil { + return "", fmt.Errorf("secure workspace root: %w", err) + } + instances := filepath.Join(root, "instances") + if err := ensureDirectory(instances); err != nil { + return "", fmt.Errorf("secure workspace instances: %w", err) + } + serverDir := filepath.Join(instances, serverInstanceID) + if err := ensureDirectory(serverDir); err != nil { + return "", fmt.Errorf("secure server workspace: %w", err) + } + scope := filepath.Join(serverDir, profileKey) + if err := ensureDirectory(scope); err != nil { + return "", fmt.Errorf("secure profile workspace: %w", err) + } + return scope, nil +} + +func (resolver WorkspaceResolver) ExistingTarget(scope string, key string) (string, error) { + path, err := resolver.target(scope, key, false) + if err != nil { + return "", err + } + info, err := os.Lstat(path) + if err != nil { + return "", err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return "", fmt.Errorf("target must be a regular file") + } + return path, nil +} + +func (resolver WorkspaceResolver) ExistingDirectory(scope string, key string) (string, error) { + if strings.TrimSpace(scope) == "" || !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, string(rune(92))) { + return "", fmt.Errorf("logical directory is unsafe") + } + cleanScope, err := filepath.Abs(scope) + if err != nil { + return "", err + } + root, err := filepath.Abs(resolver.root) + if err != nil { + return "", err + } + rel, err := filepath.Rel(root, cleanScope) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return "", fmt.Errorf("workspace scope escapes root") + } + current := cleanScope + for _, part := range strings.Split(filepath.ToSlash(key), "/") { + if part == "" || part == "." || part == ".." { + return "", fmt.Errorf("logical directory contains unsafe component") + } + current = filepath.Join(current, part) + info, statErr := os.Lstat(current) + if statErr != nil { + return "", statErr + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return "", fmt.Errorf("logical directory is not a real directory") + } + } + return current, nil +} + +func (resolver WorkspaceResolver) WritableTarget(scope string, key string) (string, string, error) { + if strings.HasPrefix(key, "actions/") || strings.HasPrefix(key, "state/") || key == "actions" || key == "state" { + return "", "", fmt.Errorf("target is reserved") + } + parts := strings.Split(filepath.ToSlash(key), "/") + parent := scope + for _, part := range parts[:len(parts)-1] { + parent = filepath.Join(parent, part) + if err := ensureDirectory(parent); err != nil { + return "", "", err + } + } + path, err := resolver.target(scope, key, true) + if err != nil { + return "", "", err + } + return path, filepath.Dir(path), nil +} + +func (resolver WorkspaceResolver) target(scope string, key string, allowMissingFinal bool) (string, error) { + if strings.TrimSpace(scope) == "" { + return "", fmt.Errorf("workspace scope is invalid") + } + if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) { + return "", fmt.Errorf("logical key is unsafe") + } + cleanScope, err := filepath.Abs(scope) + if err != nil { + return "", err + } + root, err := filepath.Abs(resolver.root) + if err != nil { + return "", err + } + rel, err := filepath.Rel(root, cleanScope) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return "", fmt.Errorf("workspace scope escapes root") + } + parts := strings.Split(filepath.ToSlash(key), "/") + current := cleanScope + for index, part := range parts { + if part == "" || part == "." || part == ".." { + return "", fmt.Errorf("logical key contains unsafe component") + } + current = filepath.Join(current, part) + info, statErr := os.Lstat(current) + if statErr != nil { + if allowMissingFinal && index == len(parts)-1 && os.IsNotExist(statErr) { + return current, nil + } + return "", statErr + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("logical key contains a symlink") + } + if index < len(parts)-1 && !info.IsDir() { + return "", fmt.Errorf("logical key parent is not a directory") + } + if index == len(parts)-1 && info.Mode()&os.ModeType != 0 { + return "", fmt.Errorf("target is a special file") + } + } + return current, nil +} + +func ensureDirectory(path string) error { + if info, err := os.Lstat(path); err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("path is not a real directory") + } + return os.Chmod(path, 0o700) + } else if !os.IsNotExist(err) { + return err + } + parent := filepath.Dir(path) + if parent != path { + if err := ensureDirectory(parent); err != nil { + return err + } + } + if err := os.Mkdir(path, 0o700); err != nil && !os.IsExist(err) { + return err + } + info, err := os.Lstat(path) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("created path is not a real directory") + } + return os.Chmod(path, 0o700) +} + +func validateWorkspaceComponent(value string, field string) error { + if strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) || value == "." || value == ".." || strings.ContainsAny(value, `/\`) || !protocol.ValidLogicalFileKey(value) { + return fmt.Errorf("%s is unsafe", field) + } + return nil +} diff --git a/runtime/workspace_seed.go b/runtime/workspace_seed.go new file mode 100644 index 0000000..b946663 --- /dev/null +++ b/runtime/workspace_seed.go @@ -0,0 +1,147 @@ +package runtime + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "browser.local/run/config" + "browser.local/run/protocol" +) + +type workspaceSeedFile struct { + Path string `json:"path"` + Content string `json:"content"` + Encoding string `json:"encoding,omitempty"` + Mode int `json:"mode,omitempty"` +} + +// MaterializeWorkspaceSeed writes platform-packaged plugin assets into the +// scoped run workspace. The seed contains plugin-owned files only; run treats +// them as opaque generic lifecycle assets. +func MaterializeWorkspaceSeed(cfg config.Config) error { + startedAt := time.Now() + encoded := strings.TrimSpace(cfg.WorkspaceSeed) + if encoded == "" { + log.Printf("RUN phase=workspace_seed status=skipped reason=empty workspace=%s", safeOptional(cfg.WorkspaceRoot)) + return nil + } + log.Printf("RUN phase=workspace_seed status=decoding workspace=%s encodedBytes=%d componentKey=%s", safeOptional(cfg.WorkspaceRoot), len(encoded), safeOptional(cfg.ComponentKey)) + payload, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + log.Printf("RUN phase=workspace_seed status=decode_failed workspace=%s error=%s", safeOptional(cfg.WorkspaceRoot), RedactText(err.Error())) + return fmt.Errorf("decode workspace seed: %w", err) + } + var files []workspaceSeedFile + if err := json.Unmarshal(payload, &files); err != nil { + log.Printf("RUN phase=workspace_seed status=manifest_failed workspace=%s payloadBytes=%d error=%s", safeOptional(cfg.WorkspaceRoot), len(payload), RedactText(err.Error())) + return fmt.Errorf("decode workspace seed manifest: %w", err) + } + log.Printf("RUN phase=workspace_seed status=decoded workspace=%s payloadBytes=%d files=%d", safeOptional(cfg.WorkspaceRoot), len(payload), len(files)) + if len(files) == 0 { + log.Printf("RUN phase=workspace_seed status=skipped reason=no_files workspace=%s durationMs=%d", safeOptional(cfg.WorkspaceRoot), time.Since(startedAt).Milliseconds()) + return nil + } + if strings.TrimSpace(cfg.ServerInstanceID) == "" { + log.Printf("RUN phase=workspace_seed status=failed reason=missing_server workspace=%s", safeOptional(cfg.WorkspaceRoot)) + return fmt.Errorf("workspace seed requires a server instance id") + } + scope, err := seededWorkspaceScope(cfg) + if err != nil { + log.Printf("RUN phase=workspace_seed status=scope_failed workspace=%s server=%s componentKey=%s error=%s", safeOptional(cfg.WorkspaceRoot), safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKey), RedactText(err.Error())) + return err + } + log.Printf("RUN phase=workspace_seed status=scope_ready workspace=%s scope=%s files=%d", safeOptional(cfg.WorkspaceRoot), safeOptional(scope), len(files)) + totalBytes := 0 + for index, file := range files { + written, err := writeWorkspaceSeedFile(scope, file, index+1, len(files)) + if err != nil { + log.Printf("RUN phase=workspace_seed.file status=failed index=%d total=%d path=%s error=%s", index+1, len(files), safeOptional(file.Path), RedactText(err.Error())) + return err + } + totalBytes += written + } + log.Printf("RUN phase=workspace_seed status=complete workspace=%s scope=%s files=%d bytes=%d durationMs=%d", safeOptional(cfg.WorkspaceRoot), safeOptional(scope), len(files), totalBytes, time.Since(startedAt).Milliseconds()) + return nil +} + +func seededWorkspaceScope(cfg config.Config) (string, error) { + if strings.TrimSpace(cfg.ComponentKey) != "" { + return NewWorkspaceResolver(cfg.WorkspaceRoot).Scope(cfg.ServerInstanceID, cfg.ComponentKey) + } + return scopedServerWorkspace(cfg.WorkspaceRoot, cfg.ServerInstanceID) +} + +func writeWorkspaceSeedFile(scope string, file workspaceSeedFile, index int, total int) (int, error) { + target, err := workspaceSeedTarget(scope, file.Path) + if err != nil { + return 0, err + } + mode := os.FileMode(file.Mode) + if mode == 0 { + mode = 0o600 + } + if mode&0o777 != mode || mode&0o022 != 0 { + return 0, fmt.Errorf("workspace seed file mode is unsafe") + } + body, err := workspaceSeedFileContent(file) + if err != nil { + return 0, err + } + log.Printf("RUN phase=workspace_seed.file status=writing index=%d total=%d path=%s target=%s bytes=%d mode=%#o", index, total, safeOptional(file.Path), safeOptional(target), len(body), mode) + if err := ensureDirectory(filepath.Dir(target)); err != nil { + return 0, err + } + log.Printf("RUN phase=workspace_seed.file status=directory_ready index=%d total=%d dir=%s", index, total, safeOptional(filepath.Dir(target))) + if err := os.WriteFile(target, body, mode); err != nil { + return 0, err + } + log.Printf("RUN phase=workspace_seed.file status=written index=%d total=%d path=%s target=%s bytes=%d mode=%#o", index, total, safeOptional(file.Path), safeOptional(target), len(body), mode) + return len(body), nil +} + +func workspaceSeedFileContent(file workspaceSeedFile) ([]byte, error) { + switch strings.TrimSpace(file.Encoding) { + case "": + return []byte(file.Content), nil + case "base64": + body, err := base64.StdEncoding.DecodeString(strings.TrimSpace(file.Content)) + if err != nil { + return nil, fmt.Errorf("workspace seed file content is not valid base64") + } + return body, nil + default: + return nil, fmt.Errorf("workspace seed file encoding is unsupported") + } +} + +func workspaceSeedTarget(scope string, key string) (string, error) { + if strings.TrimSpace(scope) == "" { + return "", fmt.Errorf("workspace scope is invalid") + } + if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) { + return "", fmt.Errorf("workspace seed path is unsafe") + } + cleanScope, err := filepath.Abs(scope) + if err != nil { + return "", err + } + parts := strings.Split(filepath.ToSlash(key), "/") + current := cleanScope + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return "", fmt.Errorf("workspace seed path contains unsafe component") + } + current = filepath.Join(current, part) + } + rel, err := filepath.Rel(cleanScope, current) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return "", fmt.Errorf("workspace seed path escapes scope") + } + return current, nil +} diff --git a/spool/README.md b/spool/README.md new file mode 100644 index 0000000..0ea1a38 --- /dev/null +++ b/spool/README.md @@ -0,0 +1,19 @@ +# run/spool + +Local durable queues live here. + +Required spool areas: + +- `logs`: unacknowledged log segments. +- `jobs`: accepted job journal for duplicate detection and reconciliation. +- `artifacts`: incomplete artifact transfer state. + +Spool pressure must be visible in run capacity reports. + +## Channel Isolation + +- Log and artifact retry state are stored in separate spool areas and are acknowledged independently. +- Acknowledging a log batch must not scan, remove, or block on artifact chunks. +- Acknowledging an artifact chunk must not scan, remove, or block on log batches. +- Control heartbeat and job ack/result payloads remain metadata-only; they must never carry spool file paths, artifact chunks, log entries, raw credentials, direct sockets, or large inline bodies. +- Artifact transfer backlog is lower priority than log flush, job lifecycle calls, and control heartbeat. diff --git a/spool/artifact_queue.go b/spool/artifact_queue.go new file mode 100644 index 0000000..af04b7a --- /dev/null +++ b/spool/artifact_queue.go @@ -0,0 +1,183 @@ +package spool + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "browser.local/run/protocol" +) + +type ArtifactQueue struct { + dir string +} + +func NewArtifactQueue(dir string) (ArtifactQueue, error) { + if strings.TrimSpace(dir) == "" { + return ArtifactQueue{}, fmt.Errorf("spool directory is required") + } + artifactDir := filepath.Join(dir, "artifacts") + if err := os.MkdirAll(artifactDir, 0o755); err != nil { + return ArtifactQueue{}, fmt.Errorf("create artifact queue: %w", err) + } + return ArtifactQueue{dir: artifactDir}, nil +} + +func (queue ArtifactQueue) Enqueue(chunk protocol.ArtifactChunkUploadRequest) error { + path := queue.chunkPath(chunk) + if existing, err := readArtifactChunk(path); err == nil { + if existing.TransferID == chunk.TransferID && existing.ArtifactID == chunk.ArtifactID && existing.ChunkIndex == chunk.ChunkIndex && existing.Checksum == chunk.Checksum { + return nil + } + return fmt.Errorf("artifact queue chunk conflicts with committed chunk") + } else if !os.IsNotExist(err) { + return err + } + tmp := path + ".tmp" + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open artifact queue chunk: %w", err) + } + encodeErr := json.NewEncoder(file).Encode(chunk) + closeErr := file.Close() + if encodeErr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("encode artifact queue chunk: %w", encodeErr) + } + if closeErr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("close artifact queue chunk: %w", closeErr) + } + if err := syncFile(tmp); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("commit artifact queue chunk: %w", err) + } + return nil +} + +type ArtifactChunkClient interface { + UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) +} + +func (queue ArtifactQueue) Flush(ctx context.Context, client ArtifactChunkClient) (int, error) { + if client == nil { + return 0, fmt.Errorf("artifact chunk client is required") + } + pending, err := queue.Pending() + if err != nil { + return 0, err + } + acknowledged := 0 + for _, chunk := range pending { + if err := ctx.Err(); err != nil { + return acknowledged, err + } + response, err := client.UploadArtifactChunk(ctx, chunk) + if err != nil { + return acknowledged, err + } + if !response.Accepted || response.TransferID != chunk.TransferID || response.ArtifactID != chunk.ArtifactID || response.ChunkIndex != chunk.ChunkIndex { + return acknowledged, fmt.Errorf("platform artifact acknowledgement does not match pending chunk") + } + if err := queue.Ack(response); err != nil { + return acknowledged, err + } + acknowledged++ + } + return acknowledged, nil +} + +func readArtifactChunk(path string) (protocol.ArtifactChunkUploadRequest, error) { + file, err := os.Open(path) + if err != nil { + return protocol.ArtifactChunkUploadRequest{}, err + } + defer file.Close() + var chunk protocol.ArtifactChunkUploadRequest + if err := json.NewDecoder(file).Decode(&chunk); err != nil { + return protocol.ArtifactChunkUploadRequest{}, fmt.Errorf("decode artifact queue chunk: %w", err) + } + return chunk, nil +} + +func (queue ArtifactQueue) Pending() ([]protocol.ArtifactChunkUploadRequest, error) { + entries, err := os.ReadDir(queue.dir) + if err != nil { + return nil, fmt.Errorf("read artifact queue: %w", err) + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + paths = append(paths, filepath.Join(queue.dir, entry.Name())) + } + sort.Strings(paths) + chunks := make([]protocol.ArtifactChunkUploadRequest, 0, len(paths)) + for _, path := range paths { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open artifact queue chunk: %w", err) + } + var chunk protocol.ArtifactChunkUploadRequest + decodeErr := json.NewDecoder(file).Decode(&chunk) + closeErr := file.Close() + if decodeErr != nil { + return nil, fmt.Errorf("decode artifact queue chunk: %w", decodeErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close artifact queue chunk: %w", closeErr) + } + chunks = append(chunks, chunk) + } + return chunks, nil +} + +func (queue ArtifactQueue) Ack(response protocol.ArtifactChunkUploadResponse) error { + if !response.Accepted { + return nil + } + entries, err := os.ReadDir(queue.dir) + if err != nil { + return fmt.Errorf("read artifact queue: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := filepath.Join(queue.dir, entry.Name()) + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("open artifact queue chunk: %w", err) + } + var chunk protocol.ArtifactChunkUploadRequest + decodeErr := json.NewDecoder(file).Decode(&chunk) + closeErr := file.Close() + if decodeErr != nil { + return fmt.Errorf("decode artifact queue chunk: %w", decodeErr) + } + if closeErr != nil { + return fmt.Errorf("close artifact queue chunk: %w", closeErr) + } + if chunk.TransferID == response.TransferID && chunk.ArtifactID == response.ArtifactID && chunk.ChunkIndex == response.ChunkIndex { + if err := os.Remove(path); err != nil { + return fmt.Errorf("remove acknowledged artifact queue chunk: %w", err) + } + } + } + return nil +} + +func (queue ArtifactQueue) chunkPath(chunk protocol.ArtifactChunkUploadRequest) string { + transferID := sanitizeSegmentName(chunk.TransferID) + artifactID := sanitizeSegmentName(chunk.ArtifactID) + return filepath.Join(queue.dir, fmt.Sprintf("%s-%s-%020d.json", transferID, artifactID, chunk.ChunkIndex)) +} diff --git a/spool/artifact_queue_test.go b/spool/artifact_queue_test.go new file mode 100644 index 0000000..45dae9b --- /dev/null +++ b/spool/artifact_queue_test.go @@ -0,0 +1,75 @@ +package spool + +import ( + "testing" + + "browser.local/run/protocol" +) + +func TestArtifactQueueRetainsPendingAndRemovesAcknowledgedChunk(t *testing.T) { + queue, err := NewArtifactQueue(t.TempDir()) + if err != nil { + t.Fatalf("new artifact queue: %v", err) + } + first := validQueuedArtifactChunk(0) + second := validQueuedArtifactChunk(1) + if err := queue.Enqueue(first); err != nil { + t.Fatalf("enqueue first: %v", err) + } + if err := queue.Enqueue(second); err != nil { + t.Fatalf("enqueue second: %v", err) + } + pending, err := queue.Pending() + if err != nil { + t.Fatalf("pending before ack: %v", err) + } + if len(pending) != 2 { + t.Fatalf("expected two pending chunks, got %+v", pending) + } + + if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil { + t.Fatalf("ack first: %v", err) + } + pending, err = queue.Pending() + if err != nil { + t.Fatalf("pending after ack: %v", err) + } + if len(pending) != 1 || pending[0].ChunkIndex != 1 { + t.Fatalf("expected second chunk pending, got %+v", pending) + } +} + +func TestArtifactQueueRetainsChunkWhenAckDoesNotMatch(t *testing.T) { + queue, err := NewArtifactQueue(t.TempDir()) + if err != nil { + t.Fatalf("new artifact queue: %v", err) + } + if err := queue.Enqueue(validQueuedArtifactChunk(0)); err != nil { + t.Fatalf("enqueue: %v", err) + } + if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 1}); err != nil { + t.Fatalf("ack mismatch: %v", err) + } + pending, err := queue.Pending() + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(pending) != 1 || pending[0].ChunkIndex != 0 { + t.Fatalf("expected original chunk pending, got %+v", pending) + } +} + +func validQueuedArtifactChunk(index int) protocol.ArtifactChunkUploadRequest { + payload := []byte{byte(index), byte(index + 1)} + return protocol.ArtifactChunkUploadRequest{ + RunEndpointID: "run-local", + SessionToken: "session-token", + TransferID: "transfer-1", + ArtifactID: "artifact-1", + ChunkIndex: index, + Offset: int64(index * len(payload)), + SizeBytes: len(payload), + Checksum: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + Payload: payload, + } +} diff --git a/spool/channel_isolation_test.go b/spool/channel_isolation_test.go new file mode 100644 index 0000000..c9c16ef --- /dev/null +++ b/spool/channel_isolation_test.go @@ -0,0 +1,85 @@ +package spool + +import ( + "testing" + + "browser.local/run/protocol" +) + +func TestLogSpoolAckIsIndependentFromArtifactBacklog(t *testing.T) { + root := t.TempDir() + logSpool, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + artifactQueue, err := NewArtifactQueue(root) + if err != nil { + t.Fatalf("new artifact queue: %v", err) + } + + if err := logSpool.Enqueue(validSpoolLogBatch(1, 1)); err != nil { + t.Fatalf("enqueue log: %v", err) + } + for i := 0; i < 3; i++ { + if err := artifactQueue.Enqueue(validQueuedArtifactChunk(i)); err != nil { + t.Fatalf("enqueue artifact chunk %d: %v", i, err) + } + } + + if err := logSpool.Ack(protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil { + t.Fatalf("ack log batch: %v", err) + } + logs, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(logs) != 0 { + t.Fatalf("expected log batch removed despite artifact backlog, got %+v", logs) + } + chunks, err := artifactQueue.Pending() + if err != nil { + t.Fatalf("pending artifacts: %v", err) + } + if len(chunks) != 3 { + t.Fatalf("artifact backlog should remain independent, got %+v", chunks) + } +} + +func TestArtifactAckIsIndependentFromLogBacklog(t *testing.T) { + root := t.TempDir() + logSpool, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + artifactQueue, err := NewArtifactQueue(root) + if err != nil { + t.Fatalf("new artifact queue: %v", err) + } + + for _, batch := range []protocol.LogBatchIngestRequest{validSpoolLogBatch(1, 1), validSpoolLogBatch(2, 2)} { + if err := logSpool.Enqueue(batch); err != nil { + t.Fatalf("enqueue log batch: %v", err) + } + } + if err := artifactQueue.Enqueue(validQueuedArtifactChunk(0)); err != nil { + t.Fatalf("enqueue artifact chunk: %v", err) + } + + if err := artifactQueue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil { + t.Fatalf("ack artifact chunk: %v", err) + } + chunks, err := artifactQueue.Pending() + if err != nil { + t.Fatalf("pending artifacts: %v", err) + } + if len(chunks) != 0 { + t.Fatalf("expected artifact chunk removed despite log backlog, got %+v", chunks) + } + logs, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending logs: %v", err) + } + if len(logs) != 2 { + t.Fatalf("log backlog should remain independent, got %+v", logs) + } +} diff --git a/spool/flush_test.go b/spool/flush_test.go new file mode 100644 index 0000000..c35c51a --- /dev/null +++ b/spool/flush_test.go @@ -0,0 +1,36 @@ +package spool + +import ( + "context" + "testing" + + "browser.local/run/protocol" +) + +type fakeLogBatchClient struct { + accepted bool +} + +func (client fakeLogBatchClient) IngestLogBatch(_ context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + return protocol.LogBatchIngestResponse{Accepted: client.accepted, LogStreamID: batch.LogStreamID, AcceptedFrom: batch.FirstSeq, AcceptedTo: batch.LastSeq}, nil +} + +func TestLogSpoolFlushRetainsRejectedBatchForRetry(t *testing.T) { + spool, err := NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new spool: %v", err) + } + if err := spool.Enqueue(validSpoolLogBatch(1, 1)); err != nil { + t.Fatalf("enqueue: %v", err) + } + if _, err := spool.Flush(context.Background(), fakeLogBatchClient{accepted: false}); err == nil { + t.Fatal("expected rejected flush") + } + pending, err := spool.Pending() + if err != nil || len(pending) != 1 { + t.Fatalf("expected batch retained after rejection, pending=%+v err=%v", pending, err) + } + if count, err := spool.Flush(context.Background(), fakeLogBatchClient{accepted: true}); err != nil || count != 1 { + t.Fatalf("expected retry success, count=%d err=%v", count, err) + } +} diff --git a/spool/log_spool.go b/spool/log_spool.go new file mode 100644 index 0000000..29c5a63 --- /dev/null +++ b/spool/log_spool.go @@ -0,0 +1,560 @@ +package spool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "browser.local/run/protocol" +) + +type LogSpool struct { + dir string + statePath string + mu *sync.Mutex + watermarks map[string]logStreamWatermark + inflight map[string]bool +} + +const maxAggregatedLogEntries = 128 + +type logStreamWatermark struct { + Allocated uint64 `json:"allocated"` + Acknowledged uint64 `json:"acknowledged"` + SourceOffset int64 `json:"sourceOffset,omitempty"` +} +type logSpoolState struct { + Streams map[string]logStreamWatermark `json:"streams"` +} + +type LogSourceCursor struct { + StartOffset int64 `json:"startOffset"` + EndOffset int64 `json:"endOffset"` +} + +type durableLogBatch struct { + protocol.LogBatchIngestRequest + SourceCursor *LogSourceCursor `json:"_sourceCursor,omitempty"` +} + +type pendingLogSegment struct { + path string + batch durableLogBatch +} + +func NewLogSpool(dir string) (LogSpool, error) { + if strings.TrimSpace(dir) == "" { + return LogSpool{}, fmt.Errorf("spool directory is required") + } + logDir := filepath.Join(dir, "logs") + if err := os.MkdirAll(logDir, 0o755); err != nil { + return LogSpool{}, fmt.Errorf("create log spool: %w", err) + } + spool := LogSpool{dir: logDir, statePath: filepath.Join(dir, "log-watermarks.json"), mu: &sync.Mutex{}, watermarks: map[string]logStreamWatermark{}, inflight: map[string]bool{}} + if err := spool.loadWatermarks(); err != nil { + return LogSpool{}, err + } + if err := spool.restorePendingWatermarks(); err != nil { + return LogSpool{}, err + } + return spool, nil +} + +// restorePendingWatermarks supports upgrades from spools created before +// per-stream watermarks existed. Pending durable segments are still an +// allocation fact and must win over a remote progress lookup. +func (spool *LogSpool) restorePendingWatermarks() error { + spool.mu.Lock() + defer spool.mu.Unlock() + pending, err := spool.pendingSegmentsLocked() + if err != nil { + return err + } + changed := false + for _, segment := range pending { + batch := segment.batch + watermark := spool.watermarks[batch.LogStreamID] + if batch.LastSeq > watermark.Allocated { + watermark.Allocated = batch.LastSeq + changed = true + } + if batch.SourceCursor != nil && batch.SourceCursor.EndOffset > watermark.SourceOffset { + watermark.SourceOffset = batch.SourceCursor.EndOffset + changed = true + } + spool.watermarks[batch.LogStreamID] = watermark + } + if changed { + return spool.persistWatermarksLocked() + } + return nil +} + +func (spool LogSpool) NextSequence(ctx context.Context, streamID string, recover func(context.Context, string) (uint64, error)) (uint64, error) { + spool.mu.Lock() + defer spool.mu.Unlock() + watermark, known := spool.watermarks[streamID] + if !known && recover != nil { + latest, err := recover(ctx, streamID) + if err != nil { + return 0, err + } + watermark = logStreamWatermark{Allocated: latest, Acknowledged: latest} + } + watermark.Allocated++ + spool.watermarks[streamID] = watermark + if err := spool.persistWatermarksLocked(); err != nil { + return 0, err + } + return watermark.Allocated, nil +} + +func (spool LogSpool) Enqueue(batch protocol.LogBatchIngestRequest) error { + return spool.enqueue(batch, nil) +} + +// EnqueueAggregated extends the newest compatible durable segment so callers +// do not create one upload request for every process-output line. +func (spool LogSpool) EnqueueAggregated(batch protocol.LogBatchIngestRequest, checksum func([]protocol.LogEntry) (string, error)) error { + return spool.enqueue(batch, checksum) +} + +// EnqueueNextAggregated allocates the next sequence and commits its batch +// under one spool lock. The durable segment is the source of truth; startup +// restores its watermark if the separate watermark snapshot was interrupted. +func (spool LogSpool) EnqueueNextAggregated(ctx context.Context, batch protocol.LogBatchIngestRequest, cursor *LogSourceCursor, recover func(context.Context, string) (uint64, error), checksum func([]protocol.LogEntry) (string, error)) (uint64, bool, error) { + if len(batch.Entries) != 1 { + return 0, false, fmt.Errorf("next aggregated log batch requires exactly one entry") + } + if cursor != nil && (cursor.StartOffset < 0 || cursor.EndOffset <= cursor.StartOffset) { + return 0, false, fmt.Errorf("source cursor range is invalid") + } + spool.mu.Lock() + defer spool.mu.Unlock() + watermark, known := spool.watermarks[batch.LogStreamID] + if !known && recover != nil { + latest, err := recover(ctx, batch.LogStreamID) + if err != nil { + return 0, false, err + } + watermark = logStreamWatermark{Allocated: latest, Acknowledged: latest} + } + if cursor != nil && cursor.EndOffset <= watermark.SourceOffset { + return watermark.Allocated, false, nil + } + sequence := watermark.Allocated + 1 + batch.FirstSeq = sequence + batch.LastSeq = sequence + batch.Entries[0].Seq = sequence + var err error + batch.Checksum, err = checksum(batch.Entries) + if err != nil { + return 0, false, err + } + if err := spool.writeAggregatedLocked(durableLogBatch{LogBatchIngestRequest: batch, SourceCursor: cursor}, checksum); err != nil { + return 0, false, err + } + watermark.Allocated = sequence + if cursor != nil { + watermark.SourceOffset = cursor.EndOffset + } + spool.watermarks[batch.LogStreamID] = watermark + // The batch is already synced and durable. A later startup reconstructs + // this watermark from pending segments if this snapshot cannot be written. + _ = spool.persistWatermarksLocked() + return sequence, true, nil +} + +func (spool LogSpool) enqueue(batch protocol.LogBatchIngestRequest, checksum func([]protocol.LogEntry) (string, error)) error { + spool.mu.Lock() + defer spool.mu.Unlock() + watermark := spool.watermarks[batch.LogStreamID] + if batch.LastSeq > watermark.Allocated { + watermark.Allocated = batch.LastSeq + spool.watermarks[batch.LogStreamID] = watermark + if err := spool.persistWatermarksLocked(); err != nil { + return err + } + } + return spool.writeAggregatedLocked(durableLogBatch{LogBatchIngestRequest: batch}, checksum) +} + +func (spool LogSpool) writeAggregatedLocked(batch durableLogBatch, checksum func([]protocol.LogEntry) (string, error)) error { + if checksum != nil && len(batch.Entries) == 1 { + if merged, previous, ok, err := spool.mergeLatest(batch, checksum); err != nil { + return err + } else if ok { + return spool.replaceBatch(previous.path, merged) + } + } + return spool.writeBatch(batch) +} + +func (spool LogSpool) mergeLatest(batch durableLogBatch, checksum func([]protocol.LogEntry) (string, error)) (durableLogBatch, pendingLogSegment, bool, error) { + pending, err := spool.pendingSegmentsLocked() + if err != nil { + return durableLogBatch{}, pendingLogSegment{}, false, err + } + for index := len(pending) - 1; index >= 0; index-- { + previous := pending[index] + if spool.inflight[previous.path] || !compatibleLogBatch(previous.batch, batch) || previous.batch.LastSeq+1 != batch.FirstSeq || len(previous.batch.Entries)+len(batch.Entries) > maxAggregatedLogEntries || !compatibleSourceCursor(previous.batch.SourceCursor, batch.SourceCursor) { + continue + } + merged := previous.batch + merged.LastSeq = batch.LastSeq + merged.Entries = append(append([]protocol.LogEntry(nil), previous.batch.Entries...), batch.Entries...) + if merged.SourceCursor != nil { + cursor := *merged.SourceCursor + cursor.EndOffset = batch.SourceCursor.EndOffset + merged.SourceCursor = &cursor + } + merged.Checksum, err = checksum(merged.Entries) + if err != nil { + return durableLogBatch{}, pendingLogSegment{}, false, err + } + return merged, previous, true, nil + } + return durableLogBatch{}, pendingLogSegment{}, false, nil +} + +func compatibleLogBatch(previous durableLogBatch, next durableLogBatch) bool { + return previous.LogStreamID == next.LogStreamID && previous.ServerInstanceID == next.ServerInstanceID && previous.StreamKey == next.StreamKey && previous.Source == next.Source && previous.Compression == next.Compression && previous.LogSessionID == next.LogSessionID && previous.SessionStartedAt.Equal(next.SessionStartedAt) +} + +func compatibleSourceCursor(previous *LogSourceCursor, next *LogSourceCursor) bool { + if previous == nil || next == nil { + return previous == nil && next == nil + } + return next.StartOffset >= previous.EndOffset +} + +func (spool LogSpool) writeBatch(batch durableLogBatch) error { + path := spool.batchPath(batch) + if existing, err := readDurableLogBatch(path); err == nil { + if existing.LogStreamID == batch.LogStreamID && existing.FirstSeq == batch.FirstSeq && existing.LastSeq == batch.LastSeq && existing.Checksum == batch.Checksum { + return nil + } + return fmt.Errorf("log spool segment conflicts with committed batch") + } else if !os.IsNotExist(err) { + return err + } + return spool.writeBatchAt(path, batch) +} + +func (spool LogSpool) replaceBatch(path string, batch durableLogBatch) error { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("log spool segment path is required") + } + return spool.writeBatchAt(path, batch) +} + +func (spool LogSpool) writeBatchAt(path string, batch durableLogBatch) error { + tmp := path + ".tmp" + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open log spool segment: %w", err) + } + encodeErr := json.NewEncoder(file).Encode(batch) + closeErr := file.Close() + if encodeErr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("encode log spool segment: %w", encodeErr) + } + if closeErr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("close log spool segment: %w", closeErr) + } + if err := syncFile(tmp); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("commit log spool segment: %w", err) + } + return nil +} + +func (spool LogSpool) Pending() ([]protocol.LogBatchIngestRequest, error) { + spool.mu.Lock() + defer spool.mu.Unlock() + segments, err := spool.pendingSegmentsLocked() + if err != nil { + return nil, err + } + batches := make([]protocol.LogBatchIngestRequest, 0, len(segments)) + for _, segment := range segments { + batches = append(batches, segment.batch.LogBatchIngestRequest) + } + return batches, nil +} + +func (spool LogSpool) pendingSegmentsLocked() ([]pendingLogSegment, error) { + entries, err := os.ReadDir(spool.dir) + if err != nil { + return nil, fmt.Errorf("read log spool: %w", err) + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + paths = append(paths, filepath.Join(spool.dir, entry.Name())) + } + sort.Strings(paths) + segments := make([]pendingLogSegment, 0, len(paths)) + for _, path := range paths { + batch, err := readDurableLogBatch(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + segments = append(segments, pendingLogSegment{path: path, batch: batch}) + } + return segments, nil +} + +func (spool LogSpool) Ack(response protocol.LogBatchIngestResponse) error { + if response.AcceptedFrom == 0 || response.AcceptedTo < response.AcceptedFrom { + return nil + } + spool.mu.Lock() + defer spool.mu.Unlock() + watermark := spool.watermarks[response.LogStreamID] + if response.AcceptedTo > watermark.Acknowledged { + watermark.Acknowledged = response.AcceptedTo + } + if watermark.Allocated < watermark.Acknowledged { + watermark.Allocated = watermark.Acknowledged + } + spool.watermarks[response.LogStreamID] = watermark + if err := spool.persistWatermarksLocked(); err != nil { + return err + } + segments, err := spool.pendingSegmentsLocked() + if err != nil { + return err + } + for _, segment := range segments { + batch := segment.batch + if batch.LogStreamID == response.LogStreamID && batch.FirstSeq >= response.AcceptedFrom && batch.LastSeq <= response.AcceptedTo { + if err := os.Remove(segment.path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove acknowledged log spool segment: %w", err) + } + } + } + return nil +} + +func (spool *LogSpool) loadWatermarks() error { + body, err := os.ReadFile(spool.statePath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("read log watermarks: %w", err) + } + var state logSpoolState + if err := json.Unmarshal(body, &state); err != nil { + return fmt.Errorf("decode log watermarks: %w", err) + } + if state.Streams != nil { + spool.watermarks = state.Streams + } + return nil +} + +func (spool LogSpool) persistWatermarksLocked() error { + body, err := json.Marshal(logSpoolState{Streams: spool.watermarks}) + if err != nil { + return err + } + temporary := spool.statePath + ".tmp" + if err := os.WriteFile(temporary, body, 0o600); err != nil { + return fmt.Errorf("write log watermarks: %w", err) + } + if err := syncFile(temporary); err != nil { + _ = os.Remove(temporary) + return err + } + if err := os.Rename(temporary, spool.statePath); err != nil { + _ = os.Remove(temporary) + return fmt.Errorf("commit log watermarks: %w", err) + } + return nil +} + +type LogBatchClient interface { + IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) +} + +type PermanentLogBatchError struct { + Reason string + Err error +} + +func (err PermanentLogBatchError) Error() string { + if err.Err == nil { + return "permanent log batch rejection: " + err.Reason + } + return "permanent log batch rejection: " + err.Reason + ": " + err.Err.Error() +} + +func (err PermanentLogBatchError) Unwrap() error { + return err.Err +} + +func PermanentLogBatchRejection(reason string, err error) error { + return PermanentLogBatchError{Reason: sanitizeSegmentName(reason), Err: err} +} + +func (spool LogSpool) Flush(ctx context.Context, client LogBatchClient) (int, error) { + if client == nil { + return 0, fmt.Errorf("log batch client is required") + } + spool.mu.Lock() + pending, err := spool.pendingSegmentsLocked() + if err != nil { + spool.mu.Unlock() + return 0, err + } + for _, segment := range pending { + spool.inflight[segment.path] = true + } + spool.mu.Unlock() + defer func() { + spool.mu.Lock() + defer spool.mu.Unlock() + for _, segment := range pending { + delete(spool.inflight, segment.path) + } + }() + acknowledged := 0 + for _, segment := range pending { + batch := segment.batch.LogBatchIngestRequest + if err := ctx.Err(); err != nil { + return acknowledged, err + } + response, err := client.IngestLogBatch(ctx, batch) + if err != nil { + var permanent PermanentLogBatchError + if errors.As(err, &permanent) { + if rejectErr := spool.Reject(batch, permanent.Reason); rejectErr != nil { + return acknowledged, rejectErr + } + acknowledged++ + continue + } + return acknowledged, err + } + if !response.Accepted || response.LogStreamID != batch.LogStreamID || response.AcceptedFrom > batch.FirstSeq || response.AcceptedTo < batch.LastSeq { + return acknowledged, fmt.Errorf("platform log acknowledgement does not cover pending batch") + } + if err := spool.Ack(response); err != nil { + return acknowledged, err + } + acknowledged++ + } + return acknowledged, nil +} + +func (spool LogSpool) Reject(batch protocol.LogBatchIngestRequest, reason string) error { + spool.mu.Lock() + defer spool.mu.Unlock() + segments, err := spool.pendingSegmentsLocked() + if err != nil { + return err + } + path := "" + for _, segment := range segments { + current := segment.batch + if current.LogStreamID == batch.LogStreamID && current.FirstSeq == batch.FirstSeq && current.LastSeq == batch.LastSeq && current.Checksum == batch.Checksum { + path = segment.path + break + } + } + if path == "" { + return nil + } + rejectedDir := filepath.Join(filepath.Dir(spool.dir), "logs-rejected") + if err := os.MkdirAll(rejectedDir, 0o755); err != nil { + return fmt.Errorf("create rejected log spool directory: %w", err) + } + rejectedPath := filepath.Join(rejectedDir, fmt.Sprintf("%s.%s", filepath.Base(path), sanitizeSegmentName(reason))) + if err := os.Rename(path, rejectedPath); err != nil { + return fmt.Errorf("move rejected log spool segment: %w", err) + } + return nil +} + +func readDurableLogBatch(path string) (durableLogBatch, error) { + file, err := os.Open(path) + if err != nil { + return durableLogBatch{}, err + } + defer file.Close() + var batch durableLogBatch + if err := json.NewDecoder(file).Decode(&batch); err != nil { + return durableLogBatch{}, fmt.Errorf("decode log spool segment: %w", err) + } + return batch, nil +} + +func syncFile(path string) error { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return fmt.Errorf("open spool file for sync: %w", err) + } + defer file.Close() + if err := file.Sync(); err != nil { + return fmt.Errorf("sync spool file: %w", err) + } + return nil +} + +func (spool LogSpool) batchPath(batch durableLogBatch) string { + streamID := sanitizeSegmentName(batch.LogStreamID) + return filepath.Join(spool.dir, fmt.Sprintf("%s-%020d-%020d.json", streamID, batch.FirstSeq, batch.LastSeq)) +} + +func (spool LogSpool) watermarkPath(streamID string) string { + return filepath.Join(filepath.Dir(spool.dir), "log-watermarks", sanitizeSegmentName(streamID)+".json") +} + +func (spool LogSpool) readWatermark(streamID string) (uint64, error) { + file, err := os.Open(spool.watermarkPath(streamID)) + if err != nil { + return 0, err + } + defer file.Close() + var watermark struct { + LatestSeq uint64 `json:"latestSeq"` + } + if err := json.NewDecoder(file).Decode(&watermark); err != nil { + return 0, fmt.Errorf("decode log watermark: %w", err) + } + return watermark.LatestSeq, nil +} + +func sanitizeSegmentName(value string) string { + var builder strings.Builder + for _, r := range value { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' { + builder.WriteRune(r) + continue + } + builder.WriteByte('_') + } + if builder.Len() == 0 { + return "stream" + } + return builder.String() +} diff --git a/spool/log_spool_test.go b/spool/log_spool_test.go new file mode 100644 index 0000000..69bf77c --- /dev/null +++ b/spool/log_spool_test.go @@ -0,0 +1,290 @@ +package spool + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "browser.local/run/protocol" +) + +func TestLogSpoolAggregatesContiguousEntriesForOneStream(t *testing.T) { + logSpool, err := NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + checksum := func(entries []protocol.LogEntry) (string, error) { return fmt.Sprintf("sha256:%d", len(entries)), nil } + for sequence := uint64(1); sequence <= 3; sequence++ { + batch := validSpoolLogBatch(sequence, sequence) + if err := logSpool.EnqueueAggregated(batch, checksum); err != nil { + t.Fatalf("enqueue sequence %d: %v", sequence, err) + } + } + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(pending) != 1 || pending[0].FirstSeq != 1 || pending[0].LastSeq != 3 || len(pending[0].Entries) != 3 || pending[0].Checksum != "sha256:3" { + t.Fatalf("expected one aggregated batch, got %+v", pending) + } + if err := logSpool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 3}); err != nil { + t.Fatalf("ack aggregated batch: %v", err) + } + pending, err = logSpool.Pending() + if err != nil || len(pending) != 0 { + t.Fatalf("expected acknowledged aggregation removed, pending=%+v err=%v", pending, err) + } +} + +func TestLogSpoolRetainsPendingAndRemovesAcknowledgedBatch(t *testing.T) { + spool, err := NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + first := validSpoolLogBatch(1, 2) + second := validSpoolLogBatch(3, 3) + if err := spool.Enqueue(first); err != nil { + t.Fatalf("enqueue first: %v", err) + } + if err := spool.Enqueue(second); err != nil { + t.Fatalf("enqueue second: %v", err) + } + pending, err := spool.Pending() + if err != nil { + t.Fatalf("pending before ack: %v", err) + } + if len(pending) != 2 { + t.Fatalf("expected two pending batches, got %+v", pending) + } + + if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 2}); err != nil { + t.Fatalf("ack first: %v", err) + } + pending, err = spool.Pending() + if err != nil { + t.Fatalf("pending after ack: %v", err) + } + if len(pending) != 1 || pending[0].FirstSeq != 3 { + t.Fatalf("expected second batch pending, got %+v", pending) + } +} + +func TestLogSpoolRetainsBatchWhenAckDoesNotCoverRange(t *testing.T) { + spool, err := NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + if err := spool.Enqueue(validSpoolLogBatch(1, 2)); err != nil { + t.Fatalf("enqueue: %v", err) + } + if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil { + t.Fatalf("partial ack: %v", err) + } + pending, err := spool.Pending() + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("expected batch to remain pending, got %+v", pending) + } +} + +func TestLogSpoolQuarantinesPermanentRejectedBatch(t *testing.T) { + root := t.TempDir() + logSpool, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + if err := logSpool.Enqueue(validSpoolLogBatch(5, 5)); err != nil { + t.Fatalf("enqueue: %v", err) + } + flushed, err := logSpool.Flush(context.Background(), permanentRejectLogBatchClient{}) + if err != nil { + t.Fatalf("flush permanent rejection: %v", err) + } + if flushed != 1 { + t.Fatalf("expected one quarantined batch, got %d", flushed) + } + pending, err := logSpool.Pending() + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(pending) != 0 { + t.Fatalf("expected no pending batches, got %+v", pending) + } + rejected, err := os.ReadDir(filepath.Join(root, "logs-rejected")) + if err != nil { + t.Fatalf("read rejected dir: %v", err) + } + if len(rejected) != 1 || !filepath.IsLocal(rejected[0].Name()) { + t.Fatalf("expected one local rejected file, got %+v", rejected) + } +} + +func TestLogSpoolRestoresPendingAllocationWithoutWatermark(t *testing.T) { + root := t.TempDir() + first, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new first spool: %v", err) + } + batch := validSpoolLogBatch(9, 9) + batch.LogStreamID = "run.endpoint.server.stdout" + if err := first.Enqueue(batch); err != nil { + t.Fatalf("enqueue pending batch: %v", err) + } + if err := os.Remove(filepath.Join(root, "log-watermarks.json")); err != nil { + t.Fatalf("remove watermark state: %v", err) + } + restarted, err := NewLogSpool(root) + if err != nil { + t.Fatalf("restart spool: %v", err) + } + called := false + sequence, err := restarted.NextSequence(context.Background(), batch.LogStreamID, func(context.Context, string) (uint64, error) { + called = true + return 3, nil + }) + if err != nil { + t.Fatalf("allocate after restart: %v", err) + } + if called || sequence != 10 { + t.Fatalf("expected pending watermark to allocate 10 without remote recovery, got sequence=%d remoteCalled=%t", sequence, called) + } +} + +func TestLogSpoolSourceCursorDeduplicatesAcknowledgedReplayAfterRestart(t *testing.T) { + root := t.TempDir() + checksum := func(entries []protocol.LogEntry) (string, error) { + return fmt.Sprintf("sha256:%d:%s", len(entries), entries[len(entries)-1].Line), nil + } + first, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + batch := validSpoolLogBatch(0, 0) + batch.FirstSeq = 0 + batch.LastSeq = 0 + batch.Entries = []protocol.LogEntry{{Timestamp: time.Now().UTC(), Line: "first line"}} + sequence, appended, err := first.EnqueueNextAggregated(context.Background(), batch, &LogSourceCursor{StartOffset: 0, EndOffset: 11}, nil, checksum) + if err != nil || !appended || sequence != 1 { + t.Fatalf("append first cursor: sequence=%d appended=%t err=%v", sequence, appended, err) + } + if err := first.Ack(protocol.LogBatchIngestResponse{LogStreamID: batch.LogStreamID, AcceptedFrom: 1, AcceptedTo: 1}); err != nil { + t.Fatalf("ack first cursor: %v", err) + } + restarted, err := NewLogSpool(root) + if err != nil { + t.Fatalf("restart log spool: %v", err) + } + sequence, appended, err = restarted.EnqueueNextAggregated(context.Background(), batch, &LogSourceCursor{StartOffset: 0, EndOffset: 11}, nil, checksum) + if err != nil || appended || sequence != 1 { + t.Fatalf("deduplicate acknowledged cursor: sequence=%d appended=%t err=%v", sequence, appended, err) + } + batch.Entries[0].Line = "second line" + sequence, appended, err = restarted.EnqueueNextAggregated(context.Background(), batch, &LogSourceCursor{StartOffset: 11, EndOffset: 23}, nil, checksum) + if err != nil || !appended || sequence != 2 { + t.Fatalf("append next cursor: sequence=%d appended=%t err=%v", sequence, appended, err) + } + pending, err := restarted.Pending() + if err != nil || len(pending) != 1 || pending[0].FirstSeq != 2 || pending[0].Entries[0].Line != "second line" { + t.Fatalf("unexpected pending cursor batches: pending=%+v err=%v", pending, err) + } +} + +func TestLogSpoolAggregatedSegmentIsReplacedInPlace(t *testing.T) { + root := t.TempDir() + logSpool, err := NewLogSpool(root) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + checksum := func(entries []protocol.LogEntry) (string, error) { return fmt.Sprintf("sha256:%d", len(entries)), nil } + if err := logSpool.EnqueueAggregated(validSpoolLogBatch(1, 1), checksum); err != nil { + t.Fatalf("enqueue first segment: %v", err) + } + before, err := os.ReadDir(filepath.Join(root, "logs")) + if err != nil || len(before) != 1 { + t.Fatalf("read first segment: entries=%+v err=%v", before, err) + } + if err := logSpool.EnqueueAggregated(validSpoolLogBatch(2, 2), checksum); err != nil { + t.Fatalf("aggregate second segment: %v", err) + } + after, err := os.ReadDir(filepath.Join(root, "logs")) + if err != nil || len(after) != 1 || after[0].Name() != before[0].Name() { + t.Fatalf("aggregation did not replace one stable path: before=%+v after=%+v err=%v", before, after, err) + } + pending, err := logSpool.Pending() + if err != nil || len(pending) != 1 || pending[0].FirstSeq != 1 || pending[0].LastSeq != 2 { + t.Fatalf("unexpected aggregate after replacement: pending=%+v err=%v", pending, err) + } +} + +func TestLogSpoolDoesNotExtendInflightAggregate(t *testing.T) { + logSpool, err := NewLogSpool(t.TempDir()) + if err != nil { + t.Fatalf("new log spool: %v", err) + } + checksum := func(entries []protocol.LogEntry) (string, error) { return fmt.Sprintf("sha256:%d", len(entries)), nil } + if err := logSpool.EnqueueAggregated(validSpoolLogBatch(1, 1), checksum); err != nil { + t.Fatalf("enqueue first segment: %v", err) + } + client := &blockingLogBatchClient{started: make(chan struct{}), release: make(chan struct{})} + done := make(chan error, 1) + go func() { + _, err := logSpool.Flush(context.Background(), client) + done <- err + }() + <-client.started + if err := logSpool.EnqueueAggregated(validSpoolLogBatch(2, 2), checksum); err != nil { + t.Fatalf("enqueue while first segment is inflight: %v", err) + } + close(client.release) + if err := <-done; err != nil { + t.Fatalf("flush inflight segment: %v", err) + } + pending, err := logSpool.Pending() + if err != nil || len(pending) != 1 || pending[0].FirstSeq != 2 || pending[0].LastSeq != 2 { + t.Fatalf("inflight segment was extended or next segment lost: pending=%+v err=%v", pending, err) + } +} + +type blockingLogBatchClient struct { + once sync.Once + started chan struct{} + release chan struct{} +} + +func (client *blockingLogBatchClient) IngestLogBatch(_ context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + client.once.Do(func() { close(client.started) }) + <-client.release + return protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: batch.LogStreamID, AcceptedFrom: batch.FirstSeq, AcceptedTo: batch.LastSeq}, nil +} + +type permanentRejectLogBatchClient struct{} + +func (permanentRejectLogBatchClient) IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { + return protocol.LogBatchIngestResponse{}, PermanentLogBatchRejection("platform_sequence_gap", nil) +} + +func validSpoolLogBatch(firstSeq uint64, lastSeq uint64) protocol.LogBatchIngestRequest { + entries := make([]protocol.LogEntry, 0, lastSeq-firstSeq+1) + for seq := firstSeq; seq <= lastSeq; seq++ { + entries = append(entries, protocol.LogEntry{Seq: seq, Timestamp: time.Date(2026, 7, 3, 12, 0, int(seq), 0, time.UTC), Level: "info", Line: "line"}) + } + return protocol.LogBatchIngestRequest{ + RunEndpointID: "run-local", + SessionToken: "session-token", + LogStreamID: "log-1", + ServerInstanceID: "server-1", + StreamKey: "stdout", + Source: "process", + FirstSeq: firstSeq, + LastSeq: lastSeq, + Compression: "none", + Checksum: "sha256:test", + Entries: entries, + } +}