first commit
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
RUN_MODE=worker
|
||||
RUN_PLATFORM_URL=http://127.0.0.1:8080
|
||||
RUN_ENDPOINT_ID=run-local
|
||||
RUN_DISPLAY_NAME=Local Run
|
||||
RUN_VERSION=0.1.0
|
||||
RUN_REGISTRATION_TOKEN=local-registration
|
||||
RUN_WORKSPACE_ROOT=.run-workspace
|
||||
RUN_SPOOL_ROOT=.run-workspace/spool
|
||||
RUN_MAX_JOBS=1
|
||||
RUN_HEARTBEAT_INTERVAL_MS=15000
|
||||
RUN_POLL_INTERVAL_MS=2000
|
||||
RUN_RETRY_BACKOFF_MS=1000
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# AGENTS.md for run
|
||||
|
||||
This file applies to `run/`.
|
||||
|
||||
## Channel Rules
|
||||
|
||||
Do not rebuild the old all-in-one WebSocket model. Keep these workloads separate:
|
||||
|
||||
- Control: hello, heartbeat, version, capabilities, capacity.
|
||||
- Job: claim, ack, progress, result, cancel, reconcile.
|
||||
- Logs: local spool, batch upload, sequence acknowledgement, retry.
|
||||
- Artifacts: chunks, checksums, resume, throttling.
|
||||
- Game client bridge: optional in-game command/snapshot channel.
|
||||
|
||||
Artifact transfer must not block control heartbeats, job result reporting, or log upload.
|
||||
|
||||
## Structure Rules
|
||||
|
||||
Protocol structs live in `protocol/`. Local runtime types live in `runtime/` or `domain/`. Shared helpers live in `shared/` only when needed by multiple packages.
|
||||
|
||||
## Safety Rules
|
||||
|
||||
Run must enforce scoped paths and never expose raw host paths, local secrets, or unrestricted command execution to platform_web or plugins.
|
||||
@@ -0,0 +1,29 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM golang:1.25.1-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY run/go.mod ./
|
||||
RUN go mod download
|
||||
COPY run/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/run ./cmd/run
|
||||
|
||||
FROM alpine:3.21
|
||||
RUN addgroup -S run && adduser -S run -G run
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/run /app/run
|
||||
RUN mkdir -p /data/run/workspace /data/run/spool && chown -R run:run /data/run
|
||||
USER run
|
||||
ENV RUN_MODE=worker \
|
||||
RUN_PLATFORM_URL=http://platform:8080 \
|
||||
RUN_ENDPOINT_ID=run-docker \
|
||||
RUN_DISPLAY_NAME="Docker Run" \
|
||||
RUN_VERSION=0.1.0 \
|
||||
RUN_REGISTRATION_TOKEN=local-registration \
|
||||
RUN_WORKSPACE_ROOT=/data/run/workspace \
|
||||
RUN_SPOOL_ROOT=/data/run/spool \
|
||||
RUN_MAX_JOBS=1 \
|
||||
RUN_HEARTBEAT_INTERVAL_MS=15000 \
|
||||
RUN_POLL_INTERVAL_MS=2000 \
|
||||
RUN_RETRY_BACKOFF_MS=1000
|
||||
ENTRYPOINT ["/app/run"]
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# run
|
||||
|
||||
Machine-side executor for scoped server operations.
|
||||
|
||||
## Responsibilities
|
||||
|
||||
- Register with platform and report heartbeat, version, capabilities, and capacity.
|
||||
- Claim and execute jobs for server lifecycle, file/config work, backups, updates, and bounded database or command work.
|
||||
- Collect server logs into local spool and upload acknowledged batches.
|
||||
- Transfer artifacts with chunking, checksums, resume, throttling, and low priority.
|
||||
- Optionally coordinate with a game client bridge when a specific game requires in-game commands or snapshots.
|
||||
|
||||
## Required Directory Plan
|
||||
|
||||
Implementation should use dedicated directories for:
|
||||
|
||||
- `api/`: platform-facing HTTP/gRPC client adapters.
|
||||
- `protocol/`: control, job, log, artifact, and game-client bridge DTOs.
|
||||
- `domain/`: executor domain types.
|
||||
- `runtime/`: local execution and server process orchestration.
|
||||
- `spool/`: local durable log/job/artifact queues.
|
||||
- `artifact/`: chunk transfer implementation.
|
||||
- `logingest/`: log collectors and uploaders.
|
||||
- `config/`: configuration structures and loading.
|
||||
- `shared/`: small shared helpers.
|
||||
|
||||
Logs and artifacts must have separate queues and priority controls.
|
||||
|
||||
## Development Baseline
|
||||
|
||||
Tooling:
|
||||
|
||||
- Go 1.25.1.
|
||||
- Module: `browser.local/run`.
|
||||
|
||||
Commands:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
go run ./cmd/run
|
||||
```
|
||||
|
||||
Runtime configuration:
|
||||
|
||||
- `RUN_MODE`: local mode, default `smoke`.
|
||||
- `RUN_PLATFORM_URL`: platform base URL, default `http://127.0.0.1:8080`.
|
||||
- `RUN_ENDPOINT_ID`, `RUN_DISPLAY_NAME`, `RUN_VERSION`, `RUN_REGISTRATION_TOKEN`: worker identity and registration metadata.
|
||||
- `RUN_WORKSPACE_ROOT`, `RUN_SPOOL_ROOT`: scoped local server workspace and separate local log/artifact queues.
|
||||
- `RUN_MAX_JOBS`, `RUN_HEARTBEAT_INTERVAL_MS`, `RUN_POLL_INTERVAL_MS`, `RUN_RETRY_BACKOFF_MS`: worker capacity and scheduling controls.
|
||||
|
||||
For local direct debugging, copy `run/.env.example` to `run/.env`, edit the values, and run:
|
||||
|
||||
```bash
|
||||
set -a
|
||||
source .env
|
||||
set +a
|
||||
go run ./cmd/run
|
||||
```
|
||||
|
||||
Use `RUN_MODE=worker` when you want the executor to register, heartbeat, claim jobs, and execute lifecycle templates. Use `RUN_MODE=smoke` for a one-shot config summary.
|
||||
|
||||
In Docker, `RUN_PLATFORM_URL` must be `http://platform:8080` because `platform` is the compose service name. Locally, keep it as `http://127.0.0.1:8080`.
|
||||
|
||||
Current executable behavior includes smoke mode plus worker mode. Worker mode registers with platform, sends lightweight heartbeat metadata, claims lifecycle jobs, acknowledges leases, reports bounded progress, executes scoped `process.install`, `process.start`, and `process.stop` command templates inside per-server workspaces, polls cancellation, submits terminal results, and reconciles active jobs.
|
||||
|
||||
Lifecycle templates are JSON files addressed by logical keys under the server workspace. They resolve to direct executable/argument vectors, not shell strings. Absolute paths, parent traversal, raw credentials, direct sockets, shell launchers, unsafe environment keys, and unsafe output are rejected or redacted. Process stdout/stderr is written to the log spool, and lifecycle result metadata is queued through artifact hooks so control heartbeat and job result submission stay independent from log and artifact work.
|
||||
@@ -0,0 +1,154 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestPlatformClientArtifactMethodsPostJSONAndDecodeResponses(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seen[r.URL.Path] = true
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/artifacts/open":
|
||||
var request protocol.ArtifactTransferOpenRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.ArtifactID != "artifact-1" || request.ChunkSizeBytes != 8 {
|
||||
t.Fatalf("unexpected artifact open request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, validArtifactOpenResponse())
|
||||
case "/api/v1/run/artifacts/chunks":
|
||||
var request protocol.ArtifactChunkUploadRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.TransferID != "transfer-1" || request.ChunkIndex != 0 || string(request.Payload) != "payload" {
|
||||
t.Fatalf("unexpected artifact chunk request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/artifacts/status":
|
||||
var request protocol.ArtifactTransferStatusRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.TransferID != "transfer-1" {
|
||||
t.Fatalf("unexpected artifact status request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.ArtifactTransferStatusResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", Direction: "upload", TotalChunks: 2, ChunkSizeBytes: 8, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/artifacts/complete":
|
||||
var request protocol.ArtifactTransferCompleteRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.Checksum == "" || request.SizeBytes != 7 {
|
||||
t.Fatalf("unexpected artifact complete request: %+v", request)
|
||||
}
|
||||
response := protocol.ArtifactTransferCompleteResponse{Accepted: true, TransferID: "transfer-1", Artifact: validArtifactMetadata("available"), Completed: true, ServerTime: fixedClientTestTime()}
|
||||
writeTestJSON(t, w, response)
|
||||
default:
|
||||
t.Fatalf("unexpected request path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
open, err := client.OpenArtifactTransfer(ctx, validClientArtifactOpen())
|
||||
if err != nil || !open.Accepted || open.TransferID != "transfer-1" {
|
||||
t.Fatalf("open artifact response=%+v err=%v", open, err)
|
||||
}
|
||||
chunk, err := client.UploadArtifactChunk(ctx, validClientArtifactChunk())
|
||||
if err != nil || chunk.NextMissingChunkIndex != 1 {
|
||||
t.Fatalf("upload artifact chunk response=%+v err=%v", chunk, err)
|
||||
}
|
||||
status, err := client.QueryArtifactTransferStatus(ctx, validClientArtifactStatus())
|
||||
if err != nil || len(status.ReceivedChunkIndexes) != 1 {
|
||||
t.Fatalf("artifact status response=%+v err=%v", status, err)
|
||||
}
|
||||
complete, err := client.CompleteArtifactTransfer(ctx, validClientArtifactComplete())
|
||||
if err != nil || !complete.Completed || complete.Artifact.State != "available" {
|
||||
t.Fatalf("complete artifact response=%+v err=%v", complete, err)
|
||||
}
|
||||
|
||||
for _, path := range []string{"/api/v1/run/artifacts/open", "/api/v1/run/artifacts/chunks", "/api/v1/run/artifacts/status", "/api/v1/run/artifacts/complete"} {
|
||||
if !seen[path] {
|
||||
t.Fatalf("expected request to %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientArtifactMethodReturnsErrorForPlatformFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"code":"validation_failed"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
if _, err := client.OpenArtifactTransfer(context.Background(), validClientArtifactOpen()); err == nil {
|
||||
t.Fatal("expected platform error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactChunkPayloadUsesJSONBase64Encoding(t *testing.T) {
|
||||
encoded, err := json.Marshal(validClientArtifactChunk())
|
||||
if err != nil {
|
||||
t.Fatalf("marshal artifact chunk: %v", err)
|
||||
}
|
||||
if !json.Valid(encoded) || !containsJSONPayloadField(encoded) {
|
||||
t.Fatalf("expected JSON encoded payload field, got %s", string(encoded))
|
||||
}
|
||||
}
|
||||
|
||||
func validClientArtifactOpen() protocol.ArtifactTransferOpenRequest {
|
||||
return protocol.ArtifactTransferOpenRequest{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: "session-token",
|
||||
ArtifactID: "artifact-1",
|
||||
Direction: "upload",
|
||||
OwnerKind: "job",
|
||||
OwnerID: "job-1",
|
||||
SizeBytes: 7,
|
||||
ChunkSizeBytes: 8,
|
||||
Checksum: validSHA256Checksum(),
|
||||
IdempotencyKey: "artifact-upload-1",
|
||||
}
|
||||
}
|
||||
|
||||
func validClientArtifactChunk() protocol.ArtifactChunkUploadRequest {
|
||||
return protocol.ArtifactChunkUploadRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0, Offset: 0, SizeBytes: 7, Checksum: validSHA256Checksum(), Payload: []byte("payload")}
|
||||
}
|
||||
|
||||
func validClientArtifactStatus() protocol.ArtifactTransferStatusRequest {
|
||||
return protocol.ArtifactTransferStatusRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1"}
|
||||
}
|
||||
|
||||
func validClientArtifactComplete() protocol.ArtifactTransferCompleteRequest {
|
||||
return protocol.ArtifactTransferCompleteRequest{RunEndpointID: "run-local", SessionToken: "session-token", TransferID: "transfer-1", ArtifactID: "artifact-1", Checksum: validSHA256Checksum(), SizeBytes: 7}
|
||||
}
|
||||
|
||||
func validArtifactOpenResponse() protocol.ArtifactTransferOpenResponse {
|
||||
return protocol.ArtifactTransferOpenResponse{Accepted: true, TransferID: "transfer-1", Direction: "upload", Artifact: validArtifactMetadata("uploading"), TotalChunks: 2, ChunkSizeBytes: 8, NextMissingChunkIndex: 0, ServerTime: fixedClientTestTime()}
|
||||
}
|
||||
|
||||
func validArtifactMetadata(state string) protocol.ArtifactMetadata {
|
||||
return protocol.ArtifactMetadata{ID: "artifact-1", OwnerKind: "job", OwnerID: "job-1", SizeBytes: 7, Checksum: validSHA256Checksum(), State: state, CreatedAt: fixedClientTestTime(), UpdatedAt: fixedClientTestTime()}
|
||||
}
|
||||
|
||||
func validSHA256Checksum() string {
|
||||
return "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
|
||||
func containsJSONPayloadField(encoded []byte) bool {
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(encoded, &body); err != nil {
|
||||
return false
|
||||
}
|
||||
_, exists := body["payload"]
|
||||
return exists
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestPlatformClientLightweightChannelsCompleteWhileArtifactChunkIsBlocked(t *testing.T) {
|
||||
artifactStarted := make(chan struct{})
|
||||
releaseArtifact := make(chan struct{})
|
||||
artifactDone := make(chan struct{})
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/artifacts/chunks":
|
||||
var request protocol.ArtifactChunkUploadRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.ChunkIndex != 0 || len(request.Payload) == 0 {
|
||||
t.Fatalf("unexpected artifact payload: %+v", request)
|
||||
}
|
||||
close(artifactStarted)
|
||||
<-releaseArtifact
|
||||
writeTestJSON(t, w, protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: request.TransferID, ArtifactID: request.ArtifactID, ChunkIndex: request.ChunkIndex, ReceivedChunkIndexes: []int{0}, NextMissingChunkIndex: 1, ServerTime: fixedClientTestTime()})
|
||||
close(artifactDone)
|
||||
case "/api/v1/run/control/heartbeat":
|
||||
var request protocol.RunHeartbeatRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
writeTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/result":
|
||||
var request protocol.RunJobResultRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
encoded, _ := json.Marshal(request)
|
||||
for _, forbidden := range []string{"payload", "entries", "/Users/", "unix://", "tcp://", "Bearer ", "sk-", "password="} {
|
||||
if strings.Contains(string(encoded), forbidden) {
|
||||
t.Fatalf("job result carried forbidden transfer content %q: %s", forbidden, string(encoded))
|
||||
}
|
||||
}
|
||||
job := validRunJobAssignment()
|
||||
job.State = request.State
|
||||
job.ResultRef = request.ResultRef
|
||||
writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/logs/batches":
|
||||
var request protocol.LogBatchIngestRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if len(request.Entries) != 1 || request.FirstSeq != 1 || request.LastSeq != 1 {
|
||||
t.Fatalf("unexpected log batch: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: request.LogStreamID, AcceptedFrom: 1, AcceptedTo: 1, LatestSeq: 1, ServerTime: fixedClientTestTime()})
|
||||
default:
|
||||
t.Fatalf("unexpected request path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.UploadArtifactChunk(context.Background(), validClientArtifactChunk())
|
||||
errCh <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-artifactStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("artifact request did not start")
|
||||
}
|
||||
|
||||
lightCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, err := client.Heartbeat(lightCtx, validRunHeartbeatRequest("session-token")); err != nil {
|
||||
t.Fatalf("heartbeat should not wait for artifact chunk: %v", err)
|
||||
}
|
||||
if _, err := client.CompleteJob(lightCtx, validRunJobResultRequest()); err != nil {
|
||||
t.Fatalf("job result should not wait for artifact chunk: %v", err)
|
||||
}
|
||||
if _, err := client.IngestLogBatch(lightCtx, validClientLogBatch()); err != nil {
|
||||
t.Fatalf("log ingest should not wait for artifact chunk: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-artifactDone:
|
||||
t.Fatal("artifact chunk completed before release")
|
||||
default:
|
||||
}
|
||||
close(releaseArtifact)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
t.Fatalf("artifact chunk upload: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("artifact chunk did not finish after release")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestPlatformClientJobMethodsPostJSONAndDecodeResponses(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seen[r.URL.Path] = true
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/jobs/claim":
|
||||
var request protocol.RunJobClaimRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.RunEndpointID != "run-local" || request.SessionToken != "session-token" {
|
||||
t.Fatalf("unexpected claim request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, validRunJobClaimResponse())
|
||||
case "/api/v1/run/jobs/ack":
|
||||
var request protocol.RunJobAckRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.JobID != "job-1" || request.LeaseToken != "lease-1" {
|
||||
t.Fatalf("unexpected ack request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: validRunJobAssignment(), ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/progress":
|
||||
var request protocol.RunJobProgressRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.Progress.Percent != 40 {
|
||||
t.Fatalf("unexpected progress request: %+v", request)
|
||||
}
|
||||
assignment := validRunJobAssignment()
|
||||
assignment.Progress.Percent = 40
|
||||
writeTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/result":
|
||||
var request protocol.RunJobResultRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.State != "succeeded" || request.ResultRef == "" {
|
||||
t.Fatalf("unexpected result request: %+v", request)
|
||||
}
|
||||
assignment := validRunJobAssignment()
|
||||
assignment.State = "succeeded"
|
||||
assignment.ResultRef = request.ResultRef
|
||||
writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/cancel":
|
||||
var request protocol.RunJobCancelPollRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if request.JobID != "job-1" {
|
||||
t.Fatalf("unexpected cancel request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-local", HasCancel: true, JobID: "job-1", Reason: "stop", ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/reconcile":
|
||||
var request protocol.RunJobReconcileRequest
|
||||
decodeTestRequest(t, r, &request)
|
||||
if len(request.ActiveJobIDs) != 1 || request.ActiveJobIDs[0] != "job-1" {
|
||||
t.Fatalf("unexpected reconcile request: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-local", ActiveJobs: []protocol.RunJobAssignment{validRunJobAssignment()}, ServerTime: fixedClientTestTime()})
|
||||
default:
|
||||
t.Fatalf("unexpected request path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
claim, err := client.ClaimJob(ctx, validRunJobClaimRequest())
|
||||
if err != nil || !claim.HasJob {
|
||||
t.Fatalf("claim job response=%+v err=%v", claim, err)
|
||||
}
|
||||
if _, err := client.AckJob(ctx, validRunJobAckRequest()); err != nil {
|
||||
t.Fatalf("ack job: %v", err)
|
||||
}
|
||||
if _, err := client.UpdateJobProgress(ctx, validRunJobProgressRequest()); err != nil {
|
||||
t.Fatalf("progress job: %v", err)
|
||||
}
|
||||
if _, err := client.CompleteJob(ctx, validRunJobResultRequest()); err != nil {
|
||||
t.Fatalf("complete job: %v", err)
|
||||
}
|
||||
if _, err := client.PollJobCancel(ctx, validRunJobCancelPollRequest()); err != nil {
|
||||
t.Fatalf("poll cancel: %v", err)
|
||||
}
|
||||
if _, err := client.ReconcileJobs(ctx, validRunJobReconcileRequest()); err != nil {
|
||||
t.Fatalf("reconcile jobs: %v", err)
|
||||
}
|
||||
|
||||
for _, path := range []string{"/api/v1/run/jobs/claim", "/api/v1/run/jobs/ack", "/api/v1/run/jobs/progress", "/api/v1/run/jobs/result", "/api/v1/run/jobs/cancel", "/api/v1/run/jobs/reconcile"} {
|
||||
if !seen[path] {
|
||||
t.Fatalf("expected request to %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientJobMethodReturnsErrorForPlatformFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"code":"validation_failed"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
if _, err := client.ClaimJob(context.Background(), validRunJobClaimRequest()); err == nil {
|
||||
t.Fatal("expected platform error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientJobLifecycleFlow(t *testing.T) {
|
||||
assignment := validRunJobAssignment()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/jobs/claim":
|
||||
assignment.State = "accepted"
|
||||
writeTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: "run-local", HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/ack":
|
||||
assignment.State = "running"
|
||||
writeTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/progress":
|
||||
assignment.Progress.Percent = 70
|
||||
writeTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/result":
|
||||
assignment.State = "succeeded"
|
||||
assignment.Progress.Percent = 100
|
||||
assignment.ResultRef = "artifact://jobs/job-1/result"
|
||||
writeTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: fixedClientTestTime()})
|
||||
case "/api/v1/run/jobs/reconcile":
|
||||
writeTestJSON(t, w, protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-local", UnknownJobIDs: []string{"local-only"}, ServerTime: fixedClientTestTime()})
|
||||
default:
|
||||
t.Fatalf("unexpected request path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
claim, err := client.ClaimJob(ctx, validRunJobClaimRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("claim job: %v", err)
|
||||
}
|
||||
ack, err := client.AckJob(ctx, protocol.RunJobAckRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
||||
if err != nil {
|
||||
t.Fatalf("ack job: %v", err)
|
||||
}
|
||||
progress, err := client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: ack.Job.JobID, LeaseToken: ack.Job.LeaseToken, Attempt: ack.Job.Attempt, Progress: protocol.RunJobProgressReport{Percent: 70}})
|
||||
if err != nil {
|
||||
t.Fatalf("progress job: %v", err)
|
||||
}
|
||||
result, err := client.CompleteJob(ctx, protocol.RunJobResultRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: progress.Job.JobID, LeaseToken: progress.Job.LeaseToken, Attempt: progress.Job.Attempt, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100}, ResultRef: "artifact://jobs/job-1/result"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete job: %v", err)
|
||||
}
|
||||
reconcile, err := client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{RunEndpointID: "run-local", SessionToken: "session-token", ActiveJobIDs: []string{"local-only"}})
|
||||
if err != nil {
|
||||
t.Fatalf("reconcile jobs: %v", err)
|
||||
}
|
||||
if claim.Job.State != "accepted" || ack.Job.State != "running" || progress.Job.Progress.Percent != 70 || result.Job.State != "succeeded" || len(reconcile.UnknownJobIDs) != 1 {
|
||||
t.Fatalf("unexpected lifecycle responses: claim=%+v ack=%+v progress=%+v result=%+v reconcile=%+v", claim, ack, progress, result, reconcile)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeTestRequest(t *testing.T, r *http.Request, target any) {
|
||||
t.Helper()
|
||||
if r.Method != http.MethodPost {
|
||||
t.Fatalf("expected POST, got %s", r.Method)
|
||||
}
|
||||
if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
|
||||
t.Fatalf("expected JSON content type, got %q", contentType)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(target); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func fixedClientTestTime() time.Time {
|
||||
return time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func validRunJobClaimRequest() protocol.RunJobClaimRequest {
|
||||
return protocol.RunJobClaimRequest{RunEndpointID: "run-local", SessionToken: "session-token", Capabilities: []string{"process.start"}, Capacity: protocol.RunCapacityReport{MaxJobs: 4}}
|
||||
}
|
||||
|
||||
func validRunJobAckRequest() protocol.RunJobAckRequest {
|
||||
return protocol.RunJobAckRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, Message: "started"}
|
||||
}
|
||||
|
||||
func validRunJobProgressRequest() protocol.RunJobProgressRequest {
|
||||
return protocol.RunJobProgressRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, Progress: protocol.RunJobProgressReport{Percent: 40, Message: "working"}}
|
||||
}
|
||||
|
||||
func validRunJobResultRequest() protocol.RunJobResultRequest {
|
||||
return protocol.RunJobResultRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1", Attempt: 1, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100}, ResultRef: "artifact://jobs/job-1/result", Message: "done"}
|
||||
}
|
||||
|
||||
func validRunJobCancelPollRequest() protocol.RunJobCancelPollRequest {
|
||||
return protocol.RunJobCancelPollRequest{RunEndpointID: "run-local", SessionToken: "session-token", JobID: "job-1", LeaseToken: "lease-1"}
|
||||
}
|
||||
|
||||
func validRunJobReconcileRequest() protocol.RunJobReconcileRequest {
|
||||
return protocol.RunJobReconcileRequest{RunEndpointID: "run-local", SessionToken: "session-token", ActiveJobIDs: []string{"job-1"}}
|
||||
}
|
||||
|
||||
func validRunJobClaimResponse() protocol.RunJobClaimResponse {
|
||||
job := validRunJobAssignment()
|
||||
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: "run-local", HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: fixedClientTestTime()}
|
||||
}
|
||||
|
||||
func validRunJobAssignment() protocol.RunJobAssignment {
|
||||
return protocol.RunJobAssignment{
|
||||
JobID: "job-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: "process.start",
|
||||
IdempotencyKey: "idem-1",
|
||||
State: "accepted",
|
||||
LeaseToken: "lease-1",
|
||||
Attempt: 1,
|
||||
CreatedAt: fixedClientTestTime(),
|
||||
UpdatedAt: fixedClientTestTime(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestPlatformClientIngestLogBatchPostsJSONAndDecodesResponse(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/logs/batches" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
|
||||
t.Fatalf("expected JSON content type, got %q", contentType)
|
||||
}
|
||||
var request protocol.LogBatchIngestRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatalf("decode log ingest request: %v", err)
|
||||
}
|
||||
if request.LogStreamID != "log-1" || request.FirstSeq != 1 || request.LastSeq != 1 || len(request.Entries) != 1 {
|
||||
t.Fatalf("unexpected log ingest payload: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1, LatestSeq: 1, ServerTime: fixedClientTestTime()})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
response, err := client.IngestLogBatch(context.Background(), validClientLogBatch())
|
||||
if err != nil {
|
||||
t.Fatalf("ingest log batch: %v", err)
|
||||
}
|
||||
if !response.Accepted || response.LatestSeq != 1 {
|
||||
t.Fatalf("unexpected log ingest response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientIngestLogBatchReturnsErrorForPlatformFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"code":"validation_failed"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
if _, err := client.IngestLogBatch(context.Background(), validClientLogBatch()); err == nil {
|
||||
t.Fatal("expected platform error")
|
||||
}
|
||||
}
|
||||
|
||||
func validClientLogBatch() protocol.LogBatchIngestRequest {
|
||||
return protocol.LogBatchIngestRequest{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: "session-token",
|
||||
LogStreamID: "log-1",
|
||||
ServerInstanceID: "server-1",
|
||||
StreamKey: "stdout",
|
||||
Source: "process",
|
||||
FirstSeq: 1,
|
||||
LastSeq: 1,
|
||||
Compression: "none",
|
||||
Checksum: "sha256:test",
|
||||
Entries: []protocol.LogEntry{{
|
||||
Seq: 1,
|
||||
Timestamp: time.Date(2026, 7, 3, 12, 0, 1, 0, time.UTC),
|
||||
Level: "info",
|
||||
Line: "line",
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type PlatformClient struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewPlatformClient(rawURL string) (PlatformClient, error) {
|
||||
return NewPlatformClientWithHTTPClient(rawURL, http.DefaultClient)
|
||||
}
|
||||
|
||||
func NewPlatformClientWithHTTPClient(rawURL string, httpClient *http.Client) (PlatformClient, error) {
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return PlatformClient{}, err
|
||||
}
|
||||
if parsed.Scheme == "" || parsed.Host == "" {
|
||||
return PlatformClient{}, fmt.Errorf("platform URL must include scheme and host")
|
||||
}
|
||||
if httpClient == nil {
|
||||
httpClient = http.DefaultClient
|
||||
}
|
||||
|
||||
return PlatformClient{baseURL: strings.TrimRight(parsed.String(), "/"), httpClient: httpClient}, nil
|
||||
}
|
||||
|
||||
func (c PlatformClient) BaseURL() string {
|
||||
return c.baseURL
|
||||
}
|
||||
|
||||
func (c PlatformClient) Hello(ctx context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) {
|
||||
return postPlatformJSON[protocol.RunHelloRequest, protocol.RunHelloResponse](ctx, c, "/api/v1/run/control/hello", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) Heartbeat(ctx context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) {
|
||||
return postPlatformJSON[protocol.RunHeartbeatRequest, protocol.RunHeartbeatResponse](ctx, c, "/api/v1/run/control/heartbeat", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) ClaimJob(ctx context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobClaimRequest, protocol.RunJobClaimResponse](ctx, c, "/api/v1/run/jobs/claim", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) AckJob(ctx context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobAckRequest, protocol.RunJobAckResponse](ctx, c, "/api/v1/run/jobs/ack", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) UpdateJobProgress(ctx context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobProgressRequest, protocol.RunJobProgressResponse](ctx, c, "/api/v1/run/jobs/progress", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) CompleteJob(ctx context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobResultRequest, protocol.RunJobResultResponse](ctx, c, "/api/v1/run/jobs/result", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) PollJobCancel(ctx context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobCancelPollRequest, protocol.RunJobCancelPollResponse](ctx, c, "/api/v1/run/jobs/cancel", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) ReconcileJobs(ctx context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) {
|
||||
return postPlatformJSON[protocol.RunJobReconcileRequest, protocol.RunJobReconcileResponse](ctx, c, "/api/v1/run/jobs/reconcile", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) IngestLogBatch(ctx context.Context, request protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) {
|
||||
return postPlatformJSON[protocol.LogBatchIngestRequest, protocol.LogBatchIngestResponse](ctx, c, "/api/v1/run/logs/batches", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) OpenArtifactTransfer(ctx context.Context, request protocol.ArtifactTransferOpenRequest) (protocol.ArtifactTransferOpenResponse, error) {
|
||||
return postPlatformJSON[protocol.ArtifactTransferOpenRequest, protocol.ArtifactTransferOpenResponse](ctx, c, "/api/v1/run/artifacts/open", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) UploadArtifactChunk(ctx context.Context, request protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) {
|
||||
return postPlatformJSON[protocol.ArtifactChunkUploadRequest, protocol.ArtifactChunkUploadResponse](ctx, c, "/api/v1/run/artifacts/chunks", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) QueryArtifactTransferStatus(ctx context.Context, request protocol.ArtifactTransferStatusRequest) (protocol.ArtifactTransferStatusResponse, error) {
|
||||
return postPlatformJSON[protocol.ArtifactTransferStatusRequest, protocol.ArtifactTransferStatusResponse](ctx, c, "/api/v1/run/artifacts/status", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) CompleteArtifactTransfer(ctx context.Context, request protocol.ArtifactTransferCompleteRequest) (protocol.ArtifactTransferCompleteResponse, error) {
|
||||
return postPlatformJSON[protocol.ArtifactTransferCompleteRequest, protocol.ArtifactTransferCompleteResponse](ctx, c, "/api/v1/run/artifacts/complete", request)
|
||||
}
|
||||
|
||||
func postPlatformJSON[Request any, Response any](ctx context.Context, client PlatformClient, path string, request Request) (Response, error) {
|
||||
var response Response
|
||||
|
||||
var body bytes.Buffer
|
||||
if err := json.NewEncoder(&body).Encode(request); err != nil {
|
||||
return response, fmt.Errorf("encode platform request: %w", err)
|
||||
}
|
||||
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, client.baseURL+path, &body)
|
||||
if err != nil {
|
||||
return response, fmt.Errorf("build platform request: %w", err)
|
||||
}
|
||||
httpRequest.Header.Set("Content-Type", "application/json")
|
||||
httpRequest.Header.Set("Accept", "application/json")
|
||||
|
||||
httpResponse, err := client.httpClient.Do(httpRequest)
|
||||
if err != nil {
|
||||
return response, fmt.Errorf("send platform request: %w", err)
|
||||
}
|
||||
defer httpResponse.Body.Close()
|
||||
|
||||
if httpResponse.StatusCode < http.StatusOK || httpResponse.StatusCode >= http.StatusMultipleChoices {
|
||||
message, _ := io.ReadAll(io.LimitReader(httpResponse.Body, 4096))
|
||||
return response, fmt.Errorf("platform request failed: status=%d body=%s", httpResponse.StatusCode, strings.TrimSpace(string(message)))
|
||||
}
|
||||
if err := json.NewDecoder(httpResponse.Body).Decode(&response); err != nil {
|
||||
return response, fmt.Errorf("decode platform response: %w", err)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestNewPlatformClientNormalizesBaseURL(t *testing.T) {
|
||||
client, err := NewPlatformClient("http://platform.test/")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if client.BaseURL() != "http://platform.test" {
|
||||
t.Fatalf("expected normalized base URL, got %q", client.BaseURL())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPlatformClientRequiresAbsoluteURL(t *testing.T) {
|
||||
if _, err := NewPlatformClient("platform.local"); err == nil {
|
||||
t.Fatal("expected error for URL without scheme and host")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientHelloPostsJSONAndDecodesResponse(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/control/hello" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
|
||||
t.Fatalf("expected JSON content type, got %q", contentType)
|
||||
}
|
||||
var request protocol.RunHelloRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatalf("decode hello request: %v", err)
|
||||
}
|
||||
if request.RunEndpointID != "run-local" || request.CapabilityReport.Fingerprint != "cap-v1" {
|
||||
t.Fatalf("unexpected hello payload: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunHelloResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: "session-token",
|
||||
ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
HeartbeatIntervalSeconds: 15,
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
response, err := client.Hello(context.Background(), validRunHelloRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("hello: %v", err)
|
||||
}
|
||||
if !response.Accepted || response.SessionToken != "session-token" || response.HeartbeatIntervalSeconds != 15 {
|
||||
t.Fatalf("unexpected hello response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientHeartbeatPostsJSONAndDecodesResponse(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/run/control/heartbeat" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
var request protocol.RunHeartbeatRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatalf("decode heartbeat request: %v", err)
|
||||
}
|
||||
if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 1 {
|
||||
t.Fatalf("unexpected heartbeat payload: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunHeartbeatResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: "run-local",
|
||||
NextHeartbeatSeconds: 15,
|
||||
RefreshCapabilities: true,
|
||||
ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
response, err := client.Heartbeat(context.Background(), validRunHeartbeatRequest("session-token"))
|
||||
if err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if !response.Accepted || !response.RefreshCapabilities || response.NextHeartbeatSeconds != 15 {
|
||||
t.Fatalf("unexpected heartbeat response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientReturnsErrorForPlatformFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"code":"validation_failed"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
if _, err := client.Hello(context.Background(), validRunHelloRequest()); err == nil {
|
||||
t.Fatal("expected platform error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformClientHelloThenHeartbeatFlow(t *testing.T) {
|
||||
var activeSessionToken string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/control/hello":
|
||||
var request protocol.RunHelloRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatalf("decode hello request: %v", err)
|
||||
}
|
||||
if request.RegistrationToken == "" || request.RunEndpointID != "run-local" {
|
||||
t.Fatalf("unexpected hello request: %+v", request)
|
||||
}
|
||||
activeSessionToken = "session-token"
|
||||
writeTestJSON(t, w, protocol.RunHelloResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: request.RunEndpointID,
|
||||
SessionToken: activeSessionToken,
|
||||
ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
HeartbeatIntervalSeconds: 15,
|
||||
})
|
||||
case "/api/v1/run/control/heartbeat":
|
||||
var request protocol.RunHeartbeatRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatalf("decode heartbeat request: %v", err)
|
||||
}
|
||||
if request.SessionToken != activeSessionToken {
|
||||
t.Fatalf("heartbeat did not use active session token: %+v", request)
|
||||
}
|
||||
writeTestJSON(t, w, protocol.RunHeartbeatResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: request.RunEndpointID,
|
||||
NextHeartbeatSeconds: 15,
|
||||
ServerTime: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected request path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
hello, err := client.Hello(context.Background(), validRunHelloRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("hello: %v", err)
|
||||
}
|
||||
heartbeat, err := client.Heartbeat(context.Background(), validRunHeartbeatRequest(hello.SessionToken))
|
||||
if err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if !hello.Accepted || !heartbeat.Accepted {
|
||||
t.Fatalf("expected accepted hello and heartbeat, got %+v %+v", hello, heartbeat)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Fatalf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func validRunHelloRequest() protocol.RunHelloRequest {
|
||||
return protocol.RunHelloRequest{
|
||||
RegistrationToken: "registration-token",
|
||||
RunEndpointID: "run-local",
|
||||
DisplayName: "Local Run",
|
||||
Version: "0.1.0",
|
||||
Status: "online",
|
||||
Platform: "darwin/arm64",
|
||||
CapabilityReport: protocol.RunCapabilityReport{
|
||||
Capabilities: []string{"control.hello", "control.heartbeat"},
|
||||
Fingerprint: "cap-v1",
|
||||
},
|
||||
Capacity: protocol.RunCapacityReport{MaxJobs: 4},
|
||||
}
|
||||
}
|
||||
|
||||
func validRunHeartbeatRequest(sessionToken string) protocol.RunHeartbeatRequest {
|
||||
return protocol.RunHeartbeatRequest{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: sessionToken,
|
||||
Version: "0.1.0",
|
||||
Status: "online",
|
||||
CapabilityFingerprint: "cap-v1",
|
||||
Capacity: protocol.RunCapacityReport{
|
||||
MaxJobs: 4,
|
||||
RunningJobs: 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# run/artifact
|
||||
|
||||
Artifact transfer implementation lives here.
|
||||
|
||||
Artifact transfer is lower priority than control, job ack/result, and log ingest. Implement concurrency and bandwidth limits before enabling large transfers.
|
||||
|
||||
Artifact uploads must remain chunked and resumable. Slow, queued, or retrying artifact chunks must not prevent:
|
||||
|
||||
- control hello/heartbeat calls,
|
||||
- job claim/ack/progress/result/cancel/reconcile calls,
|
||||
- durable log batch selection, upload, acknowledgement, or cleanup.
|
||||
|
||||
Artifact queue entries may contain bounded transfer metadata and chunk bytes only. They must not expose host paths, raw credentials, direct sockets, run session transport details, or plugin/browser storage credentials.
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"browser.local/run/api"
|
||||
"browser.local/run/config"
|
||||
runruntime "browser.local/run/runtime"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
client, err := api.NewPlatformClient(cfg.PlatformURL)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid platform URL: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if cfg.Mode == "worker" {
|
||||
logSpool, err := spool.NewLogSpool(cfg.SpoolRoot)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "initialize log spool: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
artifactQueue, err := spool.NewArtifactQueue(cfg.SpoolRoot)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "initialize artifact queue: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
worker, err := runruntime.NewWorker(
|
||||
cfg,
|
||||
client,
|
||||
runruntime.WithProcessLogSink(&runruntime.SpoolLogSink{Spool: logSpool}),
|
||||
runruntime.WithLifecycleArtifactHook(&runruntime.QueueArtifactHook{Queue: artifactQueue}),
|
||||
)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "initialize worker: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
if err := worker.Run(ctx); err != nil && err != context.Canceled {
|
||||
fmt.Fprintf(os.Stderr, "run worker stopped: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
summary := runruntime.SmokeSummary(cfg)
|
||||
summary.PlatformURL = client.BaseURL()
|
||||
|
||||
if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "encode smoke summary: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultMode = "smoke"
|
||||
DefaultPlatformURL = "http://127.0.0.1:8080"
|
||||
DefaultEndpointID = "run-local"
|
||||
DefaultDisplayName = "Local Run"
|
||||
DefaultVersion = "0.1.0"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Mode string
|
||||
PlatformURL string
|
||||
RunEndpointID string
|
||||
DisplayName string
|
||||
Version string
|
||||
RegistrationToken string
|
||||
WorkspaceRoot string
|
||||
SpoolRoot string
|
||||
MaxJobs int
|
||||
HeartbeatInterval time.Duration
|
||||
PollInterval time.Duration
|
||||
RetryBackoff time.Duration
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
mode := os.Getenv("RUN_MODE")
|
||||
if mode == "" {
|
||||
mode = DefaultMode
|
||||
}
|
||||
|
||||
platformURL := os.Getenv("RUN_PLATFORM_URL")
|
||||
if platformURL == "" {
|
||||
platformURL = DefaultPlatformURL
|
||||
}
|
||||
|
||||
workspaceRoot := envOrDefault("RUN_WORKSPACE_ROOT", filepath.Join(".", ".run-workspace"))
|
||||
return Config{
|
||||
Mode: mode,
|
||||
PlatformURL: platformURL,
|
||||
RunEndpointID: envOrDefault("RUN_ENDPOINT_ID", DefaultEndpointID),
|
||||
DisplayName: envOrDefault("RUN_DISPLAY_NAME", DefaultDisplayName),
|
||||
Version: envOrDefault("RUN_VERSION", DefaultVersion),
|
||||
RegistrationToken: envOrDefault("RUN_REGISTRATION_TOKEN", "local-registration"),
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
SpoolRoot: envOrDefault("RUN_SPOOL_ROOT", filepath.Join(workspaceRoot, "spool")),
|
||||
MaxJobs: intEnvOrDefault("RUN_MAX_JOBS", 1),
|
||||
HeartbeatInterval: durationEnvOrDefault("RUN_HEARTBEAT_INTERVAL_MS", 15*time.Second),
|
||||
PollInterval: durationEnvOrDefault("RUN_POLL_INTERVAL_MS", 2*time.Second),
|
||||
RetryBackoff: durationEnvOrDefault("RUN_RETRY_BACKOFF_MS", time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
func envOrDefault(key string, fallback string) string {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intEnvOrDefault(key string, fallback int) int {
|
||||
value, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || value <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func durationEnvOrDefault(key string, fallback time.Duration) time.Duration {
|
||||
value, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || value <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return time.Duration(value) * time.Millisecond
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLoadUsesDefaults(t *testing.T) {
|
||||
t.Setenv("RUN_MODE", "")
|
||||
t.Setenv("RUN_PLATFORM_URL", "")
|
||||
|
||||
cfg := Load()
|
||||
if cfg.Mode != DefaultMode {
|
||||
t.Fatalf("expected mode %q, got %q", DefaultMode, cfg.Mode)
|
||||
}
|
||||
if cfg.PlatformURL != DefaultPlatformURL {
|
||||
t.Fatalf("expected platform URL %q, got %q", DefaultPlatformURL, cfg.PlatformURL)
|
||||
}
|
||||
if cfg.RunEndpointID != DefaultEndpointID || cfg.DisplayName != DefaultDisplayName || cfg.Version != DefaultVersion {
|
||||
t.Fatalf("expected worker defaults, got %+v", cfg)
|
||||
}
|
||||
if cfg.MaxJobs != 1 || cfg.HeartbeatInterval <= 0 || cfg.PollInterval <= 0 || cfg.RetryBackoff <= 0 {
|
||||
t.Fatalf("expected positive worker scheduling defaults, got %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadUsesEnvironment(t *testing.T) {
|
||||
t.Setenv("RUN_MODE", "worker")
|
||||
t.Setenv("RUN_PLATFORM_URL", "http://platform.test")
|
||||
t.Setenv("RUN_ENDPOINT_ID", "run-edge")
|
||||
t.Setenv("RUN_DISPLAY_NAME", "Edge Run")
|
||||
t.Setenv("RUN_VERSION", "1.2.3")
|
||||
t.Setenv("RUN_REGISTRATION_TOKEN", "registration-token")
|
||||
t.Setenv("RUN_WORKSPACE_ROOT", "/tmp/run-workspace")
|
||||
t.Setenv("RUN_SPOOL_ROOT", "/tmp/run-spool")
|
||||
t.Setenv("RUN_MAX_JOBS", "3")
|
||||
t.Setenv("RUN_HEARTBEAT_INTERVAL_MS", "250")
|
||||
t.Setenv("RUN_POLL_INTERVAL_MS", "125")
|
||||
t.Setenv("RUN_RETRY_BACKOFF_MS", "75")
|
||||
|
||||
cfg := Load()
|
||||
if cfg.Mode != "worker" {
|
||||
t.Fatalf("expected configured mode, got %q", cfg.Mode)
|
||||
}
|
||||
if cfg.PlatformURL != "http://platform.test" {
|
||||
t.Fatalf("expected configured platform URL, got %q", cfg.PlatformURL)
|
||||
}
|
||||
if cfg.RunEndpointID != "run-edge" || cfg.DisplayName != "Edge Run" || cfg.Version != "1.2.3" || cfg.RegistrationToken != "registration-token" {
|
||||
t.Fatalf("expected configured worker identity, got %+v", cfg)
|
||||
}
|
||||
if cfg.WorkspaceRoot != "/tmp/run-workspace" || cfg.SpoolRoot != "/tmp/run-spool" || cfg.MaxJobs != 3 {
|
||||
t.Fatalf("expected configured worker paths/capacity, got %+v", cfg)
|
||||
}
|
||||
if cfg.HeartbeatInterval.Milliseconds() != 250 || cfg.PollInterval.Milliseconds() != 125 || cfg.RetryBackoff.Milliseconds() != 75 {
|
||||
t.Fatalf("expected configured durations, got %+v", cfg)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package domain
|
||||
|
||||
type ExecutorStatus struct {
|
||||
Mode string `json:"mode"`
|
||||
PlatformURL string `json:"platformUrl"`
|
||||
Status string `json:"status"`
|
||||
ExposedHostPath bool `json:"exposedHostPath"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module browser.local/run
|
||||
|
||||
go 1.25.1
|
||||
@@ -0,0 +1,7 @@
|
||||
# run/logingest
|
||||
|
||||
Log collectors and uploaders live here.
|
||||
|
||||
Collectors read process output and server log files, assign stream IDs and sequence numbers, write to local spool, and upload batches to platform ingest APIs.
|
||||
|
||||
Do not send primary run-to-platform logs through the UI realtime channel.
|
||||
@@ -0,0 +1,101 @@
|
||||
package protocol
|
||||
|
||||
import "time"
|
||||
|
||||
type ArtifactMetadata struct {
|
||||
ID string `json:"id"`
|
||||
OwnerKind string `json:"ownerKind"`
|
||||
OwnerID string `json:"ownerId"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
Checksum string `json:"checksum"`
|
||||
State string `json:"state"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type ArtifactTransferOpenRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
Direction string `json:"direction"`
|
||||
OwnerKind string `json:"ownerKind"`
|
||||
OwnerID string `json:"ownerId"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
ChunkSizeBytes int `json:"chunkSizeBytes"`
|
||||
Checksum string `json:"checksum"`
|
||||
IdempotencyKey string `json:"idempotencyKey"`
|
||||
}
|
||||
|
||||
type ArtifactTransferOpenResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
TransferID string `json:"transferId"`
|
||||
Direction string `json:"direction"`
|
||||
Artifact ArtifactMetadata `json:"artifact"`
|
||||
TotalChunks int `json:"totalChunks"`
|
||||
ChunkSizeBytes int `json:"chunkSizeBytes"`
|
||||
ReceivedChunkIndexes []int `json:"receivedChunkIndexes"`
|
||||
NextMissingChunkIndex int `json:"nextMissingChunkIndex"`
|
||||
Completed bool `json:"completed"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type ArtifactChunkUploadRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
TransferID string `json:"transferId"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
ChunkIndex int `json:"chunkIndex"`
|
||||
Offset int64 `json:"offset"`
|
||||
SizeBytes int `json:"sizeBytes"`
|
||||
Checksum string `json:"checksum"`
|
||||
Payload []byte `json:"payload"`
|
||||
}
|
||||
|
||||
type ArtifactChunkUploadResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
TransferID string `json:"transferId"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
ChunkIndex int `json:"chunkIndex"`
|
||||
ReceivedChunkIndexes []int `json:"receivedChunkIndexes"`
|
||||
NextMissingChunkIndex int `json:"nextMissingChunkIndex"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type ArtifactTransferStatusRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
TransferID string `json:"transferId"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
}
|
||||
|
||||
type ArtifactTransferStatusResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
TransferID string `json:"transferId"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
Direction string `json:"direction"`
|
||||
TotalChunks int `json:"totalChunks"`
|
||||
ChunkSizeBytes int `json:"chunkSizeBytes"`
|
||||
ReceivedChunkIndexes []int `json:"receivedChunkIndexes"`
|
||||
NextMissingChunkIndex int `json:"nextMissingChunkIndex"`
|
||||
Completed bool `json:"completed"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type ArtifactTransferCompleteRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
TransferID string `json:"transferId"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
Checksum string `json:"checksum"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
}
|
||||
|
||||
type ArtifactTransferCompleteResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
TransferID string `json:"transferId"`
|
||||
Artifact ArtifactMetadata `json:"artifact"`
|
||||
Completed bool `json:"completed"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Run Artifact Contract
|
||||
|
||||
Artifacts move files and large payloads between platform and run without blocking logs or control.
|
||||
|
||||
## Implemented Routes
|
||||
|
||||
- `POST /api/v1/run/artifacts/open`: opens a run-to-platform upload transfer and returns resume state.
|
||||
- `POST /api/v1/run/artifacts/chunks`: uploads one bounded chunk with byte range and checksum metadata.
|
||||
- `POST /api/v1/run/artifacts/status`: queries received chunks and the next missing chunk index.
|
||||
- `POST /api/v1/run/artifacts/complete`: verifies all chunks and final checksum before marking the artifact available.
|
||||
|
||||
Browser-facing artifact downloads are implemented through platform-owned routes after a run upload completes:
|
||||
|
||||
- `POST /api/v1/artifacts/{id}/download`: returns safe download metadata and a platform content route.
|
||||
- `GET /api/v1/artifacts/{id}/content`: returns bounded byte ranges for authorized browser or plugin-page reads.
|
||||
|
||||
## Payloads
|
||||
|
||||
- `ArtifactTransferOpenRequest`: run ID, session token, artifact ID, upload direction, owner scope, size, chunk size, checksum, and idempotency key.
|
||||
- `ArtifactTransferOpenResponse`: transfer ID, artifact metadata, total chunks, received chunk indexes, next missing chunk index, duplicate flag, and server time.
|
||||
- `ArtifactChunkUploadRequest`: transfer ID, artifact ID, chunk index, byte offset, size, checksum, and JSON byte payload.
|
||||
- `ArtifactChunkUploadResponse`: accepted chunk index, received chunk indexes, next missing chunk index, duplicate flag, and server time.
|
||||
- `ArtifactTransferStatusRequest`: run ID, session token, transfer ID, and artifact ID.
|
||||
- `ArtifactTransferStatusResponse`: transfer direction, total chunks, received chunk indexes, next missing chunk index, completion flag, and server time.
|
||||
- `ArtifactTransferCompleteRequest`: transfer ID, artifact ID, final checksum, and final size.
|
||||
- `ArtifactTransferCompleteResponse`: completed artifact metadata and server time.
|
||||
|
||||
## Local Queue
|
||||
|
||||
Run stores unacknowledged `ArtifactChunkUploadRequest` payloads in the local artifact queue. A queued chunk may be removed only after the platform acknowledges the same transfer ID, artifact ID, and chunk index. The queue must not store or expose raw host paths.
|
||||
|
||||
## Rules
|
||||
|
||||
- Transfers must be resumable.
|
||||
- Transfers must be checksummed.
|
||||
- Artifact concurrency must be limited.
|
||||
- Artifact transfer must not block control heartbeat, job ack/result, or log upload.
|
||||
- Artifact transfer is lower priority than control, job lifecycle metadata, and durable log ingest.
|
||||
- Slow or retrying artifact chunks must not prevent log spool acknowledgement cleanup or terminal job result submission.
|
||||
- Control, job, and log routes must reject artifact chunk payloads or transport details rather than accepting them through lightweight channel payloads.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Platform-to-run download, browser artifact upload, external object storage, presigned URLs, and production throttling policies remain separate future work.
|
||||
@@ -0,0 +1,52 @@
|
||||
package protocol
|
||||
|
||||
import "time"
|
||||
|
||||
type RunCapacityReport struct {
|
||||
MaxJobs int `json:"maxJobs"`
|
||||
RunningJobs int `json:"runningJobs"`
|
||||
QueuedJobs int `json:"queuedJobs"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
}
|
||||
|
||||
type RunCapabilityReport struct {
|
||||
Capabilities []string `json:"capabilities"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
}
|
||||
|
||||
type RunHelloRequest struct {
|
||||
RegistrationToken string `json:"registrationToken"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
DisplayName string `json:"displayName"`
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
Platform string `json:"platform,omitempty"`
|
||||
CapabilityReport RunCapabilityReport `json:"capabilityReport"`
|
||||
Capacity RunCapacityReport `json:"capacity"`
|
||||
}
|
||||
|
||||
type RunHelloResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds"`
|
||||
FeatureFlags []string `json:"featureFlags,omitempty"`
|
||||
}
|
||||
|
||||
type RunHeartbeatRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
CapabilityFingerprint string `json:"capabilityFingerprint"`
|
||||
Capacity RunCapacityReport `json:"capacity"`
|
||||
}
|
||||
|
||||
type RunHeartbeatResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
NextHeartbeatSeconds int `json:"nextHeartbeatSeconds"`
|
||||
RefreshCapabilities bool `json:"refreshCapabilities"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
# Run Control Contract
|
||||
|
||||
Control is the lightweight high-priority channel between run and platform.
|
||||
|
||||
## Implemented Routes
|
||||
|
||||
- `POST /api/v1/run/control/hello`: registers run metadata and receives a platform-issued session token.
|
||||
- `POST /api/v1/run/control/heartbeat`: reports status, capacity, and capability fingerprint using the active session token.
|
||||
|
||||
## Payloads
|
||||
|
||||
- `RunHelloRequest`: registration token, run ID, display name, version, status, platform, capability summary, and capacity summary.
|
||||
- `RunHelloResponse`: session token, server time, polling hints, and feature flags.
|
||||
- `RunHeartbeatRequest`: session token, version, status, capacity, and current capability fingerprint.
|
||||
- `RunHeartbeatResponse`: accepted status, next heartbeat interval, and optional capability refresh request.
|
||||
- `RunCapabilityReport`: capability names and compact fingerprint metadata.
|
||||
- `RunCapacityReport`: max jobs, active jobs, queued jobs, and local resource summary.
|
||||
|
||||
## Rules
|
||||
|
||||
- Control payloads must be small.
|
||||
- Control must not carry logs, artifact chunks, or long job result bodies.
|
||||
- Control must have priority over job execution, log upload, and artifact transfer.
|
||||
- Heartbeat capacity summaries must remain metadata-only and must not mention or carry heavy channel payloads.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Durable log ingest, artifact chunk transfer, and the optional game client bridge remain separate channels. The job channel is separate from control and uses `/api/v1/run/jobs/*` routes.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Game Client Bridge Contract
|
||||
|
||||
The game client bridge is optional and exists only for games that need in-game command execution or snapshots.
|
||||
|
||||
## Payloads
|
||||
|
||||
- `ClientHello`: game client credential, server instance ID, version, and display name.
|
||||
- `ClientHeartbeat`: session token, version, status, and game connection status.
|
||||
- `ClientCommandPoll`: session token and batch limit.
|
||||
- `ClientCommandResult`: command ID, status, bounded output, and timestamp.
|
||||
- `ClientSnapshot`: snapshot mode, raw bounded text or structured data reference, and timestamp.
|
||||
|
||||
## Rules
|
||||
|
||||
- The bridge must not carry run lifecycle jobs.
|
||||
- The bridge must not carry run log ingest batches.
|
||||
- Games without in-game bridge needs should not enable this channel.
|
||||
@@ -0,0 +1,132 @@
|
||||
package protocol
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
RunCapabilityProcessInstall = "process.install"
|
||||
RunCapabilityProcessStart = "process.start"
|
||||
RunCapabilityProcessStop = "process.stop"
|
||||
RunCapabilityLogsRead = "logs.read"
|
||||
RunCapabilityConfigWrite = "config.write"
|
||||
RunCapabilityFilesRead = "files.read"
|
||||
RunCapabilityFilesWrite = "files.write"
|
||||
)
|
||||
|
||||
type RunJobProgressReport struct {
|
||||
Percent int `json:"percent"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type RunJobAssignment struct {
|
||||
JobID string `json:"jobId"`
|
||||
ServerInstanceID string `json:"serverInstanceId,omitempty"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
Capability string `json:"capability"`
|
||||
TargetKey string `json:"targetKey,omitempty"`
|
||||
InputRef string `json:"inputRef,omitempty"`
|
||||
IdempotencyKey string `json:"idempotencyKey"`
|
||||
State string `json:"state"`
|
||||
Progress RunJobProgressReport `json:"progress"`
|
||||
ResultRef string `json:"resultRef,omitempty"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type RunJobClaimRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
Capacity RunCapacityReport `json:"capacity"`
|
||||
}
|
||||
|
||||
type RunJobClaimResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
HasJob bool `json:"hasJob"`
|
||||
Job *RunJobAssignment `json:"job,omitempty"`
|
||||
NextPollSeconds int `json:"nextPollSeconds"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type RunJobAckRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
JobID string `json:"jobId"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
Attempt int `json:"attempt"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type RunJobAckResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Job RunJobAssignment `json:"job"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type RunJobProgressRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
JobID string `json:"jobId"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
Attempt int `json:"attempt"`
|
||||
Progress RunJobProgressReport `json:"progress"`
|
||||
Sequence uint64 `json:"sequence,omitempty"`
|
||||
}
|
||||
|
||||
type RunJobProgressResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Job RunJobAssignment `json:"job"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type RunJobResultRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
JobID string `json:"jobId"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
Attempt int `json:"attempt"`
|
||||
State string `json:"state"`
|
||||
Progress RunJobProgressReport `json:"progress"`
|
||||
ResultRef string `json:"resultRef,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ErrorCode string `json:"errorCode,omitempty"`
|
||||
}
|
||||
|
||||
type RunJobResultResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Job RunJobAssignment `json:"job"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type RunJobCancelPollRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
LeaseToken string `json:"leaseToken,omitempty"`
|
||||
}
|
||||
|
||||
type RunJobCancelPollResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
HasCancel bool `json:"hasCancel"`
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RequestedAt time.Time `json:"requestedAt,omitempty"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type RunJobReconcileRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
ActiveJobIDs []string `json:"activeJobIds"`
|
||||
}
|
||||
|
||||
type RunJobReconcileResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
ActiveJobs []RunJobAssignment `json:"activeJobs"`
|
||||
UnknownJobIDs []string `json:"unknownJobIds"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# Run Job Contract
|
||||
|
||||
Jobs execute bounded server management work.
|
||||
|
||||
## Implemented Routes
|
||||
|
||||
- `POST /api/v1/run/jobs/claim`: claims one queued job for the registered run endpoint.
|
||||
- `POST /api/v1/run/jobs/ack`: acknowledges an active leased job before execution.
|
||||
- `POST /api/v1/run/jobs/progress`: reports bounded progress for an active leased job.
|
||||
- `POST /api/v1/run/jobs/result`: submits a bounded terminal result for an active leased job.
|
||||
- `POST /api/v1/run/jobs/cancel`: polls platform cancellation requests for active leased jobs.
|
||||
- `POST /api/v1/run/jobs/reconcile`: reconciles platform-known active jobs after run restart or reconnect.
|
||||
|
||||
## Payloads
|
||||
|
||||
- `RunJobClaimRequest`: session token, run ID, capacity, and supported capabilities.
|
||||
- `RunJobClaimResponse`: optional job assignment with identity, capability, server instance, logical target key, scoped input ref, idempotency key, lease token, attempt, and polling hint.
|
||||
- `RunJobAckRequest`: job ID, run ID, session token, lease token, attempt, and bounded message.
|
||||
- `RunJobProgressRequest`: job ID, run ID, session token, lease token, attempt, percent, sequence, and bounded message.
|
||||
- `RunJobResultRequest`: job ID, run ID, session token, lease token, attempt, terminal state, progress, bounded message, error code, and result reference.
|
||||
- `RunJobCancelPollRequest`: run ID, session token, and optional job lease identity.
|
||||
- `RunJobReconcileRequest`: run ID, session token, and active local job IDs.
|
||||
|
||||
## Local Journal
|
||||
|
||||
Run must keep a local short-term journal for accepted jobs so duplicate delivery, reconnect, and restart can be reconciled.
|
||||
|
||||
## Lifecycle Executor
|
||||
|
||||
The runtime worker executes these bounded lifecycle job capabilities:
|
||||
|
||||
- `process.install`
|
||||
- `process.start`
|
||||
- `process.stop`
|
||||
|
||||
Platform-dispatched config/file jobs are now represented in the run job payload and validated before execution by later worker implementations:
|
||||
|
||||
- `config.write`: writes approved config content addressed by a logical config key plus scoped `input://...` ref.
|
||||
- `files.read`: reads a declared logical file key and returns results through bounded metadata or artifact refs.
|
||||
- `files.write`: writes content addressed by a logical file key plus scoped `input://...` or `artifact://...` ref.
|
||||
|
||||
The executor resolves lifecycle action templates under the scoped server workspace and runs direct command/argument vectors through the process supervisor. It does not run unrestricted shell strings, execute arbitrary plugin code, expose host paths, return raw credentials, open direct sockets, or embed logs/artifacts in job result payloads.
|
||||
|
||||
## Rules
|
||||
|
||||
- Job ack must be sent before execution.
|
||||
- Terminal result must be replayable while the journal retains the job.
|
||||
- Large files must be passed as artifact references, not embedded in job payloads.
|
||||
- Config/file job payloads must use logical target keys and scoped input/artifact refs.
|
||||
- Job payloads must not include logs, artifact chunks, raw host paths, raw credentials, direct sockets, or large inline result bodies.
|
||||
- Process stdout/stderr must be redacted and written to the log spool rather than embedded in progress/result bodies.
|
||||
- Job ack/progress/result/cancel/reconcile calls are lightweight lifecycle metadata and must be able to complete while artifact/file transfer work is active or retrying.
|
||||
- Terminal results must remain idempotent under log and artifact retry pressure and must reference artifacts by safe `artifact://...` refs rather than embedding transfer payloads.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Durable log ingest, artifact chunk transfer, and optional game client bridge traffic remain separate channels and must not be multiplexed through job result payloads. Artifact transfer carries chunk payloads only through `/api/v1/run/artifacts/*` routes.
|
||||
@@ -0,0 +1,58 @@
|
||||
package protocol
|
||||
|
||||
import "strings"
|
||||
|
||||
const maxRunLogicalFileKeyLength = 160
|
||||
|
||||
func ValidateRunJobAssignment(assignment RunJobAssignment) error {
|
||||
if assignment.JobID == "" || assignment.RunEndpointID == "" || assignment.Capability == "" {
|
||||
return ValidationError("jobId, runEndpointId, and capability are required")
|
||||
}
|
||||
switch assignment.Capability {
|
||||
case RunCapabilityConfigWrite, RunCapabilityFilesRead, RunCapabilityFilesWrite:
|
||||
if assignment.ServerInstanceID == "" {
|
||||
return ValidationError("serverInstanceId is required for scoped file jobs")
|
||||
}
|
||||
if !ValidLogicalFileKey(assignment.TargetKey) {
|
||||
return ValidationError("targetKey is not allowed")
|
||||
}
|
||||
}
|
||||
switch assignment.Capability {
|
||||
case RunCapabilityConfigWrite, RunCapabilityFilesWrite:
|
||||
if !ValidScopedInputRef(assignment.InputRef) {
|
||||
return ValidationError("inputRef is not allowed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ValidationError string
|
||||
|
||||
func (err ValidationError) Error() string { return string(err) }
|
||||
|
||||
func ValidLogicalFileKey(key string) bool {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if trimmed == "" || trimmed != key || len([]rune(key)) > maxRunLogicalFileKeyLength {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(key)
|
||||
if strings.HasPrefix(key, "/") || strings.Contains(key, "..") || strings.Contains(key, `\`) || strings.Contains(key, "://") || strings.Contains(lower, "/users/") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "sk-") || strings.Contains(lower, "bearer ") {
|
||||
return false
|
||||
}
|
||||
for _, char := range key {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '_' || char == '-' || char == '.' || char == '/' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func ValidScopedInputRef(ref string) bool {
|
||||
trimmed := strings.TrimSpace(ref)
|
||||
lower := strings.ToLower(ref)
|
||||
if trimmed == "" || trimmed != ref || strings.Contains(lower, "/users/") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "sk-") || strings.Contains(lower, "bearer ") {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(ref, "input://") || strings.HasPrefix(ref, "artifact://")
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateRunJobAssignmentScopedFilePayloads(t *testing.T) {
|
||||
assignment := RunJobAssignment{
|
||||
JobID: "job-config-write",
|
||||
ServerInstanceID: "server-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: RunCapabilityConfigWrite,
|
||||
TargetKey: "server.properties",
|
||||
InputRef: "input://server-config/server-1/server.properties/v1",
|
||||
IdempotencyKey: "idem-config",
|
||||
}
|
||||
if err := ValidateRunJobAssignment(assignment); err != nil {
|
||||
t.Fatalf("expected valid config write assignment: %v", err)
|
||||
}
|
||||
|
||||
assignment.TargetKey = "/Users/tasia/server.properties"
|
||||
if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "targetKey") {
|
||||
t.Fatalf("expected raw host path rejection, got %v", err)
|
||||
}
|
||||
|
||||
assignment.TargetKey = "server.properties"
|
||||
assignment.InputRef = "sk-raw-secret"
|
||||
if err := ValidateRunJobAssignment(assignment); err == nil || !strings.Contains(err.Error(), "inputRef") {
|
||||
t.Fatalf("expected raw credential ref rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRunJobAssignmentScopedReadDoesNotRequireInputRef(t *testing.T) {
|
||||
assignment := RunJobAssignment{
|
||||
JobID: "job-files-read",
|
||||
ServerInstanceID: "server-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: RunCapabilityFilesRead,
|
||||
TargetKey: "logs/latest.log",
|
||||
IdempotencyKey: "idem-read",
|
||||
}
|
||||
if err := ValidateRunJobAssignment(assignment); err != nil {
|
||||
t.Fatalf("expected valid file read assignment: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# Run Log Ingest Contract
|
||||
|
||||
Logs are durable historical data. They are not transported as best-effort UI messages.
|
||||
|
||||
## Implemented Routes
|
||||
|
||||
- `POST /api/v1/run/logs/batches`: uploads one bounded log batch and receives an acknowledgement range.
|
||||
- `POST /api/v1/log-streams/query`: queries stored log entries after a stream sequence cursor.
|
||||
|
||||
## Payloads
|
||||
|
||||
- `LogBatchIngestRequest`: run ID, session token, server instance ID, stream ID, source, sequence range, compression metadata, checksum, and bounded entries.
|
||||
- `LogEntry`: sequence, timestamp, level, line, parser metadata, and redaction state.
|
||||
- `LogBatchIngestResponse`: accepted sequence range, latest acknowledged sequence, duplicate flag, retry hint, and server time.
|
||||
- `LogStreamCursorRequest`: stream ID, sequence cursor, and limit.
|
||||
- `LogStreamCursorResponse`: ordered entries, next cursor, and latest acknowledged sequence.
|
||||
|
||||
## Local Spool
|
||||
|
||||
Run must write unacknowledged logs to a local spool/WAL before upload. Segments may be removed only after platform acknowledgement.
|
||||
|
||||
## Priority
|
||||
|
||||
Log flush has higher priority than artifact transfer. Artifact work must slow down when log spool pressure rises.
|
||||
|
||||
Log spool retry state is independent from artifact/file retry state. Acknowledged log batches may be removed even when artifact chunks are still pending, and artifact chunk acknowledgement must not alter log sequence state. Log ingest payloads carry bounded entries only and must not include artifact chunks, file bodies, host paths, raw credentials, or direct socket details.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Browser live tail, external log storage backends, and optional game client bridge traffic remain separate future channels. Artifact transfer uses its own lower-priority channel and must not be multiplexed through log ingest.
|
||||
@@ -0,0 +1,50 @@
|
||||
package protocol
|
||||
|
||||
import "time"
|
||||
|
||||
type LogEntry struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Line string `json:"line"`
|
||||
Fields map[string]string `json:"fields,omitempty"`
|
||||
Redacted bool `json:"redacted"`
|
||||
}
|
||||
|
||||
type LogBatchIngestRequest struct {
|
||||
RunEndpointID string `json:"runEndpointId"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
LogStreamID string `json:"logStreamId"`
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
Source string `json:"source"`
|
||||
FirstSeq uint64 `json:"firstSeq"`
|
||||
LastSeq uint64 `json:"lastSeq"`
|
||||
Compression string `json:"compression"`
|
||||
Checksum string `json:"checksum"`
|
||||
Entries []LogEntry `json:"entries"`
|
||||
}
|
||||
|
||||
type LogBatchIngestResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
LogStreamID string `json:"logStreamId"`
|
||||
AcceptedFrom uint64 `json:"acceptedFrom"`
|
||||
AcceptedTo uint64 `json:"acceptedTo"`
|
||||
LatestSeq uint64 `json:"latestSeq"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
RetryAfterSec int `json:"retryAfterSec,omitempty"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type LogStreamCursorRequest struct {
|
||||
LogStreamID string `json:"logStreamId"`
|
||||
AfterSeq uint64 `json:"afterSeq"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
type LogStreamCursorResponse struct {
|
||||
LogStreamID string `json:"logStreamId"`
|
||||
Entries []LogEntry `json:"entries"`
|
||||
NextSeq uint64 `json:"nextSeq"`
|
||||
LatestSeq uint64 `json:"latestSeq"`
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
lifecycleResultStateSucceeded = "succeeded"
|
||||
lifecycleResultStateFailed = "failed"
|
||||
lifecycleResultStateCancelled = "cancelled"
|
||||
|
||||
defaultLifecycleTimeout = 30 * time.Second
|
||||
maxLifecycleOutputBytes = 4096
|
||||
)
|
||||
|
||||
var (
|
||||
commandNamePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||
envNamePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,63}$`)
|
||||
disallowedExecutables = map[string]struct{}{
|
||||
"bash": {},
|
||||
"cmd": {},
|
||||
"fish": {},
|
||||
"powershell": {},
|
||||
"pwsh": {},
|
||||
"sh": {},
|
||||
"zsh": {},
|
||||
}
|
||||
)
|
||||
|
||||
type LifecycleExecutor struct {
|
||||
workspaceRoot string
|
||||
supervisor ProcessSupervisor
|
||||
logSink ProcessLogSink
|
||||
artifactHook LifecycleArtifactHook
|
||||
}
|
||||
|
||||
type LifecycleExecutionResult struct {
|
||||
State string
|
||||
Progress protocol.RunJobProgressReport
|
||||
ResultRef string
|
||||
Message string
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
type LifecycleExecutorOption func(*LifecycleExecutor)
|
||||
|
||||
func NewLifecycleExecutor(options ...LifecycleExecutorOption) LifecycleExecutor {
|
||||
executor := LifecycleExecutor{
|
||||
workspaceRoot: filepath.Join(".", ".run-workspace"),
|
||||
supervisor: OSProcessSupervisor{},
|
||||
logSink: NoopProcessLogSink{},
|
||||
artifactHook: StaticLifecycleArtifactHook{},
|
||||
}
|
||||
for _, option := range options {
|
||||
option(&executor)
|
||||
}
|
||||
return executor
|
||||
}
|
||||
|
||||
func WithLifecycleWorkspaceRoot(root string) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if strings.TrimSpace(root) != "" {
|
||||
executor.workspaceRoot = root
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithProcessSupervisor(supervisor ProcessSupervisor) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if supervisor != nil {
|
||||
executor.supervisor = supervisor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithProcessLogSink(sink ProcessLogSink) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if sink != nil {
|
||||
executor.logSink = sink
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithLifecycleArtifactHook(hook LifecycleArtifactHook) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if hook != nil {
|
||||
executor.artifactHook = hook
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SupportedLifecycleCapabilities() []string {
|
||||
return []string{
|
||||
protocol.RunCapabilityProcessInstall,
|
||||
protocol.RunCapabilityProcessStart,
|
||||
protocol.RunCapabilityProcessStop,
|
||||
}
|
||||
}
|
||||
|
||||
func SupportedRunCapabilities() []string {
|
||||
capabilities := append([]string(nil), SupportedLifecycleCapabilities()...)
|
||||
capabilities = append(capabilities, protocol.RunCapabilityLogsRead)
|
||||
return capabilities
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) SupportedCapabilities() []string {
|
||||
return SupportedLifecycleCapabilities()
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) Execute(assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
return executor.ExecuteContext(context.Background(), assignment)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if !isSupportedLifecycleCapability(assignment.Capability) {
|
||||
return lifecycleFailure("unsupported_lifecycle_capability", "unsupported lifecycle capability")
|
||||
}
|
||||
command, err := executor.ResolveCommand(assignment)
|
||||
if err != nil {
|
||||
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
||||
}
|
||||
result, err := executor.supervisor.Run(ctx, command)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action cancelled"},
|
||||
Message: "lifecycle action cancelled",
|
||||
ErrorCode: "lifecycle_cancelled",
|
||||
}
|
||||
}
|
||||
executor.writeProcessLogs(ctx, assignment, result)
|
||||
if err != nil {
|
||||
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode))
|
||||
}
|
||||
artifactRef, err := executor.artifactHook.QueueLifecycleResult(ctx, assignment, result)
|
||||
if err != nil {
|
||||
return lifecycleFailure("lifecycle_artifact_hook_failed", err.Error())
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action completed"},
|
||||
ResultRef: artifactRef,
|
||||
Message: fmt.Sprintf("%s completed", assignment.Capability),
|
||||
}
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ResolveCommand(assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
||||
workdir, err := scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, err
|
||||
}
|
||||
if err := os.MkdirAll(workdir, 0o755); err != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("create scoped workspace: %w", err)
|
||||
}
|
||||
template := LifecycleActionTemplate{
|
||||
Command: []string{"true"},
|
||||
TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond),
|
||||
}
|
||||
if assignment.TargetKey != "" {
|
||||
path, err := scopedPath(workdir, assignment.TargetKey)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("open lifecycle action template: %w", err)
|
||||
}
|
||||
decodeErr := json.NewDecoder(file).Decode(&template)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("decode lifecycle action template: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("close lifecycle action template: %w", closeErr)
|
||||
}
|
||||
}
|
||||
return template.ToProcessCommand(workdir)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) writeProcessLogs(ctx context.Context, assignment protocol.RunJobAssignment, result ProcessResult) {
|
||||
for _, item := range []struct {
|
||||
stream string
|
||||
body string
|
||||
}{
|
||||
{stream: "stdout", body: result.Stdout},
|
||||
{stream: "stderr", body: result.Stderr},
|
||||
} {
|
||||
for _, line := range splitBoundedLines(item.body) {
|
||||
_ = executor.logSink.Append(ctx, assignment, item.stream, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type LifecycleActionTemplate struct {
|
||||
Command []string `json:"command"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
TimeoutMS int `json:"timeoutMs,omitempty"`
|
||||
}
|
||||
|
||||
func (template LifecycleActionTemplate) ToProcessCommand(workdir string) (ProcessCommand, error) {
|
||||
if len(template.Command) == 0 {
|
||||
return ProcessCommand{}, fmt.Errorf("command is required")
|
||||
}
|
||||
for i, part := range template.Command {
|
||||
if strings.TrimSpace(part) == "" {
|
||||
return ProcessCommand{}, fmt.Errorf("command part is required")
|
||||
}
|
||||
if containsUnsafeRuntimeText(part) {
|
||||
return ProcessCommand{}, fmt.Errorf("command contains unsafe content")
|
||||
}
|
||||
if i == 0 {
|
||||
if !commandNamePattern.MatchString(part) || strings.Contains(part, "/") || filepath.IsAbs(part) {
|
||||
return ProcessCommand{}, fmt.Errorf("command executable must be an allowlisted name")
|
||||
}
|
||||
if _, disallowed := disallowedExecutables[strings.ToLower(part)]; disallowed {
|
||||
return ProcessCommand{}, fmt.Errorf("command executable must not be a shell")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.ContainsAny(part, "|;&`$<>") {
|
||||
return ProcessCommand{}, fmt.Errorf("command arguments must not contain shell metacharacters")
|
||||
}
|
||||
}
|
||||
env := make(map[string]string, len(template.Env))
|
||||
for key, value := range template.Env {
|
||||
if !envNamePattern.MatchString(key) || !strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_") {
|
||||
return ProcessCommand{}, fmt.Errorf("env key is not allowlisted")
|
||||
}
|
||||
if containsUnsafeRuntimeText(value) {
|
||||
return ProcessCommand{}, fmt.Errorf("env value contains unsafe content")
|
||||
}
|
||||
env[key] = value
|
||||
}
|
||||
timeout := defaultLifecycleTimeout
|
||||
if template.TimeoutMS > 0 {
|
||||
timeout = time.Duration(template.TimeoutMS) * time.Millisecond
|
||||
}
|
||||
if timeout > 5*time.Minute {
|
||||
return ProcessCommand{}, fmt.Errorf("timeout is too large")
|
||||
}
|
||||
return ProcessCommand{WorkDir: workdir, Args: append([]string(nil), template.Command...), Env: env, Timeout: timeout}, nil
|
||||
}
|
||||
|
||||
type ProcessCommand struct {
|
||||
WorkDir string
|
||||
Args []string
|
||||
Env map[string]string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type ProcessResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
}
|
||||
|
||||
type ProcessSupervisor interface {
|
||||
Run(context.Context, ProcessCommand) (ProcessResult, error)
|
||||
}
|
||||
|
||||
type OSProcessSupervisor struct{}
|
||||
|
||||
func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) {
|
||||
if len(command.Args) == 0 {
|
||||
return ProcessResult{ExitCode: -1}, fmt.Errorf("command is required")
|
||||
}
|
||||
if command.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, command.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, command.Args[0], command.Args[1:]...)
|
||||
cmd.Dir = command.WorkDir
|
||||
cmd.Env = os.Environ()
|
||||
for key, value := range command.Env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = ioLimitWriter{Writer: &stdout, Limit: maxLifecycleOutputBytes}
|
||||
cmd.Stderr = ioLimitWriter{Writer: &stderr, Limit: maxLifecycleOutputBytes}
|
||||
err := cmd.Run()
|
||||
result := ProcessResult{Stdout: RedactText(stdout.String()), Stderr: RedactText(stderr.String())}
|
||||
if cmd.ProcessState != nil {
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type ioLimitWriter struct {
|
||||
Writer *bytes.Buffer
|
||||
Limit int
|
||||
}
|
||||
|
||||
func (writer ioLimitWriter) Write(p []byte) (int, error) {
|
||||
remaining := writer.Limit - writer.Writer.Len()
|
||||
if remaining > 0 {
|
||||
if len(p) > remaining {
|
||||
_, _ = writer.Writer.Write(p[:remaining])
|
||||
} else {
|
||||
_, _ = writer.Writer.Write(p)
|
||||
}
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
type ProcessLogSink interface {
|
||||
Append(context.Context, protocol.RunJobAssignment, string, string) error
|
||||
}
|
||||
|
||||
type NoopProcessLogSink struct{}
|
||||
|
||||
func (NoopProcessLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type LifecycleArtifactHook interface {
|
||||
QueueLifecycleResult(context.Context, protocol.RunJobAssignment, ProcessResult) (string, error)
|
||||
}
|
||||
|
||||
type StaticLifecycleArtifactHook struct{}
|
||||
|
||||
func (StaticLifecycleArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
||||
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", url.PathEscape(assignment.JobID)), nil
|
||||
}
|
||||
|
||||
func LifecycleResultRequest(assignment protocol.RunJobAssignment, sessionToken string, result LifecycleExecutionResult) protocol.RunJobResultRequest {
|
||||
return protocol.RunJobResultRequest{
|
||||
RunEndpointID: assignment.RunEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
State: result.State,
|
||||
Progress: result.Progress,
|
||||
ResultRef: result.ResultRef,
|
||||
Message: result.Message,
|
||||
ErrorCode: result.ErrorCode,
|
||||
}
|
||||
}
|
||||
|
||||
func isSupportedLifecycleCapability(capability string) bool {
|
||||
for _, supported := range SupportedLifecycleCapabilities() {
|
||||
if capability == supported {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lifecycleFailure(code string, message string) LifecycleExecutionResult {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateFailed,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)},
|
||||
Message: RedactText(message),
|
||||
ErrorCode: code,
|
||||
}
|
||||
}
|
||||
|
||||
func scopedServerWorkspace(root string, serverInstanceID string) (string, error) {
|
||||
if strings.TrimSpace(serverInstanceID) == "" {
|
||||
return "", fmt.Errorf("server instance id is required")
|
||||
}
|
||||
if containsUnsafeRuntimeText(serverInstanceID) || strings.ContainsAny(serverInstanceID, `/\`) || serverInstanceID == "." || serverInstanceID == ".." {
|
||||
return "", fmt.Errorf("server instance id is unsafe")
|
||||
}
|
||||
return scopedPath(root, serverInstanceID)
|
||||
}
|
||||
|
||||
func scopedPath(root string, key string) (string, error) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return "", fmt.Errorf("workspace root is required")
|
||||
}
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return "", fmt.Errorf("logical key is required")
|
||||
}
|
||||
if filepath.IsAbs(key) || strings.Contains(key, "..") || strings.Contains(key, `\`) || containsUnsafeRuntimeText(key) {
|
||||
return "", fmt.Errorf("logical key is unsafe")
|
||||
}
|
||||
cleanRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
candidate := filepath.Clean(filepath.Join(cleanRoot, filepath.FromSlash(key)))
|
||||
rel, err := filepath.Rel(cleanRoot, candidate)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("logical key escapes workspace")
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func containsUnsafeRuntimeText(value string) bool {
|
||||
normalized := strings.ToLower(value)
|
||||
for _, marker := range []string{"/users/", "/.ssh/", "password=", "apikey", "api_key", "secret=", "bearer ", "sk-", "unix://", "tcp://", "://"} {
|
||||
if strings.Contains(normalized, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func RedactText(value string) string {
|
||||
redacted := value
|
||||
replacements := []string{"/Users/", "[host]/", "Bearer ", "Bearer [redacted] ", "sk-", "sk-[redacted]", "password=", "password=[redacted]", "api_key=", "api_key=[redacted]", "secret=", "secret=[redacted]", "unix://", "socket://"}
|
||||
for i := 0; i+1 < len(replacements); i += 2 {
|
||||
redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1])
|
||||
}
|
||||
if len(redacted) > maxLifecycleOutputBytes {
|
||||
return redacted[:maxLifecycleOutputBytes]
|
||||
}
|
||||
return redacted
|
||||
}
|
||||
|
||||
func splitBoundedLines(value string) []string {
|
||||
value = RedactText(value)
|
||||
lines := strings.Split(value, "\n")
|
||||
out := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checksumForText(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestLifecycleExecutorHandlesSupportedJobs(t *testing.T) {
|
||||
executor := NewLifecycleExecutor()
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
assignment := lifecycleAssignment(capability)
|
||||
result := executor.Execute(assignment)
|
||||
if result.State != "succeeded" || result.Progress.Percent != 100 || result.ResultRef == "" {
|
||||
t.Fatalf("expected successful bounded result for %s, got %+v", capability, result)
|
||||
}
|
||||
for _, forbidden := range []string{"host path", "/Users/", "run socket", "api_key", "sk-"} {
|
||||
if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) {
|
||||
t.Fatalf("lifecycle result exposed forbidden content %q: %+v", forbidden, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsupportedJobs(t *testing.T) {
|
||||
result := NewLifecycleExecutor().Execute(lifecycleAssignment("files.write"))
|
||||
if result.State != "failed" || result.ErrorCode != "unsupported_lifecycle_capability" || result.ResultRef != "" {
|
||||
t.Fatalf("expected unsupported lifecycle failure, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleResultRequestUsesAssignmentLease(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
execution := NewLifecycleExecutor().Execute(assignment)
|
||||
request := LifecycleResultRequest(assignment, "session-token", execution)
|
||||
|
||||
if request.RunEndpointID != assignment.RunEndpointID || request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken || request.Attempt != assignment.Attempt {
|
||||
t.Fatalf("expected result request to use assignment lease, got %+v", request)
|
||||
}
|
||||
if request.SessionToken != "session-token" || request.State != "succeeded" {
|
||||
t.Fatalf("unexpected result request: %+v", request)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRunsScopedCommandTemplateAndHooks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
template := LifecycleActionTemplate{
|
||||
Command: []string{"echo", "server-ready"},
|
||||
Env: map[string]string{"GAME_MODE": "test"},
|
||||
}
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "start.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template: %v", err)
|
||||
}
|
||||
assignment.TargetKey = "actions/start.json"
|
||||
logSink := &recordingLogSink{}
|
||||
artifactHook := &recordingArtifactHook{}
|
||||
|
||||
result := NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithProcessLogSink(logSink),
|
||||
WithLifecycleArtifactHook(artifactHook),
|
||||
).Execute(assignment)
|
||||
|
||||
if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/lifecycle-result" {
|
||||
t.Fatalf("expected scoped lifecycle success, got %+v", result)
|
||||
}
|
||||
if len(logSink.lines) != 1 || logSink.lines[0] != "stdout:server-ready" {
|
||||
t.Fatalf("expected process stdout to be logged, got %+v", logSink.lines)
|
||||
}
|
||||
if !artifactHook.called {
|
||||
t.Fatal("expected artifact hook to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsafeTemplates(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
for name, template := range map[string]LifecycleActionTemplate{
|
||||
"absolute": {Command: []string{"/bin/echo", "nope"}},
|
||||
"shell": {Command: []string{"sh", "-c", "echo nope"}},
|
||||
"secret": {Command: []string{"echo", "sk-secret"}},
|
||||
"env": {Command: []string{"echo", "ok"}, Env: map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}},
|
||||
} {
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
actionPath := filepath.Join(serverRoot, "actions", name+".json")
|
||||
if err := os.WriteFile(actionPath, body, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
unsafeAssignment := assignment
|
||||
unsafeAssignment.TargetKey = "actions/" + name + ".json"
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(unsafeAssignment)
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected unsafe command rejection for %s, got %+v", name, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsWorkspaceEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.TargetKey = "../outside.json"
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment)
|
||||
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected workspace escape rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorKeepsSiblingInstanceWorkspacesIsolated(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
first := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
first.ServerInstanceID = "server-alpha"
|
||||
second := lifecycleAssignment(protocol.RunCapabilityProcessStop)
|
||||
second.JobID = "job-2"
|
||||
second.ServerInstanceID = "server-beta"
|
||||
for _, assignment := range []protocol.RunJobAssignment{first, second} {
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
body, err := json.Marshal(LifecycleActionTemplate{Command: []string{"echo", assignment.ServerInstanceID}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "lifecycle.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
}
|
||||
first.TargetKey = "actions/lifecycle.json"
|
||||
second.TargetKey = "actions/lifecycle.json"
|
||||
logSink := &recordingLogSink{}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink))
|
||||
|
||||
firstResult := executor.Execute(first)
|
||||
secondResult := executor.Execute(second)
|
||||
|
||||
if firstResult.State != "succeeded" || secondResult.State != "succeeded" {
|
||||
t.Fatalf("expected both lifecycle jobs to succeed, got first=%+v second=%+v", firstResult, secondResult)
|
||||
}
|
||||
joined := strings.Join(logSink.lines, "\n")
|
||||
if !strings.Contains(joined, "stdout:server-alpha") || !strings.Contains(joined, "stdout:server-beta") {
|
||||
t.Fatalf("expected instance-specific output, got %q", joined)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-alpha", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected alpha template to remain scoped: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-beta", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected beta template to remain scoped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorCancelsRunningCommand(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
result := NewLifecycleExecutor(WithProcessSupervisor(blockingSupervisor{})).ExecuteContext(ctx, lifecycleAssignment(protocol.RunCapabilityProcessStart))
|
||||
|
||||
if result.State != "cancelled" || result.ErrorCode != "lifecycle_cancelled" {
|
||||
t.Fatalf("expected cancelled lifecycle result, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLifecycleCapabilities(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
if !containsCapability(summary.Capabilities, capability) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLogReadCapability(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
if !containsCapability(summary.Capabilities, protocol.RunCapabilityLogsRead) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", protocol.RunCapabilityLogsRead, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingLogSink struct {
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (sink *recordingLogSink) Append(_ context.Context, _ protocol.RunJobAssignment, stream string, line string) error {
|
||||
sink.lines = append(sink.lines, stream+":"+line)
|
||||
return nil
|
||||
}
|
||||
|
||||
type recordingArtifactHook struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
func (hook *recordingArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
||||
hook.called = true
|
||||
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID), nil
|
||||
}
|
||||
|
||||
type blockingSupervisor struct{}
|
||||
|
||||
func (blockingSupervisor) Run(ctx context.Context, _ ProcessCommand) (ProcessResult, error) {
|
||||
<-ctx.Done()
|
||||
return ProcessResult{ExitCode: -1}, ctx.Err()
|
||||
}
|
||||
|
||||
func lifecycleAssignment(capability string) protocol.RunJobAssignment {
|
||||
now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
|
||||
return protocol.RunJobAssignment{
|
||||
JobID: "job-1",
|
||||
ServerInstanceID: "server-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: capability,
|
||||
IdempotencyKey: "idem-1",
|
||||
State: "accepted",
|
||||
LeaseToken: "lease-1",
|
||||
Attempt: 1,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func containsCapability(capabilities []string, capability string) bool {
|
||||
for _, item := range capabilities {
|
||||
if item == capability {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/domain"
|
||||
)
|
||||
|
||||
func SmokeSummary(cfg config.Config) domain.ExecutorStatus {
|
||||
return domain.ExecutorStatus{
|
||||
Mode: cfg.Mode,
|
||||
PlatformURL: cfg.PlatformURL,
|
||||
Status: "ok",
|
||||
ExposedHostPath: false,
|
||||
Capabilities: append([]string{
|
||||
"control.hello",
|
||||
"control.heartbeat",
|
||||
}, SupportedRunCapabilities()...),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
)
|
||||
|
||||
func TestSmokeSummaryDoesNotExposeHostPaths(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{
|
||||
Mode: "smoke",
|
||||
PlatformURL: "http://platform.test",
|
||||
})
|
||||
|
||||
if summary.Status != "ok" {
|
||||
t.Fatalf("expected ok status, got %q", summary.Status)
|
||||
}
|
||||
if summary.ExposedHostPath {
|
||||
t.Fatal("smoke summary must not expose host paths")
|
||||
}
|
||||
if len(summary.Capabilities) == 0 {
|
||||
t.Fatal("expected baseline capabilities")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
type WorkerClient interface {
|
||||
Hello(context.Context, protocol.RunHelloRequest) (protocol.RunHelloResponse, error)
|
||||
Heartbeat(context.Context, protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error)
|
||||
ClaimJob(context.Context, protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error)
|
||||
AckJob(context.Context, protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error)
|
||||
UpdateJobProgress(context.Context, protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error)
|
||||
CompleteJob(context.Context, protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error)
|
||||
PollJobCancel(context.Context, protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error)
|
||||
ReconcileJobs(context.Context, protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error)
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
cfg config.Config
|
||||
client WorkerClient
|
||||
executor LifecycleExecutor
|
||||
state WorkerState
|
||||
journal *JobJournal
|
||||
}
|
||||
|
||||
type WorkerState struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Capabilities []string
|
||||
Capacity protocol.RunCapacityReport
|
||||
LastHeartbeat time.Time
|
||||
Sequence uint64
|
||||
}
|
||||
|
||||
func NewWorker(cfg config.Config, client WorkerClient, options ...LifecycleExecutorOption) (*Worker, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("worker client is required")
|
||||
}
|
||||
if cfg.RunEndpointID == "" {
|
||||
cfg.RunEndpointID = config.DefaultEndpointID
|
||||
}
|
||||
if cfg.DisplayName == "" {
|
||||
cfg.DisplayName = config.DefaultDisplayName
|
||||
}
|
||||
if cfg.Version == "" {
|
||||
cfg.Version = config.DefaultVersion
|
||||
}
|
||||
if cfg.MaxJobs <= 0 {
|
||||
cfg.MaxJobs = 1
|
||||
}
|
||||
executorOptions := append([]LifecycleExecutorOption{
|
||||
WithLifecycleWorkspaceRoot(cfg.WorkspaceRoot),
|
||||
}, options...)
|
||||
return &Worker{
|
||||
cfg: cfg,
|
||||
client: client,
|
||||
executor: NewLifecycleExecutor(executorOptions...),
|
||||
state: WorkerState{
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
Capabilities: SupportedRunCapabilities(),
|
||||
Capacity: protocol.RunCapacityReport{MaxJobs: cfg.MaxJobs},
|
||||
},
|
||||
journal: NewJobJournal(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (worker *Worker) Register(ctx context.Context) error {
|
||||
response, err := worker.client.Hello(ctx, protocol.RunHelloRequest{
|
||||
RegistrationToken: worker.cfg.RegistrationToken,
|
||||
RunEndpointID: worker.cfg.RunEndpointID,
|
||||
DisplayName: worker.cfg.DisplayName,
|
||||
Version: worker.cfg.Version,
|
||||
Status: "online",
|
||||
Platform: runtime.GOOS,
|
||||
CapabilityReport: protocol.RunCapabilityReport{
|
||||
Capabilities: worker.state.Capabilities,
|
||||
Fingerprint: capabilityFingerprint(worker.state.Capabilities),
|
||||
},
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted || response.SessionToken == "" {
|
||||
return fmt.Errorf("run hello was not accepted")
|
||||
}
|
||||
worker.state.SessionToken = response.SessionToken
|
||||
if sink, ok := worker.executor.logSink.(*SpoolLogSink); ok {
|
||||
sink.RunEndpointID = worker.state.RunEndpointID
|
||||
sink.SessionToken = worker.state.SessionToken
|
||||
}
|
||||
if hook, ok := worker.executor.artifactHook.(*QueueArtifactHook); ok {
|
||||
hook.RunEndpointID = worker.state.RunEndpointID
|
||||
hook.SessionToken = worker.state.SessionToken
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) HeartbeatOnce(ctx context.Context) error {
|
||||
if worker.state.SessionToken == "" {
|
||||
return fmt.Errorf("worker is not registered")
|
||||
}
|
||||
response, err := worker.client.Heartbeat(ctx, protocol.RunHeartbeatRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
Version: worker.cfg.Version,
|
||||
Status: "online",
|
||||
CapabilityFingerprint: capabilityFingerprint(worker.state.Capabilities),
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted {
|
||||
return fmt.Errorf("heartbeat was not accepted")
|
||||
}
|
||||
worker.state.LastHeartbeat = response.ServerTime
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) ClaimAndRunOnce(ctx context.Context) (bool, error) {
|
||||
if worker.state.SessionToken == "" {
|
||||
return false, fmt.Errorf("worker is not registered")
|
||||
}
|
||||
claim, err := worker.client.ClaimJob(ctx, protocol.RunJobClaimRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
Capabilities: worker.state.Capabilities,
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !claim.Accepted || !claim.HasJob || claim.Job == nil {
|
||||
return false, nil
|
||||
}
|
||||
assignment := *claim.Job
|
||||
worker.journal.MarkActive(assignment)
|
||||
ack, err := worker.client.AckJob(ctx, protocol.RunJobAckRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
Message: "job accepted by run worker",
|
||||
})
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
assignment = ack.Job
|
||||
worker.journal.MarkActive(assignment)
|
||||
worker.state.Sequence++
|
||||
if _, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 10, Message: "lifecycle execution started"},
|
||||
Sequence: worker.state.Sequence,
|
||||
}); err != nil {
|
||||
return true, err
|
||||
}
|
||||
jobCtx, cancel := context.WithCancel(ctx)
|
||||
cancelled := make(chan protocol.RunJobCancelPollResponse, 1)
|
||||
go func() {
|
||||
cancelPoll, pollErr := worker.client.PollJobCancel(ctx, protocol.RunJobCancelPollRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
})
|
||||
if pollErr == nil && cancelPoll.HasCancel {
|
||||
cancel()
|
||||
cancelled <- cancelPoll
|
||||
return
|
||||
}
|
||||
cancelled <- protocol.RunJobCancelPollResponse{Accepted: true}
|
||||
}()
|
||||
execution := worker.executor.ExecuteContext(jobCtx, assignment)
|
||||
cancel()
|
||||
select {
|
||||
case poll := <-cancelled:
|
||||
if poll.HasCancel && execution.State == lifecycleResultStateSucceeded {
|
||||
execution = LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "cancelled by platform"},
|
||||
Message: "cancelled by platform",
|
||||
ErrorCode: "lifecycle_cancelled",
|
||||
}
|
||||
}
|
||||
default:
|
||||
}
|
||||
if _, err := worker.client.CompleteJob(ctx, LifecycleResultRequest(assignment, worker.state.SessionToken, execution)); err != nil {
|
||||
return true, err
|
||||
}
|
||||
worker.journal.MarkTerminal(assignment.JobID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (worker *Worker) ReconcileOnce(ctx context.Context) error {
|
||||
if worker.state.SessionToken == "" {
|
||||
return fmt.Errorf("worker is not registered")
|
||||
}
|
||||
response, err := worker.client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
ActiveJobIDs: worker.journal.ActiveJobIDs(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range response.ActiveJobs {
|
||||
worker.journal.MarkActive(job)
|
||||
}
|
||||
for _, unknown := range response.UnknownJobIDs {
|
||||
worker.journal.MarkTerminal(unknown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) Run(ctx context.Context) error {
|
||||
if err := worker.Register(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
heartbeatInterval := durationOrDefault(worker.cfg.HeartbeatInterval, 15*time.Second)
|
||||
jobInterval := durationOrDefault(worker.cfg.PollInterval, 2*time.Second)
|
||||
heartbeatTicker := time.NewTicker(heartbeatInterval)
|
||||
jobTicker := time.NewTicker(jobInterval)
|
||||
defer heartbeatTicker.Stop()
|
||||
defer jobTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-heartbeatTicker.C:
|
||||
if err := worker.HeartbeatOnce(ctx); err != nil {
|
||||
heartbeatTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
|
||||
continue
|
||||
}
|
||||
heartbeatTicker.Reset(heartbeatInterval)
|
||||
case <-jobTicker.C:
|
||||
if _, err := worker.ClaimAndRunOnce(ctx); err != nil {
|
||||
jobTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
|
||||
continue
|
||||
}
|
||||
jobTicker.Reset(jobInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) capacityReport() protocol.RunCapacityReport {
|
||||
return protocol.RunCapacityReport{
|
||||
MaxJobs: worker.state.Capacity.MaxJobs,
|
||||
RunningJobs: worker.journal.ActiveCount(),
|
||||
QueuedJobs: 0,
|
||||
Summary: "worker control active; job capacity reported separately",
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) State() WorkerState {
|
||||
state := worker.state
|
||||
state.Capabilities = append([]string(nil), state.Capabilities...)
|
||||
return state
|
||||
}
|
||||
|
||||
type JobJournal struct {
|
||||
mu sync.Mutex
|
||||
active map[string]protocol.RunJobAssignment
|
||||
}
|
||||
|
||||
func NewJobJournal() *JobJournal {
|
||||
return &JobJournal{active: map[string]protocol.RunJobAssignment{}}
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkActive(job protocol.RunJobAssignment) {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
journal.active[job.JobID] = job
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkTerminal(jobID string) {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
delete(journal.active, jobID)
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveJobIDs() []string {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
ids := make([]string, 0, len(journal.active))
|
||||
for id := range journal.active {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveCount() int {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
return len(journal.active)
|
||||
}
|
||||
|
||||
type SpoolLogSink struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Spool spool.LogSpool
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (sink *SpoolLogSink) Append(_ context.Context, assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
sink.seq++
|
||||
entry := protocol.LogEntry{Seq: sink.seq, Timestamp: time.Now().UTC(), Level: "info", Line: RedactText(line), Redacted: line != RedactText(line)}
|
||||
logStreamID := fmt.Sprintf("job.%s.%s", assignment.JobID, stream)
|
||||
return sink.Spool.Enqueue(protocol.LogBatchIngestRequest{
|
||||
RunEndpointID: sink.RunEndpointID,
|
||||
SessionToken: sink.SessionToken,
|
||||
LogStreamID: logStreamID,
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
StreamKey: stream,
|
||||
Source: "process",
|
||||
FirstSeq: sink.seq,
|
||||
LastSeq: sink.seq,
|
||||
Checksum: checksumForText(entry.Line),
|
||||
Entries: []protocol.LogEntry{entry},
|
||||
})
|
||||
}
|
||||
|
||||
type QueueArtifactHook struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Queue spool.ArtifactQueue
|
||||
}
|
||||
|
||||
func (hook QueueArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, result ProcessResult) (string, error) {
|
||||
ref := fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID)
|
||||
payload := []byte(RedactText(result.Stdout + result.Stderr))
|
||||
if len(payload) == 0 {
|
||||
payload = []byte("lifecycle result metadata")
|
||||
}
|
||||
artifactID := "artifact-" + assignment.JobID + "-lifecycle"
|
||||
if err := hook.Queue.Enqueue(protocol.ArtifactChunkUploadRequest{
|
||||
RunEndpointID: hook.RunEndpointID,
|
||||
SessionToken: hook.SessionToken,
|
||||
TransferID: "transfer-" + assignment.JobID,
|
||||
ArtifactID: artifactID,
|
||||
ChunkIndex: 0,
|
||||
Offset: 0,
|
||||
SizeBytes: len(payload),
|
||||
Checksum: checksumForText(string(payload)),
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func capabilityFingerprint(capabilities []string) string {
|
||||
return checksumForText(strings.Join(capabilities, ","))
|
||||
}
|
||||
|
||||
func durationOrDefault(value time.Duration, fallback time.Duration) time.Duration {
|
||||
if value <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func boundedRetryBackoff(value time.Duration) time.Duration {
|
||||
value = durationOrDefault(value, time.Second)
|
||||
if value > 30*time.Second {
|
||||
return 30 * time.Second
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/api"
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
func TestWorkerRegistersHeartbeatsAndStoresSession(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
worker, err := NewWorker(workerTestConfig(t), client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if worker.State().SessionToken != "session-token" {
|
||||
t.Fatalf("expected session token stored, got %+v", worker.State())
|
||||
}
|
||||
if len(client.helloRequests) != 1 || client.helloRequests[0].RegistrationToken != "registration-token" || len(client.helloRequests[0].CapabilityReport.Capabilities) == 0 {
|
||||
t.Fatalf("unexpected hello request: %+v", client.helloRequests)
|
||||
}
|
||||
|
||||
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if len(client.heartbeatRequests) != 1 {
|
||||
t.Fatalf("expected heartbeat request")
|
||||
}
|
||||
heartbeat := client.heartbeatRequests[0]
|
||||
if heartbeat.SessionToken != "session-token" || heartbeat.Capacity.MaxJobs != 2 || heartbeat.Capacity.RunningJobs != 0 {
|
||||
t.Fatalf("unexpected heartbeat request: %+v", heartbeat)
|
||||
}
|
||||
for _, forbidden := range []string{"/Users/", "unix://", "Bearer ", "sk-", "password=", "artifact", "log"} {
|
||||
if containsText(heartbeat.Capacity.Summary, forbidden) {
|
||||
t.Fatalf("heartbeat summary exposed forbidden fragment %q: %+v", forbidden, heartbeat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerClaimsAcksProgressAndCompletesJob(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(staticSupervisor{stdout: "server ready\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
handled, err := worker.ClaimAndRunOnce(context.Background())
|
||||
if err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
if len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || len(client.resultRequests) != 1 || len(client.cancelPollRequests) != 1 {
|
||||
t.Fatalf("expected ack/progress/result/cancel calls, got ack=%d progress=%d result=%d cancel=%d", len(client.ackRequests), len(client.progressRequests), len(client.resultRequests), len(client.cancelPollRequests))
|
||||
}
|
||||
if client.progressRequests[0].Progress.Percent != 10 || client.resultRequests[0].State != "succeeded" || client.resultRequests[0].ResultRef == "" {
|
||||
t.Fatalf("unexpected job channel payloads: progress=%+v result=%+v", client.progressRequests[0], client.resultRequests[0])
|
||||
}
|
||||
if worker.journal.ActiveCount() != 0 {
|
||||
t.Fatalf("expected terminal job removed from journal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlesCancellationAndReconcile(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
client.cancelResponse = protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", HasCancel: true, JobID: "job-worker", Reason: "operator requested", ServerTime: workerTestTime()}
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(blockingSupervisor{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
handled, err := worker.ClaimAndRunOnce(context.Background())
|
||||
if err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
if len(client.resultRequests) != 1 || client.resultRequests[0].State != "cancelled" || client.resultRequests[0].ErrorCode != "lifecycle_cancelled" {
|
||||
t.Fatalf("expected cancelled terminal result, got %+v", client.resultRequests)
|
||||
}
|
||||
|
||||
worker.journal.MarkActive(workerJobAssignment(protocol.RunCapabilityProcessStart))
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: "run-test",
|
||||
ActiveJobs: []protocol.RunJobAssignment{workerJobAssignment(protocol.RunCapabilityProcessStop)},
|
||||
UnknownJobIDs: []string{"job-worker"},
|
||||
ServerTime: workerTestTime(),
|
||||
}
|
||||
if err := worker.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if ids := worker.journal.ActiveJobIDs(); !reflect.DeepEqual(ids, []string{"job-worker-stop"}) {
|
||||
t.Fatalf("expected reconcile to replace active job ids, got %+v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
logSpool, err := spool.NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("log spool: %v", err)
|
||||
}
|
||||
artifactQueue, err := spool.NewArtifactQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("artifact queue: %v", err)
|
||||
}
|
||||
worker, err := NewWorker(
|
||||
workerTestConfig(t),
|
||||
client,
|
||||
WithProcessSupervisor(staticSupervisor{stdout: "started password=hidden\n"}),
|
||||
WithProcessLogSink(&SpoolLogSink{Spool: logSpool}),
|
||||
WithLifecycleArtifactHook(&QueueArtifactHook{Queue: artifactQueue}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if _, err := worker.ClaimAndRunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("claim/run: %v", err)
|
||||
}
|
||||
logs, err := logSpool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending logs: %v", err)
|
||||
}
|
||||
if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || containsText(logs[0].Entries[0].Line, "password=hidden") {
|
||||
t.Fatalf("unexpected spooled logs: %+v", logs)
|
||||
}
|
||||
chunks, err := artifactQueue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending artifact chunks: %v", err)
|
||||
}
|
||||
if len(chunks) != 1 || chunks[0].RunEndpointID != "run-test" || chunks[0].SessionToken != "session-token" {
|
||||
t.Fatalf("unexpected artifact chunks: %+v", chunks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRetryBackoffIsBounded(t *testing.T) {
|
||||
if got := boundedRetryBackoff(75 * time.Millisecond); got != 75*time.Millisecond {
|
||||
t.Fatalf("expected configured backoff, got %s", got)
|
||||
}
|
||||
if got := boundedRetryBackoff(time.Minute); got != 30*time.Second {
|
||||
t.Fatalf("expected capped backoff, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerIntegrationWithPlatformLikeServer(t *testing.T) {
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
seen := []string{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seen = append(seen, r.URL.Path)
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/control/hello":
|
||||
var request protocol.RunHelloRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.RunEndpointID != "run-test" || request.Capacity.MaxJobs != 2 {
|
||||
t.Fatalf("unexpected hello: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", HeartbeatIntervalSeconds: 15, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/control/heartbeat":
|
||||
var request protocol.RunHeartbeatRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 0 {
|
||||
t.Fatalf("unexpected heartbeat: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/claim":
|
||||
var request protocol.RunJobClaimRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.SessionToken != "session-token" || len(request.Capabilities) == 0 {
|
||||
t.Fatalf("unexpected claim: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/ack":
|
||||
var request protocol.RunJobAckRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken {
|
||||
t.Fatalf("unexpected ack: %+v", request)
|
||||
}
|
||||
assignment.State = "running"
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/progress":
|
||||
var request protocol.RunJobProgressRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.Progress.Percent != 10 {
|
||||
t.Fatalf("unexpected progress: %+v", request)
|
||||
}
|
||||
assignment.Progress = request.Progress
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/cancel":
|
||||
var request protocol.RunJobCancelPollRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.JobID != assignment.JobID {
|
||||
t.Fatalf("unexpected cancel poll: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/result":
|
||||
var request protocol.RunJobResultRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.State != "succeeded" || request.ResultRef == "" {
|
||||
t.Fatalf("unexpected result: %+v", request)
|
||||
}
|
||||
assignment.State = request.State
|
||||
assignment.ResultRef = request.ResultRef
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
default:
|
||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := api.NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("platform client: %v", err)
|
||||
}
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.PlatformURL = server.URL
|
||||
worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "integration ok\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
expected := []string{
|
||||
"/api/v1/run/control/hello",
|
||||
"/api/v1/run/control/heartbeat",
|
||||
"/api/v1/run/jobs/claim",
|
||||
"/api/v1/run/jobs/ack",
|
||||
"/api/v1/run/jobs/progress",
|
||||
"/api/v1/run/jobs/cancel",
|
||||
"/api/v1/run/jobs/result",
|
||||
}
|
||||
if !reflect.DeepEqual(seen, expected) {
|
||||
t.Fatalf("unexpected platform flow: %+v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeWorkerClient struct {
|
||||
helloRequests []protocol.RunHelloRequest
|
||||
heartbeatRequests []protocol.RunHeartbeatRequest
|
||||
claimRequests []protocol.RunJobClaimRequest
|
||||
ackRequests []protocol.RunJobAckRequest
|
||||
progressRequests []protocol.RunJobProgressRequest
|
||||
resultRequests []protocol.RunJobResultRequest
|
||||
cancelPollRequests []protocol.RunJobCancelPollRequest
|
||||
reconcileRequests []protocol.RunJobReconcileRequest
|
||||
claimJob protocol.RunJobAssignment
|
||||
cancelResponse protocol.RunJobCancelPollResponse
|
||||
reconcileResponse protocol.RunJobReconcileResponse
|
||||
}
|
||||
|
||||
func newFakeWorkerClient() *fakeWorkerClient {
|
||||
return &fakeWorkerClient{
|
||||
cancelResponse: protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
||||
reconcileResponse: protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
||||
}
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) Hello(_ context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) {
|
||||
client.helloRequests = append(client.helloRequests, request)
|
||||
return protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", ServerTime: workerTestTime(), HeartbeatIntervalSeconds: 15}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) Heartbeat(_ context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) {
|
||||
client.heartbeatRequests = append(client.heartbeatRequests, request)
|
||||
return protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) ClaimJob(_ context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) {
|
||||
client.claimRequests = append(client.claimRequests, request)
|
||||
if client.claimJob.JobID == "" {
|
||||
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: false, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
job := client.claimJob
|
||||
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) AckJob(_ context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) {
|
||||
client.ackRequests = append(client.ackRequests, request)
|
||||
job := client.claimJob
|
||||
job.State = "running"
|
||||
return protocol.RunJobAckResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) UpdateJobProgress(_ context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) {
|
||||
client.progressRequests = append(client.progressRequests, request)
|
||||
job := client.claimJob
|
||||
job.Progress = request.Progress
|
||||
return protocol.RunJobProgressResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) CompleteJob(_ context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) {
|
||||
client.resultRequests = append(client.resultRequests, request)
|
||||
job := client.claimJob
|
||||
job.State = request.State
|
||||
job.Progress = request.Progress
|
||||
job.ResultRef = request.ResultRef
|
||||
return protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) PollJobCancel(_ context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) {
|
||||
client.cancelPollRequests = append(client.cancelPollRequests, request)
|
||||
return client.cancelResponse, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) ReconcileJobs(_ context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) {
|
||||
client.reconcileRequests = append(client.reconcileRequests, request)
|
||||
return client.reconcileResponse, nil
|
||||
}
|
||||
|
||||
type staticSupervisor struct {
|
||||
stdout string
|
||||
stderr string
|
||||
err error
|
||||
}
|
||||
|
||||
func (supervisor staticSupervisor) Run(context.Context, ProcessCommand) (ProcessResult, error) {
|
||||
return ProcessResult{ExitCode: 0, Stdout: supervisor.stdout, Stderr: supervisor.stderr}, supervisor.err
|
||||
}
|
||||
|
||||
func workerTestConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
return config.Config{
|
||||
Mode: "worker",
|
||||
PlatformURL: "http://platform.test",
|
||||
RunEndpointID: "run-test",
|
||||
DisplayName: "Run Test",
|
||||
Version: "0.1.0-test",
|
||||
RegistrationToken: "registration-token",
|
||||
WorkspaceRoot: t.TempDir(),
|
||||
SpoolRoot: t.TempDir(),
|
||||
MaxJobs: 2,
|
||||
HeartbeatInterval: time.Second,
|
||||
PollInterval: time.Second,
|
||||
RetryBackoff: time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func workerJobAssignment(capability string) protocol.RunJobAssignment {
|
||||
job := lifecycleAssignment(capability)
|
||||
job.JobID = "job-worker"
|
||||
if capability == protocol.RunCapabilityProcessStop {
|
||||
job.JobID = "job-worker-stop"
|
||||
}
|
||||
job.RunEndpointID = "run-test"
|
||||
job.ServerInstanceID = "server-worker"
|
||||
return job
|
||||
}
|
||||
|
||||
func workerTestTime() time.Time {
|
||||
return time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func containsText(value string, needle string) bool {
|
||||
return strings.Contains(value, needle)
|
||||
}
|
||||
|
||||
func decodeWorkerTestJSON(t *testing.T, r *http.Request, target any) {
|
||||
t.Helper()
|
||||
if r.Method != http.MethodPost {
|
||||
t.Fatalf("expected POST, got %s", r.Method)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(target); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeWorkerTestJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Fatalf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# run/spool
|
||||
|
||||
Local durable queues live here.
|
||||
|
||||
Required spool areas:
|
||||
|
||||
- `logs`: unacknowledged log segments.
|
||||
- `jobs`: accepted job journal for duplicate detection and reconciliation.
|
||||
- `artifacts`: incomplete artifact transfer state.
|
||||
|
||||
Spool pressure must be visible in run capacity reports.
|
||||
|
||||
## Channel Isolation
|
||||
|
||||
- Log and artifact retry state are stored in separate spool areas and are acknowledged independently.
|
||||
- Acknowledging a log batch must not scan, remove, or block on artifact chunks.
|
||||
- Acknowledging an artifact chunk must not scan, remove, or block on log batches.
|
||||
- Control heartbeat and job ack/result payloads remain metadata-only; they must never carry spool file paths, artifact chunks, log entries, raw credentials, direct sockets, or large inline bodies.
|
||||
- Artifact transfer backlog is lower priority than log flush, job lifecycle calls, and control heartbeat.
|
||||
@@ -0,0 +1,125 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type ArtifactQueue struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func NewArtifactQueue(dir string) (ArtifactQueue, error) {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return ArtifactQueue{}, fmt.Errorf("spool directory is required")
|
||||
}
|
||||
artifactDir := filepath.Join(dir, "artifacts")
|
||||
if err := os.MkdirAll(artifactDir, 0o755); err != nil {
|
||||
return ArtifactQueue{}, fmt.Errorf("create artifact queue: %w", err)
|
||||
}
|
||||
return ArtifactQueue{dir: artifactDir}, nil
|
||||
}
|
||||
|
||||
func (queue ArtifactQueue) Enqueue(chunk protocol.ArtifactChunkUploadRequest) error {
|
||||
path := queue.chunkPath(chunk)
|
||||
tmp := path + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open artifact queue chunk: %w", err)
|
||||
}
|
||||
encodeErr := json.NewEncoder(file).Encode(chunk)
|
||||
closeErr := file.Close()
|
||||
if encodeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("encode artifact queue chunk: %w", encodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("close artifact queue chunk: %w", closeErr)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("commit artifact queue chunk: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue ArtifactQueue) Pending() ([]protocol.ArtifactChunkUploadRequest, error) {
|
||||
entries, err := os.ReadDir(queue.dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact queue: %w", err)
|
||||
}
|
||||
paths := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, filepath.Join(queue.dir, entry.Name()))
|
||||
}
|
||||
sort.Strings(paths)
|
||||
chunks := make([]protocol.ArtifactChunkUploadRequest, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open artifact queue chunk: %w", err)
|
||||
}
|
||||
var chunk protocol.ArtifactChunkUploadRequest
|
||||
decodeErr := json.NewDecoder(file).Decode(&chunk)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return nil, fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("close artifact queue chunk: %w", closeErr)
|
||||
}
|
||||
chunks = append(chunks, chunk)
|
||||
}
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
func (queue ArtifactQueue) Ack(response protocol.ArtifactChunkUploadResponse) error {
|
||||
if !response.Accepted {
|
||||
return nil
|
||||
}
|
||||
entries, err := os.ReadDir(queue.dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read artifact queue: %w", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(queue.dir, entry.Name())
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open artifact queue chunk: %w", err)
|
||||
}
|
||||
var chunk protocol.ArtifactChunkUploadRequest
|
||||
decodeErr := json.NewDecoder(file).Decode(&chunk)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close artifact queue chunk: %w", closeErr)
|
||||
}
|
||||
if chunk.TransferID == response.TransferID && chunk.ArtifactID == response.ArtifactID && chunk.ChunkIndex == response.ChunkIndex {
|
||||
if err := os.Remove(path); err != nil {
|
||||
return fmt.Errorf("remove acknowledged artifact queue chunk: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue ArtifactQueue) chunkPath(chunk protocol.ArtifactChunkUploadRequest) string {
|
||||
transferID := sanitizeSegmentName(chunk.TransferID)
|
||||
artifactID := sanitizeSegmentName(chunk.ArtifactID)
|
||||
return filepath.Join(queue.dir, fmt.Sprintf("%s-%s-%020d.json", transferID, artifactID, chunk.ChunkIndex))
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestArtifactQueueRetainsPendingAndRemovesAcknowledgedChunk(t *testing.T) {
|
||||
queue, err := NewArtifactQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new artifact queue: %v", err)
|
||||
}
|
||||
first := validQueuedArtifactChunk(0)
|
||||
second := validQueuedArtifactChunk(1)
|
||||
if err := queue.Enqueue(first); err != nil {
|
||||
t.Fatalf("enqueue first: %v", err)
|
||||
}
|
||||
if err := queue.Enqueue(second); err != nil {
|
||||
t.Fatalf("enqueue second: %v", err)
|
||||
}
|
||||
pending, err := queue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending before ack: %v", err)
|
||||
}
|
||||
if len(pending) != 2 {
|
||||
t.Fatalf("expected two pending chunks, got %+v", pending)
|
||||
}
|
||||
|
||||
if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil {
|
||||
t.Fatalf("ack first: %v", err)
|
||||
}
|
||||
pending, err = queue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending after ack: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ChunkIndex != 1 {
|
||||
t.Fatalf("expected second chunk pending, got %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactQueueRetainsChunkWhenAckDoesNotMatch(t *testing.T) {
|
||||
queue, err := NewArtifactQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new artifact queue: %v", err)
|
||||
}
|
||||
if err := queue.Enqueue(validQueuedArtifactChunk(0)); err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 1}); err != nil {
|
||||
t.Fatalf("ack mismatch: %v", err)
|
||||
}
|
||||
pending, err := queue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ChunkIndex != 0 {
|
||||
t.Fatalf("expected original chunk pending, got %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func validQueuedArtifactChunk(index int) protocol.ArtifactChunkUploadRequest {
|
||||
payload := []byte{byte(index), byte(index + 1)}
|
||||
return protocol.ArtifactChunkUploadRequest{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: "session-token",
|
||||
TransferID: "transfer-1",
|
||||
ArtifactID: "artifact-1",
|
||||
ChunkIndex: index,
|
||||
Offset: int64(index * len(payload)),
|
||||
SizeBytes: len(payload),
|
||||
Checksum: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
Payload: payload,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestLogSpoolAckIsIndependentFromArtifactBacklog(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logSpool, err := NewLogSpool(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
artifactQueue, err := NewArtifactQueue(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new artifact queue: %v", err)
|
||||
}
|
||||
|
||||
if err := logSpool.Enqueue(validSpoolLogBatch(1, 1)); err != nil {
|
||||
t.Fatalf("enqueue log: %v", err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := artifactQueue.Enqueue(validQueuedArtifactChunk(i)); err != nil {
|
||||
t.Fatalf("enqueue artifact chunk %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := logSpool.Ack(protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
|
||||
t.Fatalf("ack log batch: %v", err)
|
||||
}
|
||||
logs, err := logSpool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending logs: %v", err)
|
||||
}
|
||||
if len(logs) != 0 {
|
||||
t.Fatalf("expected log batch removed despite artifact backlog, got %+v", logs)
|
||||
}
|
||||
chunks, err := artifactQueue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending artifacts: %v", err)
|
||||
}
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("artifact backlog should remain independent, got %+v", chunks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactAckIsIndependentFromLogBacklog(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logSpool, err := NewLogSpool(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
artifactQueue, err := NewArtifactQueue(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new artifact queue: %v", err)
|
||||
}
|
||||
|
||||
for _, batch := range []protocol.LogBatchIngestRequest{validSpoolLogBatch(1, 1), validSpoolLogBatch(2, 2)} {
|
||||
if err := logSpool.Enqueue(batch); err != nil {
|
||||
t.Fatalf("enqueue log batch: %v", err)
|
||||
}
|
||||
}
|
||||
if err := artifactQueue.Enqueue(validQueuedArtifactChunk(0)); err != nil {
|
||||
t.Fatalf("enqueue artifact chunk: %v", err)
|
||||
}
|
||||
|
||||
if err := artifactQueue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil {
|
||||
t.Fatalf("ack artifact chunk: %v", err)
|
||||
}
|
||||
chunks, err := artifactQueue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending artifacts: %v", err)
|
||||
}
|
||||
if len(chunks) != 0 {
|
||||
t.Fatalf("expected artifact chunk removed despite log backlog, got %+v", chunks)
|
||||
}
|
||||
logs, err := logSpool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending logs: %v", err)
|
||||
}
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("log backlog should remain independent, got %+v", logs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type LogSpool struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func NewLogSpool(dir string) (LogSpool, error) {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return LogSpool{}, fmt.Errorf("spool directory is required")
|
||||
}
|
||||
logDir := filepath.Join(dir, "logs")
|
||||
if err := os.MkdirAll(logDir, 0o755); err != nil {
|
||||
return LogSpool{}, fmt.Errorf("create log spool: %w", err)
|
||||
}
|
||||
return LogSpool{dir: logDir}, nil
|
||||
}
|
||||
|
||||
func (spool LogSpool) Enqueue(batch protocol.LogBatchIngestRequest) error {
|
||||
path := spool.batchPath(batch)
|
||||
tmp := path + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open log spool segment: %w", err)
|
||||
}
|
||||
encodeErr := json.NewEncoder(file).Encode(batch)
|
||||
closeErr := file.Close()
|
||||
if encodeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("encode log spool segment: %w", encodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("close log spool segment: %w", closeErr)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("commit log spool segment: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (spool LogSpool) Pending() ([]protocol.LogBatchIngestRequest, error) {
|
||||
entries, err := os.ReadDir(spool.dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read log spool: %w", err)
|
||||
}
|
||||
paths := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, filepath.Join(spool.dir, entry.Name()))
|
||||
}
|
||||
sort.Strings(paths)
|
||||
batches := make([]protocol.LogBatchIngestRequest, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log spool segment: %w", err)
|
||||
}
|
||||
var batch protocol.LogBatchIngestRequest
|
||||
decodeErr := json.NewDecoder(file).Decode(&batch)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return nil, fmt.Errorf("decode log spool segment: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("close log spool segment: %w", closeErr)
|
||||
}
|
||||
batches = append(batches, batch)
|
||||
}
|
||||
return batches, nil
|
||||
}
|
||||
|
||||
func (spool LogSpool) Ack(response protocol.LogBatchIngestResponse) error {
|
||||
entries, err := os.ReadDir(spool.dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read log spool: %w", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(spool.dir, entry.Name())
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open log spool segment: %w", err)
|
||||
}
|
||||
var batch protocol.LogBatchIngestRequest
|
||||
decodeErr := json.NewDecoder(file).Decode(&batch)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return fmt.Errorf("decode log spool segment: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close log spool segment: %w", closeErr)
|
||||
}
|
||||
if batch.LogStreamID == response.LogStreamID && batch.FirstSeq >= response.AcceptedFrom && batch.LastSeq <= response.AcceptedTo {
|
||||
if err := os.Remove(path); err != nil {
|
||||
return fmt.Errorf("remove acknowledged log spool segment: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (spool LogSpool) batchPath(batch protocol.LogBatchIngestRequest) string {
|
||||
streamID := sanitizeSegmentName(batch.LogStreamID)
|
||||
return filepath.Join(spool.dir, fmt.Sprintf("%s-%020d-%020d.json", streamID, batch.FirstSeq, batch.LastSeq))
|
||||
}
|
||||
|
||||
func sanitizeSegmentName(value string) string {
|
||||
var builder strings.Builder
|
||||
for _, r := range value {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
|
||||
builder.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
builder.WriteByte('_')
|
||||
}
|
||||
if builder.Len() == 0 {
|
||||
return "stream"
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestLogSpoolRetainsPendingAndRemovesAcknowledgedBatch(t *testing.T) {
|
||||
spool, err := NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
first := validSpoolLogBatch(1, 2)
|
||||
second := validSpoolLogBatch(3, 3)
|
||||
if err := spool.Enqueue(first); err != nil {
|
||||
t.Fatalf("enqueue first: %v", err)
|
||||
}
|
||||
if err := spool.Enqueue(second); err != nil {
|
||||
t.Fatalf("enqueue second: %v", err)
|
||||
}
|
||||
pending, err := spool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending before ack: %v", err)
|
||||
}
|
||||
if len(pending) != 2 {
|
||||
t.Fatalf("expected two pending batches, got %+v", pending)
|
||||
}
|
||||
|
||||
if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 2}); err != nil {
|
||||
t.Fatalf("ack first: %v", err)
|
||||
}
|
||||
pending, err = spool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending after ack: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].FirstSeq != 3 {
|
||||
t.Fatalf("expected second batch pending, got %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogSpoolRetainsBatchWhenAckDoesNotCoverRange(t *testing.T) {
|
||||
spool, err := NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
if err := spool.Enqueue(validSpoolLogBatch(1, 2)); err != nil {
|
||||
t.Fatalf("enqueue: %v", err)
|
||||
}
|
||||
if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
|
||||
t.Fatalf("partial ack: %v", err)
|
||||
}
|
||||
pending, err := spool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending: %v", err)
|
||||
}
|
||||
if len(pending) != 1 {
|
||||
t.Fatalf("expected batch to remain pending, got %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func validSpoolLogBatch(firstSeq uint64, lastSeq uint64) protocol.LogBatchIngestRequest {
|
||||
entries := make([]protocol.LogEntry, 0, lastSeq-firstSeq+1)
|
||||
for seq := firstSeq; seq <= lastSeq; seq++ {
|
||||
entries = append(entries, protocol.LogEntry{Seq: seq, Timestamp: time.Date(2026, 7, 3, 12, 0, int(seq), 0, time.UTC), Level: "info", Line: "line"})
|
||||
}
|
||||
return protocol.LogBatchIngestRequest{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: "session-token",
|
||||
LogStreamID: "log-1",
|
||||
ServerInstanceID: "server-1",
|
||||
StreamKey: "stdout",
|
||||
Source: "process",
|
||||
FirstSeq: firstSeq,
|
||||
LastSeq: lastSeq,
|
||||
Compression: "none",
|
||||
Checksum: "sha256:test",
|
||||
Entries: entries,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user