447 lines
18 KiB
Go
447 lines
18 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/run/api"
|
|
"browser.local/run/config"
|
|
"browser.local/run/protocol"
|
|
"browser.local/run/spool"
|
|
)
|
|
|
|
func TestWorkerRegistersHeartbeatsAndStoresSession(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
worker, err := NewWorker(workerTestConfig(t), client)
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
if worker.State().SessionToken != "session-token" {
|
|
t.Fatalf("expected session token stored, got %+v", worker.State())
|
|
}
|
|
if len(client.helloRequests) != 1 || client.helloRequests[0].RegistrationToken != "registration-token" || len(client.helloRequests[0].CapabilityReport.Capabilities) == 0 {
|
|
t.Fatalf("unexpected hello request: %+v", client.helloRequests)
|
|
}
|
|
|
|
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
|
t.Fatalf("heartbeat: %v", err)
|
|
}
|
|
if len(client.heartbeatRequests) != 1 {
|
|
t.Fatalf("expected heartbeat request")
|
|
}
|
|
heartbeat := client.heartbeatRequests[0]
|
|
if heartbeat.SessionToken != "session-token" || heartbeat.Capacity.MaxJobs != 2 || heartbeat.Capacity.RunningJobs != 0 {
|
|
t.Fatalf("unexpected heartbeat request: %+v", heartbeat)
|
|
}
|
|
for _, forbidden := range []string{"/Users/", "unix://", "Bearer ", "sk-", "password=", "artifact", "log"} {
|
|
if containsText(heartbeat.Capacity.Summary, forbidden) {
|
|
t.Fatalf("heartbeat summary exposed forbidden fragment %q: %+v", forbidden, heartbeat)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestWorkerClaimsAcksProgressAndCompletesJob(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
|
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(staticSupervisor{stdout: "server ready\n"}))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
handled, err := worker.ClaimAndRunOnce(context.Background())
|
|
if err != nil || !handled {
|
|
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
|
}
|
|
if len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || len(client.resultRequests) != 1 || len(client.cancelPollRequests) != 1 {
|
|
t.Fatalf("expected ack/progress/result/cancel calls, got ack=%d progress=%d result=%d cancel=%d", len(client.ackRequests), len(client.progressRequests), len(client.resultRequests), len(client.cancelPollRequests))
|
|
}
|
|
if client.progressRequests[0].Progress.Percent != 10 || client.resultRequests[0].State != "succeeded" || client.resultRequests[0].ResultRef == "" {
|
|
t.Fatalf("unexpected job channel payloads: progress=%+v result=%+v", client.progressRequests[0], client.resultRequests[0])
|
|
}
|
|
if worker.journal.ActiveCount() != 0 {
|
|
t.Fatalf("expected terminal job removed from journal")
|
|
}
|
|
}
|
|
|
|
func TestWorkerDispatchesSelfUpdateJob(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate)
|
|
assignment.TargetKey = "run/update"
|
|
assignment.InputRef = "artifact://artifact-run-latest"
|
|
client.claimJob = assignment
|
|
worker, err := NewWorker(workerTestConfig(t), client)
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
handled, err := worker.ClaimAndRunOnce(context.Background())
|
|
if err != nil || !handled {
|
|
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
|
}
|
|
if len(client.resultRequests) != 1 || client.resultRequests[0].State != "succeeded" || !strings.Contains(client.resultRequests[0].ResultRef, "run-update-staged") {
|
|
t.Fatalf("expected self-update result, got %+v", client.resultRequests)
|
|
}
|
|
}
|
|
|
|
func TestWorkerRegistersPackageIdentity(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
cfg := workerTestConfig(t)
|
|
cfg.RegistrationToken = "current-run-key"
|
|
cfg.ServerInstanceID = "server-worker"
|
|
cfg.PluginID = "game.minecraft"
|
|
cfg.ComponentKind = "run"
|
|
cfg.KeyGeneration = 7
|
|
worker, err := NewWorker(cfg, client)
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
hello := client.helloRequests[0]
|
|
if hello.RegistrationToken != "current-run-key" || hello.ServerInstanceID != "server-worker" || hello.ComponentKind != "run" || hello.KeyGeneration != 7 {
|
|
t.Fatalf("expected package identity in hello request, got %+v", hello)
|
|
}
|
|
}
|
|
|
|
func TestWorkerHandlesCancellationAndReconcile(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
|
client.cancelResponse = protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", HasCancel: true, JobID: "job-worker", Reason: "operator requested", ServerTime: workerTestTime()}
|
|
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(blockingSupervisor{}))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
handled, err := worker.ClaimAndRunOnce(context.Background())
|
|
if err != nil || !handled {
|
|
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
|
}
|
|
if len(client.resultRequests) != 1 || client.resultRequests[0].State != "cancelled" || client.resultRequests[0].ErrorCode != "lifecycle_cancelled" {
|
|
t.Fatalf("expected cancelled terminal result, got %+v", client.resultRequests)
|
|
}
|
|
|
|
worker.journal.MarkActive(workerJobAssignment(protocol.RunCapabilityProcessStart))
|
|
client.reconcileResponse = protocol.RunJobReconcileResponse{
|
|
Accepted: true,
|
|
RunEndpointID: "run-test",
|
|
ActiveJobs: []protocol.RunJobAssignment{workerJobAssignment(protocol.RunCapabilityProcessStop)},
|
|
UnknownJobIDs: []string{"job-worker"},
|
|
ServerTime: workerTestTime(),
|
|
}
|
|
if err := worker.ReconcileOnce(context.Background()); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if ids := worker.journal.ActiveJobIDs(); !reflect.DeepEqual(ids, []string{"job-worker-stop"}) {
|
|
t.Fatalf("expected reconcile to replace active job ids, got %+v", ids)
|
|
}
|
|
}
|
|
|
|
func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) {
|
|
client := newFakeWorkerClient()
|
|
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
|
logSpool, err := spool.NewLogSpool(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("log spool: %v", err)
|
|
}
|
|
artifactQueue, err := spool.NewArtifactQueue(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("artifact queue: %v", err)
|
|
}
|
|
worker, err := NewWorker(
|
|
workerTestConfig(t),
|
|
client,
|
|
WithProcessSupervisor(staticSupervisor{stdout: "started password=hidden\n"}),
|
|
WithProcessLogSink(&SpoolLogSink{Spool: logSpool}),
|
|
WithLifecycleArtifactHook(&QueueArtifactHook{Queue: artifactQueue}),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
if _, err := worker.ClaimAndRunOnce(context.Background()); err != nil {
|
|
t.Fatalf("claim/run: %v", err)
|
|
}
|
|
logs, err := logSpool.Pending()
|
|
if err != nil {
|
|
t.Fatalf("pending logs: %v", err)
|
|
}
|
|
if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || containsText(logs[0].Entries[0].Line, "password=hidden") {
|
|
t.Fatalf("unexpected spooled logs: %+v", logs)
|
|
}
|
|
chunks, err := artifactQueue.Pending()
|
|
if err != nil {
|
|
t.Fatalf("pending artifact chunks: %v", err)
|
|
}
|
|
if len(chunks) != 1 || chunks[0].RunEndpointID != "run-test" || chunks[0].SessionToken != "session-token" {
|
|
t.Fatalf("unexpected artifact chunks: %+v", chunks)
|
|
}
|
|
}
|
|
|
|
func TestWorkerRetryBackoffIsBounded(t *testing.T) {
|
|
if got := boundedRetryBackoff(75 * time.Millisecond); got != 75*time.Millisecond {
|
|
t.Fatalf("expected configured backoff, got %s", got)
|
|
}
|
|
if got := boundedRetryBackoff(time.Minute); got != 30*time.Second {
|
|
t.Fatalf("expected capped backoff, got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestWorkerIntegrationWithPlatformLikeServer(t *testing.T) {
|
|
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
|
seen := []string{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
seen = append(seen, r.URL.Path)
|
|
switch r.URL.Path {
|
|
case "/api/v1/run/control/hello":
|
|
var request protocol.RunHelloRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.RunEndpointID != "run-test" || request.Capacity.MaxJobs != 2 {
|
|
t.Fatalf("unexpected hello: %+v", request)
|
|
}
|
|
writeWorkerTestJSON(t, w, protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", HeartbeatIntervalSeconds: 15, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/control/heartbeat":
|
|
var request protocol.RunHeartbeatRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 0 {
|
|
t.Fatalf("unexpected heartbeat: %+v", request)
|
|
}
|
|
writeWorkerTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/jobs/claim":
|
|
var request protocol.RunJobClaimRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.SessionToken != "session-token" || len(request.Capabilities) == 0 {
|
|
t.Fatalf("unexpected claim: %+v", request)
|
|
}
|
|
writeWorkerTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/jobs/ack":
|
|
var request protocol.RunJobAckRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken {
|
|
t.Fatalf("unexpected ack: %+v", request)
|
|
}
|
|
assignment.State = "running"
|
|
writeWorkerTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/jobs/progress":
|
|
var request protocol.RunJobProgressRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.Progress.Percent != 10 {
|
|
t.Fatalf("unexpected progress: %+v", request)
|
|
}
|
|
assignment.Progress = request.Progress
|
|
writeWorkerTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/jobs/cancel":
|
|
var request protocol.RunJobCancelPollRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.JobID != assignment.JobID {
|
|
t.Fatalf("unexpected cancel poll: %+v", request)
|
|
}
|
|
writeWorkerTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ServerTime: workerTestTime()})
|
|
case "/api/v1/run/jobs/result":
|
|
var request protocol.RunJobResultRequest
|
|
decodeWorkerTestJSON(t, r, &request)
|
|
if request.State != "succeeded" || request.ResultRef == "" {
|
|
t.Fatalf("unexpected result: %+v", request)
|
|
}
|
|
assignment.State = request.State
|
|
assignment.ResultRef = request.ResultRef
|
|
writeWorkerTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
|
default:
|
|
t.Fatalf("unexpected path: %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
client, err := api.NewPlatformClient(server.URL)
|
|
if err != nil {
|
|
t.Fatalf("platform client: %v", err)
|
|
}
|
|
cfg := workerTestConfig(t)
|
|
cfg.PlatformURL = server.URL
|
|
worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "integration ok\n"}))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
|
t.Fatalf("heartbeat: %v", err)
|
|
}
|
|
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
|
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
|
}
|
|
expected := []string{
|
|
"/api/v1/run/control/hello",
|
|
"/api/v1/run/control/heartbeat",
|
|
"/api/v1/run/jobs/claim",
|
|
"/api/v1/run/jobs/ack",
|
|
"/api/v1/run/jobs/progress",
|
|
"/api/v1/run/jobs/cancel",
|
|
"/api/v1/run/jobs/result",
|
|
}
|
|
if !reflect.DeepEqual(seen, expected) {
|
|
t.Fatalf("unexpected platform flow: %+v", seen)
|
|
}
|
|
}
|
|
|
|
type fakeWorkerClient struct {
|
|
helloRequests []protocol.RunHelloRequest
|
|
heartbeatRequests []protocol.RunHeartbeatRequest
|
|
claimRequests []protocol.RunJobClaimRequest
|
|
ackRequests []protocol.RunJobAckRequest
|
|
progressRequests []protocol.RunJobProgressRequest
|
|
resultRequests []protocol.RunJobResultRequest
|
|
cancelPollRequests []protocol.RunJobCancelPollRequest
|
|
reconcileRequests []protocol.RunJobReconcileRequest
|
|
claimJob protocol.RunJobAssignment
|
|
cancelResponse protocol.RunJobCancelPollResponse
|
|
reconcileResponse protocol.RunJobReconcileResponse
|
|
}
|
|
|
|
func newFakeWorkerClient() *fakeWorkerClient {
|
|
return &fakeWorkerClient{
|
|
cancelResponse: protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
|
reconcileResponse: protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
|
}
|
|
}
|
|
|
|
func (client *fakeWorkerClient) Hello(_ context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) {
|
|
client.helloRequests = append(client.helloRequests, request)
|
|
return protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", ServerTime: workerTestTime(), HeartbeatIntervalSeconds: 15}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) Heartbeat(_ context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) {
|
|
client.heartbeatRequests = append(client.heartbeatRequests, request)
|
|
return protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) ClaimJob(_ context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) {
|
|
client.claimRequests = append(client.claimRequests, request)
|
|
if client.claimJob.JobID == "" {
|
|
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: false, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
|
}
|
|
job := client.claimJob
|
|
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) AckJob(_ context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) {
|
|
client.ackRequests = append(client.ackRequests, request)
|
|
job := client.claimJob
|
|
job.State = "running"
|
|
return protocol.RunJobAckResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) UpdateJobProgress(_ context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) {
|
|
client.progressRequests = append(client.progressRequests, request)
|
|
job := client.claimJob
|
|
job.Progress = request.Progress
|
|
return protocol.RunJobProgressResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) CompleteJob(_ context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) {
|
|
client.resultRequests = append(client.resultRequests, request)
|
|
job := client.claimJob
|
|
job.State = request.State
|
|
job.Progress = request.Progress
|
|
job.ResultRef = request.ResultRef
|
|
return protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) PollJobCancel(_ context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) {
|
|
client.cancelPollRequests = append(client.cancelPollRequests, request)
|
|
return client.cancelResponse, nil
|
|
}
|
|
|
|
func (client *fakeWorkerClient) ReconcileJobs(_ context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) {
|
|
client.reconcileRequests = append(client.reconcileRequests, request)
|
|
return client.reconcileResponse, nil
|
|
}
|
|
|
|
type staticSupervisor struct {
|
|
stdout string
|
|
stderr string
|
|
err error
|
|
}
|
|
|
|
func (supervisor staticSupervisor) Run(context.Context, ProcessCommand) (ProcessResult, error) {
|
|
return ProcessResult{ExitCode: 0, Stdout: supervisor.stdout, Stderr: supervisor.stderr}, supervisor.err
|
|
}
|
|
|
|
func workerTestConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
return config.Config{
|
|
Mode: "worker",
|
|
PlatformURL: "http://platform.test",
|
|
RunEndpointID: "run-test",
|
|
DisplayName: "Run Test",
|
|
Version: "0.1.0-test",
|
|
RegistrationToken: "registration-token",
|
|
WorkspaceRoot: t.TempDir(),
|
|
SpoolRoot: t.TempDir(),
|
|
MaxJobs: 2,
|
|
HeartbeatInterval: time.Second,
|
|
PollInterval: time.Second,
|
|
RetryBackoff: time.Millisecond,
|
|
}
|
|
}
|
|
|
|
func workerJobAssignment(capability string) protocol.RunJobAssignment {
|
|
job := lifecycleAssignment(capability)
|
|
job.JobID = "job-worker"
|
|
if capability == protocol.RunCapabilityProcessStop {
|
|
job.JobID = "job-worker-stop"
|
|
}
|
|
job.RunEndpointID = "run-test"
|
|
job.ServerInstanceID = "server-worker"
|
|
return job
|
|
}
|
|
|
|
func workerTestTime() time.Time {
|
|
return time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC)
|
|
}
|
|
|
|
func containsText(value string, needle string) bool {
|
|
return strings.Contains(value, needle)
|
|
}
|
|
|
|
func decodeWorkerTestJSON(t *testing.T, r *http.Request, target any) {
|
|
t.Helper()
|
|
if r.Method != http.MethodPost {
|
|
t.Fatalf("expected POST, got %s", r.Method)
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(target); err != nil {
|
|
t.Fatalf("decode request: %v", err)
|
|
}
|
|
}
|
|
|
|
func writeWorkerTestJSON(t *testing.T, w http.ResponseWriter, value any) {
|
|
t.Helper()
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if err := json.NewEncoder(w).Encode(value); err != nil {
|
|
t.Fatalf("encode response: %v", err)
|
|
}
|
|
}
|