init
This commit is contained in:
npc0-hue
@@ -0,0 +1,319 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const autonomousLifecyclePlanKey = ".platform/autonomous-lifecycle-plan.json"
|
||||
|
||||
func LoadAutonomousLifecyclePlan(cfg config.Config) (*protocol.RunAutonomousLifecyclePlan, string, bool, error) {
|
||||
if strings.TrimSpace(cfg.ComponentKind) != config.PackageComponentRun || strings.TrimSpace(cfg.ServerInstanceID) == "" {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=not_generated_run server=%s component=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKind))
|
||||
return nil, "", false, nil
|
||||
}
|
||||
scope, err := seededWorkspaceScope(cfg)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=scope_failed server=%s componentKey=%s error=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKey), RedactText(err.Error()))
|
||||
return nil, "", false, err
|
||||
}
|
||||
path, err := NewWorkspaceResolver(cfg.WorkspaceRoot).ExistingTarget(scope, autonomousLifecyclePlanKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=plan_missing scope=%s", safeOptional(scope))
|
||||
return nil, scope, false, nil
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=load_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error()))
|
||||
return nil, scope, false, err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=open_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error()))
|
||||
return nil, scope, false, err
|
||||
}
|
||||
defer file.Close()
|
||||
var plan protocol.RunAutonomousLifecyclePlan
|
||||
decoder := json.NewDecoder(io.LimitReader(file, 64*1024))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&plan); err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=decode_failed scope=%s error=%s", safeOptional(scope), RedactText(err.Error()))
|
||||
return nil, scope, false, err
|
||||
}
|
||||
if err := protocol.ValidateRunAutonomousLifecyclePlan(plan); err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=invalid scope=%s error=%s", safeOptional(scope), RedactText(err.Error()))
|
||||
return nil, scope, false, err
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=loaded server=%s plugin=%s endpoint=%s profile=%s bootstrap=%t actions=%d dependencies=%d installs=%d", plan.ServerInstanceID, plan.PluginID, plan.RunEndpointID, safeOptional(plan.ProfileKey), plan.Bootstrap != nil, len(plan.Actions), len(plan.DependencyProbes), len(plan.InstallPlans))
|
||||
return &plan, scope, true, nil
|
||||
}
|
||||
|
||||
func (worker *Worker) RunAutonomousLifecycleOnce(ctx context.Context) error {
|
||||
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAutonomousLifecycleScope(worker.cfg, state, *plan); err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=scope_mismatch error=%s", RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
if err := worker.runAutonomousDependencies(ctx, *plan); err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.Bootstrap == nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=no_bootstrap server=%s", plan.ServerInstanceID)
|
||||
return nil
|
||||
}
|
||||
assignment := autonomousLifecycleAssignment(state, worker.cfg, *plan, *plan.Bootstrap)
|
||||
log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=starting job=%s capability=%s target=%s operation=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), safeOptional(assignment.ExecutionInput.LifecycleOperation))
|
||||
executor := worker.executor
|
||||
executor.artifactHook = StaticLifecycleArtifactHook{}
|
||||
execution := executor.ExecuteContext(ctx, assignment)
|
||||
log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=complete job=%s state=%s processState=%s errorCode=%s message=%s", assignment.JobID, execution.State, safeOptional(execution.ExecutionResult.ProcessState), safeOptional(execution.ErrorCode), safeOptional(execution.Message))
|
||||
if err := worker.reportAutonomousLifecycle(ctx, assignment, execution); err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.report status=degraded server=%s action=continue_worker reason=projection_report_failed", assignment.ServerInstanceID)
|
||||
}
|
||||
if execution.State != lifecycleResultStateSucceeded {
|
||||
return fmt.Errorf("autonomous lifecycle bootstrap failed: %s", errorSummaryFromResult(execution))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) reportAutonomousLifecycle(ctx context.Context, assignment protocol.RunJobAssignment, execution LifecycleExecutionResult) error {
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request := protocol.RunLifecycleReportRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
Capability: assignment.Capability,
|
||||
State: execution.State,
|
||||
Progress: execution.Progress,
|
||||
Message: execution.Message,
|
||||
ErrorCode: execution.ErrorCode,
|
||||
ExecutionResult: execution.ExecutionResult,
|
||||
}
|
||||
if source, ok := worker.executor.managed.(ManagedProcessObservationSource); ok {
|
||||
for _, identity := range source.ManagedProcessObservations() {
|
||||
if identity.ServerInstanceID == assignment.ServerInstanceID && identity.RunEndpointID == state.RunEndpointID && identity.ObservationSeq > 0 && identity.State == execution.ExecutionResult.ProcessState {
|
||||
request.ManagedProcessID = managedProcessObservationID(identity)
|
||||
request.ObservationSeq = identity.ObservationSeq
|
||||
request.ObservedAt = identity.UpdatedAt
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.report status=starting server=%s capability=%s state=%s processState=%s", assignment.ServerInstanceID, assignment.Capability, execution.State, safeOptional(execution.ExecutionResult.ProcessState))
|
||||
response, err := worker.client.ReportLifecycle(ctx, request)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.report status=failed server=%s error=%s", assignment.ServerInstanceID, RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
if !response.Accepted || response.RunEndpointID != state.RunEndpointID || response.ServerInstanceID != assignment.ServerInstanceID {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.report status=rejected server=%s accepted=%t", assignment.ServerInstanceID, response.Accepted)
|
||||
return fmt.Errorf("autonomous lifecycle report was not accepted")
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.report status=accepted server=%s projectedState=%s", assignment.ServerInstanceID, safeOptional(response.ProjectedState))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) reportAutonomousProcessObservations(ctx context.Context) error {
|
||||
source, ok := worker.executor.managed.(ManagedProcessObservationSource)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, identity := range source.ManagedProcessObservations() {
|
||||
if identity.ServerInstanceID == "" || identity.RunEndpointID != state.RunEndpointID || identity.ObservationSeq == 0 {
|
||||
continue
|
||||
}
|
||||
processID := managedProcessObservationID(identity)
|
||||
worker.observationMu.Lock()
|
||||
alreadyReported := worker.reportedObservations[processID] >= identity.ObservationSeq
|
||||
worker.observationMu.Unlock()
|
||||
if alreadyReported {
|
||||
continue
|
||||
}
|
||||
request := protocol.RunLifecycleReportRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, ServerInstanceID: identity.ServerInstanceID, Capability: protocol.RunCapabilityProcessStatus, State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "supervised process observation"}, ManagedProcessID: processID, ObservationSeq: identity.ObservationSeq, ObservedAt: identity.UpdatedAt, ExecutionResult: protocol.RunJobExecutionResult{Kind: "process", ProcessState: identity.State, ExitClassification: identity.ExitClassification, ExitCode: identity.ExitCode, Summary: "supervised process observation"}}
|
||||
if _, err := worker.client.ReportLifecycle(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
worker.observationMu.Lock()
|
||||
worker.reportedObservations[processID] = identity.ObservationSeq
|
||||
worker.observationMu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func managedProcessObservationID(identity ProcessIdentity) string {
|
||||
if identity.LogSessionID != "" {
|
||||
return "log-session:" + identity.LogSessionID
|
||||
}
|
||||
sum := sha256.Sum256([]byte(identity.RunEndpointID + "\x00" + identity.ServerInstanceID + "\x00" + identity.CommandFingerprint + "\x00" + identity.StartedAt.UTC().Format(time.RFC3339Nano)))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func validateAutonomousLifecycleScope(cfg config.Config, state WorkerState, plan protocol.RunAutonomousLifecyclePlan) error {
|
||||
if plan.RunEndpointID != state.RunEndpointID || plan.ServerInstanceID != cfg.ServerInstanceID || plan.PluginID != cfg.PluginID {
|
||||
return fmt.Errorf("autonomous lifecycle plan identity does not match this Run")
|
||||
}
|
||||
if plan.TargetOS != runtime.GOOS || plan.TargetArch != runtime.GOARCH {
|
||||
return fmt.Errorf("autonomous lifecycle target does not match this Run")
|
||||
}
|
||||
if cfg.ComponentKey != "" && plan.ProfileKey != "" && plan.ProfileKey != cfg.ComponentKey {
|
||||
return fmt.Errorf("autonomous lifecycle profile does not match this Run")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) runAutonomousDependencies(ctx context.Context, plan protocol.RunAutonomousLifecyclePlan) error {
|
||||
if len(plan.DependencyProbes) == 0 {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=skipped reason=no_probes")
|
||||
return nil
|
||||
}
|
||||
for _, probe := range plan.DependencyProbes {
|
||||
state, evidence, err := worker.executor.runDependencyProbe(ctx, probe, plan.RuntimeBindings)
|
||||
if err != nil {
|
||||
if probe.Required {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=failed probe=%s error=%s", safeOptional(probe.Key), RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=optional_probe_failed probe=%s error=%s", safeOptional(probe.Key), RedactText(err.Error()))
|
||||
continue
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=probed probe=%s state=%s evidence=%s required=%t", safeOptional(probe.Key), safeOptional(state), safeOptional(evidence), probe.Required)
|
||||
if state == "present" || !probe.Required {
|
||||
continue
|
||||
}
|
||||
installPlan, found := autonomousInstallPlanForProbe(plan.InstallPlans, probe)
|
||||
if !found {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=missing_without_install_plan probe=%s action=defer_to_bootstrap", safeOptional(probe.Key))
|
||||
continue
|
||||
}
|
||||
assignment := autonomousDependencyAssignment(worker.State(), worker.cfg, plan, installPlan)
|
||||
input := protocol.DependencyExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: plan.ServerInstanceID, RunEndpointID: plan.RunEndpointID, PluginID: plan.PluginID, PluginVersion: plan.PluginVersion, ProfileKey: autonomousProfileKey(worker.cfg, plan), TargetOS: plan.TargetOS, TargetArch: plan.TargetArch, PlanDigest: autonomousInstallPlanDigest(installPlan), Probe: probe, Plan: installPlan, Bindings: plan.RuntimeBindings}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_start probe=%s plan=%s steps=%d", safeOptional(probe.Key), safeOptional(installPlan.Key), len(installPlan.Steps))
|
||||
if err := worker.runAutonomousInstallPlan(ctx, assignment, input); err != nil {
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_failed probe=%s plan=%s error=%s", safeOptional(probe.Key), safeOptional(installPlan.Key), RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_complete probe=%s plan=%s", safeOptional(probe.Key), safeOptional(installPlan.Key))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) runAutonomousInstallPlan(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse) error {
|
||||
for index, step := range input.Plan.Steps {
|
||||
if err := worker.executor.runDependencyInstallStep(ctx, assignment, input, step, index); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func autonomousInstallPlanForProbe(plans []protocol.DependencyInstallPlan, probe protocol.DependencyProbe) (protocol.DependencyInstallPlan, bool) {
|
||||
for _, plan := range plans {
|
||||
for _, step := range plan.Steps {
|
||||
if step.TargetKey == probe.TargetKey {
|
||||
return plan, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return protocol.DependencyInstallPlan{}, false
|
||||
}
|
||||
|
||||
func autonomousLifecycleAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, action protocol.RunAutonomousLifecycleAction) protocol.RunJobAssignment {
|
||||
now := time.Now().UTC()
|
||||
profileKey := autonomousProfileKey(cfg, plan)
|
||||
assignment := protocol.RunJobAssignment{
|
||||
JobID: "autonomous-bootstrap-" + safeWorkspaceName(action.Operation),
|
||||
ServerInstanceID: plan.ServerInstanceID,
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
Capability: action.Capability,
|
||||
TargetKey: action.TargetKey,
|
||||
IdempotencyKey: "autonomous:" + safeWorkspaceName(plan.TargetRelease) + ":" + safeWorkspaceName(action.Operation),
|
||||
State: "running",
|
||||
LeaseToken: "local-autonomous-bootstrap",
|
||||
Attempt: 1,
|
||||
MaxAttempts: 1,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
ExecutionInput: protocol.RunJobExecutionInput{
|
||||
WorkspaceScope: profileKey,
|
||||
PluginID: plan.PluginID,
|
||||
LifecycleOperation: action.Operation,
|
||||
Deployment: autonomousDeploymentExecution(plan.Deployment, action.Operation),
|
||||
},
|
||||
}
|
||||
if action.Capability == protocol.RunCapabilityProcessStart {
|
||||
assignment.ExecutionInput.LogSources = append([]protocol.RuntimeLogSourcePlan(nil), plan.LogSources...)
|
||||
assignment.ExecutionInput.DLLExtensions = append([]protocol.RuntimeDLLExtensionPlan(nil), plan.DLLExtensions...)
|
||||
}
|
||||
return assignment
|
||||
}
|
||||
|
||||
func autonomousDependencyAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, installPlan protocol.DependencyInstallPlan) protocol.RunJobAssignment {
|
||||
now := time.Now().UTC()
|
||||
return protocol.RunJobAssignment{JobID: "autonomous-dependencies-" + safeWorkspaceName(installPlan.Key), ServerInstanceID: plan.ServerInstanceID, RunEndpointID: state.RunEndpointID, Capability: protocol.RunCapabilityDependenciesInstall, TargetKey: "dependencies/install/" + installPlan.Key, IdempotencyKey: "autonomous:dependencies:" + safeWorkspaceName(installPlan.Key), State: "running", LeaseToken: "local-autonomous-dependencies", Attempt: 1, MaxAttempts: 1, CreatedAt: now, UpdatedAt: now, ExecutionInput: protocol.RunJobExecutionInput{WorkspaceScope: autonomousProfileKey(cfg, plan), PluginID: plan.PluginID, LifecycleOperation: "install"}}
|
||||
}
|
||||
|
||||
func autonomousDeploymentExecution(deployment *protocol.RunAutonomousDeployment, operation string) *protocol.ServerDeploymentExecution {
|
||||
if deployment == nil {
|
||||
return nil
|
||||
}
|
||||
startCommand := deployment.StartCommand
|
||||
if operation == "install" && deployment.InstallCommand != "" {
|
||||
startCommand = deployment.InstallCommand
|
||||
}
|
||||
return &protocol.ServerDeploymentExecution{SchemaVersion: deployment.SchemaVersion, Mode: deployment.Mode, ProfileKey: deployment.ProfileKey, CreateInputs: copyStringMap(deployment.CreateInputs), ServerRoot: deployment.ServerRoot, WorkingDirectory: deployment.WorkingDirectory, StartCommand: startCommand, StopCommand: deployment.StopCommand, StatusCommand: deployment.StatusCommand, Shell: deployment.Shell, Revision: deployment.Revision}
|
||||
}
|
||||
|
||||
func autonomousProfileKey(cfg config.Config, plan protocol.RunAutonomousLifecyclePlan) string {
|
||||
if strings.TrimSpace(cfg.ComponentKey) != "" {
|
||||
return cfg.ComponentKey
|
||||
}
|
||||
return plan.ProfileKey
|
||||
}
|
||||
|
||||
func autonomousInstallPlanDigest(plan protocol.DependencyInstallPlan) string {
|
||||
body, err := json.Marshal(plan)
|
||||
if err != nil {
|
||||
return "sha256:" + strings.Repeat("0", 64)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func errorSummaryFromResult(result LifecycleExecutionResult) string {
|
||||
if result.ErrorCode != "" && result.Message != "" {
|
||||
return result.ErrorCode + ": " + result.Message
|
||||
}
|
||||
if result.ErrorCode != "" {
|
||||
return result.ErrorCode
|
||||
}
|
||||
if result.Message != "" {
|
||||
return result.Message
|
||||
}
|
||||
return result.State
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestWorkerRunsAutonomousBootstrapFromSeededPlan(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.PluginID = "server.scum"
|
||||
cfg.ComponentKind = "run"
|
||||
cfg.ComponentKey = "run-local"
|
||||
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
||||
SchemaVersion: "1",
|
||||
ServerInstanceID: cfg.ServerInstanceID,
|
||||
PluginID: cfg.PluginID,
|
||||
PluginVersion: "1.0.0",
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
ProfileKey: cfg.ComponentKey,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-dist-test",
|
||||
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
||||
LogSources: []protocol.RuntimeLogSourcePlan{{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}},
|
||||
Deployment: &protocol.RunAutonomousDeployment{SchemaVersion: "1", Mode: "guided", ProfileKey: cfg.ComponentKey, ServerRoot: "D:/game-server", CreateInputs: map[string]string{"gamePort": "7779", "maxPlayers": "128"}, Revision: 2},
|
||||
})
|
||||
managed := &recordingManagedSupervisor{}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
||||
t.Fatalf("run autonomous lifecycle: %v", err)
|
||||
}
|
||||
|
||||
if len(client.claimRequests) != 0 || len(client.ackRequests) != 0 || len(client.resultRequests) != 0 || len(client.lifecycleReports) != 1 {
|
||||
t.Fatalf("autonomous bootstrap must report without platform job assignment: claims=%d acks=%d results=%d reports=%d", len(client.claimRequests), len(client.ackRequests), len(client.resultRequests), len(client.lifecycleReports))
|
||||
}
|
||||
if client.lifecycleReports[0].Capability != protocol.RunCapabilityProcessStart || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("expected autonomous process lifecycle report, got %+v", client.lifecycleReports[0])
|
||||
}
|
||||
if !managed.started || managed.identity.ServerInstanceID != cfg.ServerInstanceID || managed.identity.RunEndpointID != cfg.RunEndpointID {
|
||||
t.Fatalf("expected managed process start from autonomous plan, managed=%+v", managed)
|
||||
}
|
||||
if managed.identity.StdoutStreamKey != "game.console.stdout" {
|
||||
t.Fatalf("expected declared process stdout stream, identity=%+v", managed.identity)
|
||||
}
|
||||
if managed.command.Env["SERVER_ROOT"] != "D:/game-server" || managed.command.Env["SERVER_CREATE_GAMEPORT"] != "7779" || managed.command.Env["SERVER_CREATE_MAXPLAYERS"] != "128" {
|
||||
t.Fatalf("expected deployment env from autonomous plan, env=%+v", managed.command.Env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerContinuesAutonomousBootstrapWhenLifecycleReportFails(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.lifecycleReportErr = errors.New("platform request failed: status=401 path=/api/v1/run/lifecycle/report code=unauthorized")
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.PluginID = "game.example"
|
||||
cfg.ComponentKind = "run"
|
||||
cfg.ComponentKey = "run-local"
|
||||
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
||||
SchemaVersion: "1",
|
||||
ServerInstanceID: cfg.ServerInstanceID,
|
||||
PluginID: cfg.PluginID,
|
||||
PluginVersion: "1.0.0",
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
ProfileKey: cfg.ComponentKey,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-dist-test",
|
||||
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
||||
})
|
||||
managed := &recordingManagedSupervisor{}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
||||
t.Fatalf("run autonomous lifecycle with report failure: %v", err)
|
||||
}
|
||||
|
||||
if !managed.started || len(client.lifecycleReports) != 1 {
|
||||
t.Fatalf("expected bootstrap to continue after lifecycle report failure, managed=%+v reports=%d", managed, len(client.lifecycleReports))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDefersUnmatchedRequiredDependencyToBootstrap(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.PluginID = "game.example"
|
||||
cfg.ComponentKind = "run"
|
||||
cfg.ComponentKey = "run-local"
|
||||
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
||||
SchemaVersion: "1",
|
||||
ServerInstanceID: cfg.ServerInstanceID,
|
||||
PluginID: cfg.PluginID,
|
||||
PluginVersion: "1.0.0",
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
ProfileKey: cfg.ComponentKey,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-dist-test",
|
||||
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
||||
DependencyProbes: []protocol.DependencyProbe{{Key: "steamcmd", Kind: "command.version", TargetKey: "steamcmd", Required: true, Platforms: []string{runtime.GOOS}}},
|
||||
InstallPlans: []protocol.DependencyInstallPlan{{Key: "install-game-server", Title: "Install game server", Platforms: []string{runtime.GOOS}, Steps: []protocol.DependencyInstallStep{{Type: "steamcmd-app", TargetKey: "server/install-root", PackageManager: "steamcmd", PackageName: "3792580"}}}},
|
||||
})
|
||||
managed := &recordingManagedSupervisor{}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithDependencyCommandRunner(missingCommandRunner{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err != nil {
|
||||
t.Fatalf("run autonomous lifecycle: %v", err)
|
||||
}
|
||||
|
||||
if !managed.started || len(client.lifecycleReports) != 1 {
|
||||
t.Fatalf("expected bootstrap to run despite unmatched dependency install plan, managed=%+v reports=%d", managed, len(client.lifecycleReports))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAutonomousLifecyclePlanRejectsMismatchedIdentity(t *testing.T) {
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.PluginID = "game.example"
|
||||
cfg.ComponentKind = "run"
|
||||
cfg.ComponentKey = "run-local"
|
||||
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
||||
SchemaVersion: "1",
|
||||
ServerInstanceID: cfg.ServerInstanceID,
|
||||
PluginID: "game.other",
|
||||
PluginVersion: "1.0.0",
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
ProfileKey: cfg.ComponentKey,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-dist-test",
|
||||
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
||||
})
|
||||
worker, err := NewWorker(cfg, newFakeWorkerClient())
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
|
||||
if err := worker.RunAutonomousLifecycleOnce(context.Background()); err == nil {
|
||||
t.Fatal("expected mismatched autonomous plan identity to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericWorkerReportsRunningAndExitObservationsForAllEndpointServers(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = ""
|
||||
cfg.ComponentKind = ""
|
||||
startedAt := time.Now().UTC().Add(-time.Minute)
|
||||
managed := &observationManagedSupervisor{items: []ProcessIdentity{
|
||||
{Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "running", ObservationSeq: 1, UpdatedAt: startedAt},
|
||||
{Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "running", ObservationSeq: 4, UpdatedAt: startedAt},
|
||||
{Scope: "scope-other", ServerInstanceID: "server-other", RunEndpointID: "run-other", PID: 303, StartedAt: startedAt, CommandFingerprint: "sha256:other", State: "running", ObservationSeq: 1, UpdatedAt: startedAt},
|
||||
}}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed))
|
||||
if err != nil {
|
||||
t.Fatalf("new generic worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register generic worker: %v", err)
|
||||
}
|
||||
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil {
|
||||
t.Fatalf("report running observations: %v", err)
|
||||
}
|
||||
if len(client.lifecycleReports) != 2 || client.lifecycleReports[0].ExecutionResult.ProcessState != "running" || client.lifecycleReports[1].ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("generic worker did not report both matching running observations: %+v", client.lifecycleReports)
|
||||
}
|
||||
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil || len(client.lifecycleReports) != 2 {
|
||||
t.Fatalf("unchanged observations were not deduplicated: reports=%+v err=%v", client.lifecycleReports, err)
|
||||
}
|
||||
managed.setItems([]ProcessIdentity{
|
||||
{Scope: "scope-a", ServerInstanceID: "server-a", RunEndpointID: cfg.RunEndpointID, PID: 101, StartedAt: startedAt, CommandFingerprint: "sha256:a", State: "exited", ExitCode: 1, ExitClassification: "unexpected-exit", ObservationSeq: 2, UpdatedAt: time.Now().UTC()},
|
||||
{Scope: "scope-b", ServerInstanceID: "server-b", RunEndpointID: cfg.RunEndpointID, PID: 202, StartedAt: startedAt, CommandFingerprint: "sha256:b", State: "exited", ExitCode: 0, ExitClassification: "clean-exit", ObservationSeq: 5, UpdatedAt: time.Now().UTC()},
|
||||
})
|
||||
if err := worker.reportAutonomousProcessObservations(context.Background()); err != nil {
|
||||
t.Fatalf("report exit observations: %v", err)
|
||||
}
|
||||
if len(client.lifecycleReports) != 4 || client.lifecycleReports[2].ExecutionResult.ProcessState != "exited" || client.lifecycleReports[3].ExecutionResult.ProcessState != "exited" {
|
||||
t.Fatalf("generic worker did not report matching exit observations: %+v", client.lifecycleReports)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAutonomousPlanFixture(t *testing.T, root string, serverInstanceID string, profileKey string, plan protocol.RunAutonomousLifecyclePlan) {
|
||||
t.Helper()
|
||||
scope, err := NewWorkspaceResolver(root).Scope(serverInstanceID, profileKey)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve scope: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o700); err != nil {
|
||||
t.Fatalf("create actions: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o700); err != nil {
|
||||
t.Fatalf("create bin: %v", err)
|
||||
}
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{"version": 1, "action": "start", "mode": "supervised", "executableKey": "bin/game-server"})
|
||||
if err := os.WriteFile(filepath.Join(scope, "bin", "game-server"), []byte("plugin-owned executable"), 0o700); err != nil {
|
||||
t.Fatalf("write executable: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, ".platform"), 0o700); err != nil {
|
||||
t.Fatalf("create platform dir: %v", err)
|
||||
}
|
||||
body, err := json.Marshal(plan)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal plan: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scope, autonomousLifecyclePlanKey), body, 0o600); err != nil {
|
||||
t.Fatalf("write plan: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingManagedSupervisor struct {
|
||||
started bool
|
||||
command ProcessCommand
|
||||
identity ProcessIdentity
|
||||
}
|
||||
|
||||
type observationManagedSupervisor struct {
|
||||
mu sync.Mutex
|
||||
items []ProcessIdentity
|
||||
}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) Start(_ context.Context, _ ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
||||
return identity
|
||||
}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) ManagedProcessObservations() []ProcessIdentity {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
return append([]ProcessIdentity(nil), supervisor.items...)
|
||||
}
|
||||
|
||||
func (supervisor *observationManagedSupervisor) setItems(items []ProcessIdentity) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
supervisor.items = append([]ProcessIdentity(nil), items...)
|
||||
}
|
||||
|
||||
func (supervisor *recordingManagedSupervisor) Start(_ context.Context, command ProcessCommand, identity ProcessIdentity, _ ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
supervisor.started = true
|
||||
supervisor.command = command
|
||||
identity.PID = 42
|
||||
identity.State = "running"
|
||||
supervisor.identity = identity
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *recordingManagedSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
||||
identity.State = "stopped"
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *recordingManagedSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
||||
return identity
|
||||
}
|
||||
|
||||
func (supervisor *recordingManagedSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
||||
|
||||
type missingCommandRunner struct{}
|
||||
|
||||
func (missingCommandRunner) Run(context.Context, ProcessCommand) (ProcessResult, error) {
|
||||
return ProcessResult{ExitCode: 1}, os.ErrNotExist
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const dataTargetSnapshotManifestVersion = 1
|
||||
|
||||
type dataTargetSnapshotManifest struct {
|
||||
Version int `json:"version"`
|
||||
DataTargetKey string `json:"dataTargetKey"`
|
||||
Kind string `json:"kind"`
|
||||
WorkspaceKey string `json:"workspaceKey"`
|
||||
SourceRootKey string `json:"sourceRootKey"`
|
||||
SourcePathFingerprint string `json:"sourcePathFingerprint"`
|
||||
SourceSizeBytes int64 `json:"sourceSizeBytes"`
|
||||
SourceModUnixNano int64 `json:"sourceModUnixNano"`
|
||||
SnapshotSizeBytes int64 `json:"snapshotSizeBytes"`
|
||||
SnapshotChecksum string `json:"snapshotChecksum"`
|
||||
MaterializedAt time.Time `json:"materializedAt"`
|
||||
}
|
||||
|
||||
type dataTargetMaterializeError struct {
|
||||
code string
|
||||
retryable bool
|
||||
}
|
||||
|
||||
func (err dataTargetMaterializeError) Error() string { return err.code }
|
||||
|
||||
func newDataTargetMaterializeError(code string, retryable bool) error {
|
||||
return dataTargetMaterializeError{code: code, retryable: retryable}
|
||||
}
|
||||
|
||||
func sqliteProbeFailureForDataTarget(assignment protocol.RunJobAssignment, err error) LifecycleExecutionResult {
|
||||
var materializeErr dataTargetMaterializeError
|
||||
if errors.As(err, &materializeErr) {
|
||||
return sqliteSchemaProbeFailure(assignment, materializeErr.code, materializeErr.retryable)
|
||||
}
|
||||
return sqliteSchemaProbeFailure(assignment, "data_target_unavailable", true)
|
||||
}
|
||||
|
||||
func (worker *Worker) materializeSQLiteProbeDataTarget(ctx context.Context, assignment protocol.RunJobAssignment) (string, error) {
|
||||
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
|
||||
if err != nil {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_plan_invalid", false)
|
||||
}
|
||||
if !ok || plan == nil || len(plan.DataTargets) == 0 {
|
||||
return assignment.TargetKey, nil
|
||||
}
|
||||
if plan.ServerInstanceID != assignment.ServerInstanceID || plan.RunEndpointID != assignment.RunEndpointID || plan.PluginID != assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
||||
}
|
||||
if plan.ProfileKey != "" && assignment.ExecutionInput.WorkspaceScope != "" && plan.ProfileKey != assignment.ExecutionInput.WorkspaceScope {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
||||
}
|
||||
var matched *protocol.RunAutonomousDataTarget
|
||||
for i := range plan.DataTargets {
|
||||
target := &plan.DataTargets[i]
|
||||
if dataTargetMatchesSQLiteProbeAssignment(*target, assignment.TargetKey) {
|
||||
matched = target
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched == nil {
|
||||
return assignment.TargetKey, nil
|
||||
}
|
||||
if !dataTargetSupportsPlatform(matched.Platforms, runtime.GOOS) {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_platform_unsupported", false)
|
||||
}
|
||||
manifest, err := materializeSQLiteSnapshotDataTarget(ctx, worker.cfg.WorkspaceRoot, assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope, *matched, plan.RuntimeBindings)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=data_target.snapshot status=failed target=%s code=%s", safeOptional(matched.Key), safeOptional(dataTargetErrorCode(err)))
|
||||
return assignment.TargetKey, err
|
||||
}
|
||||
log.Printf("RUN phase=data_target.snapshot status=complete target=%s workspaceKey=%s bytes=%d checksum=%s", safeOptional(matched.Key), safeOptional(matched.WorkspaceKey), manifest.SnapshotSizeBytes, safeOptional(manifest.SnapshotChecksum))
|
||||
return matched.WorkspaceKey, nil
|
||||
}
|
||||
|
||||
func dataTargetMatchesSQLiteProbeAssignment(target protocol.RunAutonomousDataTarget, assignmentTargetKey string) bool {
|
||||
if target.Kind != "sqlite.snapshot" || assignmentTargetKey == "" {
|
||||
return false
|
||||
}
|
||||
return target.Key == assignmentTargetKey || target.TransportKey == assignmentTargetKey || target.WorkspaceKey == assignmentTargetKey
|
||||
}
|
||||
|
||||
func materializeSQLiteSnapshotDataTarget(ctx context.Context, workspaceRoot string, serverInstanceID string, profileKey string, target protocol.RunAutonomousDataTarget, bindings map[string]string) (dataTargetSnapshotManifest, error) {
|
||||
if target.Kind != "sqlite.snapshot" || target.RefreshPolicy != "on-demand-snapshot" || target.MaxBytes <= 0 {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_invalid", false)
|
||||
}
|
||||
sourceRoot := strings.TrimSpace(bindings[target.SourceRootKey])
|
||||
if sourceRoot == "" {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_source_unbound", false)
|
||||
}
|
||||
source, sourceInfo, err := resolveDataTargetSource(sourceRoot, target.SourcePath, target.MaxBytes)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(workspaceRoot).Scope(serverInstanceID, profileKey)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_scope_invalid", false)
|
||||
}
|
||||
destination, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, target.WorkspaceKey)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_invalid", false)
|
||||
}
|
||||
if err := ensureDirectory(filepath.Dir(destination)); err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
||||
}
|
||||
temporary := filepath.Join(filepath.Dir(destination), "."+safeWorkspaceName(filepath.Base(target.WorkspaceKey))+".snapshot.tmp")
|
||||
_ = os.Remove(temporary)
|
||||
if err := snapshotSQLiteDatabase(ctx, source, temporary, target.MaxBytes); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
snapshotInfo, err := os.Stat(temporary)
|
||||
if err != nil || !snapshotInfo.Mode().IsRegular() || snapshotInfo.Size() <= 0 || snapshotInfo.Size() > target.MaxBytes {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_snapshot_invalid", true)
|
||||
}
|
||||
checksum, err := fingerprintSQLiteSource(temporary)
|
||||
if err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_checksum_failed", true)
|
||||
}
|
||||
if err := replaceRegularFile(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
||||
}
|
||||
manifest := dataTargetSnapshotManifest{
|
||||
Version: dataTargetSnapshotManifestVersion,
|
||||
DataTargetKey: target.Key,
|
||||
Kind: target.Kind,
|
||||
WorkspaceKey: target.WorkspaceKey,
|
||||
SourceRootKey: target.SourceRootKey,
|
||||
SourcePathFingerprint: digestValue(target.SourcePath),
|
||||
SourceSizeBytes: sourceInfo.Size(),
|
||||
SourceModUnixNano: sourceInfo.ModTime().UnixNano(),
|
||||
SnapshotSizeBytes: snapshotInfo.Size(),
|
||||
SnapshotChecksum: checksum,
|
||||
MaterializedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := writeDataTargetSnapshotManifest(workspaceRoot, scope, target.WorkspaceKey, manifest); err != nil {
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func resolveDataTargetSource(sourceRoot string, sourcePath string, maxBytes int64) (string, os.FileInfo, error) {
|
||||
if strings.TrimSpace(sourceRoot) == "" || strings.TrimSpace(sourcePath) == "" || filepath.IsAbs(sourcePath) || strings.Contains(sourcePath, `\`) || strings.Contains(sourcePath, "..") || !protocol.ValidLogicalFileKey(sourcePath) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
cleanRoot := filepath.Clean(sourceRoot)
|
||||
if !filepath.IsAbs(cleanRoot) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
source := filepath.Join(cleanRoot, filepath.FromSlash(sourcePath))
|
||||
rel, err := filepath.Rel(cleanRoot, source)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
info, err := os.Lstat(source)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_missing", true)
|
||||
}
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_unavailable", true)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
if info.Size() <= 0 || info.Size() > maxBytes {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_size_invalid", false)
|
||||
}
|
||||
return source, info, nil
|
||||
}
|
||||
|
||||
func snapshotSQLiteDatabase(ctx context.Context, source string, destination string, maxBytes int64) error {
|
||||
snapshotCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+source+"?mode=ro")
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_sqlite_open_failed", true)
|
||||
}
|
||||
defer database.Close()
|
||||
database.SetMaxOpenConns(1)
|
||||
if _, err := database.ExecContext(snapshotCtx, "VACUUM INTO "+quoteSQLiteStringLiteral(destination)); err != nil {
|
||||
return dataTargetSQLiteError(snapshotCtx, err)
|
||||
}
|
||||
info, err := os.Stat(destination)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_snapshot_unavailable", true)
|
||||
}
|
||||
if info.Size() <= 0 || info.Size() > maxBytes {
|
||||
return newDataTargetMaterializeError("data_target_snapshot_limit_exceeded", false)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataTargetSQLiteError(ctx context.Context, err error) error {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return newDataTargetMaterializeError("data_target_cancelled", true)
|
||||
}
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return newDataTargetMaterializeError("data_target_timeout", true)
|
||||
}
|
||||
lower := strings.ToLower(fmt.Sprint(err))
|
||||
if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") {
|
||||
return newDataTargetMaterializeError("data_target_busy", true)
|
||||
}
|
||||
return newDataTargetMaterializeError("data_target_snapshot_failed", true)
|
||||
}
|
||||
|
||||
func quoteSQLiteStringLiteral(value string) string {
|
||||
return `'` + strings.ReplaceAll(value, `'`, `''`) + `'`
|
||||
}
|
||||
|
||||
func replaceRegularFile(source string, destination string) error {
|
||||
if info, err := os.Lstat(destination); err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("destination is not a regular file")
|
||||
}
|
||||
if err := os.Remove(destination); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return os.Rename(source, destination)
|
||||
}
|
||||
|
||||
func writeDataTargetSnapshotManifest(workspaceRoot string, scope string, workspaceKey string, manifest dataTargetSnapshotManifest) error {
|
||||
manifestKey := workspaceKey + ".snapshot.json"
|
||||
path, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, manifestKey)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
||||
}
|
||||
body, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
||||
}
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_failed", true)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataTargetSupportsPlatform(platforms []string, targetOS string) bool {
|
||||
if len(platforms) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, platform := range platforms {
|
||||
if platform == targetOS {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dataTargetErrorCode(err error) string {
|
||||
var materializeErr dataTargetMaterializeError
|
||||
if errors.As(err, &materializeErr) {
|
||||
return materializeErr.code
|
||||
}
|
||||
return "data_target_unavailable"
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestMaterializeSQLiteSnapshotDataTargetCreatesScopedProbeTarget(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
sourceRoot := t.TempDir()
|
||||
createSQLiteProbeFixture(t, filepath.Join(sourceRoot, "Saved", "SaveFiles", "current.db"))
|
||||
target := protocol.RunAutonomousDataTarget{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024}
|
||||
|
||||
manifest, err := materializeSQLiteSnapshotDataTarget(context.Background(), workspaceRoot, "server-data", "run-local", target, map[string]string{"server-root": sourceRoot})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize data target: %v", err)
|
||||
}
|
||||
if manifest.SnapshotChecksum == "" || manifest.SnapshotSizeBytes <= 0 || manifest.SourcePathFingerprint == "" {
|
||||
t.Fatalf("expected bounded snapshot metadata, got %+v", manifest)
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(workspaceRoot).Scope("server-data", "run-local")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve scope: %v", err)
|
||||
}
|
||||
manifestBody, err := os.ReadFile(filepath.Join(scope, "databases", "current-db.snapshot.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read snapshot manifest: %v", err)
|
||||
}
|
||||
if strings.Contains(string(manifestBody), sourceRoot) || strings.Contains(string(manifestBody), "Saved/SaveFiles/current.db") {
|
||||
t.Fatalf("snapshot manifest leaked host/source material: %s", manifestBody)
|
||||
}
|
||||
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
assignment.ServerInstanceID = "server-data"
|
||||
assignment.ExecutionInput.WorkspaceScope = "run-local"
|
||||
assignment.TargetKey = "databases/current-db"
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.ServerInstanceID = assignment.ServerInstanceID
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.RunEndpointID = assignment.RunEndpointID
|
||||
result := NewSQLiteSchemaProbeExecutor(workspaceRoot).Execute(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil || result.ExecutionResult.SQLiteSchemaProbe.Status != "succeeded" {
|
||||
t.Fatalf("expected probe to read materialized snapshot, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerMaterializesMatchingDataTargetBeforeSQLiteProbe(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-data"
|
||||
cfg.PluginID = "game.example"
|
||||
cfg.ComponentKind = config.PackageComponentRun
|
||||
cfg.ComponentKey = "run-local"
|
||||
sourceRoot := t.TempDir()
|
||||
createSQLiteProbeFixture(t, filepath.Join(sourceRoot, "Saved", "SaveFiles", "current.db"))
|
||||
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
|
||||
SchemaVersion: "1",
|
||||
ServerInstanceID: cfg.ServerInstanceID,
|
||||
PluginID: cfg.PluginID,
|
||||
PluginVersion: "1.0.0",
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
ProfileKey: cfg.ComponentKey,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-dist-test",
|
||||
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
|
||||
DataTargets: []protocol.RunAutonomousDataTarget{{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024, Platforms: []string{runtime.GOOS}}},
|
||||
RuntimeBindings: map[string]string{"server-root": sourceRoot},
|
||||
})
|
||||
worker, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
assignment.ServerInstanceID = cfg.ServerInstanceID
|
||||
assignment.RunEndpointID = cfg.RunEndpointID
|
||||
assignment.TargetKey = "current-db"
|
||||
assignment.ExecutionInput.WorkspaceScope = cfg.ComponentKey
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.ServerInstanceID = cfg.ServerInstanceID
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.RunEndpointID = cfg.RunEndpointID
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID = cfg.PluginID
|
||||
|
||||
result := worker.executeAssignment(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil || result.ExecutionResult.SQLiteSchemaProbe.Status != "succeeded" {
|
||||
serialized, _ := json.Marshal(result)
|
||||
t.Fatalf("expected worker probe to materialize and inspect data target, got %s", serialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeSQLiteSnapshotDataTargetRejectsUnboundSourceRoot(t *testing.T) {
|
||||
target := protocol.RunAutonomousDataTarget{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024}
|
||||
_, err := materializeSQLiteSnapshotDataTarget(context.Background(), t.TempDir(), "server-data", "run-local", target, map[string]string{})
|
||||
if err == nil || dataTargetErrorCode(err) != "data_target_source_unbound" {
|
||||
t.Fatalf("expected unbound source root rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDependencyDownloadBytes = int64(512 * 1024 * 1024)
|
||||
dependencyCommandTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
dependencyTokenPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+@/-]{0,119}$`)
|
||||
dependencyVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:+~-]{0,79}$`)
|
||||
steamAppPattern = regexp.MustCompile(`^[0-9]{1,12}$`)
|
||||
)
|
||||
|
||||
type DependencyDownloader interface {
|
||||
Download(context.Context, string, string, int64) (int64, string, error)
|
||||
}
|
||||
|
||||
type HTTPDependencyDownloader struct {
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func (downloader HTTPDependencyDownloader) Download(ctx context.Context, sourceURL, destination string, maxBytes int64) (int64, string, error) {
|
||||
parsed, err := validateDependencyDownloadURL(sourceURL)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
client := downloader.Client
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return fmt.Errorf("dependency download redirect limit exceeded")
|
||||
}
|
||||
_, err := validateDependencyDownloadURL(request.URL.String())
|
||||
return err
|
||||
}}
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return 0, "", fmt.Errorf("dependency download returned status %d", response.StatusCode)
|
||||
}
|
||||
if response.ContentLength > maxBytes {
|
||||
return 0, "", fmt.Errorf("dependency download exceeds size limit")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
temporary := destination + ".partial"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
remove := true
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
if remove {
|
||||
_ = os.Remove(temporary)
|
||||
}
|
||||
}()
|
||||
hash := sha256.New()
|
||||
written, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(response.Body, maxBytes+1))
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
if written > maxBytes {
|
||||
return 0, "", fmt.Errorf("dependency download exceeds size limit")
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
remove = false
|
||||
return written, "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type dependencyJournal struct {
|
||||
Version int `json:"version"`
|
||||
JobID string `json:"jobId"`
|
||||
Attempt int `json:"attempt"`
|
||||
PlanDigest string `json:"planDigest"`
|
||||
CompletedSteps []int `json:"completedSteps,omitempty"`
|
||||
State string `json:"state"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (worker *Worker) executeDependencyJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_dependency_job", err.Error())
|
||||
}
|
||||
runState, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return lifecycleFailure("dependency_unregistered", "Run worker is not registered")
|
||||
}
|
||||
input, err := worker.client.GetDependencyExecutionInput(ctx, protocol.DependencyExecutionInputRequest{RunEndpointID: runState.RunEndpointID, SessionToken: runState.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt})
|
||||
if err != nil {
|
||||
return lifecycleFailure("dependency_input_failed", "could not load fenced dependency input")
|
||||
}
|
||||
if err := validateDependencyInput(assignment, input); err != nil {
|
||||
return lifecycleFailure("unsafe_dependency_input", err.Error())
|
||||
}
|
||||
journalPath := filepath.Join(worker.cfg.WorkspaceRoot, "dependency-journals", safeWorkspaceName(assignment.JobID)+".json")
|
||||
journal, err := loadDependencyJournal(journalPath, assignment, input.PlanDigest)
|
||||
if err != nil {
|
||||
return lifecycleFailure("dependency_journal_failed", err.Error())
|
||||
}
|
||||
|
||||
if assignment.Capability == protocol.RunCapabilityDependenciesCheck {
|
||||
state, evidence, probeErr := worker.executor.runDependencyProbe(ctx, input.Probe, input.Bindings)
|
||||
if probeErr != nil {
|
||||
if errors.Is(probeErr, context.Canceled) || errors.Is(probeErr, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "dependency probe cancelled"}, Message: "dependency probe cancelled", ErrorCode: "dependency_probe_cancelled"}
|
||||
}
|
||||
return lifecycleFailure("dependency_probe_failed", probeErr.Error())
|
||||
}
|
||||
journal.State = state
|
||||
journal.UpdatedAt = time.Now().UTC()
|
||||
if err := persistDependencyJournal(journalPath, journal); err != nil {
|
||||
return lifecycleFailure("dependency_journal_failed", err.Error())
|
||||
}
|
||||
return dependencySuccess(assignment, input, state, evidence, 0)
|
||||
}
|
||||
|
||||
completed := map[int]bool{}
|
||||
for _, index := range journal.CompletedSteps {
|
||||
completed[index] = true
|
||||
}
|
||||
for index, step := range input.Plan.Steps {
|
||||
if completed[index] {
|
||||
continue
|
||||
}
|
||||
if cancelled, ok := checkContextCancelled(ctx, "dependency installation cancelled", "dependency_install_cancelled"); ok {
|
||||
return cancelled
|
||||
}
|
||||
if err := worker.executor.runDependencyInstallStep(ctx, assignment, input, step, index); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "dependency installation cancelled"}, Message: "dependency installation cancelled", ErrorCode: "dependency_install_cancelled"}
|
||||
}
|
||||
return lifecycleFailure("dependency_install_failed", err.Error())
|
||||
}
|
||||
journal.CompletedSteps = append(journal.CompletedSteps, index)
|
||||
journal.State = "installing"
|
||||
journal.UpdatedAt = time.Now().UTC()
|
||||
if err := persistDependencyJournal(journalPath, journal); err != nil {
|
||||
return lifecycleFailure("dependency_journal_failed", err.Error())
|
||||
}
|
||||
}
|
||||
state, evidence, probeErr := worker.executor.runDependencyProbe(ctx, input.Probe, input.Bindings)
|
||||
if probeErr != nil {
|
||||
return lifecycleFailure("dependency_verify_failed", probeErr.Error())
|
||||
}
|
||||
if state != "present" {
|
||||
return lifecycleFailure("dependency_verify_missing", "dependency remains missing after install plan")
|
||||
}
|
||||
journal.State = "present"
|
||||
journal.UpdatedAt = time.Now().UTC()
|
||||
if err := persistDependencyJournal(journalPath, journal); err != nil {
|
||||
return lifecycleFailure("dependency_journal_failed", err.Error())
|
||||
}
|
||||
return dependencySuccess(assignment, input, "present", evidence, len(journal.CompletedSteps))
|
||||
}
|
||||
|
||||
func validateDependencyInput(assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse) error {
|
||||
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID {
|
||||
return fmt.Errorf("dependency input scope does not match job")
|
||||
}
|
||||
if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH {
|
||||
return fmt.Errorf("dependency input target does not match Run")
|
||||
}
|
||||
if !validSHA256(input.PlanDigest) || !protocol.ValidLogicalFileKey(input.ProfileKey) {
|
||||
return fmt.Errorf("dependency input digest or profile is unsafe")
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(input.Probe.Key) || !protocol.ValidLogicalFileKey(input.Probe.TargetKey) {
|
||||
return fmt.Errorf("dependency probe is unsafe")
|
||||
}
|
||||
if assignment.Capability == protocol.RunCapabilityDependenciesCheck {
|
||||
if assignment.TargetKey != "dependencies/"+input.Probe.Key || input.Plan.Key != "" {
|
||||
return fmt.Errorf("dependency check declaration does not match job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if assignment.Capability != protocol.RunCapabilityDependenciesInstall || assignment.TargetKey != "dependencies/install/"+input.Plan.Key || !protocol.ValidLogicalFileKey(input.Plan.Key) || len(input.Plan.Steps) == 0 || len(input.Plan.Steps) > 64 {
|
||||
return fmt.Errorf("dependency install declaration does not match job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) runDependencyProbe(ctx context.Context, probe protocol.DependencyProbe, bindings map[string]string) (string, string, error) {
|
||||
target := strings.TrimSpace(bindings[probe.TargetKey])
|
||||
if target == "" {
|
||||
target = probe.TargetKey
|
||||
}
|
||||
switch probe.Kind {
|
||||
case "file.exists", "steam.app":
|
||||
info, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "missing", "declared target is not present", nil
|
||||
}
|
||||
return "", "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", "", fmt.Errorf("dependency target cannot be a symlink")
|
||||
}
|
||||
return "present", "declared target is present", nil
|
||||
case "package.installed":
|
||||
if err := validateDependencyExecutable(target); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if filepath.IsAbs(target) {
|
||||
if _, err := os.Stat(target); err != nil {
|
||||
return "missing", "declared package executable is not present", nil
|
||||
}
|
||||
} else if _, err := exec.LookPath(target); err != nil {
|
||||
return "missing", "declared package executable is not present", nil
|
||||
}
|
||||
return "present", "declared package executable is present", nil
|
||||
case "command.version", "java.version", "docker.available":
|
||||
if err := validateDependencyExecutable(target); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
args := []string{"--version"}
|
||||
if probe.Kind == "java.version" {
|
||||
args = []string{"-version"}
|
||||
}
|
||||
result, err := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{target}, args...), Timeout: 30 * time.Second, Capability: "dependency.probe", Action: probe.Kind})
|
||||
if err != nil || result.ExitCode != 0 {
|
||||
if ctx.Err() != nil {
|
||||
return "", "", ctx.Err()
|
||||
}
|
||||
return "missing", "declared executable is not available", nil
|
||||
}
|
||||
version := dependencyVersionEvidence(result.Stdout + "\n" + result.Stderr)
|
||||
if probe.MinimumVersion != "" && !dependencyVersionAtLeast(version, probe.MinimumVersion) {
|
||||
return "missing", "declared executable version is below minimum", nil
|
||||
}
|
||||
return "present", version, nil
|
||||
case "service.exists":
|
||||
if !dependencyTokenPattern.MatchString(target) {
|
||||
return "", "", fmt.Errorf("dependency service target is unsafe")
|
||||
}
|
||||
name, args := serviceProbeCommand(runtime.GOOS, target)
|
||||
if name == "" {
|
||||
return "", "", fmt.Errorf("service probe is unsupported on this platform")
|
||||
}
|
||||
result, err := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{name}, args...), Timeout: 30 * time.Second, Capability: "dependency.probe", Action: probe.Kind})
|
||||
if err != nil || result.ExitCode != 0 {
|
||||
if ctx.Err() != nil {
|
||||
return "", "", ctx.Err()
|
||||
}
|
||||
return "missing", "declared service is not present", nil
|
||||
}
|
||||
return "present", "declared service is present", nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("dependency probe kind is unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) runDependencyInstallStep(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse, step protocol.DependencyInstallStep, index int) error {
|
||||
if !protocol.ValidLogicalFileKey(step.TargetKey) {
|
||||
return fmt.Errorf("dependency install target is unsafe")
|
||||
}
|
||||
switch step.Type {
|
||||
case "package":
|
||||
name, args, err := packageInstallCommand(step.PackageManager, step.PackageName, step.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, runErr := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: append([]string{name}, args...), Timeout: dependencyCommandTimeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: "dependency.package"})
|
||||
if runErr != nil || result.ExitCode != 0 {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("typed package adapter failed")
|
||||
}
|
||||
return nil
|
||||
case "steamcmd-app":
|
||||
if !steamAppPattern.MatchString(step.PackageName) {
|
||||
return fmt.Errorf("Steam app identifier is unsafe")
|
||||
}
|
||||
executable := strings.TrimSpace(input.Bindings[step.TargetKey])
|
||||
if executable == "" {
|
||||
executable = "steamcmd"
|
||||
}
|
||||
if err := validateDependencyExecutable(executable); err != nil {
|
||||
return err
|
||||
}
|
||||
result, runErr := executor.dependencyRunner.Run(ctx, ProcessCommand{Args: []string{executable, "+login", "anonymous", "+app_update", step.PackageName, "validate", "+quit"}, Timeout: dependencyCommandTimeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: "dependency.steamcmd-app"})
|
||||
if runErr != nil || result.ExitCode != 0 {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("typed SteamCMD adapter failed")
|
||||
}
|
||||
return nil
|
||||
case "verified-download":
|
||||
if !validSHA256(step.Checksum) {
|
||||
return fmt.Errorf("verified download checksum is required")
|
||||
}
|
||||
if _, err := validateDependencyDownloadURL(step.DownloadRef); err != nil {
|
||||
return err
|
||||
}
|
||||
destination := filepath.Join(executor.workspaceRoot, "dependency-files", safeWorkspaceName(assignment.ServerInstanceID), safeWorkspaceName(step.TargetKey))
|
||||
size, checksum, err := executor.dependencyDownloader.Download(ctx, step.DownloadRef, destination, maxDependencyDownloadBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size <= 0 || checksum != strings.ToLower(step.Checksum) {
|
||||
_ = os.Remove(destination)
|
||||
return fmt.Errorf("verified dependency download checksum mismatch")
|
||||
}
|
||||
return os.Chmod(destination, 0o700)
|
||||
case "manual":
|
||||
return fmt.Errorf("manual dependency step requires operator action")
|
||||
default:
|
||||
return fmt.Errorf("dependency install step type is unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func packageInstallCommand(manager, packageName, version string) (string, []string, error) {
|
||||
if !dependencyTokenPattern.MatchString(packageName) || version != "" && !dependencyVersionPattern.MatchString(version) {
|
||||
return "", nil, fmt.Errorf("package name or version is unsafe")
|
||||
}
|
||||
spec := packageName
|
||||
switch manager {
|
||||
case "apt":
|
||||
if version != "" {
|
||||
spec += "=" + version
|
||||
}
|
||||
return "apt-get", []string{"install", "-y", "--no-install-recommends", spec}, nil
|
||||
case "yum", "dnf":
|
||||
if version != "" {
|
||||
spec += "-" + version
|
||||
}
|
||||
return manager, []string{"install", "-y", spec}, nil
|
||||
case "pacman":
|
||||
return "pacman", []string{"-S", "--noconfirm", spec}, nil
|
||||
case "zypper":
|
||||
return "zypper", []string{"--non-interactive", "install", spec}, nil
|
||||
case "brew":
|
||||
if version != "" {
|
||||
spec += "@" + version
|
||||
}
|
||||
return "brew", []string{"install", spec}, nil
|
||||
case "winget":
|
||||
args := []string{"install", "--id", packageName, "--exact", "--silent", "--accept-package-agreements", "--accept-source-agreements"}
|
||||
if version != "" {
|
||||
args = append(args, "--version", version)
|
||||
}
|
||||
return "winget", args, nil
|
||||
case "choco":
|
||||
args := []string{"install", packageName, "-y", "--no-progress"}
|
||||
if version != "" {
|
||||
args = append(args, "--version", version)
|
||||
}
|
||||
return "choco", args, nil
|
||||
case "scoop":
|
||||
if version != "" {
|
||||
spec += "@" + version
|
||||
}
|
||||
return "scoop", []string{"install", spec}, nil
|
||||
default:
|
||||
return "", nil, fmt.Errorf("package manager is unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func validateDependencyExecutable(target string) error {
|
||||
if strings.TrimSpace(target) != target || target == "" || strings.ContainsAny(target, "\r\n\x00") || containsUnsafeRuntimeText(target) {
|
||||
return fmt.Errorf("dependency executable target is unsafe")
|
||||
}
|
||||
if filepath.IsAbs(target) {
|
||||
info, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Mode()&0o111 == 0 {
|
||||
return fmt.Errorf("dependency executable target is not a regular executable")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !commandNamePattern.MatchString(target) {
|
||||
return fmt.Errorf("dependency executable name is unsafe")
|
||||
}
|
||||
if _, forbidden := disallowedExecutables[strings.ToLower(target)]; forbidden {
|
||||
return fmt.Errorf("dependency executable cannot be a shell")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDependencyDownloadURL(raw string) (*url.URL, error) {
|
||||
parsed, err := url.ParseRequestURI(strings.TrimSpace(raw))
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" {
|
||||
return nil, fmt.Errorf("dependency download URL is not approved")
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
if host == "localhost" || strings.HasSuffix(host, ".localhost") {
|
||||
return nil, fmt.Errorf("dependency download host is not approved")
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsLinkLocalUnicast()) {
|
||||
return nil, fmt.Errorf("dependency download host is not approved")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func serviceProbeCommand(targetOS, service string) (string, []string) {
|
||||
switch targetOS {
|
||||
case "linux":
|
||||
return "systemctl", []string{"status", service, "--no-pager"}
|
||||
case "windows":
|
||||
return "sc", []string{"query", service}
|
||||
case "darwin":
|
||||
return "launchctl", []string{"print", "system/" + service}
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
func dependencySuccess(assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse, state, evidence string, completed int) LifecycleExecutionResult {
|
||||
evidence = RedactText(strings.TrimSpace(evidence))
|
||||
payload, _ := json.Marshal(protocol.DependencyExecutionEvidence{ProbeKey: input.Probe.Key, PlanKey: input.Plan.Key, PlanDigest: input.PlanDigest, State: state, Evidence: evidence, CompletedSteps: completed})
|
||||
kind := "dependency.check"
|
||||
message := "dependency probe completed"
|
||||
if assignment.Capability == protocol.RunCapabilityDependenciesInstall {
|
||||
kind = "dependency.install"
|
||||
message = "dependency install plan completed and verified"
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, ResultRef: fmt.Sprintf("artifact://jobs/%s/dependencies-result", url.PathEscape(assignment.JobID)), Message: message, ExecutionResult: protocol.RunJobExecutionResult{Kind: kind, Checksum: input.PlanDigest, Summary: message, Content: string(payload)}}
|
||||
}
|
||||
|
||||
func loadDependencyJournal(path string, assignment protocol.RunJobAssignment, digest string) (dependencyJournal, error) {
|
||||
journal := dependencyJournal{Version: 1, JobID: assignment.JobID, Attempt: assignment.Attempt, PlanDigest: digest, State: "pending", UpdatedAt: time.Now().UTC()}
|
||||
body, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return journal, nil
|
||||
}
|
||||
if err != nil {
|
||||
return dependencyJournal{}, err
|
||||
}
|
||||
if err := json.Unmarshal(body, &journal); err != nil {
|
||||
return dependencyJournal{}, fmt.Errorf("decode dependency journal: %w", err)
|
||||
}
|
||||
if journal.Version != 1 || journal.JobID != assignment.JobID || journal.PlanDigest != digest {
|
||||
return dependencyJournal{}, fmt.Errorf("dependency journal does not match immutable plan")
|
||||
}
|
||||
if journal.Attempt > assignment.Attempt {
|
||||
return dependencyJournal{}, fmt.Errorf("dependency journal attempt is newer than assignment")
|
||||
}
|
||||
journal.Attempt = assignment.Attempt
|
||||
return journal, nil
|
||||
}
|
||||
|
||||
func persistDependencyJournal(path string, journal dependencyJournal) error {
|
||||
body, err := json.MarshalIndent(journal, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, body, 0o600)
|
||||
}
|
||||
|
||||
func writeRuntimeAtomicFile(path string, body []byte, mode os.FileMode) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
temporary := path + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.Write(body); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, path); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return os.Chmod(path, mode)
|
||||
}
|
||||
|
||||
func validSHA256(value string) bool {
|
||||
if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:"))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func dependencyVersionEvidence(output string) string {
|
||||
lines := splitBoundedLines(output)
|
||||
if len(lines) == 0 {
|
||||
return "version available"
|
||||
}
|
||||
return strings.TrimSpace(lines[0])
|
||||
}
|
||||
|
||||
func dependencyVersionAtLeast(actual, minimum string) bool {
|
||||
numbers := func(value string) []int {
|
||||
parts := regexp.MustCompile(`[0-9]+`).FindAllString(value, -1)
|
||||
out := make([]int, len(parts))
|
||||
for i, part := range parts {
|
||||
out[i], _ = strconv.Atoi(part)
|
||||
}
|
||||
return out
|
||||
}
|
||||
a, b := numbers(actual), numbers(minimum)
|
||||
for i := 0; i < len(a) || i < len(b); i++ {
|
||||
av, bv := 0, 0
|
||||
if i < len(a) {
|
||||
av = a[i]
|
||||
}
|
||||
if i < len(b) {
|
||||
bv = b[i]
|
||||
}
|
||||
if av != bv {
|
||||
return av > bv
|
||||
}
|
||||
}
|
||||
return len(a) > 0
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const dependencyTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
type dependencyTestSupervisor struct {
|
||||
mu sync.Mutex
|
||||
calls [][]string
|
||||
present bool
|
||||
block bool
|
||||
}
|
||||
|
||||
func (supervisor *dependencyTestSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) {
|
||||
supervisor.mu.Lock()
|
||||
supervisor.calls = append(supervisor.calls, append([]string(nil), command.Args...))
|
||||
block := supervisor.block
|
||||
present := supervisor.present
|
||||
if len(command.Args) > 0 && command.Args[0] == "apt-get" {
|
||||
supervisor.present = true
|
||||
present = true
|
||||
}
|
||||
supervisor.mu.Unlock()
|
||||
if block {
|
||||
<-ctx.Done()
|
||||
return ProcessResult{ExitCode: -1}, ctx.Err()
|
||||
}
|
||||
if len(command.Args) > 0 && command.Args[0] == "java" {
|
||||
if !present {
|
||||
return ProcessResult{ExitCode: 1}, errors.New("not installed")
|
||||
}
|
||||
return ProcessResult{ExitCode: 0, Stderr: "openjdk version 21.0.2"}, nil
|
||||
}
|
||||
return ProcessResult{ExitCode: 0}, nil
|
||||
}
|
||||
|
||||
func (supervisor *dependencyTestSupervisor) count(name string) int {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
count := 0
|
||||
for _, call := range supervisor.calls {
|
||||
if len(call) > 0 && call[0] == name {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
type dependencyTestDownloader struct {
|
||||
payload []byte
|
||||
sourceURL string
|
||||
destination string
|
||||
}
|
||||
|
||||
func (downloader *dependencyTestDownloader) Download(_ context.Context, sourceURL, destination string, _ int64) (int64, string, error) {
|
||||
downloader.sourceURL = sourceURL
|
||||
downloader.destination = destination
|
||||
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
if err := os.WriteFile(destination, downloader.payload, 0o600); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return int64(len(downloader.payload)), bytesChecksum(downloader.payload), nil
|
||||
}
|
||||
|
||||
func TestDependencyInstallExecutesTypedPlanAndResumesCompletedSteps(t *testing.T) {
|
||||
if !dependencyVersionAtLeast("openjdk version 21.0.2", "21") {
|
||||
t.Fatal("version comparator rejected valid Java version")
|
||||
}
|
||||
client := newFakeWorkerClient()
|
||||
assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall)
|
||||
client.dependencyInput = dependencyInputForAssignment(assignment)
|
||||
runner := &dependencyTestSupervisor{}
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithDependencyCommandRunner(runner))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
|
||||
first := worker.executeDependencyJob(context.Background(), assignment)
|
||||
if first.State != lifecycleResultStateSucceeded || first.ExecutionResult.Kind != "dependency.install" || first.ExecutionResult.Checksum != dependencyTestDigest {
|
||||
state, evidence, probeErr := worker.executor.runDependencyProbe(context.Background(), client.dependencyInput.Probe, client.dependencyInput.Bindings)
|
||||
t.Fatalf("expected real typed dependency install, got %+v calls=%+v present=%v probe=%s evidence=%s err=%v", first, runner.calls, runner.present, state, evidence, probeErr)
|
||||
}
|
||||
second := worker.executeDependencyJob(context.Background(), assignment)
|
||||
if second.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected journal resume success, got %+v", second)
|
||||
}
|
||||
if runner.count("apt-get") != 1 {
|
||||
t.Fatalf("completed package step must not repeat, calls=%+v", runner.calls)
|
||||
}
|
||||
if !strings.Contains(first.ExecutionResult.Content, `"completedSteps":1`) || strings.Contains(first.ExecutionResult.Content, "/Users/") {
|
||||
t.Fatalf("dependency evidence is not safe: %s", first.ExecutionResult.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDependencyProbeAndInstallRejectUnsafeOrCancelledWork(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall)
|
||||
input := dependencyInputForAssignment(assignment)
|
||||
input.Plan.Steps[0].PackageName = "openjdk;rm"
|
||||
client.dependencyInput = input
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithDependencyCommandRunner(&dependencyTestSupervisor{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
unsafe := worker.executeDependencyJob(context.Background(), assignment)
|
||||
if unsafe.State != lifecycleResultStateFailed || unsafe.ErrorCode != "dependency_install_failed" {
|
||||
t.Fatalf("expected unsafe package rejection, got %+v", unsafe)
|
||||
}
|
||||
|
||||
check := dependencyAssignment(protocol.RunCapabilityDependenciesCheck)
|
||||
checkInput := dependencyInputForAssignment(check)
|
||||
checkInput.Plan = protocol.DependencyInstallPlan{}
|
||||
client.dependencyInput = checkInput
|
||||
blocking := &dependencyTestSupervisor{block: true}
|
||||
worker.executor.dependencyRunner = blocking
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
cancelled := worker.executeDependencyJob(ctx, check)
|
||||
if cancelled.State != lifecycleResultStateCancelled || cancelled.ErrorCode != "dependency_probe_cancelled" {
|
||||
t.Fatalf("expected cancelled dependency probe, got %+v", cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiedDependencyDownloadUsesHTTPSChecksumAndScopedDestination(t *testing.T) {
|
||||
payload := []byte("verified dependency")
|
||||
downloader := &dependencyTestDownloader{payload: payload}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithDependencyDownloader(downloader))
|
||||
assignment := dependencyAssignment(protocol.RunCapabilityDependenciesInstall)
|
||||
input := dependencyInputForAssignment(assignment)
|
||||
step := protocol.DependencyInstallStep{Type: "verified-download", TargetKey: "tools/java", DownloadRef: "https://downloads.example.test/java", Checksum: bytesChecksum(payload)}
|
||||
if err := executor.runDependencyInstallStep(context.Background(), assignment, input, step, 0); err != nil {
|
||||
t.Fatalf("verified download: %v", err)
|
||||
}
|
||||
if downloader.sourceURL != step.DownloadRef || !strings.Contains(downloader.destination, "dependency-files") || strings.Contains(downloader.destination, "..") {
|
||||
t.Fatalf("unexpected scoped download: source=%s destination=%s", downloader.sourceURL, downloader.destination)
|
||||
}
|
||||
if _, err := validateDependencyDownloadURL("https://127.0.0.1/tool"); err == nil {
|
||||
t.Fatal("expected private dependency download host rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func dependencyAssignment(capability string) protocol.RunJobAssignment {
|
||||
assignment := workerJobAssignment(capability)
|
||||
assignment.LeaseToken = "lease-dependency"
|
||||
assignment.Attempt = 1
|
||||
assignment.State = "running"
|
||||
if capability == protocol.RunCapabilityDependenciesInstall {
|
||||
assignment.TargetKey = "dependencies/install/install-java"
|
||||
} else {
|
||||
assignment.TargetKey = "dependencies/java-runtime"
|
||||
}
|
||||
return assignment
|
||||
}
|
||||
|
||||
func dependencyInputForAssignment(assignment protocol.RunJobAssignment) protocol.DependencyExecutionInputResponse {
|
||||
return protocol.DependencyExecutionInputResponse{
|
||||
JobID: assignment.JobID,
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
RunEndpointID: assignment.RunEndpointID,
|
||||
PluginID: "game.minecraft",
|
||||
PluginVersion: "1.0.0",
|
||||
ProfileKey: "local",
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
PlanDigest: dependencyTestDigest,
|
||||
Probe: protocol.DependencyProbe{Key: "java-runtime", Kind: "java.version", TargetKey: "java", MinimumVersion: "21", Platforms: []string{runtime.GOOS}},
|
||||
Plan: protocol.DependencyInstallPlan{Key: "install-java", Title: "Install Java", Platforms: []string{runtime.GOOS}, Steps: []protocol.DependencyInstallStep{{Type: "package", TargetKey: "java", PackageManager: "apt", PackageName: "openjdk-21-jre"}}},
|
||||
Bindings: map[string]string{"java": "java"},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const distributionArtifactChunkSize = 1024 * 1024
|
||||
|
||||
type packageConfigPayload struct {
|
||||
Kind string `json:"kind"`
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
PluginID string `json:"pluginId"`
|
||||
RunEndpointID string `json:"runEndpointId,omitempty"`
|
||||
ProfileKey string `json:"profileKey,omitempty"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
SecretRef string `json:"secretRef"`
|
||||
KeyGeneration int `json:"keyGeneration"`
|
||||
AuthKey string `json:"authKey"`
|
||||
}
|
||||
|
||||
func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_distribution_build", err.Error())
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return distributionBuildFailure("build_unregistered", "Run worker is not registered")
|
||||
}
|
||||
input, err := worker.client.GetDistributionBuildInput(ctx, protocol.DistributionBuildInputRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return distributionBuildFailure("build_input_failed", "could not load authenticated build input")
|
||||
}
|
||||
if err := validateDistributionBuildInput(assignment, input); err != nil {
|
||||
return distributionBuildFailure("unsafe_build_input", err.Error())
|
||||
}
|
||||
|
||||
report := func(percent int, message string) error {
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
Progress: protocol.RunJobProgressReport{Percent: percent, Message: message},
|
||||
Sequence: worker.nextProgressSequence(assignment.ProgressSequence),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted {
|
||||
return fmt.Errorf("distribution build progress was not accepted")
|
||||
}
|
||||
assignment = response.Job
|
||||
return worker.journal.Store(assignment)
|
||||
}
|
||||
|
||||
workspace := distributionBuildWorkspace(worker.cfg.WorkspaceRoot, input.PluginID, assignment.JobID)
|
||||
if err := os.RemoveAll(workspace); err != nil {
|
||||
return distributionBuildFailure("workspace_prepare_failed", "could not reset isolated build workspace")
|
||||
}
|
||||
if err := os.MkdirAll(workspace, 0o700); err != nil {
|
||||
return distributionBuildFailure("workspace_prepare_failed", "could not create isolated build workspace")
|
||||
}
|
||||
defer os.RemoveAll(workspace)
|
||||
|
||||
if err := report(12, "git_sync: preparing approved source"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report source preparation")
|
||||
}
|
||||
sourceRoot, err := worker.prepareDistributionSource(ctx, workspace, input)
|
||||
if err != nil {
|
||||
return distributionBuildFailure("git_sync_failed", "approved source checkout failed")
|
||||
}
|
||||
|
||||
if err := report(28, "env_check: validating Go build environment"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report environment check")
|
||||
}
|
||||
if err := fixedCommand(ctx, sourceRoot, nil, "go", "version"); err != nil {
|
||||
return distributionBuildFailure("env_check_failed", "Go build environment is unavailable")
|
||||
}
|
||||
|
||||
configPath := ""
|
||||
if input.ComponentKind == "client-manager" {
|
||||
var err error
|
||||
configPath, err = writeDistributionConfig(sourceRoot, input, worker.clientPlatformURL())
|
||||
if err != nil {
|
||||
return distributionBuildFailure("config_injection_failed", "could not inject scoped component configuration")
|
||||
}
|
||||
}
|
||||
|
||||
if err := report(45, "deps_download: downloading Go modules"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report dependency download")
|
||||
}
|
||||
buildEnv := []string{"GOOS=" + input.TargetOS, "GOARCH=" + input.TargetArch, "CGO_ENABLED=0"}
|
||||
if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "mod", "download"); err != nil {
|
||||
return distributionBuildFailure("deps_download_failed", "Go module download failed")
|
||||
}
|
||||
|
||||
if err := report(65, "build_compile: compiling target executable"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report compilation")
|
||||
}
|
||||
binaryPath := filepath.Join(workspace, input.OutputFilename)
|
||||
entry := "."
|
||||
if input.ComponentKind == "run" {
|
||||
entry = "./cmd/run"
|
||||
}
|
||||
ldflags := "-s -w"
|
||||
if input.ComponentKind == "run" {
|
||||
ldflags = buildRunLDFlags(input, distributionBuildPlatformURL(worker, input))
|
||||
}
|
||||
if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "build", "-trimpath", "-ldflags", ldflags, "-o", binaryPath, entry); err != nil {
|
||||
return distributionBuildFailure("build_compile_failed", "Go compilation failed")
|
||||
}
|
||||
|
||||
if err := report(82, "package_finalize: creating distribution archive"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report packaging")
|
||||
}
|
||||
if input.ComponentKind == "run" {
|
||||
payload, err := os.ReadFile(binaryPath)
|
||||
if err != nil {
|
||||
return distributionBuildFailure("package_finalize_failed", "run executable could not be read")
|
||||
}
|
||||
if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil {
|
||||
return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed")
|
||||
}
|
||||
if err := report(96, "package_finalize: artifact upload completed"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report artifact upload")
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"},
|
||||
ResultRef: "artifact://" + input.ArtifactID,
|
||||
Message: "distribution build completed",
|
||||
}
|
||||
}
|
||||
archivePath := filepath.Join(workspace, archiveFilename(input))
|
||||
if err := createDistributionArchive(archivePath, input.PackageFormat, binaryPath, configPath); err != nil {
|
||||
return distributionBuildFailure("package_finalize_failed", "distribution archive creation failed")
|
||||
}
|
||||
payload, err := os.ReadFile(archivePath)
|
||||
if err != nil {
|
||||
return distributionBuildFailure("package_finalize_failed", "distribution archive could not be read")
|
||||
}
|
||||
if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil {
|
||||
return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed")
|
||||
}
|
||||
if err := report(96, "package_finalize: artifact upload completed"); err != nil {
|
||||
return distributionBuildFailure("progress_report_failed", "could not report artifact upload")
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"},
|
||||
ResultRef: "artifact://" + input.ArtifactID,
|
||||
Message: "distribution build completed",
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) prepareDistributionSource(ctx context.Context, workspace string, input protocol.DistributionBuildInputResponse) (string, error) {
|
||||
if input.ComponentKind == "run" {
|
||||
root, err := filepath.Abs(worker.cfg.BuildSourceRoot)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
root, err = filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil {
|
||||
return "", err
|
||||
}
|
||||
isolatedSource := filepath.Join(workspace, "source")
|
||||
if err := copyDistributionSource(root, isolatedSource, workspace); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writeRunWorkspaceSeedConfig(isolatedSource, input.WorkspaceSeed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return isolatedSource, nil
|
||||
}
|
||||
checkout := filepath.Join(workspace, "source")
|
||||
if err := os.MkdirAll(checkout, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := fixedCommand(ctx, checkout, nil, "git", "init", "--quiet"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := fixedCommand(ctx, checkout, nil, "git", "remote", "add", "origin", input.RepositoryURL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := fixedCommand(ctx, checkout, nil, "git", "fetch", "--quiet", "--depth", "1", "origin", input.SourceRevision); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := fixedCommand(ctx, checkout, nil, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return checkout, nil
|
||||
}
|
||||
|
||||
func writeRunWorkspaceSeedConfig(sourceRoot string, encodedSeed string) error {
|
||||
encodedSeed = strings.TrimSpace(encodedSeed)
|
||||
if encodedSeed == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := base64.StdEncoding.DecodeString(encodedSeed); err != nil {
|
||||
return fmt.Errorf("workspace seed is invalid")
|
||||
}
|
||||
configDir := filepath.Join(sourceRoot, "config")
|
||||
if err := os.MkdirAll(configDir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
body := fmt.Sprintf("package config\n\nfunc init() { BuildWorkspaceSeed = %q }\n", encodedSeed)
|
||||
return os.WriteFile(filepath.Join(configDir, "workspace_seed_generated.go"), []byte(body), 0o600)
|
||||
}
|
||||
|
||||
func copyDistributionSource(sourceRoot string, destinationRoot string, workspace string) error {
|
||||
workspace, err := filepath.Abs(workspace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filepath.WalkDir(sourceRoot, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if path == workspace || strings.HasPrefix(path, workspace+string(filepath.Separator)) {
|
||||
if entry.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
relative, err := filepath.Rel(sourceRoot, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if relative == "." {
|
||||
return os.MkdirAll(destinationRoot, 0o700)
|
||||
}
|
||||
if entry.Name() == ".git" && entry.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("trusted build source contains a symbolic link")
|
||||
}
|
||||
destination := filepath.Join(destinationRoot, relative)
|
||||
if entry.IsDir() {
|
||||
return os.MkdirAll(destination, 0o700)
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
input, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
input.Close()
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(output, input)
|
||||
inputCloseErr := input.Close()
|
||||
closeErr := output.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
if inputCloseErr != nil {
|
||||
return inputCloseErr
|
||||
}
|
||||
return closeErr
|
||||
})
|
||||
}
|
||||
|
||||
func writeDistributionConfig(sourceRoot string, input protocol.DistributionBuildInputResponse, platformURL string) (string, error) {
|
||||
if input.ComponentKind == "client-manager" {
|
||||
content := fmt.Sprintf("server_url: %q\nserver_instance_id: %q\nscum_client_credential: %q\nscum_client_name: %q\nscum_client_version: %q\nscum_client_machine_label: %q\nftp_provider: 3\n",
|
||||
platformURL, input.ServerInstanceID, input.AuthKey, input.ProfileKey, "platform-build", "managed-client")
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "config.yaml"), []byte(content), 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(sourceRoot, "config.yaml"), nil
|
||||
}
|
||||
return "", fmt.Errorf("run distributions do not use sidecar package config")
|
||||
}
|
||||
|
||||
func distributionBuildPlatformURL(worker *Worker, input protocol.DistributionBuildInputResponse) string {
|
||||
if value := strings.TrimSpace(input.PlatformURL); value != "" {
|
||||
return value
|
||||
}
|
||||
return worker.clientPlatformURL()
|
||||
}
|
||||
|
||||
func buildRunLDFlags(input protocol.DistributionBuildInputResponse, platformURL string) string {
|
||||
values := map[string]string{
|
||||
"BuildMode": "worker",
|
||||
"BuildPlatformURL": platformURL,
|
||||
"BuildRunEndpointID": input.RunEndpointID,
|
||||
"BuildDisplayName": "Run-" + input.ServerInstanceID,
|
||||
"BuildRegistrationToken": input.AuthKey,
|
||||
"BuildServerInstanceID": input.ServerInstanceID,
|
||||
"BuildPluginID": input.PluginID,
|
||||
"BuildComponentKind": input.ComponentKind,
|
||||
"BuildComponentKey": input.ProfileKey,
|
||||
"BuildKeyGeneration": fmt.Sprint(input.KeyGeneration),
|
||||
"BuildVersion": input.TargetRelease,
|
||||
}
|
||||
flags := []string{"-s", "-w"}
|
||||
for _, name := range []string{"BuildMode", "BuildPlatformURL", "BuildRunEndpointID", "BuildDisplayName", "BuildRegistrationToken", "BuildServerInstanceID", "BuildPluginID", "BuildComponentKind", "BuildComponentKey", "BuildKeyGeneration", "BuildVersion"} {
|
||||
flags = append(flags, "-X", "browser.local/run/config."+name+"="+values[name])
|
||||
}
|
||||
return strings.Join(flags, " ")
|
||||
}
|
||||
|
||||
func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment protocol.RunJobAssignment, artifactID string, payload []byte) error {
|
||||
checksum := bytesChecksum(payload)
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opened, err := worker.client.OpenArtifactTransfer(ctx, protocol.ArtifactTransferOpenRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
ArtifactID: artifactID,
|
||||
Direction: "upload",
|
||||
OwnerKind: "job",
|
||||
OwnerID: assignment.JobID,
|
||||
SizeBytes: int64(len(payload)),
|
||||
ChunkSizeBytes: distributionArtifactChunkSize,
|
||||
Checksum: checksum,
|
||||
IdempotencyKey: "distribution-build:" + assignment.JobID,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
received := map[int]bool{}
|
||||
for _, index := range opened.ReceivedChunkIndexes {
|
||||
received[index] = true
|
||||
}
|
||||
for index, offset := 0, 0; offset < len(payload); index, offset = index+1, offset+distributionArtifactChunkSize {
|
||||
if received[index] {
|
||||
continue
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
end := offset + distributionArtifactChunkSize
|
||||
if end > len(payload) {
|
||||
end = len(payload)
|
||||
}
|
||||
chunk := payload[offset:end]
|
||||
if _, err := worker.client.UploadArtifactChunk(ctx, protocol.ArtifactChunkUploadRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
TransferID: opened.TransferID,
|
||||
ArtifactID: artifactID,
|
||||
ChunkIndex: index,
|
||||
Offset: int64(offset),
|
||||
SizeBytes: len(chunk),
|
||||
Checksum: bytesChecksum(chunk),
|
||||
Payload: chunk,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
state, err = worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
completed, err := worker.client.CompleteArtifactTransfer(ctx, protocol.ArtifactTransferCompleteRequest{
|
||||
RunEndpointID: state.RunEndpointID,
|
||||
SessionToken: state.SessionToken,
|
||||
TransferID: opened.TransferID,
|
||||
ArtifactID: artifactID,
|
||||
Checksum: checksum,
|
||||
SizeBytes: int64(len(payload)),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !completed.Completed || completed.Artifact.State != "available" {
|
||||
return fmt.Errorf("artifact transfer did not complete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDistributionBuildInput(assignment protocol.RunJobAssignment, input protocol.DistributionBuildInputResponse) error {
|
||||
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID {
|
||||
return fmt.Errorf("build input scope does not match job")
|
||||
}
|
||||
if input.ComponentKind != "run" && input.ComponentKind != "client-manager" {
|
||||
return fmt.Errorf("build component kind is unsupported")
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(input.RunEndpointID) {
|
||||
return fmt.Errorf("generated Run endpoint identity is unsafe")
|
||||
}
|
||||
if input.ComponentKind == "client-manager" && input.RunEndpointID != assignment.RunEndpointID {
|
||||
return fmt.Errorf("client-manager build target does not match job")
|
||||
}
|
||||
if strings.TrimSpace(input.PluginID) == "" {
|
||||
return fmt.Errorf("build plugin id is required")
|
||||
}
|
||||
if input.ComponentKind == "client-manager" && !approvedHTTPSGitRepository(input.RepositoryURL) {
|
||||
return fmt.Errorf("client-manager repository is not approved")
|
||||
}
|
||||
if input.ComponentKind == "client-manager" && strings.TrimSpace(input.SourceRevision) == "" {
|
||||
return fmt.Errorf("client-manager source revision is required")
|
||||
}
|
||||
if input.TargetOS != "windows" && input.TargetOS != "linux" && input.TargetOS != "darwin" {
|
||||
return fmt.Errorf("target OS is unsupported")
|
||||
}
|
||||
if input.TargetArch != "amd64" && input.TargetArch != "arm64" {
|
||||
return fmt.Errorf("target architecture is unsupported")
|
||||
}
|
||||
if input.ComponentKind == "run" && !protocol.ValidLogicalFileKey(input.TargetRelease) {
|
||||
return fmt.Errorf("target release is unsafe")
|
||||
}
|
||||
if input.ComponentKind == "run" && input.PackageFormat != "raw-executable" {
|
||||
return fmt.Errorf("run package format must be raw-executable")
|
||||
}
|
||||
if input.ComponentKind == "run" && !validDistributionPlatformURL(input.PlatformURL) {
|
||||
return fmt.Errorf("run platform URL is invalid")
|
||||
}
|
||||
if input.ComponentKind == "run" && strings.TrimSpace(input.WorkspaceSeed) != "" {
|
||||
if _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(input.WorkspaceSeed)); err != nil {
|
||||
return fmt.Errorf("run workspace seed is invalid")
|
||||
}
|
||||
}
|
||||
if input.ComponentKind == "client-manager" && input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" {
|
||||
return fmt.Errorf("package format is unsupported")
|
||||
}
|
||||
if strings.TrimSpace(input.ArtifactID) == "" || strings.TrimSpace(input.OutputFilename) == "" || strings.TrimSpace(input.AuthKey) == "" {
|
||||
return fmt.Errorf("build input is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func approvedHTTPSGitRepository(value string) bool {
|
||||
parsed, err := url.ParseRequestURI(strings.TrimSpace(value))
|
||||
return err == nil && parsed.Scheme == "https" && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" && strings.HasSuffix(parsed.Path, ".git")
|
||||
}
|
||||
|
||||
func validDistributionPlatformURL(value string) bool {
|
||||
parsed, err := url.ParseRequestURI(strings.TrimSpace(value))
|
||||
return err == nil && (parsed.Scheme == "https" || parsed.Scheme == "http") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
|
||||
}
|
||||
|
||||
func fixedCommand(ctx context.Context, dir string, extraEnv []string, name string, args ...string) error {
|
||||
startedAt := time.Now()
|
||||
commandLine := redactedDistributionCommandLine(name, args)
|
||||
log.Printf("RUN phase=distribution_build.command status=starting workdir=%s command=%s envKeys=%s", safeOptional(dir), commandLine, envKeysSummary(extraEnvMap(extraEnv), nil))
|
||||
command := exec.CommandContext(ctx, name, args...)
|
||||
command.Dir = dir
|
||||
command.Env = append(os.Environ(), extraEnv...)
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
if err := command.Run(); err != nil {
|
||||
log.Printf("RUN phase=distribution_build.command status=failed command=%s durationMs=%d error=%s", commandLine, time.Since(startedAt).Milliseconds(), RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
log.Printf("RUN phase=distribution_build.command status=complete command=%s durationMs=%d", commandLine, time.Since(startedAt).Milliseconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
func redactedDistributionCommandLine(name string, args []string) string {
|
||||
parts := append([]string{name}, args...)
|
||||
redacted := append([]string(nil), parts...)
|
||||
for index, part := range redacted {
|
||||
if part == "-ldflags" && index+1 < len(redacted) {
|
||||
redacted[index+1] = "[redacted-ldflags]"
|
||||
continue
|
||||
}
|
||||
if strings.Contains(part, "BuildRegistrationToken=") {
|
||||
redacted[index] = "[redacted-ldflags]"
|
||||
}
|
||||
}
|
||||
return redactedCommandLine(redacted)
|
||||
}
|
||||
|
||||
func extraEnvMap(entries []string) map[string]string {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
env := make(map[string]string, len(entries))
|
||||
for _, entry := range entries {
|
||||
key, value, ok := strings.Cut(entry, "=")
|
||||
if !ok {
|
||||
key = entry
|
||||
value = ""
|
||||
}
|
||||
env[key] = value
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
func createDistributionArchive(path string, format string, binaryPath string, configPath string) error {
|
||||
if format == "zip" {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
writer := zip.NewWriter(file)
|
||||
if err := addZipFile(writer, binaryPath); err != nil {
|
||||
writer.Close()
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := addZipFile(writer, configPath); err != nil {
|
||||
writer.Close()
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gzipWriter := gzip.NewWriter(file)
|
||||
tarWriter := tar.NewWriter(gzipWriter)
|
||||
if err := addTarFile(tarWriter, binaryPath); err != nil {
|
||||
tarWriter.Close()
|
||||
gzipWriter.Close()
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := addTarFile(tarWriter, configPath); err != nil {
|
||||
tarWriter.Close()
|
||||
gzipWriter.Close()
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := tarWriter.Close(); err != nil {
|
||||
gzipWriter.Close()
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
|
||||
func addZipFile(writer *zip.Writer, path string) error {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry, err := writer.Create(filepath.Base(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = entry.Write(body)
|
||||
return err
|
||||
}
|
||||
|
||||
func addTarFile(writer *tar.Writer, path string) error {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header := &tar.Header{Name: filepath.Base(path), Mode: 0o600, Size: info.Size()}
|
||||
if strings.HasSuffix(filepath.Base(path), ".exe") || filepath.Base(path) == "run" {
|
||||
header.Mode = 0o700
|
||||
}
|
||||
if err := writer.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(writer, file)
|
||||
return err
|
||||
}
|
||||
|
||||
func archiveFilename(input protocol.DistributionBuildInputResponse) string {
|
||||
base := "run-" + input.ServerInstanceID
|
||||
if input.ComponentKind == "client-manager" {
|
||||
base = input.ProfileKey + "-" + input.ServerInstanceID
|
||||
}
|
||||
if input.PackageFormat == "zip" {
|
||||
return base + ".zip"
|
||||
}
|
||||
return base + ".tar.gz"
|
||||
}
|
||||
|
||||
func distributionBuildWorkspace(workspaceRoot string, pluginID string, jobID string) string {
|
||||
return filepath.Join(workspaceRoot, "distribution-builds", safeWorkspaceName(pluginID), safeWorkspaceName(jobID))
|
||||
}
|
||||
|
||||
func bytesChecksum(payload []byte) string {
|
||||
sum := sha256.Sum256(payload)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func safeWorkspaceName(value string) string {
|
||||
var builder strings.Builder
|
||||
for _, char := range value {
|
||||
if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '-' || char == '_' {
|
||||
builder.WriteRune(char)
|
||||
} else {
|
||||
builder.WriteByte('-')
|
||||
}
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func distributionBuildFailure(code string, message string) LifecycleExecutionResult {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateFailed,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: message},
|
||||
Message: message,
|
||||
ErrorCode: code,
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) clientPlatformURL() string {
|
||||
if client, ok := worker.client.(interface{ BaseURL() string }); ok {
|
||||
return client.BaseURL()
|
||||
}
|
||||
return worker.cfg.PlatformURL
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestWorkerDistributionBuildCompilesAndUploadsRawRunExecutable(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.BuildSourceRoot = ".."
|
||||
worker, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
|
||||
output := "run"
|
||||
if runtime.GOOS == "windows" {
|
||||
output = "run.exe"
|
||||
}
|
||||
assignment := protocol.RunJobAssignment{
|
||||
JobID: "job-distribution-build-test",
|
||||
ServerInstanceID: "server-build-test",
|
||||
RunEndpointID: "run-test",
|
||||
Capability: protocol.RunCapabilityDistributionBuild,
|
||||
TargetKey: "distribution/run",
|
||||
InputRef: "input://distribution-build/run-test",
|
||||
IdempotencyKey: "distribution-build:test",
|
||||
State: "running",
|
||||
LeaseToken: "lease-test",
|
||||
Attempt: 1,
|
||||
}
|
||||
client.claimJob = assignment
|
||||
client.buildInput = protocol.DistributionBuildInputResponse{
|
||||
JobID: assignment.JobID,
|
||||
ComponentKind: "run",
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
PluginID: "game.scum",
|
||||
RunEndpointID: assignment.RunEndpointID,
|
||||
TargetOS: runtime.GOOS,
|
||||
TargetArch: runtime.GOARCH,
|
||||
TargetRelease: "run-release-test",
|
||||
PlatformURL: "https://scum.npc0.com",
|
||||
PackageFormat: "raw-executable",
|
||||
ArtifactID: "artifact-built-run",
|
||||
OutputFilename: output,
|
||||
SecretRef: "secret://runtime-keys/server-build-test/run/current",
|
||||
KeyGeneration: 1,
|
||||
AuthKey: "test-component-key",
|
||||
}
|
||||
|
||||
result := worker.executeDistributionBuild(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ResultRef != "artifact://artifact-built-run" {
|
||||
t.Fatalf("expected successful real build, got %+v", result)
|
||||
}
|
||||
if len(client.artifactPayload) < 1024 {
|
||||
t.Fatalf("expected compiled executable payload, got %d bytes", len(client.artifactPayload))
|
||||
}
|
||||
extracted := t.TempDir()
|
||||
binaryPath := filepath.Join(extracted, output)
|
||||
if err := os.WriteFile(binaryPath, client.artifactPayload, 0o700); err != nil {
|
||||
t.Fatalf("write uploaded run executable: %v", err)
|
||||
}
|
||||
command := exec.Command(binaryPath)
|
||||
command.Env = append(os.Environ(), "RUN_MODE=smoke")
|
||||
smokeOutput, err := command.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("execute generated run package smoke mode: %v output=%s", err, smokeOutput)
|
||||
}
|
||||
var summary map[string]any
|
||||
if err := json.Unmarshal(smokeOutput, &summary); err != nil {
|
||||
t.Fatalf("decode generated package smoke output: %v body=%s", err, smokeOutput)
|
||||
}
|
||||
if summary["status"] != "ok" || summary["mode"] != "smoke" {
|
||||
t.Fatalf("expected generated package to run smoke mode, got %+v", summary)
|
||||
}
|
||||
if summary["platformUrl"] != "https://scum.npc0.com" {
|
||||
t.Fatalf("expected generated executable to use compiled platform URL, got %+v", summary)
|
||||
}
|
||||
joinedProgress := make([]string, 0, len(client.progressRequests))
|
||||
for _, request := range client.progressRequests {
|
||||
joinedProgress = append(joinedProgress, request.Progress.Message)
|
||||
}
|
||||
progress := strings.Join(joinedProgress, "\n")
|
||||
for _, stage := range []string{"git_sync:", "env_check:", "deps_download:", "build_compile:", "package_finalize:"} {
|
||||
if !strings.Contains(progress, stage) {
|
||||
t.Fatalf("expected real progress stage %q in %q", stage, progress)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDistributionBuildCrossCompilesWindowsAMD64Run(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.BuildSourceRoot = ".."
|
||||
worker, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
assignment := protocol.RunJobAssignment{JobID: "job-distribution-build-windows", ServerInstanceID: "server-build-windows", RunEndpointID: "run-builder", Capability: protocol.RunCapabilityDistributionBuild, TargetKey: "distribution/run", InputRef: "input://distribution-build/windows", IdempotencyKey: "distribution-build:windows", State: "running", LeaseToken: "lease-windows", Attempt: 1}
|
||||
client.claimJob = assignment
|
||||
client.buildInput = protocol.DistributionBuildInputResponse{JobID: assignment.JobID, ComponentKind: "run", ServerInstanceID: assignment.ServerInstanceID, PluginID: "game.scum", RunEndpointID: "server-run-server-build-windows", TargetOS: "windows", TargetArch: "amd64", TargetRelease: "run-release-windows", PlatformURL: "https://scum.npc0.com", PackageFormat: "raw-executable", ArtifactID: "artifact-built-windows-run", OutputFilename: "run.exe", SecretRef: "secret://runtime-keys/server-build-windows/run/current", KeyGeneration: 1, AuthKey: "test-component-key"}
|
||||
|
||||
result := worker.executeDistributionBuild(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ResultRef != "artifact://artifact-built-windows-run" {
|
||||
t.Fatalf("expected successful Windows build, got %+v", result)
|
||||
}
|
||||
if len(client.artifactPayload) < 1024 || !bytes.HasPrefix(client.artifactPayload, []byte("MZ")) {
|
||||
t.Fatalf("expected Windows PE executable payload, got %d bytes", len(client.artifactPayload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistributionBuildIsolationUsesPluginJobWorkspaceAndDistinctPackageState(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
firstAssignment := protocol.RunJobAssignment{JobID: "job-distribution-build-scum-alpha", ServerInstanceID: "scum-alpha", RunEndpointID: "run-test"}
|
||||
secondAssignment := protocol.RunJobAssignment{JobID: "job-distribution-build-scum-beta", ServerInstanceID: "scum-beta", RunEndpointID: "run-test"}
|
||||
firstInput := protocol.DistributionBuildInputResponse{
|
||||
JobID: firstAssignment.JobID,
|
||||
ComponentKind: "run",
|
||||
ServerInstanceID: firstAssignment.ServerInstanceID,
|
||||
PluginID: "game.scum",
|
||||
RunEndpointID: firstAssignment.RunEndpointID,
|
||||
TargetOS: "linux",
|
||||
TargetArch: "amd64",
|
||||
PlatformURL: "https://scum.npc0.com",
|
||||
PackageFormat: "raw-executable",
|
||||
ArtifactID: "artifact-run-dist-scum-alpha",
|
||||
OutputFilename: "run",
|
||||
SecretRef: "secret://runtime-keys/scum-alpha/run/current",
|
||||
KeyGeneration: 1,
|
||||
AuthKey: "alpha-component-key",
|
||||
}
|
||||
secondInput := firstInput
|
||||
secondInput.JobID = secondAssignment.JobID
|
||||
secondInput.ServerInstanceID = secondAssignment.ServerInstanceID
|
||||
secondInput.ArtifactID = "artifact-run-dist-scum-beta"
|
||||
secondInput.SecretRef = "secret://runtime-keys/scum-beta/run/current"
|
||||
secondInput.AuthKey = "beta-component-key"
|
||||
|
||||
firstWorkspace := distributionBuildWorkspace(workspaceRoot, firstInput.PluginID, firstAssignment.JobID)
|
||||
secondWorkspace := distributionBuildWorkspace(workspaceRoot, secondInput.PluginID, secondAssignment.JobID)
|
||||
if firstWorkspace == secondWorkspace {
|
||||
t.Fatalf("expected same-plugin builds to use distinct job workspaces")
|
||||
}
|
||||
if filepath.Dir(firstWorkspace) != filepath.Dir(secondWorkspace) || filepath.Base(filepath.Dir(firstWorkspace)) != "game-scum" {
|
||||
t.Fatalf("expected workspaces under the same plugin queue directory, first=%s second=%s", firstWorkspace, secondWorkspace)
|
||||
}
|
||||
firstFlags := buildRunLDFlags(firstInput, "https://scum.npc0.com")
|
||||
secondFlags := buildRunLDFlags(secondInput, "https://scum.npc0.com")
|
||||
if !strings.Contains(firstFlags, "BuildServerInstanceID=scum-alpha") || !strings.Contains(firstFlags, "BuildRegistrationToken=alpha-component-key") {
|
||||
t.Fatalf("expected first build flags to carry first server identity, got %q", firstFlags)
|
||||
}
|
||||
if !strings.Contains(secondFlags, "BuildServerInstanceID=scum-beta") || !strings.Contains(secondFlags, "BuildRegistrationToken=beta-component-key") {
|
||||
t.Fatalf("expected second build flags to carry second server identity, got %q", secondFlags)
|
||||
}
|
||||
if firstFlags == secondFlags {
|
||||
t.Fatalf("expected same-plugin run builds to stay per-server")
|
||||
}
|
||||
|
||||
client := newFakeWorkerClient()
|
||||
worker := &Worker{cfg: workerTestConfig(t), client: client}
|
||||
worker.state.RunEndpointID = "run-test"
|
||||
worker.state.SessionToken = "session-token"
|
||||
client.buildInput = firstInput
|
||||
if err := worker.uploadDistributionArtifact(context.Background(), firstAssignment, firstInput.ArtifactID, []byte("alpha archive")); err != nil {
|
||||
t.Fatalf("upload first artifact: %v", err)
|
||||
}
|
||||
client.artifactPayload = nil
|
||||
client.buildInput = secondInput
|
||||
if err := worker.uploadDistributionArtifact(context.Background(), secondAssignment, secondInput.ArtifactID, []byte("beta archive")); err != nil {
|
||||
t.Fatalf("upload second artifact: %v", err)
|
||||
}
|
||||
if len(client.artifactOpenRequests) != 2 {
|
||||
t.Fatalf("expected two artifact transfer opens, got %+v", client.artifactOpenRequests)
|
||||
}
|
||||
if client.artifactOpenRequests[0].ArtifactID != firstInput.ArtifactID || client.artifactOpenRequests[0].OwnerID != firstAssignment.JobID {
|
||||
t.Fatalf("first artifact upload used wrong scope: %+v", client.artifactOpenRequests[0])
|
||||
}
|
||||
if client.artifactOpenRequests[1].ArtifactID != secondInput.ArtifactID || client.artifactOpenRequests[1].OwnerID != secondAssignment.JobID {
|
||||
t.Fatalf("second artifact upload used wrong scope: %+v", client.artifactOpenRequests[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateDistributionArchiveIncludesExecutableAndConfigForSupportedFormats(t *testing.T) {
|
||||
for _, format := range []string{"tar.gz", "zip"} {
|
||||
t.Run(format, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
executableName := "run"
|
||||
if format == "zip" {
|
||||
executableName = "run.exe"
|
||||
}
|
||||
binaryPath := filepath.Join(root, executableName)
|
||||
configPath := filepath.Join(root, "config.json")
|
||||
if err := os.WriteFile(binaryPath, []byte("binary"), 0o700); err != nil {
|
||||
t.Fatalf("write binary: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte(`{"kind":"run"}`), 0o600); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
archivePath := filepath.Join(root, "package."+strings.ReplaceAll(format, ".", ""))
|
||||
if err := createDistributionArchive(archivePath, format, binaryPath, configPath); err != nil {
|
||||
t.Fatalf("create archive: %v", err)
|
||||
}
|
||||
payload, err := os.ReadFile(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read archive: %v", err)
|
||||
}
|
||||
entries := archiveEntries(t, format, payload)
|
||||
if !entries[executableName] || !entries["config.json"] {
|
||||
t.Fatalf("expected executable and config in %s archive, got %+v", format, entries)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareDistributionSourceCopiesTrustedRunSourceIntoWorkspace(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "go.mod"), []byte("module example.test/trusted\n\ngo 1.24\n"), 0o600); err != nil {
|
||||
t.Fatalf("write go.mod: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sourceRoot, "main.go"), []byte("package main\nfunc main() {}\n"), 0o600); err != nil {
|
||||
t.Fatalf("write main.go: %v", err)
|
||||
}
|
||||
workspace := t.TempDir()
|
||||
worker := &Worker{cfg: workerTestConfig(t)}
|
||||
worker.cfg.BuildSourceRoot = sourceRoot
|
||||
seed := base64.StdEncoding.EncodeToString([]byte(`[{"path":"actions/install.json","content":"{}"}]`))
|
||||
|
||||
prepared, err := worker.prepareDistributionSource(context.Background(), workspace, protocol.DistributionBuildInputResponse{ComponentKind: "run", WorkspaceSeed: seed})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare trusted run source: %v", err)
|
||||
}
|
||||
if prepared != filepath.Join(workspace, "source") {
|
||||
t.Fatalf("expected isolated source under workspace, got %s", prepared)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(prepared, "main.go")); err != nil {
|
||||
t.Fatalf("expected copied source file: %v", err)
|
||||
}
|
||||
seedConfig, err := os.ReadFile(filepath.Join(prepared, "config", "workspace_seed_generated.go"))
|
||||
if err != nil || !strings.Contains(string(seedConfig), seed) {
|
||||
t.Fatalf("expected generated workspace seed config, body=%q err=%v", seedConfig, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(prepared, "main.go"), []byte("package main\n// isolated mutation\nfunc main() {}\n"), 0o600); err != nil {
|
||||
t.Fatalf("mutate isolated copy: %v", err)
|
||||
}
|
||||
original, err := os.ReadFile(filepath.Join(sourceRoot, "main.go"))
|
||||
if err != nil || strings.Contains(string(original), "isolated mutation") {
|
||||
t.Fatalf("trusted source was mutated, body=%q err=%v", original, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDistributionBuildInputRejectsUnapprovedClientSource(t *testing.T) {
|
||||
assignment := protocol.RunJobAssignment{JobID: "job-build", ServerInstanceID: "server-build", RunEndpointID: "run-build"}
|
||||
base := protocol.DistributionBuildInputResponse{
|
||||
JobID: assignment.JobID, ComponentKind: "client-manager", ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID,
|
||||
PluginID: "game.scum", TargetOS: "linux", TargetArch: "amd64", PackageFormat: "tar.gz", ArtifactID: "artifact-build", OutputFilename: "manager", AuthKey: "key", SourceRevision: "main",
|
||||
}
|
||||
for _, repository := range []string{"http://example.test/manager.git", "https://token@example.test/manager.git", "https://example.test/manager.git?ref=main"} {
|
||||
input := base
|
||||
input.RepositoryURL = repository
|
||||
if err := validateDistributionBuildInput(assignment, input); err == nil {
|
||||
t.Fatalf("expected repository %q to be rejected", repository)
|
||||
}
|
||||
}
|
||||
base.RepositoryURL = "https://example.test/manager.git"
|
||||
base.SourceRevision = ""
|
||||
if err := validateDistributionBuildInput(assignment, base); err == nil {
|
||||
t.Fatal("expected an unpinned client-manager source to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDistributionBuildInputAllowsDedicatedRunIdentity(t *testing.T) {
|
||||
assignment := protocol.RunJobAssignment{JobID: "job-build", ServerInstanceID: "server-build", RunEndpointID: "run-local-debug"}
|
||||
input := protocol.DistributionBuildInputResponse{JobID: assignment.JobID, ComponentKind: "run", ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: "server-run-server-build", PluginID: "game.scum", TargetOS: "windows", TargetArch: "amd64", TargetRelease: "release-1", PlatformURL: "https://scum.npc0.com", PackageFormat: "raw-executable", ArtifactID: "artifact-build", OutputFilename: "run.exe", AuthKey: "key"}
|
||||
if err := validateDistributionBuildInput(assignment, input); err != nil {
|
||||
t.Fatalf("dedicated Run identity must be accepted for a builder job: %v", err)
|
||||
}
|
||||
input.WorkspaceSeed = "not-base64"
|
||||
if err := validateDistributionBuildInput(assignment, input); err == nil || !strings.Contains(err.Error(), "workspace seed") {
|
||||
t.Fatalf("expected invalid run workspace seed rejection, got %v", err)
|
||||
}
|
||||
input.WorkspaceSeed = ""
|
||||
input.ComponentKind = "client-manager"
|
||||
input.PackageFormat = "zip"
|
||||
input.RepositoryURL = "https://example.test/manager.git"
|
||||
input.SourceRevision = "main"
|
||||
if err := validateDistributionBuildInput(assignment, input); err == nil {
|
||||
t.Fatal("client-manager build must remain bound to its assigned builder")
|
||||
}
|
||||
}
|
||||
|
||||
func archiveEntries(t *testing.T, format string, payload []byte) map[string]bool {
|
||||
t.Helper()
|
||||
entries := map[string]bool{}
|
||||
if format == "zip" {
|
||||
reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatalf("open zip: %v", err)
|
||||
}
|
||||
for _, file := range reader.File {
|
||||
entries[file.Name] = true
|
||||
}
|
||||
return entries
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("open gzip: %v", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read tar: %v", err)
|
||||
}
|
||||
entries[header.Name] = true
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func extractArchive(t *testing.T, format string, payload []byte, destination string) {
|
||||
t.Helper()
|
||||
if format == "zip" {
|
||||
reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatalf("open zip: %v", err)
|
||||
}
|
||||
for _, file := range reader.File {
|
||||
input, err := file.Open()
|
||||
if err != nil {
|
||||
t.Fatalf("open zip entry: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(input)
|
||||
closeErr := input.Close()
|
||||
if err != nil || closeErr != nil {
|
||||
t.Fatalf("read zip entry: err=%v close=%v", err, closeErr)
|
||||
}
|
||||
mode := os.FileMode(0o600)
|
||||
if file.Name == "run" || strings.HasSuffix(file.Name, ".exe") {
|
||||
mode = 0o700
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destination, file.Name), body, mode); err != nil {
|
||||
t.Fatalf("write zip entry: %v", err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("open gzip: %v", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read tar: %v", err)
|
||||
}
|
||||
mode := os.FileMode(header.Mode)
|
||||
if err := os.WriteFile(filepath.Join(destination, header.Name), mustReadAll(t, reader), mode); err != nil {
|
||||
t.Fatalf("write tar entry: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadAll(t *testing.T, reader io.Reader) []byte {
|
||||
t.Helper()
|
||||
body, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("read archive entry: %v", err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func SupportedDistributionCapabilities() []string {
|
||||
return []string{
|
||||
protocol.RunCapabilityDistributionBuild,
|
||||
protocol.RunCapabilityRunSelfUpdate,
|
||||
protocol.RunCapabilityDependenciesCheck,
|
||||
protocol.RunCapabilityDependenciesInstall,
|
||||
protocol.RunCapabilityLogsBackfill,
|
||||
}
|
||||
}
|
||||
|
||||
func ExecuteDistributionJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
switch assignment.Capability {
|
||||
case protocol.RunCapabilityDistributionBuild:
|
||||
return lifecycleFailure("distribution_build_requires_worker", "distribution build must execute through the authenticated worker")
|
||||
case protocol.RunCapabilityRunSelfUpdate:
|
||||
return lifecycleFailure("self_update_requires_worker", "Run self-update must execute through the authenticated worker")
|
||||
case protocol.RunCapabilityDependenciesCheck, protocol.RunCapabilityDependenciesInstall:
|
||||
return lifecycleFailure("dependency_execution_requires_worker", "dependency execution must execute through the authenticated worker")
|
||||
case protocol.RunCapabilityLogsBackfill:
|
||||
return ExecuteLogBackfillJob(ctx, assignment)
|
||||
default:
|
||||
return lifecycleFailure("unsupported_distribution_capability", "unsupported distribution capability")
|
||||
}
|
||||
}
|
||||
|
||||
func ExecuteSelfUpdateJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_self_update_job", err.Error())
|
||||
}
|
||||
if cancelled, ok := checkContextCancelled(ctx, "run self-update cancelled", "run_self_update_cancelled"); ok {
|
||||
return cancelled
|
||||
}
|
||||
artifactID := strings.TrimPrefix(assignment.InputRef, "artifact://")
|
||||
if strings.TrimSpace(artifactID) == "" || strings.Contains(artifactID, "..") {
|
||||
return lifecycleFailure("unsafe_self_update_artifact", "update artifact ref is unsafe")
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "run self-update staged"},
|
||||
ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", url.PathEscape(assignment.JobID)),
|
||||
Message: "run self-update artifact verified and staged through rollback-safe hook",
|
||||
}
|
||||
}
|
||||
|
||||
func ExecuteDependencyJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_dependency_job", err.Error())
|
||||
}
|
||||
if cancelled, ok := checkContextCancelled(ctx, "dependency action cancelled", "dependency_action_cancelled"); ok {
|
||||
return cancelled
|
||||
}
|
||||
operation := "dependency probe"
|
||||
if assignment.Capability == protocol.RunCapabilityDependenciesInstall {
|
||||
if !strings.HasPrefix(assignment.TargetKey, "dependencies/install/") {
|
||||
return lifecycleFailure("unsafe_dependency_install_plan", "dependency install target must reference a typed install plan")
|
||||
}
|
||||
operation = "dependency install plan"
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: operation + " completed"},
|
||||
ResultRef: fmt.Sprintf("artifact://jobs/%s/dependencies-result", url.PathEscape(assignment.JobID)),
|
||||
Message: operation + " executed through bounded typed envelope",
|
||||
}
|
||||
}
|
||||
|
||||
func ExecuteLogBackfillJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_log_backfill_job", err.Error())
|
||||
}
|
||||
if cancelled, ok := checkContextCancelled(ctx, "log backfill cancelled", "logs_backfill_cancelled"); ok {
|
||||
return cancelled
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "historical log cursor updated"},
|
||||
ResultRef: fmt.Sprintf("artifact://jobs/%s/log-backfill-cursor", url.PathEscape(assignment.JobID)),
|
||||
Message: "historical log backfill cursor stored; log bodies remain on log/artifact channels",
|
||||
}
|
||||
}
|
||||
|
||||
func isSupportedDistributionCapability(capability string) bool {
|
||||
for _, supported := range SupportedDistributionCapabilities() {
|
||||
if capability == supported {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkContextCancelled(ctx context.Context, message string, code string) (LifecycleExecutionResult, bool) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: message},
|
||||
Message: message,
|
||||
ErrorCode: code,
|
||||
}, true
|
||||
default:
|
||||
return LifecycleExecutionResult{}, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,889 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
func TestRunHelperProcess(t *testing.T) {
|
||||
if os.Getenv("RUN_TEST_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
if os.Getenv("RUN_LOG_LINES") == "1" {
|
||||
marker := os.Getenv("RUN_LOG_MARKER")
|
||||
_, _ = os.Stdout.WriteString("managed stdout ready " + marker + "\n")
|
||||
_, _ = os.Stderr.WriteString("managed stderr ready " + marker + "\n")
|
||||
return
|
||||
}
|
||||
if trigger := os.Getenv("RUN_LOG_TRIGGER_FILE"); trigger != "" {
|
||||
for {
|
||||
if _, err := os.Stat(trigger); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
marker := os.Getenv("RUN_LOG_MARKER")
|
||||
_, _ = os.Stdout.WriteString("managed stdout triggered " + marker + "\n")
|
||||
_, _ = os.Stderr.WriteString("managed stderr triggered " + marker + "\n")
|
||||
if os.Getenv("RUN_HOLD_AFTER_LOG") != "1" {
|
||||
return
|
||||
}
|
||||
for {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if os.Getenv("RUN_LOG_TICKS") == "1" {
|
||||
marker := os.Getenv("RUN_LOG_MARKER")
|
||||
for {
|
||||
_, _ = os.Stdout.WriteString("managed stdout tick " + marker + "\n")
|
||||
_, _ = os.Stderr.WriteString("managed stderr tick " + marker + "\n")
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if os.Getenv("RUN_EXIT_NOW") == "1" {
|
||||
return
|
||||
}
|
||||
for {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedProcessStartStreamsManagedOutput(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
setupProcessWorkspace(t, root, assignment, true)
|
||||
scope := processScope(root, assignment)
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{
|
||||
"version": 1,
|
||||
"action": "start",
|
||||
"mode": "supervised",
|
||||
"executableKey": "bin/game-server",
|
||||
"arguments": []string{"-test.run=TestRunHelperProcess"},
|
||||
"environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_LINES": "1"},
|
||||
})
|
||||
logSink := &recordingLogSink{}
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink)).Execute(assignment)
|
||||
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("expected managed process start, got %+v", result)
|
||||
}
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
lines := logSink.snapshot()
|
||||
for len(lines) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
lines = logSink.snapshot()
|
||||
}
|
||||
joined := strings.Join(lines, "\n")
|
||||
if !strings.Contains(joined, "stdout:managed stdout ready") || !strings.Contains(joined, "stderr:managed stderr ready") {
|
||||
t.Fatalf("expected managed stdout/stderr to stream, got %+v", lines)
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
|
||||
func TestTypedProcessOutputCaptureResumesAfterRunRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
setupProcessWorkspace(t, root, assignment, false)
|
||||
scope := processScope(root, assignment)
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{
|
||||
"version": 1,
|
||||
"action": "start",
|
||||
"mode": "supervised",
|
||||
"executableKey": "bin/game-server",
|
||||
"arguments": []string{"-test.run=TestRunHelperProcess"},
|
||||
"environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_TICKS": "1"},
|
||||
})
|
||||
|
||||
started := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment)
|
||||
if started.State != lifecycleResultStateSucceeded || started.ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("expected managed process start, got %+v", started)
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
logSink := &recordingLogSink{}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink))
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
lines := logSink.snapshot()
|
||||
for len(lines) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
lines = logSink.snapshot()
|
||||
}
|
||||
joined := strings.Join(lines, "\n")
|
||||
if !strings.Contains(joined, "stdout:managed stdout tick") || !strings.Contains(joined, "stderr:managed stderr tick") {
|
||||
t.Fatalf("expected restarted run to resume captured output, got %+v", lines)
|
||||
}
|
||||
stop := executionAssignment(protocol.RunCapabilityProcessStop)
|
||||
stop.TargetKey = "actions/stop.json"
|
||||
if stopped := restarted.Execute(stop); stopped.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("stop resumed process: %+v", stopped)
|
||||
}
|
||||
waitForManagedTailers(t, restarted.managed.(*OSManagedProcessSupervisor))
|
||||
}
|
||||
|
||||
func TestManagedProcessOutputAfterCanceledRunContextIsSpooledOnRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{
|
||||
{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"},
|
||||
{Key: "console-stderr", Kind: "process.stderr", StreamKey: "game.console.stderr", CursorKind: "sequence"},
|
||||
}
|
||||
setupProcessWorkspace(t, root, assignment, false)
|
||||
scope := processScope(root, assignment)
|
||||
trigger := filepath.Join(root, "emit-after-cancel")
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{
|
||||
"version": 1,
|
||||
"action": "start",
|
||||
"mode": "supervised",
|
||||
"executableKey": "bin/game-server",
|
||||
"arguments": []string{"-test.run=TestRunHelperProcess"},
|
||||
"environment": map[string]string{
|
||||
"RUN_TEST_HELPER": "1",
|
||||
"RUN_LOG_TRIGGER_FILE": trigger,
|
||||
"RUN_LOG_MARKER": "after-cancel",
|
||||
},
|
||||
})
|
||||
logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
oldCtx, cancelOldRun := context.WithCancel(context.Background())
|
||||
oldSink := &contextRejectingLogSink{delegate: &SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "old-session", Spool: logSpool}}
|
||||
oldExecutor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(oldSink))
|
||||
started := oldExecutor.ExecuteContext(oldCtx, assignment)
|
||||
if started.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start managed process: %+v", started)
|
||||
}
|
||||
oldManaged := oldExecutor.managed.(*OSManagedProcessSupervisor)
|
||||
identity := oldManaged.Status(ProcessIdentity{Scope: scope})
|
||||
if identity.LogSessionID == "" {
|
||||
t.Fatalf("expected managed log session: %+v", identity)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = oldManaged.Stop(context.Background(), identity)
|
||||
})
|
||||
|
||||
cancelOldRun()
|
||||
if err := os.WriteFile(trigger, []byte("emit"), 0o600); err != nil {
|
||||
t.Fatalf("trigger post-cancel output: %v", err)
|
||||
}
|
||||
waitForFileText(t, filepath.Join(root, "state", "process-output", identity.StdoutLogRef), "after-cancel")
|
||||
beforeRestart := waitForManagedState(t, oldManaged, scope, "exited")
|
||||
if beforeRestart.StdoutOffset != identity.StdoutOffset || beforeRestart.StderrOffset != identity.StderrOffset {
|
||||
t.Fatalf("canceled sink advanced output offsets: before=%+v after=%+v", identity, beforeRestart)
|
||||
}
|
||||
if pending, err := logSpool.Pending(); err != nil || len(pending) != 0 {
|
||||
t.Fatalf("canceled sink unexpectedly committed spool entries: pending=%+v err=%v", pending, err)
|
||||
}
|
||||
oldManaged.mu.Lock()
|
||||
oldManaged.stopTailersLocked(identity)
|
||||
oldManaged.mu.Unlock()
|
||||
|
||||
restartedSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("reopen log spool: %v", err)
|
||||
}
|
||||
restarted := NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "new-session", Spool: restartedSpool}),
|
||||
)
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
waitForSpooledText(t, restartedSpool, "managed stdout triggered after-cancel", "managed stderr triggered after-cancel")
|
||||
restartedManaged := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
resumed := waitForManagedOffsets(t, restartedManaged, scope, identity.StdoutOffset, identity.StderrOffset)
|
||||
if resumed.LogSessionID != identity.LogSessionID || resumed.StdoutOffset <= identity.StdoutOffset || resumed.StderrOffset <= identity.StderrOffset {
|
||||
t.Fatalf("restart did not retain session and commit offsets: before=%+v after=%+v", identity, resumed)
|
||||
}
|
||||
for _, batch := range mustPendingLogs(t, restartedSpool) {
|
||||
if batch.LogSessionID != identity.LogSessionID || !batch.SessionStartedAt.Equal(identity.StartedAt) {
|
||||
t.Fatalf("spooled batch lost process session metadata: %+v", batch)
|
||||
}
|
||||
if batch.FirstSeq != 1 || len(batch.Entries) == 0 || batch.Entries[0].Seq != 1 {
|
||||
t.Fatalf("fresh generation stream did not start durably at sequence 1: %+v", batch)
|
||||
}
|
||||
for index, entry := range batch.Entries {
|
||||
if entry.Seq != uint64(index+1) {
|
||||
t.Fatalf("fresh generation stream sequence is not contiguous: %+v", batch)
|
||||
}
|
||||
}
|
||||
}
|
||||
stop := executionAssignment(protocol.RunCapabilityProcessStop)
|
||||
stop.TargetKey = "actions/stop.json"
|
||||
if stopped := restarted.Execute(stop); stopped.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("stop resumed process: %+v", stopped)
|
||||
}
|
||||
waitForManagedTailers(t, restartedManaged)
|
||||
}
|
||||
|
||||
func TestImmediateManagedProcessRestartTailsNewGeneration(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Key: "console-stdout", Kind: "process.stdout", StreamKey: "game.console.stdout", CursorKind: "sequence"}}
|
||||
setupProcessWorkspace(t, root, assignment, false)
|
||||
scope := processScope(root, assignment)
|
||||
writeTickAction := func(marker string) {
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{
|
||||
"version": 1,
|
||||
"action": "start",
|
||||
"mode": "supervised",
|
||||
"executableKey": "bin/game-server",
|
||||
"arguments": []string{"-test.run=TestRunHelperProcess"},
|
||||
"environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_TICKS": "1", "RUN_LOG_MARKER": marker},
|
||||
})
|
||||
}
|
||||
logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
executor := NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "session-token", Spool: logSpool}),
|
||||
)
|
||||
writeTickAction("generation-a")
|
||||
if started := executor.Execute(assignment); started.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start generation A: %+v", started)
|
||||
}
|
||||
managed := executor.managed.(*OSManagedProcessSupervisor)
|
||||
first := managed.Status(ProcessIdentity{Scope: scope})
|
||||
waitForSpooledText(t, logSpool, "generation-a")
|
||||
stop := executionAssignment(protocol.RunCapabilityProcessStop)
|
||||
stop.TargetKey = "actions/stop.json"
|
||||
if stopped := executor.Execute(stop); stopped.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("stop generation A: %+v", stopped)
|
||||
}
|
||||
writeTickAction("generation-b")
|
||||
if started := executor.Execute(assignment); started.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start generation B: %+v", started)
|
||||
}
|
||||
second := managed.Status(ProcessIdentity{Scope: scope})
|
||||
if second.LogSessionID == "" || second.LogSessionID == first.LogSessionID {
|
||||
t.Fatalf("expected a new process log session: first=%+v second=%+v", first, second)
|
||||
}
|
||||
waitForSpooledText(t, logSpool, "generation-b")
|
||||
time.Sleep(managedProcessOutputDrainDelay + 300*time.Millisecond)
|
||||
waitForSpooledTextCount(t, logSpool, "generation-b", 4)
|
||||
sessions := map[string]bool{}
|
||||
for _, batch := range mustPendingLogs(t, logSpool) {
|
||||
sessions[batch.LogSessionID] = true
|
||||
}
|
||||
if !sessions[first.LogSessionID] || !sessions[second.LogSessionID] {
|
||||
t.Fatalf("durable spool did not retain both process generations: sessions=%+v", sessions)
|
||||
}
|
||||
current := managed.Status(ProcessIdentity{Scope: scope})
|
||||
if err := managed.updateOutputOffset(first, "stdout", current.StdoutOffset+1_000_000); err != nil {
|
||||
t.Fatalf("update stale generation offset: %v", err)
|
||||
}
|
||||
if afterOldDrain := managed.Status(ProcessIdentity{Scope: scope}); afterOldDrain.StdoutOffset != current.StdoutOffset {
|
||||
t.Fatalf("old generation corrupted current output offset: before=%+v after=%+v", current, afterOldDrain)
|
||||
}
|
||||
if stopped := executor.Execute(stop); stopped.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("stop generation B: %+v", stopped)
|
||||
}
|
||||
waitForManagedTailers(t, managed)
|
||||
}
|
||||
|
||||
func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "game.console.stdout"}, {Kind: "process.stderr", StreamKey: "game.console.stderr"}}
|
||||
setupProcessWorkspace(t, root, assignment, false)
|
||||
scope := processScope(root, assignment)
|
||||
writeAction := func(marker string) {
|
||||
writeJSONFixture(t, filepath.Join(scope, "actions", "start.json"), map[string]any{
|
||||
"version": 1,
|
||||
"action": "start",
|
||||
"mode": "supervised",
|
||||
"executableKey": "bin/game-server",
|
||||
"arguments": []string{"-test.run=TestRunHelperProcess"},
|
||||
"environment": map[string]string{"RUN_TEST_HELPER": "1", "RUN_LOG_LINES": "1", "RUN_LOG_MARKER": marker},
|
||||
})
|
||||
}
|
||||
oldExecutor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(alwaysRejectingLogSink{}))
|
||||
writeAction("retired-a")
|
||||
if started := oldExecutor.Execute(assignment); started.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start generation A: %+v", started)
|
||||
}
|
||||
oldManaged := oldExecutor.managed.(*OSManagedProcessSupervisor)
|
||||
first := waitForManagedState(t, oldManaged, scope, "exited")
|
||||
waitForFileText(t, filepath.Join(root, "state", "process-output", first.StdoutLogRef), "retired-a")
|
||||
|
||||
writeAction("current-b")
|
||||
if started := oldExecutor.Execute(assignment); started.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start generation B: %+v", started)
|
||||
}
|
||||
second := waitForManagedState(t, oldManaged, scope, "exited")
|
||||
waitForFileText(t, filepath.Join(root, "state", "process-output", second.StdoutLogRef), "current-b")
|
||||
if first.LogSessionID == second.LogSessionID {
|
||||
t.Fatalf("process restart reused log session: first=%+v second=%+v", first, second)
|
||||
}
|
||||
oldManaged.mu.Lock()
|
||||
if len(oldManaged.retired) != 1 {
|
||||
oldManaged.mu.Unlock()
|
||||
t.Fatalf("expected one durable retired generation, got %+v", oldManaged.retired)
|
||||
}
|
||||
oldManaged.stopTailersLocked(first)
|
||||
oldManaged.stopTailersLocked(second)
|
||||
oldManaged.mu.Unlock()
|
||||
|
||||
logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("open restart spool: %v", err)
|
||||
}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "restart-session", Spool: logSpool}))
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
waitForSpooledText(t, logSpool, "retired-a", "current-b")
|
||||
restartedManaged := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
waitForManagedTailers(t, restartedManaged)
|
||||
restartedManaged.mu.Lock()
|
||||
retiredCount := len(restartedManaged.retired)
|
||||
restartedManaged.mu.Unlock()
|
||||
if retiredCount != 0 {
|
||||
t.Fatalf("drained retired generation remained in journal: count=%d", retiredCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedProcessSourceCursorDeduplicatesCommittedLineWithStaleJournalOffset(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
stateDir := filepath.Join(root, "state")
|
||||
outputDir := filepath.Join(stateDir, "process-output")
|
||||
if err := os.MkdirAll(outputDir, 0o700); err != nil {
|
||||
t.Fatalf("create process output dir: %v", err)
|
||||
}
|
||||
line := "committed before offset\n"
|
||||
identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "server-execution", "local"), ServerInstanceID: "server-execution", RunEndpointID: "run-execution", JobID: "execution-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", LogSessionID: "session-stale-offset", PID: 12345, StartedAt: time.Now().UTC(), State: "exited", StdoutLogRef: "stale.stdout.log", StderrLogRef: "stale.stderr.log", StdoutStreamKey: "game.console.stdout", StderrStreamKey: "game.console.stderr"}
|
||||
if err := os.WriteFile(filepath.Join(outputDir, identity.StdoutLogRef), []byte(line), 0o600); err != nil {
|
||||
t.Fatalf("write stale stdout: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(outputDir, identity.StderrLogRef), nil, 0o600); err != nil {
|
||||
t.Fatalf("write stale stderr: %v", err)
|
||||
}
|
||||
body, err := json.Marshal(processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{identity.Scope: identity}, Retired: map[string]ProcessIdentity{}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal process journal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(stateDir, "processes.json"), body, 0o600); err != nil {
|
||||
t.Fatalf("write process journal: %v", err)
|
||||
}
|
||||
logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
assignment := assignmentFromProcessIdentity(identity)
|
||||
sink := &SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "old-run-session", Spool: logSpool}
|
||||
if err := sink.AppendWithCursor(context.Background(), assignment, "stdout", strings.TrimSpace(line), ProcessLogCursor{StartOffset: 0, EndOffset: int64(len(line))}); err != nil {
|
||||
t.Fatalf("commit line before offset: %v", err)
|
||||
}
|
||||
streamID := logStreamIDForAssignment(assignment, identity.StdoutStreamKey)
|
||||
if err := logSpool.Ack(protocol.LogBatchIngestResponse{LogStreamID: streamID, AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
|
||||
t.Fatalf("ack committed line: %v", err)
|
||||
}
|
||||
restartedSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("restart log spool: %v", err)
|
||||
}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "new-run-session", Spool: restartedSpool}))
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
managed := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
resumed := waitForManagedOffsets(t, managed, identity.Scope, 0, -1)
|
||||
waitForManagedTailers(t, managed)
|
||||
if resumed.StdoutOffset != int64(len(line)) {
|
||||
t.Fatalf("stale journal offset was not advanced: %+v", resumed)
|
||||
}
|
||||
if pending := mustPendingLogs(t, restartedSpool); len(pending) != 0 {
|
||||
t.Fatalf("committed source cursor was enqueued twice: %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedProcessSupervisorMigratesLiveLegacySession(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
stateDir := filepath.Join(root, "state")
|
||||
outputDir := filepath.Join(stateDir, "process-output")
|
||||
if err := os.MkdirAll(outputDir, 0o700); err != nil {
|
||||
t.Fatalf("create state dirs: %v", err)
|
||||
}
|
||||
stdout, err := os.OpenFile(filepath.Join(outputDir, "legacy.stdout.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy stdout: %v", err)
|
||||
}
|
||||
stderr, err := os.OpenFile(filepath.Join(outputDir, "legacy.stderr.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
t.Fatalf("open legacy stderr: %v", err)
|
||||
}
|
||||
cmd := exec.Command(mustExecutable(t), "-test.run=TestRunHelperProcess")
|
||||
cmd.Env = append(os.Environ(), "RUN_TEST_HELPER=1", "RUN_LOG_TICKS=1", "RUN_LOG_MARKER=legacy")
|
||||
cmd.Stdout = stdout
|
||||
cmd.Stderr = stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
t.Fatalf("start legacy managed process: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
})
|
||||
startedAt := time.Now().UTC()
|
||||
identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "legacy-server", "local"), ServerInstanceID: "legacy-server", RunEndpointID: "run-execution", JobID: "legacy-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", PID: cmd.Process.Pid, StartedAt: startedAt, CommandFingerprint: "sha256:legacy", State: "running", ObservationSeq: 1, StdoutLogRef: "legacy.stdout.log", StderrLogRef: "legacy.stderr.log", UpdatedAt: startedAt}
|
||||
body, err := json.Marshal(processJournal{Version: 1, Items: map[string]ProcessIdentity{identity.Scope: identity}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal legacy journal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(stateDir, "processes.json"), body, 0o600); err != nil {
|
||||
t.Fatalf("write legacy journal: %v", err)
|
||||
}
|
||||
supervisor, err := NewOSManagedProcessSupervisor(root)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate legacy supervisor: %v", err)
|
||||
}
|
||||
migrated := supervisor.Status(ProcessIdentity{Scope: identity.Scope})
|
||||
if migrated.LogSessionID == "" || migrated.StartedAt.IsZero() {
|
||||
t.Fatalf("live legacy process did not receive a persisted session: %+v", migrated)
|
||||
}
|
||||
persistedBody, err := os.ReadFile(filepath.Join(stateDir, "processes.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read migrated journal: %v", err)
|
||||
}
|
||||
var persisted processJournal
|
||||
if err := json.Unmarshal(persistedBody, &persisted); err != nil {
|
||||
t.Fatalf("decode migrated journal: %v", err)
|
||||
}
|
||||
if persisted.Version != managedProcessJournalVersion || persisted.Items[identity.Scope].LogSessionID != migrated.LogSessionID {
|
||||
t.Fatalf("legacy session migration was not durable: %+v", persisted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedProcessStartStopIsIdempotentAndReconciles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
setupProcessWorkspace(t, root, assignment, false)
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root))
|
||||
started := executor.Execute(assignment)
|
||||
if started.State != lifecycleResultStateSucceeded || started.ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("expected running process, got %+v", started)
|
||||
}
|
||||
managed := executor.managed.(*OSManagedProcessSupervisor)
|
||||
first := managed.Status(ProcessIdentity{Scope: processScope(root, assignment)})
|
||||
if first.LogSessionID == "" {
|
||||
t.Fatalf("expected a persisted supervised log session, got %+v", first)
|
||||
}
|
||||
second := executor.Execute(assignment)
|
||||
current := managed.Status(ProcessIdentity{Scope: processScope(root, assignment)})
|
||||
if second.State != lifecycleResultStateSucceeded || current.PID != first.PID || current.LogSessionID != first.LogSessionID {
|
||||
t.Fatalf("expected idempotent start, first=%+v second=%+v", first, second)
|
||||
}
|
||||
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root))
|
||||
status := executionAssignment(protocol.RunCapabilityProcessStatus)
|
||||
status.TargetKey = "actions/status.json"
|
||||
statusResult := restarted.Execute(status)
|
||||
if statusResult.State != lifecycleResultStateSucceeded || statusResult.ExecutionResult.ProcessState != "running" {
|
||||
t.Fatalf("expected restart reconciliation to retain process, got %+v", statusResult)
|
||||
}
|
||||
resumed := restarted.managed.(*OSManagedProcessSupervisor).Status(ProcessIdentity{Scope: processScope(root, assignment)})
|
||||
if resumed.LogSessionID != first.LogSessionID {
|
||||
t.Fatalf("expected Run restart to retain session, before=%q after=%q", first.LogSessionID, resumed.LogSessionID)
|
||||
}
|
||||
stop := executionAssignment(protocol.RunCapabilityProcessStop)
|
||||
stop.TargetKey = "actions/stop.json"
|
||||
stopped := restarted.Execute(stop)
|
||||
if stopped.State != lifecycleResultStateSucceeded || stopped.ExecutionResult.ProcessState != "stopped" {
|
||||
t.Fatalf("expected stopped process, got %+v", stopped)
|
||||
}
|
||||
if again := restarted.Execute(stop); again.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected idempotent stop, got %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedProcessUnexpectedExitIsReported(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
setupProcessWorkspace(t, root, assignment, true)
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root))
|
||||
if result := executor.Execute(assignment); result.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("start unexpected-exit fixture: %+v", result)
|
||||
}
|
||||
status := executionAssignment(protocol.RunCapabilityProcessStatus)
|
||||
status.TargetKey = "actions/status.json"
|
||||
result := executor.Execute(status)
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for result.ExecutionResult.ProcessState == "running" && time.Now().Before(deadline) {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
result = executor.Execute(status)
|
||||
}
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.ProcessState != "exited" || result.ExecutionResult.ExitClassification == "" {
|
||||
t.Fatalf("expected unexpected exit evidence, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedFileExecutorRejectsEscapesAndWritesAtomically(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
executor, err := NewFileExecutor(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new file executor: %v", err)
|
||||
}
|
||||
assignment := executionAssignment(protocol.RunCapabilityFilesWrite)
|
||||
assignment.TargetKey = "config/server.properties"
|
||||
assignment.ExecutionInput.Content = "name=alpha\n"
|
||||
assignment.ExecutionInput.MaxReadBytes = 64 * 1024
|
||||
first := executor.Execute(context.Background(), assignment)
|
||||
if first.State != lifecycleResultStateSucceeded || first.ExecutionResult.Version != 1 {
|
||||
t.Fatalf("expected first atomic write, got %+v", first)
|
||||
}
|
||||
assignment.ExecutionInput.ExpectedVersion = 99
|
||||
conflict := executor.Execute(context.Background(), assignment)
|
||||
if conflict.State != lifecycleResultStateFailed || conflict.ErrorCode != "file_version_conflict" {
|
||||
t.Fatalf("expected version conflict, got %+v", conflict)
|
||||
}
|
||||
assignment.ExecutionInput.ExpectedVersion = 1
|
||||
assignment.ExecutionInput.ExpectedChecksum = first.ExecutionResult.Checksum
|
||||
assignment.ExecutionInput.Content = "name=beta\n"
|
||||
second := executor.Execute(context.Background(), assignment)
|
||||
if second.State != lifecycleResultStateSucceeded || second.ExecutionResult.Version != 2 {
|
||||
t.Fatalf("expected compare-and-swap write, got %+v", second)
|
||||
}
|
||||
|
||||
symlinkTarget := filepath.Join(root, "outside.txt")
|
||||
if err := os.WriteFile(symlinkTarget, []byte("outside"), 0o600); err != nil {
|
||||
t.Fatalf("write outside fixture: %v", err)
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("scope: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "config"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir config: %v", err)
|
||||
}
|
||||
if err := os.Symlink(symlinkTarget, filepath.Join(scope, "config", "link")); err != nil {
|
||||
t.Fatalf("symlink fixture: %v", err)
|
||||
}
|
||||
assignment.TargetKey = "config/link"
|
||||
if result := executor.Execute(context.Background(), assignment); result.State != lifecycleResultStateFailed {
|
||||
t.Fatalf("expected symlink rejection, got %+v", result)
|
||||
}
|
||||
assignment.TargetKey = "../outside"
|
||||
if result := executor.Execute(context.Background(), assignment); result.State != lifecycleResultStateFailed {
|
||||
t.Fatalf("expected traversal rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedFileExecutorBoundsReads(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
executor, err := NewFileExecutor(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new file executor: %v", err)
|
||||
}
|
||||
assignment := executionAssignment(protocol.RunCapabilityFilesRead)
|
||||
assignment.TargetKey = "logs/latest.log"
|
||||
assignment.ExecutionInput.MaxReadBytes = 4
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("scope: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "logs"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir logs: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scope, assignment.TargetKey), []byte("too large"), 0o600); err != nil {
|
||||
t.Fatalf("write log fixture: %v", err)
|
||||
}
|
||||
result := executor.Execute(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != "file_read_too_large" {
|
||||
t.Fatalf("expected bounded read failure, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedFileExecutorListsDirectories(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
executor, err := NewFileExecutor(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new file executor: %v", err)
|
||||
}
|
||||
assignment := executionAssignment(protocol.RunCapabilityFilesList)
|
||||
assignment.TargetKey = "server-root"
|
||||
assignment.ExecutionInput.Inputs = map[string]string{"path": "", "recursive": "false", "query": ""}
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("scope: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "config", "nested"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scope, "config", "server.properties"), []byte("name=example\n"), 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
result := executor.Execute(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "file.list" {
|
||||
t.Fatalf("expected directory listing, got %+v", result)
|
||||
}
|
||||
var envelope fileListEnvelope
|
||||
if err := json.Unmarshal([]byte(result.ExecutionResult.Content), &envelope); err != nil {
|
||||
t.Fatalf("decode listing: %v", err)
|
||||
}
|
||||
if envelope.DirectoryKey != "server-root" || len(envelope.Entries) != 1 || envelope.Entries[0].LogicalKey != "server-root/config" || envelope.Entries[0].Kind != "directory" {
|
||||
t.Fatalf("unexpected root listing: %+v", envelope)
|
||||
}
|
||||
assignment.ExecutionInput.Inputs["path"] = "config"
|
||||
assignment.ExecutionInput.Inputs["recursive"] = "true"
|
||||
result = executor.Execute(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected recursive listing, got %+v", result)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.ExecutionResult.Content), &envelope); err != nil || len(envelope.Entries) != 2 {
|
||||
t.Fatalf("unexpected recursive listing: %+v err=%v", envelope, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedFileExecutorCancellationLeavesTargetUnchanged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
executor, err := NewFileExecutor(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new file executor: %v", err)
|
||||
}
|
||||
assignment := executionAssignment(protocol.RunCapabilityFilesWrite)
|
||||
assignment.TargetKey = "config/server.properties"
|
||||
assignment.ExecutionInput.Content = "cancelled=true\n"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
result := executor.Execute(ctx, assignment)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != "file_cancelled" {
|
||||
t.Fatalf("expected cancelled write, got %+v", result)
|
||||
}
|
||||
scope, _ := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if _, err := os.Stat(filepath.Join(scope, assignment.TargetKey)); !os.IsNotExist(err) {
|
||||
t.Fatalf("cancelled write changed target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func executionAssignment(capability string) protocol.RunJobAssignment {
|
||||
return protocol.RunJobAssignment{JobID: "execution-job", ServerInstanceID: "server-execution", RunEndpointID: "run-local", Capability: capability, TargetKey: "actions/start.json", InputRef: "input://server-execution/execution", LeaseToken: "lease", Attempt: 1, ExecutionInput: protocol.RunJobExecutionInput{WorkspaceScope: "local", MaxReadBytes: maxExecutionContentBytes}}
|
||||
}
|
||||
|
||||
func processScope(root string, assignment protocol.RunJobAssignment) string {
|
||||
scope, _ := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
return scope
|
||||
}
|
||||
|
||||
func setupProcessWorkspace(t *testing.T, root string, assignment protocol.RunJobAssignment, exits bool) {
|
||||
t.Helper()
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("scope process fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir process fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir executable fixture: %v", err)
|
||||
}
|
||||
binary, err := os.Open(filepath.Join(filepath.Dir(mustExecutable(t)), filepath.Base(mustExecutable(t))))
|
||||
if err != nil {
|
||||
t.Fatalf("open test binary: %v", err)
|
||||
}
|
||||
defer binary.Close()
|
||||
target := filepath.Join(scope, "bin", "game-server")
|
||||
output, err := os.Create(target)
|
||||
if err != nil {
|
||||
t.Fatalf("create test binary: %v", err)
|
||||
}
|
||||
if _, err := io.Copy(output, binary); err != nil {
|
||||
t.Fatalf("copy test binary: %v", err)
|
||||
}
|
||||
if err := output.Chmod(0o700); err != nil {
|
||||
t.Fatalf("chmod test binary: %v", err)
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
t.Fatalf("close test binary: %v", err)
|
||||
}
|
||||
actionDir := filepath.Join(scope, "actions")
|
||||
start := map[string]any{"version": 1, "action": "start", "mode": "supervised", "executableKey": "bin/game-server", "arguments": []string{"-test.run=TestRunHelperProcess"}, "environment": map[string]string{"RUN_TEST_HELPER": "1"}}
|
||||
if exits {
|
||||
start["environment"] = map[string]string{"RUN_TEST_HELPER": "1", "RUN_EXIT_NOW": "1"}
|
||||
}
|
||||
writeJSONFixture(t, filepath.Join(actionDir, "start.json"), start)
|
||||
writeJSONFixture(t, filepath.Join(actionDir, "stop.json"), map[string]any{"version": 1, "action": "stop", "mode": "control"})
|
||||
writeJSONFixture(t, filepath.Join(actionDir, "status.json"), map[string]any{"version": 1, "action": "status", "mode": "control"})
|
||||
}
|
||||
|
||||
func writeJSONFixture(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type contextRejectingLogSink struct {
|
||||
delegate ProcessLogSink
|
||||
}
|
||||
|
||||
type alwaysRejectingLogSink struct{}
|
||||
|
||||
func (alwaysRejectingLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error {
|
||||
return context.Canceled
|
||||
}
|
||||
|
||||
func (sink *contextRejectingLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return sink.delegate.Append(ctx, assignment, stream, line)
|
||||
}
|
||||
|
||||
func waitForFileText(t *testing.T, path string, expected string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
body, err := os.ReadFile(path)
|
||||
if err == nil && strings.Contains(string(body), expected) {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %q in managed output file", expected)
|
||||
}
|
||||
|
||||
func waitForSpooledText(t *testing.T, logSpool spool.LogSpool, expected ...string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(4 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
joined := ""
|
||||
for _, batch := range mustPendingLogs(t, logSpool) {
|
||||
for _, entry := range batch.Entries {
|
||||
joined += entry.Line + "\n"
|
||||
}
|
||||
}
|
||||
matched := true
|
||||
for _, value := range expected {
|
||||
matched = matched && strings.Contains(joined, value)
|
||||
}
|
||||
if matched {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for durable spool entries %q", expected)
|
||||
}
|
||||
|
||||
func mustPendingLogs(t *testing.T, logSpool spool.LogSpool) []protocol.LogBatchIngestRequest {
|
||||
t.Helper()
|
||||
pending, err := logSpool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("read pending log spool: %v", err)
|
||||
}
|
||||
return pending
|
||||
}
|
||||
|
||||
func waitForSpooledTextCount(t *testing.T, logSpool spool.LogSpool, expected string, minimum int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(4 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
count := 0
|
||||
for _, batch := range mustPendingLogs(t, logSpool) {
|
||||
for _, entry := range batch.Entries {
|
||||
if strings.Contains(entry.Line, expected) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
if count >= minimum {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %d durable lines containing %q: %+v", minimum, expected, mustPendingLogs(t, logSpool))
|
||||
}
|
||||
|
||||
func waitForManagedOffsets(t *testing.T, supervisor *OSManagedProcessSupervisor, scope string, stdoutAfter int64, stderrAfter int64) ProcessIdentity {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
identity := supervisor.Status(ProcessIdentity{Scope: scope})
|
||||
if identity.StdoutOffset > stdoutAfter && identity.StderrOffset > stderrAfter {
|
||||
return identity
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
identity := supervisor.Status(ProcessIdentity{Scope: scope})
|
||||
t.Fatalf("timed out waiting for managed output offsets: %+v", identity)
|
||||
return ProcessIdentity{}
|
||||
}
|
||||
|
||||
func waitForManagedState(t *testing.T, supervisor *OSManagedProcessSupervisor, scope string, expected string) ProcessIdentity {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
identity := supervisor.Status(ProcessIdentity{Scope: scope})
|
||||
if identity.State == expected {
|
||||
return identity
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
identity := supervisor.Status(ProcessIdentity{Scope: scope})
|
||||
t.Fatalf("timed out waiting for managed process state %q: %+v", expected, identity)
|
||||
return ProcessIdentity{}
|
||||
}
|
||||
|
||||
func waitForManagedTailers(t *testing.T, supervisor *OSManagedProcessSupervisor) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
supervisor.mu.Lock()
|
||||
count := len(supervisor.tailers)
|
||||
supervisor.mu.Unlock()
|
||||
if count == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
supervisor.mu.Lock()
|
||||
count := len(supervisor.tailers)
|
||||
supervisor.mu.Unlock()
|
||||
t.Fatalf("timed out waiting for managed output tailers to drain: %d active", count)
|
||||
}
|
||||
|
||||
func mustExecutable(t *testing.T) string {
|
||||
t.Helper()
|
||||
path, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("find test executable: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestExecutionResultDoesNotContainPrivateIdentity(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
setupProcessWorkspace(t, root, assignment, true)
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment)
|
||||
encoded, _ := json.Marshal(result)
|
||||
for _, forbidden := range []string{"\"pid\"", "/Users/", "lease-token", "session-token"} {
|
||||
if strings.Contains(string(encoded), forbidden) {
|
||||
t.Fatalf("execution result exposed %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerAdvertisesAndRoutesExecutionCapabilities(t *testing.T) {
|
||||
capabilities := SupportedRunCapabilities()
|
||||
for _, capability := range []string{protocol.RunCapabilityProcessStart, protocol.RunCapabilityProcessStop, protocol.RunCapabilityProcessStatus, protocol.RunCapabilityConfigWrite, protocol.RunCapabilityFilesRead, protocol.RunCapabilityFilesWrite} {
|
||||
if !supportedCapability(capabilities, capability) {
|
||||
t.Fatalf("expected worker capability %s, got %v", capability, capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type FileMetadata struct {
|
||||
Scope string `json:"scope"`
|
||||
Key string `json:"key"`
|
||||
Version int `json:"version"`
|
||||
Checksum string `json:"checksum"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type fileMetadataJournal struct {
|
||||
Version int `json:"version"`
|
||||
Records map[string]FileMetadata `json:"records"`
|
||||
}
|
||||
|
||||
type FileExecutor struct {
|
||||
resolver WorkspaceResolver
|
||||
path string
|
||||
mu sync.Mutex
|
||||
records map[string]FileMetadata
|
||||
}
|
||||
|
||||
func NewFileExecutor(workspaceRoot string) (*FileExecutor, error) {
|
||||
resolver := NewWorkspaceResolver(workspaceRoot)
|
||||
root, err := filepath.Abs(workspaceRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stateDir := filepath.Join(root, "state")
|
||||
if err := ensureDirectory(stateDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
executor := &FileExecutor{resolver: resolver, path: filepath.Join(stateDir, "files.json"), records: map[string]FileMetadata{}}
|
||||
if err := executor.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return executor, nil
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) Execute(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" {
|
||||
return lifecycleExecutionFailure("file_workspace_invalid", "file workspace scope is required", false)
|
||||
}
|
||||
scope, err := executor.resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_workspace_invalid", err.Error(), false)
|
||||
}
|
||||
if assignment.Capability == protocol.RunCapabilityFilesRead {
|
||||
result := executor.read(ctx, scope, assignment)
|
||||
if result.ExecutionResult.Kind == "file" {
|
||||
result.ExecutionResult.Kind = "file.read"
|
||||
}
|
||||
return result
|
||||
}
|
||||
if assignment.Capability == protocol.RunCapabilityFilesList {
|
||||
return executor.list(ctx, scope, assignment)
|
||||
}
|
||||
if assignment.Capability == protocol.RunCapabilityConfigWrite || assignment.Capability == protocol.RunCapabilityFilesWrite {
|
||||
result := executor.write(ctx, scope, assignment)
|
||||
if result.ExecutionResult.Kind == "file" {
|
||||
result.ExecutionResult.Kind = "file.write"
|
||||
}
|
||||
return result
|
||||
}
|
||||
return lifecycleExecutionFailure("unsupported_file_capability", "unsupported file capability", false)
|
||||
}
|
||||
|
||||
type fileListEntry struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
LogicalKey string `json:"logicalKey"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
ModifiedAt string `json:"modifiedAt"`
|
||||
}
|
||||
|
||||
type fileListEnvelope struct {
|
||||
DirectoryKey string `json:"directoryKey"`
|
||||
Path string `json:"path"`
|
||||
Entries []fileListEntry `json:"entries"`
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) list(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file list cancelled", false)
|
||||
}
|
||||
directoryKey := assignment.TargetKey
|
||||
relativePath := strings.TrimSpace(assignment.ExecutionInput.Inputs["path"])
|
||||
if relativePath == "" {
|
||||
relativePath = "."
|
||||
}
|
||||
if relativePath != "." && (!protocol.ValidLogicalFileKey(relativePath) || strings.Contains(relativePath, string(rune(92)))) {
|
||||
return lifecycleExecutionFailure("file_list_failed", "directory path is unsafe", false)
|
||||
}
|
||||
directory := scope
|
||||
var err error
|
||||
if relativePath != "." {
|
||||
directory, err = executor.resolver.ExistingDirectory(scope, relativePath)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_list_failed", err.Error(), false)
|
||||
}
|
||||
}
|
||||
info, err := os.Stat(directory)
|
||||
if err != nil || !info.IsDir() {
|
||||
return lifecycleExecutionFailure("file_list_failed", "target is not a directory", false)
|
||||
}
|
||||
query := strings.ToLower(strings.TrimSpace(assignment.ExecutionInput.Inputs["query"]))
|
||||
recursive := strings.EqualFold(assignment.ExecutionInput.Inputs["recursive"], "true")
|
||||
entries := make([]fileListEntry, 0, 32)
|
||||
resultLimit := assignment.ExecutionInput.MaxReadBytes
|
||||
if resultLimit <= 0 || resultLimit > maxExecutionContentBytes {
|
||||
resultLimit = maxExecutionContentBytes
|
||||
}
|
||||
visit := func(current string, item os.DirEntry) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
name := item.Name()
|
||||
full := filepath.Join(current, name)
|
||||
rel, err := filepath.Rel(directory, full)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if query != "" && !strings.Contains(strings.ToLower(rel), query) {
|
||||
return nil
|
||||
}
|
||||
kind := "file"
|
||||
if item.IsDir() {
|
||||
kind = "directory"
|
||||
} else if !item.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
entryInfo, err := item.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logicalKey := path.Join(directoryKey, rel)
|
||||
if relativePath != "." {
|
||||
logicalKey = path.Join(directoryKey, relativePath, rel)
|
||||
}
|
||||
candidate := append(entries, fileListEntry{Name: name, Kind: kind, RelativePath: rel, LogicalKey: logicalKey, SizeBytes: entryInfo.Size(), ModifiedAt: entryInfo.ModTime().UTC().Format(time.RFC3339Nano)})
|
||||
body, marshalErr := json.Marshal(fileListEnvelope{DirectoryKey: directoryKey, Path: strings.TrimPrefix(relativePath, "."), Entries: candidate})
|
||||
if marshalErr != nil {
|
||||
return marshalErr
|
||||
}
|
||||
if len(body) > resultLimit {
|
||||
return fmt.Errorf("file list exceeds approved read limit")
|
||||
}
|
||||
entries = candidate
|
||||
return nil
|
||||
}
|
||||
if recursive {
|
||||
err = filepath.WalkDir(directory, func(current string, item os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if current == directory {
|
||||
return nil
|
||||
}
|
||||
return visit(filepath.Dir(current), item)
|
||||
})
|
||||
} else {
|
||||
var items []os.DirEntry
|
||||
items, err = os.ReadDir(directory)
|
||||
for _, item := range items {
|
||||
if err == nil {
|
||||
err = visit(directory, item)
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file list cancelled", false)
|
||||
}
|
||||
return lifecycleExecutionFailure("file_list_failed", err.Error(), false)
|
||||
}
|
||||
body, err := json.Marshal(fileListEnvelope{DirectoryKey: directoryKey, Path: strings.TrimPrefix(relativePath, "."), Entries: entries})
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_list_failed", "file list encoding failed", false)
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file list completed"}, Message: "file list completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.list", SizeBytes: int64(len(body)), Content: string(body), Summary: "bounded logical file listing"}}
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) read(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false)
|
||||
}
|
||||
path, err := executor.resolver.ExistingTarget(scope, assignment.TargetKey)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
|
||||
}
|
||||
limit := assignment.ExecutionInput.MaxReadBytes
|
||||
if limit <= 0 || limit > maxExecutionContentBytes {
|
||||
limit = maxExecutionContentBytes
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
|
||||
}
|
||||
if info.Size() > int64(limit) {
|
||||
return lifecycleExecutionFailure("file_read_too_large", "file exceeds approved read limit", false)
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
|
||||
}
|
||||
defer file.Close()
|
||||
content, err := io.ReadAll(io.LimitReader(file, int64(limit)+1))
|
||||
if err != nil || len(content) > limit {
|
||||
return lifecycleExecutionFailure("file_read_too_large", "file exceeds approved read limit", false)
|
||||
}
|
||||
checksum := bytesChecksum(content)
|
||||
metadata := executor.metadata(scope, assignment.TargetKey, checksum, int64(len(content)))
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file read completed"}, Message: "file read completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.read", Version: metadata.Version, Checksum: checksum, SizeBytes: int64(len(content)), Content: string(content), Summary: "bounded regular-file read"}}
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) write(ctx context.Context, scope string, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
content := []byte(assignment.ExecutionInput.Content)
|
||||
if len(content) > maxExecutionContentBytes {
|
||||
return lifecycleExecutionFailure("file_write_too_large", "approved content is too large", false)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false)
|
||||
}
|
||||
path, parent, err := executor.resolver.WritableTarget(scope, assignment.TargetKey)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_target_invalid", err.Error(), false)
|
||||
}
|
||||
executor.mu.Lock()
|
||||
defer executor.mu.Unlock()
|
||||
current, err := executor.currentMetadataLocked(scope, assignment.TargetKey, path)
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_metadata_failed", err.Error(), false)
|
||||
}
|
||||
if current.Version == 0 && assignment.ExecutionInput.ExpectedVersion > 0 {
|
||||
current.Version = assignment.ExecutionInput.ExpectedVersion
|
||||
current.Checksum = assignment.ExecutionInput.ExpectedChecksum
|
||||
}
|
||||
if assignment.ExecutionInput.ExpectedVersion > 0 && current.Version != assignment.ExecutionInput.ExpectedVersion {
|
||||
return lifecycleExecutionFailure("file_version_conflict", "expected version does not match current file", false)
|
||||
}
|
||||
if assignment.ExecutionInput.ExpectedChecksum != "" && current.Checksum != assignment.ExecutionInput.ExpectedChecksum {
|
||||
return lifecycleExecutionFailure("file_checksum_conflict", "expected checksum does not match current file", false)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false)
|
||||
}
|
||||
if err := ensureDirectory(parent); err != nil {
|
||||
return lifecycleExecutionFailure("file_target_invalid", err.Error(), false)
|
||||
}
|
||||
temporary, err := os.CreateTemp(parent, ".run-write-*")
|
||||
if err != nil {
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
defer os.Remove(temporaryName)
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
if _, err := temporary.Write(content); err != nil {
|
||||
_ = temporary.Close()
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
_ = temporary.Close()
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lifecycleExecutionFailure("file_cancelled", "file write cancelled", false)
|
||||
}
|
||||
if err := os.Rename(temporaryName, path); err != nil {
|
||||
return lifecycleExecutionFailure("file_write_failed", err.Error(), false)
|
||||
}
|
||||
checksum := bytesChecksum(content)
|
||||
next := FileMetadata{Scope: scope, Key: assignment.TargetKey, Version: current.Version + 1, Checksum: checksum, SizeBytes: int64(len(content)), UpdatedAt: time.Now().UTC()}
|
||||
executor.records[metadataKey(scope, assignment.TargetKey)] = next
|
||||
if err := executor.persistLocked(); err != nil {
|
||||
return lifecycleExecutionFailure("file_metadata_failed", err.Error(), false)
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file write completed"}, Message: "file write completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.write", Version: next.Version, Checksum: checksum, SizeBytes: int64(len(content)), Summary: "atomic compare-and-swap file write"}}
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) metadata(scope string, key string, checksum string, size int64) FileMetadata {
|
||||
executor.mu.Lock()
|
||||
defer executor.mu.Unlock()
|
||||
item := executor.records[metadataKey(scope, key)]
|
||||
if item.Version == 0 {
|
||||
item = FileMetadata{Scope: scope, Key: key, Version: 1}
|
||||
}
|
||||
item.Checksum, item.SizeBytes, item.UpdatedAt = checksum, size, time.Now().UTC()
|
||||
executor.records[metadataKey(scope, key)] = item
|
||||
_ = executor.persistLocked()
|
||||
return item
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) currentMetadataLocked(scope string, key string, path string) (FileMetadata, error) {
|
||||
item := executor.records[metadataKey(scope, key)]
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return FileMetadata{}, err
|
||||
}
|
||||
if err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return FileMetadata{}, fmt.Errorf("target must be a regular file")
|
||||
}
|
||||
body, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return FileMetadata{}, readErr
|
||||
}
|
||||
checksum := bytesChecksum(body)
|
||||
if item.Version == 0 {
|
||||
item.Version = 1
|
||||
}
|
||||
item.Scope, item.Key, item.Checksum, item.SizeBytes = scope, key, checksum, int64(len(body))
|
||||
} else if item.Version == 0 {
|
||||
item.Scope, item.Key = scope, key
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) load() error {
|
||||
body, err := os.ReadFile(executor.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var file fileMetadataJournal
|
||||
if err := json.Unmarshal(body, &file); err != nil {
|
||||
return fmt.Errorf("decode file metadata journal: %w", err)
|
||||
}
|
||||
for key, item := range file.Records {
|
||||
executor.records[key] = item
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (executor *FileExecutor) persistLocked() error {
|
||||
body, err := json.Marshal(fileMetadataJournal{Version: 1, Records: executor.records})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporary := executor.path + ".tmp"
|
||||
if err := os.WriteFile(temporary, body, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, executor.path); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func metadataKey(scope string, key string) string { return scope + "\x00" + key }
|
||||
@@ -0,0 +1,293 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const jobJournalVersion = 1
|
||||
|
||||
type JobJournal struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
active map[string]protocol.RunJobAssignment
|
||||
pendingResults map[string]protocol.RunJobResultRequest
|
||||
pendingActivations map[string]string
|
||||
}
|
||||
|
||||
type jobJournalFile struct {
|
||||
Version int `json:"version"`
|
||||
Active []protocol.RunJobAssignment `json:"active"`
|
||||
PendingResults []protocol.RunJobResultRequest `json:"pendingResults,omitempty"`
|
||||
PendingActivations map[string]string `json:"pendingActivations,omitempty"`
|
||||
}
|
||||
|
||||
func NewJobJournal() *JobJournal {
|
||||
return &JobJournal{active: map[string]protocol.RunJobAssignment{}, pendingResults: map[string]protocol.RunJobResultRequest{}, pendingActivations: map[string]string{}}
|
||||
}
|
||||
|
||||
func NewPersistentJobJournal(workspaceRoot string) (*JobJournal, error) {
|
||||
if workspaceRoot == "" {
|
||||
workspaceRoot = filepath.Join(".", ".run-workspace")
|
||||
}
|
||||
dir := filepath.Join(workspaceRoot, "state")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create job journal directory: %w", err)
|
||||
}
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("secure job journal directory: %w", err)
|
||||
}
|
||||
journal := &JobJournal{path: filepath.Join(dir, "jobs.json"), active: map[string]protocol.RunJobAssignment{}, pendingResults: map[string]protocol.RunJobResultRequest{}, pendingActivations: map[string]string{}}
|
||||
if err := journal.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return journal, nil
|
||||
}
|
||||
|
||||
func (journal *JobJournal) Store(job protocol.RunJobAssignment) error {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
if err := validateJournalAssignment(job); err != nil {
|
||||
return err
|
||||
}
|
||||
previous, existed := journal.active[job.JobID]
|
||||
journal.active[job.JobID] = job
|
||||
if err := journal.persistLocked(); err != nil {
|
||||
if existed {
|
||||
journal.active[job.JobID] = previous
|
||||
} else {
|
||||
delete(journal.active, job.JobID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (journal *JobJournal) Delete(jobID string) error {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
previous, existed := journal.active[jobID]
|
||||
previousResult, hadResult := journal.pendingResults[jobID]
|
||||
previousActivation, hadActivation := journal.pendingActivations[jobID]
|
||||
delete(journal.active, jobID)
|
||||
delete(journal.pendingResults, jobID)
|
||||
delete(journal.pendingActivations, jobID)
|
||||
if err := journal.persistLocked(); err != nil {
|
||||
if existed {
|
||||
journal.active[jobID] = previous
|
||||
}
|
||||
if hadResult {
|
||||
journal.pendingResults[jobID] = previousResult
|
||||
}
|
||||
if hadActivation {
|
||||
journal.pendingActivations[jobID] = previousActivation
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (journal *JobJournal) StorePendingResult(result protocol.RunJobResultRequest, activationManifest string) error {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
assignment, exists := journal.active[result.JobID]
|
||||
if !exists || assignment.Attempt != result.Attempt || assignment.LeaseToken != result.LeaseToken || assignment.RunEndpointID != result.RunEndpointID {
|
||||
return fmt.Errorf("pending result does not match active journal attempt")
|
||||
}
|
||||
if result.State != "succeeded" && result.State != "failed" && result.State != "cancelled" {
|
||||
return fmt.Errorf("pending result state is not terminal")
|
||||
}
|
||||
result.SessionToken = ""
|
||||
previous, hadPrevious := journal.pendingResults[result.JobID]
|
||||
previousActivation, hadActivation := journal.pendingActivations[result.JobID]
|
||||
journal.pendingResults[result.JobID] = result
|
||||
if activationManifest != "" {
|
||||
journal.pendingActivations[result.JobID] = activationManifest
|
||||
} else {
|
||||
delete(journal.pendingActivations, result.JobID)
|
||||
}
|
||||
if err := journal.persistLocked(); err != nil {
|
||||
if hadPrevious {
|
||||
journal.pendingResults[result.JobID] = previous
|
||||
} else {
|
||||
delete(journal.pendingResults, result.JobID)
|
||||
}
|
||||
if hadActivation {
|
||||
journal.pendingActivations[result.JobID] = previousActivation
|
||||
} else {
|
||||
delete(journal.pendingActivations, result.JobID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (journal *JobJournal) PendingActivation(jobID string) string {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
return journal.pendingActivations[jobID]
|
||||
}
|
||||
|
||||
func (journal *JobJournal) PendingResult(jobID string) (protocol.RunJobResultRequest, bool) {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
result, exists := journal.pendingResults[jobID]
|
||||
return result, exists
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkActive(job protocol.RunJobAssignment) {
|
||||
_ = journal.Store(job)
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkTerminal(jobID string) {
|
||||
_ = journal.Delete(jobID)
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveJobs() []protocol.RunJobAssignment {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
jobs := make([]protocol.RunJobAssignment, 0, len(journal.active))
|
||||
for _, job := range journal.active {
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
sort.Slice(jobs, func(i, j int) bool { return jobs[i].JobID < jobs[j].JobID })
|
||||
return jobs
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ReconcileEntries() []protocol.RunJobReconcileEntry {
|
||||
jobs := journal.ActiveJobs()
|
||||
entries := make([]protocol.RunJobReconcileEntry, len(jobs))
|
||||
for i, job := range jobs {
|
||||
entries[i] = protocol.RunJobReconcileEntry{JobID: job.JobID, LeaseToken: job.LeaseToken, Attempt: job.Attempt}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveJobIDs() []string {
|
||||
jobs := journal.ActiveJobs()
|
||||
ids := make([]string, len(jobs))
|
||||
for i, job := range jobs {
|
||||
ids[i] = job.JobID
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveCount() int {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
return len(journal.active)
|
||||
}
|
||||
|
||||
func (journal *JobJournal) load() error {
|
||||
payload, err := os.ReadFile(journal.path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read job journal: %w", err)
|
||||
}
|
||||
var snapshot jobJournalFile
|
||||
if err := json.Unmarshal(payload, &snapshot); err != nil {
|
||||
return fmt.Errorf("decode job journal: %w", err)
|
||||
}
|
||||
if snapshot.Version != jobJournalVersion {
|
||||
return fmt.Errorf("unsupported job journal version %d", snapshot.Version)
|
||||
}
|
||||
for _, job := range snapshot.Active {
|
||||
if err := validateJournalAssignment(job); err != nil {
|
||||
return fmt.Errorf("invalid job journal entry: %w", err)
|
||||
}
|
||||
if _, exists := journal.active[job.JobID]; exists {
|
||||
return fmt.Errorf("duplicate job journal entry %q", job.JobID)
|
||||
}
|
||||
journal.active[job.JobID] = job
|
||||
}
|
||||
for _, result := range snapshot.PendingResults {
|
||||
assignment, exists := journal.active[result.JobID]
|
||||
if !exists || result.SessionToken != "" || assignment.Attempt != result.Attempt || assignment.LeaseToken != result.LeaseToken || assignment.RunEndpointID != result.RunEndpointID {
|
||||
return fmt.Errorf("invalid pending result journal entry for %q", result.JobID)
|
||||
}
|
||||
if _, duplicate := journal.pendingResults[result.JobID]; duplicate {
|
||||
return fmt.Errorf("duplicate pending result journal entry %q", result.JobID)
|
||||
}
|
||||
journal.pendingResults[result.JobID] = result
|
||||
}
|
||||
for jobID, manifest := range snapshot.PendingActivations {
|
||||
if _, exists := journal.pendingResults[jobID]; !exists || manifest == "" {
|
||||
return fmt.Errorf("invalid pending activation journal entry for %q", jobID)
|
||||
}
|
||||
journal.pendingActivations[jobID] = manifest
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (journal *JobJournal) persistLocked() error {
|
||||
if journal.path == "" {
|
||||
return nil
|
||||
}
|
||||
jobs := make([]protocol.RunJobAssignment, 0, len(journal.active))
|
||||
for _, job := range journal.active {
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
sort.Slice(jobs, func(i, j int) bool { return jobs[i].JobID < jobs[j].JobID })
|
||||
results := make([]protocol.RunJobResultRequest, 0, len(journal.pendingResults))
|
||||
for _, result := range journal.pendingResults {
|
||||
result.SessionToken = ""
|
||||
results = append(results, result)
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].JobID < results[j].JobID })
|
||||
activations := make(map[string]string, len(journal.pendingActivations))
|
||||
for jobID, manifest := range journal.pendingActivations {
|
||||
activations[jobID] = manifest
|
||||
}
|
||||
payload, err := json.MarshalIndent(jobJournalFile{Version: jobJournalVersion, Active: jobs, PendingResults: results, PendingActivations: activations}, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode job journal: %w", err)
|
||||
}
|
||||
temporary := journal.path + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open temporary job journal: %w", err)
|
||||
}
|
||||
removeTemporary := true
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
if removeTemporary {
|
||||
_ = os.Remove(temporary)
|
||||
}
|
||||
}()
|
||||
if _, err := file.Write(payload); err != nil {
|
||||
return fmt.Errorf("write job journal: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return fmt.Errorf("sync job journal: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("close job journal: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporary, journal.path); err != nil {
|
||||
return fmt.Errorf("replace job journal: %w", err)
|
||||
}
|
||||
removeTemporary = false
|
||||
if err := os.Chmod(journal.path, 0o600); err != nil {
|
||||
return fmt.Errorf("secure job journal: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateJournalAssignment(job protocol.RunJobAssignment) error {
|
||||
if err := protocol.ValidateRunJobAssignment(job); err != nil {
|
||||
return err
|
||||
}
|
||||
if job.Attempt <= 0 || job.LeaseToken == "" {
|
||||
return fmt.Errorf("job attempt and lease token are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestPersistentJobJournalReloadsAtomicallyWithOwnerOnlyPermissions(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
journal, err := NewPersistentJobJournal(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new journal: %v", err)
|
||||
}
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
if err := journal.Store(assignment); err != nil {
|
||||
t.Fatalf("store assignment: %v", err)
|
||||
}
|
||||
|
||||
journalPath := filepath.Join(root, "state", "jobs.json")
|
||||
info, err := os.Stat(journalPath)
|
||||
if err != nil {
|
||||
t.Fatalf("stat journal: %v", err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("expected journal mode 0600, got %o", info.Mode().Perm())
|
||||
}
|
||||
dirInfo, err := os.Stat(filepath.Dir(journalPath))
|
||||
if err != nil || dirInfo.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("expected state directory mode 0700, info=%+v err=%v", dirInfo, err)
|
||||
}
|
||||
|
||||
reloaded, err := NewPersistentJobJournal(root)
|
||||
if err != nil {
|
||||
t.Fatalf("reload journal: %v", err)
|
||||
}
|
||||
if jobs := reloaded.ActiveJobs(); len(jobs) != 1 || jobs[0].JobID != assignment.JobID || jobs[0].LeaseToken != assignment.LeaseToken || jobs[0].Attempt != assignment.Attempt {
|
||||
t.Fatalf("unexpected reloaded assignments: %+v", jobs)
|
||||
}
|
||||
if err := reloaded.Delete(assignment.JobID); err != nil {
|
||||
t.Fatalf("delete assignment: %v", err)
|
||||
}
|
||||
third, err := NewPersistentJobJournal(root)
|
||||
if err != nil || third.ActiveCount() != 0 {
|
||||
t.Fatalf("terminal delete did not persist: count=%d err=%v", third.ActiveCount(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentJobJournalRejectsCorruptState(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "state")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatalf("create state dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "jobs.json"), []byte(`{"version":1,"active":[`), 0o600); err != nil {
|
||||
t.Fatalf("write corrupt journal: %v", err)
|
||||
}
|
||||
if _, err := NewPersistentJobJournal(root); err == nil {
|
||||
t.Fatal("expected corrupt journal to fail closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRestartReconcilesAndRecoversConfirmedAttempt(t *testing.T) {
|
||||
cfg := workerTestConfig(t)
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ProgressSequence = 5
|
||||
journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("new journal: %v", err)
|
||||
}
|
||||
if err := journal.Store(assignment); err != nil {
|
||||
t.Fatalf("store interrupted assignment: %v", err)
|
||||
}
|
||||
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = assignment
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{
|
||||
Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{assignment}, ServerTime: workerTestTime(),
|
||||
}
|
||||
restarted, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "recovered\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("restart worker: %v", err)
|
||||
}
|
||||
if err := restarted.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register restarted worker: %v", err)
|
||||
}
|
||||
if err := restarted.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile restarted worker: %v", err)
|
||||
}
|
||||
if len(client.reconcileRequests) != 1 || !reflect.DeepEqual(client.reconcileRequests[0].ActiveJobs, []protocol.RunJobReconcileEntry{{JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt}}) {
|
||||
t.Fatalf("reconcile omitted attempt evidence: %+v", client.reconcileRequests)
|
||||
}
|
||||
if err := restarted.RecoverActiveJobs(context.Background()); err != nil {
|
||||
t.Fatalf("recover confirmed assignment: %v", err)
|
||||
}
|
||||
if restarted.journal.ActiveCount() != 0 || len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || client.progressRequests[0].Sequence != 6 || len(client.resultRequests) != 1 {
|
||||
t.Fatalf("confirmed attempt was not recovered: journal=%d ack=%d result=%d", restarted.journal.ActiveCount(), len(client.ackRequests), len(client.resultRequests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerReconcileDiscardsStaleAttemptWithoutExecuting(t *testing.T) {
|
||||
cfg := workerTestConfig(t)
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("new journal: %v", err)
|
||||
}
|
||||
if err := journal.Store(assignment); err != nil {
|
||||
t.Fatalf("store assignment: %v", err)
|
||||
}
|
||||
client := newFakeWorkerClient()
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, DiscardJobIDs: []string{assignment.JobID}, ServerTime: workerTestTime()}
|
||||
worker, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if worker.journal.ActiveCount() != 0 || len(client.ackRequests) != 0 {
|
||||
t.Fatalf("stale assignment was not discarded safely: journal=%d ack=%d", worker.journal.ActiveCount(), len(client.ackRequests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRetainsJournalWhenPlatformRejectsResultTransport(t *testing.T) {
|
||||
cfg := workerTestConfig(t)
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
client.resultErr = context.DeadlineExceeded
|
||||
worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "completed locally\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if handled, err := worker.ClaimAndRunOnce(context.Background()); !handled || err == nil {
|
||||
t.Fatalf("expected retained failed result transport, handled=%v err=%v", handled, err)
|
||||
}
|
||||
if worker.journal.ActiveCount() != 1 {
|
||||
t.Fatalf("result transport failure removed journal entry")
|
||||
}
|
||||
reloaded, err := NewPersistentJobJournal(cfg.WorkspaceRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("reload retained journal: %v", err)
|
||||
}
|
||||
if reloaded.ActiveCount() != 1 {
|
||||
t.Fatalf("retained journal did not survive restart: count=%d", reloaded.ActiveCount())
|
||||
}
|
||||
if pending, ok := reloaded.PendingResult(client.claimJob.JobID); !ok || pending.SessionToken != "" || pending.State != "succeeded" {
|
||||
t.Fatalf("pending terminal result was not retained safely: result=%+v ok=%v", pending, ok)
|
||||
}
|
||||
payload, err := os.ReadFile(filepath.Join(cfg.WorkspaceRoot, "state", "jobs.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read retained journal: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), "session-token") {
|
||||
t.Fatalf("journal persisted raw Run session token: %s", payload)
|
||||
}
|
||||
|
||||
client.resultErr = nil
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{client.claimJob}, ServerTime: workerTestTime()}
|
||||
restarted, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("restart result worker: %v", err)
|
||||
}
|
||||
if err := restarted.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register result worker: %v", err)
|
||||
}
|
||||
if err := restarted.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile result worker: %v", err)
|
||||
}
|
||||
ackCount := len(client.ackRequests)
|
||||
if err := restarted.RecoverActiveJobs(context.Background()); err != nil {
|
||||
t.Fatalf("replay pending result: %v", err)
|
||||
}
|
||||
if len(client.ackRequests) != ackCount || restarted.journal.ActiveCount() != 0 {
|
||||
t.Fatalf("pending result replay re-executed work: ack before=%d after=%d journal=%d", ackCount, len(client.ackRequests), restarted.journal.ActiveCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRecoversAcceptedSelfUpdateResultAndActivatesOnce(t *testing.T) {
|
||||
cfg := workerTestConfig(t)
|
||||
client := newFakeWorkerClient()
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate)
|
||||
assignment.TargetKey = "run/update"
|
||||
assignment.InputRef = "artifact://artifact-run-recovery"
|
||||
client.claimJob = assignment
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: cfg.RunEndpointID, ConfirmedJobs: []protocol.RunJobAssignment{assignment}, ServerTime: workerTestTime()}
|
||||
|
||||
journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("create self-update journal: %v", err)
|
||||
}
|
||||
if err := journal.Store(assignment); err != nil {
|
||||
t.Fatalf("store self-update assignment: %v", err)
|
||||
}
|
||||
pending := protocol.RunJobResultRequest{RunEndpointID: assignment.RunEndpointID, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, State: "succeeded", Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update staged"}, ResultRef: "artifact://jobs/run-update/staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: bytesChecksum([]byte("archive")), Summary: "verified update staged"}}
|
||||
manifestPath := filepath.Join(cfg.WorkspaceRoot, "self-updates", assignment.JobID, "manifest.json")
|
||||
if err := journal.StorePendingResult(pending, manifestPath); err != nil {
|
||||
t.Fatalf("store pending self-update result: %v", err)
|
||||
}
|
||||
|
||||
activator := &recordingSelfUpdateActivator{}
|
||||
restarted, err := NewWorker(cfg, client, WithSelfUpdateActivator(activator))
|
||||
if err != nil {
|
||||
t.Fatalf("restart worker: %v", err)
|
||||
}
|
||||
if err := restarted.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register restarted worker: %v", err)
|
||||
}
|
||||
if err := restarted.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile restarted worker: %v", err)
|
||||
}
|
||||
if err := restarted.RecoverActiveJobs(context.Background()); err != nil {
|
||||
t.Fatalf("recover accepted self-update result: %v", err)
|
||||
}
|
||||
if activator.manifestPath != manifestPath || restarted.journal.ActiveCount() != 0 || restarted.journal.PendingActivation(assignment.JobID) != "" {
|
||||
t.Fatalf("self-update activation was not recovered exactly once: path=%q active=%d pending=%q", activator.manifestPath, restarted.journal.ActiveCount(), restarted.journal.PendingActivation(assignment.JobID))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,583 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestLifecycleExecutorHandlesSupportedJobs(t *testing.T) {
|
||||
executor := NewLifecycleExecutor()
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
assignment := lifecycleAssignment(capability)
|
||||
result := executor.Execute(assignment)
|
||||
if result.State != "succeeded" || result.Progress.Percent != 100 || result.ResultRef == "" {
|
||||
t.Fatalf("expected successful bounded result for %s, got %+v", capability, result)
|
||||
}
|
||||
for _, forbidden := range []string{"host path", "/Users/", "run socket", "api_key", "sk-"} {
|
||||
if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) {
|
||||
t.Fatalf("lifecycle result exposed forbidden content %q: %+v", forbidden, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedRunOmitsBuildOnlyCapabilities(t *testing.T) {
|
||||
generated := SupportedRunCapabilitiesForComponent("run")
|
||||
if containsCapability(generated, protocol.RunCapabilityDistributionBuild) {
|
||||
t.Fatalf("generated Run must not advertise %s: %v", protocol.RunCapabilityDistributionBuild, generated)
|
||||
}
|
||||
if !containsCapability(generated, protocol.RunCapabilityRunSelfUpdate) {
|
||||
t.Fatalf("generated Run must advertise %s: %v", protocol.RunCapabilityRunSelfUpdate, generated)
|
||||
}
|
||||
for _, capability := range []string{"deployment.scum.v1"} {
|
||||
if containsCapability(generated, capability) {
|
||||
t.Fatalf("generated Run must not advertise %s: %v", capability, generated)
|
||||
}
|
||||
}
|
||||
for _, capability := range []string{protocol.RunCapabilityProcessStart, protocol.RunCapabilityDependenciesInstall} {
|
||||
if !containsCapability(generated, capability) {
|
||||
t.Fatalf("generated Run must retain %s: %v", capability, generated)
|
||||
}
|
||||
}
|
||||
if !containsCapability(SupportedRunCapabilitiesForComponent(""), protocol.RunCapabilityDistributionBuild) {
|
||||
t.Fatal("generic build worker must retain distribution.build")
|
||||
}
|
||||
if containsCapability(SupportedRunCapabilitiesForComponent(""), "deployment.scum.v1") {
|
||||
t.Fatal("generic run must not advertise game-specific SCUM deployment capability")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsLegacyGameSpecificDeploymentPlan(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessInstall)
|
||||
assignment.ExecutionInput.Deployment = &protocol.ServerDeploymentExecution{SchemaVersion: "1", Mode: "guided-install", ServerRoot: "C:/scumserver", Revision: 1}
|
||||
assignment.ExecutionInput.ServerDeploymentPlan = &protocol.ServerDeploymentPlan{SchemaVersion: "1", PluginID: "game.scum", TemplateKey: "scum-steamcmd-windows"}
|
||||
|
||||
result := NewLifecycleExecutor().Execute(assignment)
|
||||
|
||||
if result.State != "failed" || result.ErrorCode != "unsupported_legacy_deployment_plan" {
|
||||
t.Fatalf("expected legacy deployment plan rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsupportedJobs(t *testing.T) {
|
||||
result := NewLifecycleExecutor().Execute(lifecycleAssignment("files.write"))
|
||||
if result.State != "failed" || result.ErrorCode != "unsupported_lifecycle_capability" || result.ResultRef != "" {
|
||||
t.Fatalf("expected unsupported lifecycle failure, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleResultRequestUsesAssignmentLease(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
execution := NewLifecycleExecutor().Execute(assignment)
|
||||
request := LifecycleResultRequest(assignment, "session-token", execution)
|
||||
|
||||
if request.RunEndpointID != assignment.RunEndpointID || request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken || request.Attempt != assignment.Attempt {
|
||||
t.Fatalf("expected result request to use assignment lease, got %+v", request)
|
||||
}
|
||||
if request.SessionToken != "session-token" || request.State != "succeeded" {
|
||||
t.Fatalf("unexpected result request: %+v", request)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRunsScopedCommandTemplateAndHooks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
template := LifecycleActionTemplate{
|
||||
Command: []string{"echo", "server-ready"},
|
||||
Env: map[string]string{"GAME_MODE": "test"},
|
||||
}
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "start.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template: %v", err)
|
||||
}
|
||||
assignment.TargetKey = "actions/start.json"
|
||||
logSink := &recordingLogSink{}
|
||||
artifactHook := &recordingArtifactHook{}
|
||||
|
||||
result := NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithProcessLogSink(logSink),
|
||||
WithLifecycleArtifactHook(artifactHook),
|
||||
).Execute(assignment)
|
||||
|
||||
if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/lifecycle-result" {
|
||||
t.Fatalf("expected scoped lifecycle success, got %+v", result)
|
||||
}
|
||||
if len(logSink.lines) != 1 || logSink.lines[0] != "stdout:server-ready" {
|
||||
t.Fatalf("expected process stdout to be logged, got %+v", logSink.lines)
|
||||
}
|
||||
if !artifactHook.called {
|
||||
t.Fatal("expected artifact hook to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRunsTypedInstallActionWithDeploymentInputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessInstall)
|
||||
assignment.TargetKey = "actions/install.json"
|
||||
assignment.ExecutionInput.WorkspaceScope = "run-local"
|
||||
assignment.ExecutionInput.PluginID = "game.example"
|
||||
assignment.ExecutionInput.LifecycleOperation = "install"
|
||||
assignment.ExecutionInput.Deployment = &protocol.ServerDeploymentExecution{
|
||||
SchemaVersion: "1",
|
||||
Mode: "guided-install",
|
||||
ProfileKey: "run-local",
|
||||
ServerRoot: "D:/game-server",
|
||||
CreateInputs: map[string]string{"gamePort": "27000", "maxPlayers": "128"},
|
||||
Revision: 3,
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve scope: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(scope, "bin"), 0o755); err != nil {
|
||||
t.Fatalf("create bin dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scope, "bin", "install-server"), []byte("plugin-owned helper"), 0o700); err != nil {
|
||||
t.Fatalf("write helper: %v", err)
|
||||
}
|
||||
body, err := json.Marshal(LifecycleActionTemplate{Version: 1, Action: "install", Mode: "oneshot", ExecutableKey: "bin/install-server", Environment: map[string]string{"GAME_ID": "example"}, OutputMode: "console", TimeoutMS: int((90 * time.Minute) / time.Millisecond)})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal action: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scope, "actions", "install.json"), body, 0o600); err != nil {
|
||||
t.Fatalf("write action: %v", err)
|
||||
}
|
||||
supervisor := &recordingSupervisor{}
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessSupervisor(supervisor)).Execute(assignment)
|
||||
|
||||
if result.State != "succeeded" || !strings.HasSuffix(supervisor.command.Args[0], filepath.Join("bin", "install-server")) {
|
||||
t.Fatalf("expected typed install helper execution, result=%+v command=%+v", result, supervisor.command)
|
||||
}
|
||||
if supervisor.command.Env["SERVER_ROOT"] != "D:/game-server" || supervisor.command.Env["SERVER_CREATE_GAMEPORT"] != "27000" || supervisor.command.Env["SERVER_CREATE_MAXPLAYERS"] != "128" || supervisor.command.Env["SERVER_REVISION"] != "3" {
|
||||
t.Fatalf("expected deployment inputs in typed action environment, got %+v", supervisor.command.Env)
|
||||
}
|
||||
if supervisor.command.Timeout != 90*time.Minute {
|
||||
t.Fatalf("expected plugin-declared long lifecycle timeout, got %s", supervisor.command.Timeout)
|
||||
}
|
||||
if supervisor.command.OutputMode != "console" {
|
||||
t.Fatalf("expected plugin-declared output mode, got %q", supervisor.command.OutputMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsafeTemplates(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
for name, template := range map[string]LifecycleActionTemplate{
|
||||
"absolute": {Command: []string{"/bin/echo", "nope"}},
|
||||
"shell": {Command: []string{"sh", "-c", "echo nope"}},
|
||||
"secret": {Command: []string{"echo", "sk-secret"}},
|
||||
"env": {Command: []string{"echo", "ok"}, Env: map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}},
|
||||
} {
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
actionPath := filepath.Join(serverRoot, "actions", name+".json")
|
||||
if err := os.WriteFile(actionPath, body, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
unsafeAssignment := assignment
|
||||
unsafeAssignment.TargetKey = "actions/" + name + ".json"
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(unsafeAssignment)
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected unsafe command rejection for %s, got %+v", name, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsWorkspaceEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.TargetKey = "../outside.json"
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment)
|
||||
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected workspace escape rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeWorkspaceSeedWritesPluginAssetsToProfileScope(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seed, err := json.Marshal([]workspaceSeedFile{
|
||||
{Path: "actions/install.json", Content: `{"version":1,"action":"install","mode":"oneshot"}`, Mode: 0o600},
|
||||
{Path: "bin/install-server", Content: "echo install\n", Mode: 0o700},
|
||||
{Path: "assets/map.bin", Content: base64.StdEncoding.EncodeToString([]byte{0xff, 0x00, 0x7f}), Encoding: "base64", Mode: 0o600},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal seed: %v", err)
|
||||
}
|
||||
|
||||
err = MaterializeWorkspaceSeed(config.Config{
|
||||
WorkspaceRoot: root,
|
||||
ServerInstanceID: "server-seeded",
|
||||
ComponentKey: "run-local",
|
||||
WorkspaceSeed: base64.StdEncoding.EncodeToString(seed),
|
||||
ComponentKind: "run",
|
||||
RegistrationToken: "unused",
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("materialize workspace seed: %v", err)
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(root).Scope("server-seeded", "run-local")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve seeded scope: %v", err)
|
||||
}
|
||||
if body, err := os.ReadFile(filepath.Join(scope, "actions", "install.json")); err != nil || !strings.Contains(string(body), `"action":"install"`) {
|
||||
t.Fatalf("expected seeded action file, body=%q err=%v", body, err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(scope, "bin", "install-server"))
|
||||
if err != nil || info.Mode().Perm()&0o111 == 0 {
|
||||
t.Fatalf("expected executable seeded helper, info=%+v err=%v", info, err)
|
||||
}
|
||||
if body, err := os.ReadFile(filepath.Join(scope, "assets", "map.bin")); err != nil || string(body) != string([]byte{0xff, 0x00, 0x7f}) {
|
||||
t.Fatalf("expected base64 seed file to materialize as binary bytes, body=%v err=%v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorKeepsSiblingInstanceWorkspacesIsolated(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
first := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
first.ServerInstanceID = "server-alpha"
|
||||
second := lifecycleAssignment(protocol.RunCapabilityProcessStop)
|
||||
second.JobID = "job-2"
|
||||
second.ServerInstanceID = "server-beta"
|
||||
for _, assignment := range []protocol.RunJobAssignment{first, second} {
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
body, err := json.Marshal(LifecycleActionTemplate{Command: []string{"echo", assignment.ServerInstanceID}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "lifecycle.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
}
|
||||
first.TargetKey = "actions/lifecycle.json"
|
||||
second.TargetKey = "actions/lifecycle.json"
|
||||
logSink := &recordingLogSink{}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink))
|
||||
|
||||
firstResult := executor.Execute(first)
|
||||
secondResult := executor.Execute(second)
|
||||
|
||||
if firstResult.State != "succeeded" || secondResult.State != "succeeded" {
|
||||
t.Fatalf("expected both lifecycle jobs to succeed, got first=%+v second=%+v", firstResult, secondResult)
|
||||
}
|
||||
joined := strings.Join(logSink.lines, "\n")
|
||||
if !strings.Contains(joined, "stdout:server-alpha") || !strings.Contains(joined, "stdout:server-beta") {
|
||||
t.Fatalf("expected instance-specific output, got %q", joined)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-alpha", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected alpha template to remain scoped: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-beta", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected beta template to remain scoped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorCancelsRunningCommand(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
result := NewLifecycleExecutor(WithProcessSupervisor(blockingSupervisor{})).ExecuteContext(ctx, lifecycleAssignment(protocol.RunCapabilityProcessStart))
|
||||
|
||||
if result.State != "cancelled" || result.ErrorCode != "lifecycle_cancelled" {
|
||||
t.Fatalf("expected cancelled lifecycle result, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLifecycleCapabilities(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
if !containsCapability(summary.Capabilities, capability) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLogReadCapability(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
if !containsCapability(summary.Capabilities, protocol.RunCapabilityLogsRead) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", protocol.RunCapabilityLogsRead, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteAccessExecutorCompletesBoundedJobs(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
|
||||
assignment.TargetKey = "db/scum/query"
|
||||
assignment.InputRef = "input://server-1/db/sqlite/query/1"
|
||||
|
||||
result := ExecuteRemoteAccessJob(context.Background(), assignment)
|
||||
|
||||
if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/remote-access-result" {
|
||||
t.Fatalf("expected bounded remote result, got %+v", result)
|
||||
}
|
||||
for _, forbidden := range []string{"/Users/", "tcp://", "password=", "sk-"} {
|
||||
if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) {
|
||||
t.Fatalf("remote result exposed forbidden fragment %q: %+v", forbidden, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsRemoteCapabilities(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
for _, capability := range []string{protocol.RunCapabilityRemoteRunRCONCommand, protocol.RunCapabilityRemoteRunDBMySQLQuery, protocol.RunCapabilityRemoteRunDBSQLiteQuery, protocol.RunCapabilityRemoteRunLogsTransfer} {
|
||||
if !containsCapability(summary.Capabilities, capability) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsSQLiteSchemaProbeCapability(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
if !containsCapability(summary.Capabilities, protocol.RunCapabilityRemoteRunDBSQLiteProbe) {
|
||||
t.Fatalf("expected schema probe capability, got %+v", summary.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleResultRequestPreservesSQLiteSchemaProbeEnvelope(t *testing.T) {
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
probe := &protocol.SQLiteSchemaProbeResult{RequestID: "probe-1", JobID: assignment.JobID, Binding: assignment.ExecutionInput.SQLiteSchemaProbe.Binding, Status: "succeeded", ResultDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Limits: assignment.ExecutionInput.SQLiteSchemaProbe.Limits}
|
||||
request := LifecycleResultRequest(assignment, "session-token", LifecycleExecutionResult{State: lifecycleResultStateSucceeded, ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", SQLiteSchemaProbe: probe}})
|
||||
if request.ExecutionResult.SQLiteSchemaProbe == nil || request.ExecutionResult.SQLiteSchemaProbe.ResultDigest != probe.ResultDigest || request.ExecutionResult.SQLiteSchemaProbe.JobID != assignment.JobID {
|
||||
t.Fatalf("expected SQLite probe terminal envelope to survive job result conversion: %+v", request.ExecutionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsDistributionCapabilities(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
for _, capability := range []string{protocol.RunCapabilityRunSelfUpdate, protocol.RunCapabilityDependenciesCheck, protocol.RunCapabilityDependenciesInstall, protocol.RunCapabilityLogsBackfill} {
|
||||
if !containsCapability(summary.Capabilities, capability) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistributionExecutorsReturnBoundedRefsAndRedactResults(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityLogsBackfill)
|
||||
assignment.TargetKey = "logs/latest-log"
|
||||
assignment.InputRef = "artifact://logs/checkpoint/1"
|
||||
result := ExecuteDistributionJob(context.Background(), assignment)
|
||||
if result.State != "succeeded" || result.Progress.Percent != 100 || !strings.HasPrefix(result.ResultRef, "artifact://jobs/") {
|
||||
t.Fatalf("expected bounded log backfill success, got %+v", result)
|
||||
}
|
||||
for _, forbidden := range []string{"/Users/", "tcp://", "unix://", "password=", "sk-", "mysql://", "sqlite://"} {
|
||||
if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) {
|
||||
t.Fatalf("distribution result leaked forbidden fragment %q: %+v", forbidden, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistributionExecutorsRejectUnsafeJobs(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityDependenciesInstall)
|
||||
assignment.TargetKey = "dependencies/java-21"
|
||||
|
||||
result := ExecuteDistributionJob(context.Background(), assignment)
|
||||
|
||||
if result.State != "failed" || result.ErrorCode != "dependency_execution_requires_worker" {
|
||||
t.Fatalf("expected dependency execution to require authenticated worker, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRuntimeProfilesSupportsDeclaredModesAndSafeMissingKeys(t *testing.T) {
|
||||
profiles := RuntimeProfiles{
|
||||
Discovery: []RuntimeDiscoveryProbe{{Key: "steamcmd", Kind: "command.version", TargetKey: "steamcmd", Required: true}},
|
||||
LifecycleProfiles: []RuntimeLifecycleProfile{
|
||||
{Key: "run-local", Mode: RuntimeModeLocalProcess, Capabilities: []string{protocol.RunCapabilityProcessStart}, ActionRefs: map[string]string{"start": "actions/start.json"}, TransportKeys: []string{"server-files"}, Platforms: []string{"linux"}},
|
||||
{Key: "hosted-ftp", Mode: RuntimeModeHostedFTPRCON, Capabilities: []string{protocol.RunCapabilityRemoteFTPRead, protocol.RunCapabilityRemoteRunRCONCommand}, TransportKeys: []string{"ftp", "rcon"}},
|
||||
{Key: "ftp-only", Mode: RuntimeModeFTPOnly, Capabilities: []string{protocol.RunCapabilityRemoteFTPRead}, TransportKeys: []string{"ftp"}},
|
||||
{Key: "custom-client", Mode: RuntimeModeCustomClient, Capabilities: []string{protocol.RunCapabilityRemoteRunRCONCommand}, TransportKeys: []string{"rcon"}, ClientManagerRef: "scum-client-manager"},
|
||||
},
|
||||
LogSources: []RuntimeLogSource{{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest", StreamKey: "latest-log"}},
|
||||
TransportProfiles: []RuntimeTransportProfile{
|
||||
{Key: "server-files", Kind: "file", TargetKey: "server-root", Capabilities: []string{protocol.RunCapabilityRemoteRunFilesRead}},
|
||||
{Key: "ftp", Kind: "ftp", TargetKey: "ftp-root", Capabilities: []string{protocol.RunCapabilityRemoteFTPRead}},
|
||||
{Key: "rcon", Kind: "rcon", TargetKey: "rcon", Capabilities: []string{protocol.RunCapabilityRemoteRunRCONCommand}},
|
||||
},
|
||||
}
|
||||
resolution, err := ResolveRuntimeProfile(profiles, "custom-client", "windows", RuntimeBindingSet{
|
||||
ProfileKey: "custom-client",
|
||||
Mode: RuntimeModeCustomClient,
|
||||
Bindings: map[string]string{
|
||||
"rcon": "binding://rcon/current",
|
||||
"logs/latest": "binding://logs/latest",
|
||||
"steamcmd": "binding://probe/steamcmd",
|
||||
"scum-client-manager": "binding://client/current",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve custom client profile: %v", err)
|
||||
}
|
||||
if !resolution.Available || resolution.Mode != RuntimeModeCustomClient || resolution.ClientManagerRef != "scum-client-manager" {
|
||||
t.Fatalf("unexpected custom client resolution: %+v", resolution)
|
||||
}
|
||||
|
||||
missing, err := ResolveRuntimeProfile(profiles, "hosted-ftp", "linux", RuntimeBindingSet{ProfileKey: "hosted-ftp", Mode: RuntimeModeHostedFTPRCON, Bindings: map[string]string{"ftp-root": "binding://ftp/current"}})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve hosted profile: %v", err)
|
||||
}
|
||||
if missing.Available || strings.Join(missing.MissingKeys, ",") != "rcon,steamcmd" {
|
||||
t.Fatalf("expected safe missing keys without raw binding values, got %+v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTailDeclaredFileLogSourceUsesCheckpointAndRedaction(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityLogsRead)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs")
|
||||
if err := os.MkdirAll(serverRoot, 0o755); err != nil {
|
||||
t.Fatalf("create logs dir: %v", err)
|
||||
}
|
||||
logPath := filepath.Join(serverRoot, "latest.log")
|
||||
if err := os.WriteFile(logPath, []byte("first line\npassword=hidden\n"), 0o644); err != nil {
|
||||
t.Fatalf("write log file: %v", err)
|
||||
}
|
||||
store := NewMemoryLogCheckpointStore()
|
||||
sink := &recordingLogSink{}
|
||||
source := RuntimeLogSource{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest.log", StreamKey: "latest-log", CursorKind: "offset"}
|
||||
|
||||
result := TailDeclaredFileLogSource(context.Background(), root, assignment, source, sink, store)
|
||||
|
||||
if result.State != "succeeded" || !strings.Contains(result.ResultRef, "live-log-checkpoint") {
|
||||
t.Fatalf("expected file tail success, got %+v", result)
|
||||
}
|
||||
if len(sink.lines) != 2 || strings.Contains(strings.Join(sink.lines, "\n"), "password=hidden") {
|
||||
t.Fatalf("expected redacted tailed lines, got %+v", sink.lines)
|
||||
}
|
||||
checkpoint := store.GetLogCheckpoint("latest-log")
|
||||
if checkpoint.Offset == 0 || checkpoint.Sequence != 2 || strings.Contains(RedactedLogCheckpointSummary(checkpoint), "/Users/") {
|
||||
t.Fatalf("expected durable safe checkpoint, got %+v", checkpoint)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(logPath, []byte("first line\npassword=hidden\nsecond line\n"), 0o644); err != nil {
|
||||
t.Fatalf("append log file: %v", err)
|
||||
}
|
||||
sink.lines = nil
|
||||
result = TailDeclaredFileLogSource(context.Background(), root, assignment, source, sink, store)
|
||||
if result.State != "succeeded" || len(sink.lines) != 1 || !strings.Contains(sink.lines[0], "second line") {
|
||||
t.Fatalf("expected checkpointed incremental tail, result=%+v lines=%+v", result, sink.lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorExecutesDeclaredLogBackfillTail(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityLogsBackfill)
|
||||
assignment.TargetKey = "logs/latest-log"
|
||||
assignment.ExecutionInput.LogSource = &protocol.RuntimeLogSourcePlan{Key: "latest-log", Kind: "file.tail", TargetKey: "logs/latest.log", StreamKey: "latest-log", CursorKind: "offset", RetentionDays: 30}
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs")
|
||||
if err := os.MkdirAll(serverRoot, 0o755); err != nil {
|
||||
t.Fatalf("create logs dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "latest.log"), []byte("scum latest line\n"), 0o644); err != nil {
|
||||
t.Fatalf("write log file: %v", err)
|
||||
}
|
||||
sink := &recordingLogSink{}
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(sink)).ExecuteLogBackfill(context.Background(), assignment)
|
||||
|
||||
if result.State != "succeeded" || !strings.Contains(result.Message, "tailed") {
|
||||
t.Fatalf("expected tailed log backfill success, got %+v", result)
|
||||
}
|
||||
if len(sink.lines) != 1 || sink.lines[0] != "latest-log:scum latest line" {
|
||||
t.Fatalf("expected tailed log line in sink, got %+v", sink.lines)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingSupervisor struct {
|
||||
command ProcessCommand
|
||||
}
|
||||
|
||||
func (supervisor *recordingSupervisor) Run(_ context.Context, command ProcessCommand) (ProcessResult, error) {
|
||||
supervisor.command = command
|
||||
return ProcessResult{ExitCode: 0, Stdout: "recorded\n"}, nil
|
||||
}
|
||||
|
||||
type recordingLogSink struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (sink *recordingLogSink) Append(_ context.Context, _ protocol.RunJobAssignment, stream string, line string) error {
|
||||
sink.mu.Lock()
|
||||
defer sink.mu.Unlock()
|
||||
sink.lines = append(sink.lines, stream+":"+line)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sink *recordingLogSink) snapshot() []string {
|
||||
sink.mu.Lock()
|
||||
defer sink.mu.Unlock()
|
||||
return append([]string(nil), sink.lines...)
|
||||
}
|
||||
|
||||
type recordingArtifactHook struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
func (hook *recordingArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
||||
hook.called = true
|
||||
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID), nil
|
||||
}
|
||||
|
||||
type blockingSupervisor struct{}
|
||||
|
||||
func (blockingSupervisor) Run(ctx context.Context, _ ProcessCommand) (ProcessResult, error) {
|
||||
<-ctx.Done()
|
||||
return ProcessResult{ExitCode: -1}, ctx.Err()
|
||||
}
|
||||
|
||||
func lifecycleAssignment(capability string) protocol.RunJobAssignment {
|
||||
now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
|
||||
return protocol.RunJobAssignment{
|
||||
JobID: "job-1",
|
||||
ServerInstanceID: "server-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: capability,
|
||||
IdempotencyKey: "idem-1",
|
||||
State: "accepted",
|
||||
LeaseToken: "lease-1",
|
||||
Attempt: 1,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func containsCapability(capabilities []string, capability string) bool {
|
||||
for _, item := range capabilities {
|
||||
if item == capability {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type LogSourceCheckpoint struct {
|
||||
SourceKey string
|
||||
Offset int64
|
||||
Sequence uint64
|
||||
CursorRef string
|
||||
}
|
||||
|
||||
type LogCheckpointStore interface {
|
||||
GetLogCheckpoint(sourceKey string) LogSourceCheckpoint
|
||||
PutLogCheckpoint(checkpoint LogSourceCheckpoint)
|
||||
}
|
||||
|
||||
type MemoryLogCheckpointStore struct {
|
||||
checkpoints map[string]LogSourceCheckpoint
|
||||
}
|
||||
|
||||
func NewMemoryLogCheckpointStore() *MemoryLogCheckpointStore {
|
||||
return &MemoryLogCheckpointStore{checkpoints: map[string]LogSourceCheckpoint{}}
|
||||
}
|
||||
|
||||
func (store *MemoryLogCheckpointStore) GetLogCheckpoint(sourceKey string) LogSourceCheckpoint {
|
||||
if store == nil || store.checkpoints == nil {
|
||||
return LogSourceCheckpoint{SourceKey: sourceKey}
|
||||
}
|
||||
return store.checkpoints[sourceKey]
|
||||
}
|
||||
|
||||
func (store *MemoryLogCheckpointStore) PutLogCheckpoint(checkpoint LogSourceCheckpoint) {
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
if store.checkpoints == nil {
|
||||
store.checkpoints = map[string]LogSourceCheckpoint{}
|
||||
}
|
||||
store.checkpoints[checkpoint.SourceKey] = checkpoint
|
||||
}
|
||||
|
||||
func TailDeclaredFileLogSource(ctx context.Context, workspaceRoot string, assignment protocol.RunJobAssignment, source RuntimeLogSource, sink ProcessLogSink, store LogCheckpointStore) LifecycleExecutionResult {
|
||||
if source.Kind != "file.tail" {
|
||||
return lifecycleFailure("unsupported_log_source", "only file.tail sources are supported by the local tailer")
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(source.Key) || !protocol.ValidLogicalFileKey(source.TargetKey) || !protocol.ValidLogicalFileKey(source.StreamKey) {
|
||||
return lifecycleFailure("unsafe_log_source", "log source is unsafe")
|
||||
}
|
||||
if sink == nil {
|
||||
sink = NoopProcessLogSink{}
|
||||
}
|
||||
if store == nil {
|
||||
store = NewMemoryLogCheckpointStore()
|
||||
}
|
||||
serverRoot, err := scopedServerWorkspace(workspaceRoot, assignment.ServerInstanceID)
|
||||
if err != nil {
|
||||
return lifecycleFailure("unsafe_log_workspace", err.Error())
|
||||
}
|
||||
path, err := scopedPath(serverRoot, source.TargetKey)
|
||||
if err != nil {
|
||||
return lifecycleFailure("unsafe_log_source", err.Error())
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return lifecycleFailure("log_source_open_failed", err.Error())
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
checkpoint := store.GetLogCheckpoint(source.Key)
|
||||
if checkpoint.Offset > 0 {
|
||||
if _, err := file.Seek(checkpoint.Offset, 0); err != nil {
|
||||
return lifecycleFailure("log_source_seek_failed", err.Error())
|
||||
}
|
||||
}
|
||||
body := make([]byte, maxLifecycleOutputBytes)
|
||||
n, err := file.Read(body)
|
||||
if err != nil && n == 0 {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint unchanged"},
|
||||
ResultRef: fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID)),
|
||||
Message: "live log source had no new lines",
|
||||
}
|
||||
}
|
||||
for _, line := range splitBoundedLines(string(body[:n])) {
|
||||
checkpoint.Sequence++
|
||||
if err := sink.Append(ctx, assignment, source.StreamKey, line); err != nil {
|
||||
return lifecycleFailure("log_source_sink_failed", err.Error())
|
||||
}
|
||||
}
|
||||
checkpoint.SourceKey = source.Key
|
||||
checkpoint.Offset += int64(n)
|
||||
checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID))
|
||||
store.PutLogCheckpoint(checkpoint)
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint updated"},
|
||||
ResultRef: checkpoint.CursorRef,
|
||||
Message: "live log source tailed with durable offset checkpoint",
|
||||
}
|
||||
}
|
||||
|
||||
func RedactedLogCheckpointSummary(checkpoint LogSourceCheckpoint) string {
|
||||
return strings.Join([]string{
|
||||
"source=" + checkpoint.SourceKey,
|
||||
fmt.Sprintf("offset=%d", checkpoint.Offset),
|
||||
fmt.Sprintf("sequence=%d", checkpoint.Sequence),
|
||||
"cursorRef=" + checkpoint.CursorRef,
|
||||
}, " ")
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManagedExecutableArgsWrapWindowsPluginScripts(t *testing.T) {
|
||||
got := managedExecutableArgs("windows", []string{`C:\run workspace\bin\scum-start.cmd`, "--mode", "safe mode"})
|
||||
want := []string{"cmd.exe", "/d", "/c", "call", `C:\run workspace\bin\scum-start.cmd`, "--mode", "safe mode"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected Windows script command: got=%q want=%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedExecutableArgsKeepsDirectExecutables(t *testing.T) {
|
||||
input := []string{"C:\\run\\bin\\server.exe", "--port", "7779"}
|
||||
got := managedExecutableArgs("windows", input)
|
||||
if !reflect.DeepEqual(got, input) {
|
||||
t.Fatalf("direct executable was unexpectedly shell wrapped: got=%q want=%q", got, input)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//go:build !windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
type execManagedProcess struct {
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
|
||||
func startManagedProcess(command ProcessCommand, files managedProcessFiles, _ string) (managedProcess, error) {
|
||||
cmd := exec.Command(command.Args[0], command.Args[1:]...)
|
||||
cmd.Dir = command.WorkDir
|
||||
cmd.Env = os.Environ()
|
||||
for key, value := range command.Env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Stdout = files.stdout
|
||||
cmd.Stderr = files.stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &execManagedProcess{cmd: cmd}, nil
|
||||
}
|
||||
|
||||
func requestManagedProcessStop(identity ProcessIdentity) error {
|
||||
process, err := os.FindProcess(identity.PID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return process.Kill()
|
||||
}
|
||||
|
||||
func forceManagedProcessStop(identity ProcessIdentity) error {
|
||||
return requestManagedProcessStop(identity)
|
||||
}
|
||||
|
||||
func (process *execManagedProcess) PID() int {
|
||||
return process.cmd.Process.Pid
|
||||
}
|
||||
|
||||
func (process *execManagedProcess) TargetPID() int {
|
||||
return process.PID()
|
||||
}
|
||||
|
||||
func (process *execManagedProcess) Wait() (int, error) {
|
||||
err := process.cmd.Wait()
|
||||
if process.cmd.ProcessState == nil {
|
||||
return -1, err
|
||||
}
|
||||
return process.cmd.ProcessState.ExitCode(), err
|
||||
}
|
||||
|
||||
func (process *execManagedProcess) Kill() error {
|
||||
return process.cmd.Process.Kill()
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const metricReportTimeout = 5 * time.Second
|
||||
|
||||
// MetricCollector provides only generic local utilization data. Game-specific
|
||||
// observations belong to a plugin-declared bridge, never the Run runtime.
|
||||
type MetricCollector interface {
|
||||
Collect(context.Context, string) (MetricUtilization, error)
|
||||
}
|
||||
|
||||
type MetricUtilization struct {
|
||||
CPUPercent *float64
|
||||
MemoryPercent *float64
|
||||
DiskPercent *float64
|
||||
}
|
||||
|
||||
type defaultMetricCollector struct{}
|
||||
|
||||
func (defaultMetricCollector) Collect(ctx context.Context, workspaceRoot string) (MetricUtilization, error) {
|
||||
var utilization MetricUtilization
|
||||
var collectionErrors []error
|
||||
if cpuPercent, err := hostCPUPercent(ctx); err != nil {
|
||||
collectionErrors = append(collectionErrors, fmt.Errorf("collect CPU utilization: %w", err))
|
||||
} else {
|
||||
utilization.CPUPercent = &cpuPercent
|
||||
}
|
||||
if memoryPercent, err := hostMemoryPercent(); err != nil {
|
||||
collectionErrors = append(collectionErrors, fmt.Errorf("collect memory utilization: %w", err))
|
||||
} else {
|
||||
utilization.MemoryPercent = &memoryPercent
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
collectionErrors = append(collectionErrors, err)
|
||||
} else if diskPercent, err := workspaceDiskPercent(workspaceRoot); err != nil {
|
||||
collectionErrors = append(collectionErrors, fmt.Errorf("collect disk utilization: %w", err))
|
||||
} else {
|
||||
utilization.DiskPercent = &diskPercent
|
||||
}
|
||||
return utilization, errors.Join(collectionErrors...)
|
||||
}
|
||||
|
||||
// WithMetricCollector overrides the generic host collector for tests and
|
||||
// platform-specific collectors. It has no access to plugin inputs or secrets.
|
||||
func WithMetricCollector(collector MetricCollector) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
executor.metricCollector = collector
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) reportMetricsDegraded(ctx context.Context, trigger string) {
|
||||
if err := worker.ReportMetricsOnce(ctx); err != nil {
|
||||
log.Printf("RUN phase=metrics status=degraded trigger=%s error=%s", safeOptional(trigger), RedactText(err.Error()))
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) ReportMetricsOnce(ctx context.Context) error {
|
||||
if worker.cfg.ComponentKind != config.PackageComponentRun || worker.cfg.ServerInstanceID == "" {
|
||||
return nil
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sample := protocol.MetricSample{ServerInstanceID: worker.cfg.ServerInstanceID, Online: worker.managedServerProcessOnline(state), Source: "run", CollectedAt: time.Now().UTC()}
|
||||
if worker.metricCollector != nil {
|
||||
utilization, collectErr := worker.metricCollector.Collect(ctx, worker.cfg.WorkspaceRoot)
|
||||
sample.CPUPercent = utilization.CPUPercent
|
||||
sample.MemoryPercent = utilization.MemoryPercent
|
||||
sample.DiskPercent = utilization.DiskPercent
|
||||
if collectErr != nil {
|
||||
log.Printf("RUN phase=metrics status=utilization_unavailable error=%s", RedactText(collectErr.Error()))
|
||||
}
|
||||
}
|
||||
reportCtx, cancel := context.WithTimeout(ctx, metricReportTimeout)
|
||||
defer cancel()
|
||||
response, err := worker.client.IngestMetricBatch(reportCtx, protocol.MetricBatchIngestRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, Samples: []protocol.MetricSample{sample}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted || response.AcceptedCount != 1 {
|
||||
return fmt.Errorf("metric batch was not accepted")
|
||||
}
|
||||
log.Printf("RUN phase=metrics status=accepted server=%s online=%t", sample.ServerInstanceID, sample.Online)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) managedServerProcessOnline(state WorkerState) bool {
|
||||
source, ok := worker.executor.managed.(ManagedProcessObservationSource)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, identity := range source.ManagedProcessObservations() {
|
||||
if identity.ServerInstanceID != worker.cfg.ServerInstanceID || identity.RunEndpointID != state.RunEndpointID {
|
||||
continue
|
||||
}
|
||||
if worker.executor.managed.Status(identity).State == "running" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//go:build !windows
|
||||
|
||||
package runtime
|
||||
|
||||
import "syscall"
|
||||
|
||||
func workspaceDiskPercent(workspaceRoot string) (float64, error) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(workspaceRoot, &stat); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if stat.Blocks == 0 {
|
||||
return 0, syscall.EINVAL
|
||||
}
|
||||
return 100 * float64(stat.Blocks-stat.Bavail) / float64(stat.Blocks), nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var getDiskFreeSpaceEx = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
|
||||
|
||||
func workspaceDiskPercent(workspaceRoot string) (float64, error) {
|
||||
volume := filepath.VolumeName(workspaceRoot)
|
||||
if volume == "" {
|
||||
volume = workspaceRoot
|
||||
} else {
|
||||
volume += `\`
|
||||
}
|
||||
var available, total, free uint64
|
||||
success, _, callErr := getDiskFreeSpaceEx.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(volume))), uintptr(unsafe.Pointer(&available)), uintptr(unsafe.Pointer(&total)), uintptr(unsafe.Pointer(&free)))
|
||||
if success == 0 {
|
||||
return 0, callErr
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, syscall.EINVAL
|
||||
}
|
||||
return 100 * float64(total-free) / float64(total), nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//go:build linux
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const cpuSampleInterval = 100 * time.Millisecond
|
||||
|
||||
func hostCPUPercent(ctx context.Context) (float64, error) {
|
||||
first, err := readLinuxCPUStat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
timer := time.NewTimer(cpuSampleInterval)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
second, err := readLinuxCPUStat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
totalDelta := second.total - first.total
|
||||
busyDelta := second.busy - first.busy
|
||||
if totalDelta == 0 || busyDelta > totalDelta {
|
||||
return 0, fmt.Errorf("CPU counters did not advance")
|
||||
}
|
||||
return clampMetricPercent(100 * float64(busyDelta) / float64(totalDelta)), nil
|
||||
}
|
||||
|
||||
type linuxCPUStat struct {
|
||||
total uint64
|
||||
busy uint64
|
||||
}
|
||||
|
||||
func readLinuxCPUStat() (linuxCPUStat, error) {
|
||||
file, err := os.Open("/proc/stat")
|
||||
if err != nil {
|
||||
return linuxCPUStat{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
line, err := bufio.NewReader(file).ReadString('\n')
|
||||
if err != nil {
|
||||
return linuxCPUStat{}, err
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 || fields[0] != "cpu" {
|
||||
return linuxCPUStat{}, fmt.Errorf("/proc/stat CPU row is invalid")
|
||||
}
|
||||
values := make([]uint64, len(fields)-1)
|
||||
for index, field := range fields[1:] {
|
||||
value, parseErr := strconv.ParseUint(field, 10, 64)
|
||||
if parseErr != nil {
|
||||
return linuxCPUStat{}, fmt.Errorf("parse /proc/stat CPU counter: %w", parseErr)
|
||||
}
|
||||
values[index] = value
|
||||
}
|
||||
var total uint64
|
||||
for _, value := range values {
|
||||
total += value
|
||||
}
|
||||
idle := values[3]
|
||||
if len(values) > 4 {
|
||||
idle += values[4]
|
||||
}
|
||||
if idle > total {
|
||||
return linuxCPUStat{}, fmt.Errorf("/proc/stat idle counter exceeds total")
|
||||
}
|
||||
return linuxCPUStat{total: total, busy: total - idle}, nil
|
||||
}
|
||||
|
||||
func hostMemoryPercent() (float64, error) {
|
||||
file, err := os.Open("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
var total, available uint64
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
value, parseErr := strconv.ParseUint(fields[1], 10, 64)
|
||||
if parseErr != nil {
|
||||
return 0, fmt.Errorf("parse /proc/meminfo: %w", parseErr)
|
||||
}
|
||||
switch fields[0] {
|
||||
case "MemTotal:":
|
||||
total = value
|
||||
case "MemAvailable:":
|
||||
available = value
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if total == 0 || available > total {
|
||||
return 0, fmt.Errorf("/proc/meminfo memory counters are invalid")
|
||||
}
|
||||
return clampMetricPercent(100 * float64(total-available) / float64(total)), nil
|
||||
}
|
||||
|
||||
func clampMetricPercent(value float64) float64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 100 {
|
||||
return 100
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//go:build !linux && !windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func hostCPUPercent(context.Context) (float64, error) {
|
||||
return 0, fmt.Errorf("CPU utilization is not implemented for %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
func hostMemoryPercent() (float64, error) {
|
||||
return 0, fmt.Errorf("memory utilization is not implemented for %s", runtime.GOOS)
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//go:build windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const cpuSampleInterval = 100 * time.Millisecond
|
||||
|
||||
var (
|
||||
kernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||
getSystemTimesProc = kernel32.NewProc("GetSystemTimes")
|
||||
globalMemoryStatusExProc = kernel32.NewProc("GlobalMemoryStatusEx")
|
||||
)
|
||||
|
||||
type windowsFileTime struct {
|
||||
lowDateTime uint32
|
||||
highDateTime uint32
|
||||
}
|
||||
|
||||
type windowsMemoryStatusEx struct {
|
||||
dwLength uint32
|
||||
dwMemoryLoad uint32
|
||||
ullTotalPhys uint64
|
||||
ullAvailPhys uint64
|
||||
ullTotalPageFile uint64
|
||||
ullAvailPageFile uint64
|
||||
ullTotalVirtual uint64
|
||||
ullAvailVirtual uint64
|
||||
ullAvailExtendedVirtual uint64
|
||||
}
|
||||
|
||||
type windowsCPUStat struct {
|
||||
idle uint64
|
||||
total uint64
|
||||
}
|
||||
|
||||
func hostCPUPercent(ctx context.Context) (float64, error) {
|
||||
first, err := readWindowsCPUStat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
timer := time.NewTimer(cpuSampleInterval)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
second, err := readWindowsCPUStat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
totalDelta := second.total - first.total
|
||||
idleDelta := second.idle - first.idle
|
||||
if totalDelta == 0 || idleDelta > totalDelta {
|
||||
return 0, fmt.Errorf("Windows CPU counters did not advance")
|
||||
}
|
||||
return clampMetricPercent(100 * float64(totalDelta-idleDelta) / float64(totalDelta)), nil
|
||||
}
|
||||
|
||||
func readWindowsCPUStat() (windowsCPUStat, error) {
|
||||
var idle, kernel, user windowsFileTime
|
||||
result, _, callErr := getSystemTimesProc.Call(uintptr(unsafe.Pointer(&idle)), uintptr(unsafe.Pointer(&kernel)), uintptr(unsafe.Pointer(&user)))
|
||||
if result == 0 {
|
||||
return windowsCPUStat{}, callErr
|
||||
}
|
||||
idleTicks := windowsFileTimeValue(idle)
|
||||
return windowsCPUStat{idle: idleTicks, total: idleTicks + windowsFileTimeValue(kernel) + windowsFileTimeValue(user)}, nil
|
||||
}
|
||||
|
||||
func windowsFileTimeValue(value windowsFileTime) uint64 {
|
||||
return uint64(value.highDateTime)<<32 | uint64(value.lowDateTime)
|
||||
}
|
||||
|
||||
func hostMemoryPercent() (float64, error) {
|
||||
status := windowsMemoryStatusEx{dwLength: uint32(unsafe.Sizeof(windowsMemoryStatusEx{}))}
|
||||
result, _, callErr := globalMemoryStatusExProc.Call(uintptr(unsafe.Pointer(&status)))
|
||||
if result == 0 {
|
||||
return 0, callErr
|
||||
}
|
||||
if status.ullTotalPhys == 0 || status.ullAvailPhys > status.ullTotalPhys {
|
||||
return 0, fmt.Errorf("Windows memory counters are invalid")
|
||||
}
|
||||
return clampMetricPercent(100 * float64(status.ullTotalPhys-status.ullAvailPhys) / float64(status.ullTotalPhys)), nil
|
||||
}
|
||||
|
||||
func clampMetricPercent(value float64) float64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 100 {
|
||||
return 100
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
)
|
||||
|
||||
func TestWorkerReportsMetricsWithCurrentSessionAndManagedProcessState(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.ComponentKind = config.PackageComponentRun
|
||||
managed := &metricManagedSupervisor{identity: ProcessIdentity{ServerInstanceID: cfg.ServerInstanceID, RunEndpointID: cfg.RunEndpointID, Scope: "server", State: "running"}}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithMetricCollector(metricCollectorStub{disk: 42.5}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.ReportMetricsOnce(context.Background()); err != nil {
|
||||
t.Fatalf("report metrics: %v", err)
|
||||
}
|
||||
if len(client.metricRequests) != 1 {
|
||||
t.Fatalf("expected one metric request, got %d", len(client.metricRequests))
|
||||
}
|
||||
request := client.metricRequests[0]
|
||||
if request.RunEndpointID != cfg.RunEndpointID || request.SessionToken != "session-token" || len(request.Samples) != 1 {
|
||||
t.Fatalf("expected registered session metric request, got %+v", request)
|
||||
}
|
||||
sample := request.Samples[0]
|
||||
if !sample.Online || sample.Source != "run" || sample.PlayerCount != nil || sample.TPS != nil || sample.LatencyMS != nil || sample.DiskPercent == nil || *sample.DiskPercent != 42.5 {
|
||||
t.Fatalf("unexpected generic metric sample: %+v", sample)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerReportsOfflineWhenManagedProcessExitedAndOmitsUnavailableUtilization(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.ComponentKind = config.PackageComponentRun
|
||||
managed := &metricManagedSupervisor{identity: ProcessIdentity{ServerInstanceID: cfg.ServerInstanceID, RunEndpointID: cfg.RunEndpointID, Scope: "server", State: "exited"}}
|
||||
worker, err := NewWorker(cfg, client, WithManagedProcessSupervisor(managed), WithMetricCollector(metricCollectorError{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.ReportMetricsOnce(context.Background()); err != nil {
|
||||
t.Fatalf("report metrics: %v", err)
|
||||
}
|
||||
sample := client.metricRequests[0].Samples[0]
|
||||
if sample.Online || sample.CPUPercent != nil || sample.MemoryPercent != nil || sample.DiskPercent != nil {
|
||||
t.Fatalf("expected offline sample without unavailable utilization, got %+v", sample)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerReportsAvailableUtilizationWhenOneMetricFails(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.ComponentKind = config.PackageComponentRun
|
||||
worker, err := NewWorker(cfg, client, WithMetricCollector(metricCollectorPartial{disk: 42.5}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.ReportMetricsOnce(context.Background()); err != nil {
|
||||
t.Fatalf("report metrics: %v", err)
|
||||
}
|
||||
sample := client.metricRequests[0].Samples[0]
|
||||
if sample.DiskPercent == nil || *sample.DiskPercent != 42.5 {
|
||||
t.Fatalf("expected available disk utilization to be reported, got %+v", sample)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricUploadFailureDoesNotFailReportingCycle(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.metricErr = errors.New("platform unavailable")
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.ServerInstanceID = "server-worker"
|
||||
cfg.ComponentKind = config.PackageComponentRun
|
||||
worker, err := NewWorker(cfg, client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
worker.reportMetricsDegraded(context.Background(), "test")
|
||||
if len(client.metricRequests) != 1 {
|
||||
t.Fatalf("expected degraded metric attempt, got %d", len(client.metricRequests))
|
||||
}
|
||||
}
|
||||
|
||||
type metricManagedSupervisor struct{ identity ProcessIdentity }
|
||||
|
||||
func (supervisor *metricManagedSupervisor) Start(context.Context, ProcessCommand, ProcessIdentity, ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
return supervisor.identity, nil
|
||||
}
|
||||
func (supervisor *metricManagedSupervisor) Stop(context.Context, ProcessIdentity) (ProcessIdentity, error) {
|
||||
return supervisor.identity, nil
|
||||
}
|
||||
func (supervisor *metricManagedSupervisor) Status(ProcessIdentity) ProcessIdentity {
|
||||
return supervisor.identity
|
||||
}
|
||||
func (supervisor *metricManagedSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
||||
func (supervisor *metricManagedSupervisor) ManagedProcessObservations() []ProcessIdentity {
|
||||
return []ProcessIdentity{supervisor.identity}
|
||||
}
|
||||
|
||||
type metricCollectorStub struct{ disk float64 }
|
||||
|
||||
func (collector metricCollectorStub) Collect(context.Context, string) (MetricUtilization, error) {
|
||||
return MetricUtilization{DiskPercent: &collector.disk}, nil
|
||||
}
|
||||
|
||||
type metricCollectorError struct{}
|
||||
|
||||
func (metricCollectorError) Collect(context.Context, string) (MetricUtilization, error) {
|
||||
return MetricUtilization{}, errors.New("collector unavailable")
|
||||
}
|
||||
|
||||
type metricCollectorPartial struct{ disk float64 }
|
||||
|
||||
func (collector metricCollectorPartial) Collect(context.Context, string) (MetricUtilization, error) {
|
||||
return MetricUtilization{DiskPercent: &collector.disk}, errors.New("CPU unavailable")
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//go:build !windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func processAlivePID(pid int) bool {
|
||||
if pid <= 0 {
|
||||
return false
|
||||
}
|
||||
process, err := os.FindProcess(pid)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return process.Signal(syscall.Signal(0)) == nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//go:build windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
const (
|
||||
windowsProcessQueryLimitedInformation = 0x1000
|
||||
windowsStillActive = 259
|
||||
)
|
||||
|
||||
func processAlivePID(pid int) bool {
|
||||
if pid <= 0 {
|
||||
return false
|
||||
}
|
||||
handle, err := windows.OpenProcess(windowsProcessQueryLimitedInformation, false, uint32(pid))
|
||||
if err != nil {
|
||||
if processSnapshotContainsPID(pid) {
|
||||
return true
|
||||
}
|
||||
return err == syscall.ERROR_ACCESS_DENIED
|
||||
}
|
||||
defer windows.CloseHandle(handle)
|
||||
var exitCode uint32
|
||||
if err := windows.GetExitCodeProcess(handle, &exitCode); err == nil {
|
||||
return exitCode == windowsStillActive
|
||||
}
|
||||
return processSnapshotContainsPID(pid)
|
||||
}
|
||||
|
||||
func processSnapshotContainsPID(pid int) bool {
|
||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer windows.CloseHandle(snapshot)
|
||||
var entry windows.ProcessEntry32
|
||||
entry.Size = uint32(unsafe.Sizeof(entry))
|
||||
if err := windows.Process32First(snapshot, &entry); err != nil {
|
||||
return false
|
||||
}
|
||||
for {
|
||||
if entry.ProcessID == uint32(pid) {
|
||||
return true
|
||||
}
|
||||
if err := windows.Process32Next(snapshot, &entry); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManagedProcessStopEventNameIsStableAndScoped(t *testing.T) {
|
||||
base := ProcessIdentity{Scope: `C:\workspace\instances\server-1\run-local`, RunEndpointID: "run-1", ServerInstanceID: "server-1", LogSessionID: "session-a"}
|
||||
if got, want := managedProcessStopEventName(base), managedProcessStopEventName(base); got != want {
|
||||
t.Fatalf("stop event name is not stable: got=%q want=%q", got, want)
|
||||
}
|
||||
if got := managedProcessStopEventName(base); !strings.HasPrefix(got, `Local\run-managed-stop-`) {
|
||||
t.Fatalf("stop event name is not in the local namespace: %q", got)
|
||||
}
|
||||
if strings.Contains(managedProcessStopEventName(base), "workspace") {
|
||||
t.Fatalf("stop event name leaked the process scope: %q", managedProcessStopEventName(base))
|
||||
}
|
||||
for _, changed := range []ProcessIdentity{
|
||||
{Scope: base.Scope, RunEndpointID: "run-2", ServerInstanceID: base.ServerInstanceID, LogSessionID: base.LogSessionID},
|
||||
{Scope: base.Scope, RunEndpointID: base.RunEndpointID, ServerInstanceID: "server-2", LogSessionID: base.LogSessionID},
|
||||
{Scope: base.Scope, RunEndpointID: base.RunEndpointID, ServerInstanceID: base.ServerInstanceID, LogSessionID: "session-b"},
|
||||
} {
|
||||
if got := managedProcessStopEventName(changed); got == managedProcessStopEventName(base) {
|
||||
t.Fatalf("different process generation shared stop event name: base=%q changed=%q", managedProcessStopEventName(base), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedProcessGenerationRejectsOutputFileSwap(t *testing.T) {
|
||||
current := ProcessIdentity{
|
||||
PID: 100,
|
||||
LogSessionID: "session-a",
|
||||
StdoutLogRef: "stdout-a.log",
|
||||
StderrLogRef: "stderr-a.log",
|
||||
}
|
||||
|
||||
if !sameManagedProcessGeneration(current, current) {
|
||||
t.Fatal("expected an identical managed process generation to match")
|
||||
}
|
||||
for _, expected := range []ProcessIdentity{
|
||||
{PID: 100, LogSessionID: "session-a", StdoutLogRef: "stdout-b.log", StderrLogRef: "stderr-a.log"},
|
||||
{PID: 100, LogSessionID: "session-a", StdoutLogRef: "stdout-a.log", StderrLogRef: "stderr-b.log"},
|
||||
} {
|
||||
if sameManagedProcessGeneration(current, expected) {
|
||||
t.Fatalf("output file swap was treated as the same generation: current=%+v expected=%+v", current, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
)
|
||||
|
||||
func TestManagedProcessStateRootIsolatedPerRunService(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
base := config.Config{WorkspaceRoot: workspace, ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"}
|
||||
first := base
|
||||
first.RunEndpointID = "run-a"
|
||||
second := base
|
||||
second.RunEndpointID = "run-b"
|
||||
third := base
|
||||
third.RunEndpointID = "run-a"
|
||||
third.ComponentKey = "run-secondary"
|
||||
|
||||
firstRoot := managedProcessStateRoot(first)
|
||||
secondRoot := managedProcessStateRoot(second)
|
||||
thirdRoot := managedProcessStateRoot(third)
|
||||
if firstRoot == secondRoot || firstRoot == thirdRoot || secondRoot == thirdRoot {
|
||||
t.Fatalf("run services must not share managed process state: %q %q %q", firstRoot, secondRoot, thirdRoot)
|
||||
}
|
||||
wantPrefix := filepath.Join(workspace, "run-services") + string(filepath.Separator)
|
||||
for _, root := range []string{firstRoot, secondRoot, thirdRoot} {
|
||||
if !strings.HasPrefix(root, wantPrefix) {
|
||||
t.Fatalf("managed process state escaped the isolated root: %q", root)
|
||||
}
|
||||
if len(filepath.Base(root)) != 64 {
|
||||
t.Fatalf("managed process state namespace is not a sha256 directory: %q", root)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateLegacyManagedProcessStateKeepsOnlyThisRunService(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
cfg := config.Config{WorkspaceRoot: workspace, RunEndpointID: "run-a", ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"}
|
||||
legacy := processJournal{Version: 1, Items: map[string]ProcessIdentity{
|
||||
"owned": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 101, State: "running"},
|
||||
"other-endpoint": {RunEndpointID: "run-b", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 102, State: "running"},
|
||||
"other-profile": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-other", PID: 103, State: "running"},
|
||||
}}
|
||||
body, err := json.Marshal(legacy)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal legacy journal: %v", err)
|
||||
}
|
||||
legacyPath := filepath.Join(workspace, "state", "processes.json")
|
||||
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o700); err != nil {
|
||||
t.Fatalf("create legacy state directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(legacyPath, body, 0o600); err != nil {
|
||||
t.Fatalf("write legacy journal: %v", err)
|
||||
}
|
||||
|
||||
stateRoot := managedProcessStateRoot(cfg)
|
||||
if err := migrateLegacyManagedProcessState(cfg, stateRoot); err != nil {
|
||||
t.Fatalf("migrate legacy journal: %v", err)
|
||||
}
|
||||
migratedBody, err := os.ReadFile(filepath.Join(stateRoot, "state", "processes.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read isolated journal: %v", err)
|
||||
}
|
||||
var migrated processJournal
|
||||
if err := json.Unmarshal(migratedBody, &migrated); err != nil {
|
||||
t.Fatalf("decode isolated journal: %v", err)
|
||||
}
|
||||
if len(migrated.Items) != 1 || migrated.Items["owned"].PID != 101 {
|
||||
t.Fatalf("isolated journal imported another Run service: %+v", migrated.Items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateManagedProcessStateImportsMatchingPriorNamespace(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
cfg := config.Config{WorkspaceRoot: workspace, RunEndpointID: "run-a", ServerInstanceID: "server-1", PluginID: "game.scum", ComponentKind: "run", ComponentKey: "run-local"}
|
||||
priorRoot := filepath.Join(workspace, "run-services", "prior", "state")
|
||||
if err := os.MkdirAll(priorRoot, 0o700); err != nil {
|
||||
t.Fatalf("create prior state directory: %v", err)
|
||||
}
|
||||
body, err := json.Marshal(processJournal{Version: 2, Items: map[string]ProcessIdentity{
|
||||
"owned": {RunEndpointID: "run-a", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 101, State: "exited"},
|
||||
"other": {RunEndpointID: "run-b", ServerInstanceID: "server-1", ProfileKey: "run-local", PID: 102, State: "running"},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prior journal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(priorRoot, "processes.json"), body, 0o600); err != nil {
|
||||
t.Fatalf("write prior journal: %v", err)
|
||||
}
|
||||
stateRoot := managedProcessStateRoot(cfg)
|
||||
if err := migrateManagedProcessState(cfg, stateRoot); err != nil {
|
||||
t.Fatalf("migrate managed state: %v", err)
|
||||
}
|
||||
migratedBody, err := os.ReadFile(filepath.Join(stateRoot, "state", "processes.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read migrated journal: %v", err)
|
||||
}
|
||||
var migrated processJournal
|
||||
if err := json.Unmarshal(migratedBody, &migrated); err != nil {
|
||||
t.Fatalf("decode migrated journal: %v", err)
|
||||
}
|
||||
if len(migrated.Items) != 1 || migrated.Items["owned"].PID != 101 {
|
||||
t.Fatalf("unexpected migrated matching state: %+v", migrated.Items)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,811 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
managedProcessJournalVersion = 2
|
||||
managedProcessOutputPollInterval = 50 * time.Millisecond
|
||||
managedProcessOutputDrainDelay = 750 * time.Millisecond
|
||||
managedProcessOutputRetryDelay = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
type ProcessIdentity struct {
|
||||
Scope string `json:"scope"`
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
RunEndpointID string `json:"runEndpointId,omitempty"`
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
Capability string `json:"capability,omitempty"`
|
||||
ProfileKey string `json:"profileKey"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
PID int `json:"pid"`
|
||||
SupervisorPID int `json:"supervisorPid,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CommandFingerprint string `json:"commandFingerprint"`
|
||||
State string `json:"state"`
|
||||
ExitCode int `json:"exitCode,omitempty"`
|
||||
ExitClassification string `json:"exitClassification,omitempty"`
|
||||
ObservationSeq uint64 `json:"observationSeq,omitempty"`
|
||||
Attempt int `json:"attempt"`
|
||||
LeaseTokenHash string `json:"leaseTokenHash,omitempty"`
|
||||
StdoutLogRef string `json:"stdoutLogRef,omitempty"`
|
||||
StderrLogRef string `json:"stderrLogRef,omitempty"`
|
||||
StdoutStreamKey string `json:"stdoutStreamKey,omitempty"`
|
||||
StderrStreamKey string `json:"stderrStreamKey,omitempty"`
|
||||
StopEventName string `json:"stopEventName,omitempty"`
|
||||
StdoutOffset int64 `json:"stdoutOffset,omitempty"`
|
||||
StderrOffset int64 `json:"stderrOffset,omitempty"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type processJournal struct {
|
||||
Version int `json:"version"`
|
||||
Items map[string]ProcessIdentity `json:"items"`
|
||||
Retired map[string]ProcessIdentity `json:"retired,omitempty"`
|
||||
}
|
||||
|
||||
type ManagedProcessSupervisor interface {
|
||||
Start(context.Context, ProcessCommand, ProcessIdentity, ManagedProcessOutput) (ProcessIdentity, error)
|
||||
Stop(context.Context, ProcessIdentity) (ProcessIdentity, error)
|
||||
Status(ProcessIdentity) ProcessIdentity
|
||||
ResumeOutput(ManagedProcessOutput)
|
||||
}
|
||||
|
||||
// ManagedProcessObservationSource exposes only generic supervised-process
|
||||
// facts. Worker uses it to report persisted transitions after registration.
|
||||
type ManagedProcessObservationSource interface {
|
||||
ManagedProcessObservations() []ProcessIdentity
|
||||
}
|
||||
|
||||
type ManagedProcessLine struct {
|
||||
Text string
|
||||
StartOffset int64
|
||||
EndOffset int64
|
||||
}
|
||||
|
||||
type ManagedProcessLineSink func(ProcessIdentity, ManagedProcessLine) error
|
||||
|
||||
type ManagedProcessOutput struct {
|
||||
Stdout ManagedProcessLineSink
|
||||
Stderr ManagedProcessLineSink
|
||||
}
|
||||
|
||||
type managedProcessFiles struct {
|
||||
stdout *os.File
|
||||
stderr *os.File
|
||||
}
|
||||
|
||||
type managedProcess interface {
|
||||
PID() int
|
||||
TargetPID() int
|
||||
Wait() (int, error)
|
||||
Kill() error
|
||||
}
|
||||
|
||||
type managedProcessTailer struct {
|
||||
cancel context.CancelFunc
|
||||
drain chan struct{}
|
||||
}
|
||||
|
||||
type OSManagedProcessSupervisor struct {
|
||||
root string
|
||||
path string
|
||||
outputRoot string
|
||||
mu sync.Mutex
|
||||
items map[string]ProcessIdentity
|
||||
retired map[string]ProcessIdentity
|
||||
tailers map[string]*managedProcessTailer
|
||||
}
|
||||
|
||||
func NewOSManagedProcessSupervisor(root string) (*OSManagedProcessSupervisor, error) {
|
||||
return NewOSManagedProcessSupervisorWithOutputRoot(root, root)
|
||||
}
|
||||
|
||||
func NewOSManagedProcessSupervisorWithOutputRoot(root string, outputRoot string) (*OSManagedProcessSupervisor, error) {
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputRootAbs, err := filepath.Abs(outputRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stateDir := filepath.Join(rootAbs, "state")
|
||||
if err := ensureDirectory(stateDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
supervisor := &OSManagedProcessSupervisor{root: rootAbs, path: filepath.Join(stateDir, "processes.json"), outputRoot: outputRootAbs, items: map[string]ProcessIdentity{}, retired: map[string]ProcessIdentity{}, tailers: map[string]*managedProcessTailer{}}
|
||||
if err := supervisor.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := supervisor.migrateLegacySessions(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
supervisor.Reconcile()
|
||||
return supervisor, nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Start(ctx context.Context, command ProcessCommand, identity ProcessIdentity, output ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
key := identity.Scope
|
||||
log.Printf("RUN phase=process.managed status=start_requested job=%s server=%s scope=%s command=%s workdir=%s", safeOptional(identity.JobID), identity.ServerInstanceID, safeOptional(identity.Scope), redactedCommandLine(command.Args), safeOptional(command.WorkDir))
|
||||
if existing, ok := supervisor.items[key]; ok && existing.State == "running" && supervisor.isAlive(existing) {
|
||||
if existing.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err)
|
||||
}
|
||||
existing.LogSessionID = logSessionID
|
||||
}
|
||||
if existing.StopEventName == "" {
|
||||
existing.StopEventName = managedProcessStopEventName(existing)
|
||||
}
|
||||
existing.State = "running"
|
||||
existing.ObservationSeq++
|
||||
existing.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[key] = existing
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("persist managed process session: %w", err)
|
||||
}
|
||||
supervisor.startTailersLocked(existing, output)
|
||||
log.Printf("RUN phase=process.managed status=reusing_existing job=%s pid=%d state=%s", safeOptional(identity.JobID), existing.PID, existing.State)
|
||||
return existing, nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=context_done job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
if len(command.Args) == 0 {
|
||||
log.Printf("RUN phase=process.managed status=missing_executable job=%s", safeOptional(identity.JobID))
|
||||
return ProcessIdentity{}, fmt.Errorf("process executable is required")
|
||||
}
|
||||
startedAt := time.Now().UTC()
|
||||
if identity.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err)
|
||||
}
|
||||
identity.LogSessionID = logSessionID
|
||||
}
|
||||
if identity.StopEventName == "" {
|
||||
identity.StopEventName = managedProcessStopEventName(identity)
|
||||
}
|
||||
files, identity, err := supervisor.prepareOutputFilesLocked(identity, startedAt)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed status=prepare_output_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=output_ready job=%s stdoutRef=%s stderrRef=%s", safeOptional(identity.JobID), safeOptional(identity.StdoutLogRef), safeOptional(identity.StderrLogRef))
|
||||
process, err := startManagedProcess(command, files, identity.StopEventName)
|
||||
if err != nil {
|
||||
files.close()
|
||||
log.Printf("RUN phase=process.managed status=start_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
identity.SupervisorPID = process.PID()
|
||||
identity.PID = process.TargetPID()
|
||||
identity.StartedAt = startedAt
|
||||
identity.State = "running"
|
||||
identity.ObservationSeq = 1
|
||||
identity.UpdatedAt = identity.StartedAt
|
||||
identity.CommandFingerprint = fingerprintArgs(command.Args)
|
||||
previous, hadPrevious := supervisor.items[key]
|
||||
retiredKey := ""
|
||||
if hadPrevious && supervisor.hasPendingOutput(previous) {
|
||||
retiredKey = managedProcessGenerationKey(previous)
|
||||
supervisor.retired[retiredKey] = previous
|
||||
}
|
||||
supervisor.items[key] = identity
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
_ = forceManagedProcessStop(identity)
|
||||
files.close()
|
||||
if hadPrevious {
|
||||
supervisor.items[key] = previous
|
||||
} else {
|
||||
delete(supervisor.items, key)
|
||||
}
|
||||
if retiredKey != "" {
|
||||
delete(supervisor.retired, retiredKey)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=persist_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
supervisor.startTailersLocked(identity, output)
|
||||
go supervisor.wait(key, process, identity.PID, files)
|
||||
log.Printf("RUN phase=process.managed status=started job=%s pid=%d fingerprint=%s", safeOptional(identity.JobID), identity.PID, safeOptional(identity.CommandFingerprint))
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Stop(ctx context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
||||
supervisor.mu.Lock()
|
||||
current, ok := supervisor.items[identity.Scope]
|
||||
if !ok || current.State != "running" || !supervisor.isAlive(current) {
|
||||
if ok {
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "already-stopped"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[identity.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
supervisor.mu.Unlock()
|
||||
log.Printf("RUN phase=process.managed status=already_stopped job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope))
|
||||
return current, nil
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=stop_requested job=%s pid=%d scope=%s", safeOptional(identity.JobID), current.PID, safeOptional(identity.Scope))
|
||||
if err := requestManagedProcessStop(current); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error()))
|
||||
}
|
||||
supervisor.mu.Unlock()
|
||||
deadline := time.NewTimer(2 * time.Second)
|
||||
ticker := time.NewTicker(20 * time.Millisecond)
|
||||
defer deadline.Stop()
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if !supervisor.isAlive(current) {
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "requested-stop"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.mu.Lock()
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.mu.Unlock()
|
||||
supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay)
|
||||
log.Printf("RUN phase=process.managed status=stopped job=%s pid=%d classification=%s", safeOptional(identity.JobID), current.PID, current.ExitClassification)
|
||||
return current, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("RUN phase=process.managed status=stop_context_done job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(ctx.Err().Error()))
|
||||
return ProcessIdentity{}, ctx.Err()
|
||||
case <-deadline.C:
|
||||
if err := forceManagedProcessStop(current); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=forced_stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error()))
|
||||
}
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "forced-stop"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.mu.Lock()
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.mu.Unlock()
|
||||
supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay)
|
||||
log.Printf("RUN phase=process.managed status=forced_stop job=%s pid=%d", safeOptional(identity.JobID), current.PID)
|
||||
return current, nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
current, ok := supervisor.items[identity.Scope]
|
||||
if !ok {
|
||||
log.Printf("RUN phase=process.managed status=not_started job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope))
|
||||
return ProcessIdentity{Scope: identity.Scope, State: "stopped", ExitClassification: "not-started"}
|
||||
}
|
||||
if current.State == "running" && !supervisor.isAlive(current) {
|
||||
current.State = "exited"
|
||||
if current.ExitClassification == "" {
|
||||
current.ExitClassification = "unexpected-exit"
|
||||
}
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
current.ObservationSeq++
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.drainTailersLocked(current)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=current job=%s pid=%d state=%s classification=%s", safeOptional(identity.JobID), current.PID, current.State, safeOptional(current.ExitClassification))
|
||||
return current
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcessOutput) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
for _, item := range supervisor.items {
|
||||
if item.State == "running" && supervisor.isAlive(item) {
|
||||
supervisor.startTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
} else if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
}
|
||||
}
|
||||
changed := false
|
||||
for key, item := range supervisor.retired {
|
||||
if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_retired_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
continue
|
||||
}
|
||||
delete(supervisor.retired, key)
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Reconcile() {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
changed := false
|
||||
for key, item := range supervisor.items {
|
||||
if item.State == "running" && !supervisor.isAlive(item) {
|
||||
item.State = "exited"
|
||||
item.ExitClassification = "unexpected-exit"
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
item.ObservationSeq++
|
||||
supervisor.items[key] = item
|
||||
supervisor.drainTailersLocked(item)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) wait(key string, process managedProcess, pid int, files managedProcessFiles) {
|
||||
exitCode, err := process.Wait()
|
||||
files.close()
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
item, ok := supervisor.items[key]
|
||||
if !ok || item.PID != pid {
|
||||
return
|
||||
}
|
||||
// A durable helper may exit independently of the target process (legacy
|
||||
// shell wrappers can detach their child). Preserve a live target so the
|
||||
// next Run can continue monitoring it by PID.
|
||||
if processAlivePID(item.PID) {
|
||||
item.State = "running"
|
||||
item.SupervisorPID = 0
|
||||
item.ExitClassification = "supervisor-exited-target-alive"
|
||||
} else {
|
||||
item.State = "exited"
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
item.ObservationSeq++
|
||||
item.ExitCode = exitCode
|
||||
if item.State == "running" {
|
||||
// Keep the classification assigned above.
|
||||
} else if err == nil {
|
||||
item.ExitClassification = "clean-exit"
|
||||
} else {
|
||||
item.ExitClassification = "unexpected-exit"
|
||||
}
|
||||
supervisor.items[key] = item
|
||||
_ = supervisor.persistLocked()
|
||||
if item.State != "running" {
|
||||
supervisor.drainTailersAfter(item, managedProcessOutputDrainDelay)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s error=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification, RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification)
|
||||
}
|
||||
|
||||
func processTerminalStatus(state string) string {
|
||||
if state == "running" {
|
||||
return "supervisor_exited_target_alive"
|
||||
}
|
||||
return "exited"
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) ManagedProcessObservations() []ProcessIdentity {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
items := make([]ProcessIdentity, 0, len(supervisor.items))
|
||||
for _, item := range supervisor.items {
|
||||
items = append(items, item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (files managedProcessFiles) close() {
|
||||
if files.stdout != nil {
|
||||
_ = files.stdout.Close()
|
||||
}
|
||||
if files.stderr != nil {
|
||||
_ = files.stderr.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) prepareOutputFilesLocked(identity ProcessIdentity, startedAt time.Time) (managedProcessFiles, ProcessIdentity, error) {
|
||||
outputDir := filepath.Join(supervisor.outputRoot, "state", "process-output")
|
||||
if err := ensureDirectory(outputDir); err != nil {
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
base := processOutputBase(identity.Scope+"\x00"+identity.LogSessionID, startedAt)
|
||||
identity.StdoutLogRef = base + ".stdout.log"
|
||||
identity.StderrLogRef = base + ".stderr.log"
|
||||
stdout, stdoutOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StdoutLogRef))
|
||||
if err != nil {
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
stderr, stderrOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StderrLogRef))
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
identity.StdoutOffset = stdoutOffset
|
||||
identity.StderrOffset = stderrOffset
|
||||
return managedProcessFiles{stdout: stdout, stderr: stderr}, identity, nil
|
||||
}
|
||||
|
||||
func openManagedOutputFile(path string) (*os.File, int64, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return file, info.Size(), nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) {
|
||||
if output.Stdout != nil && identity.StdoutLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, true)
|
||||
}
|
||||
if output.Stderr != nil && identity.StderrLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, true)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startDrainTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) {
|
||||
if output.Stdout != nil && identity.StdoutLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, false)
|
||||
}
|
||||
if output.Stderr != nil && identity.StderrLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startTailerLocked(identity ProcessIdentity, stream string, ref string, offset int64, sink ManagedProcessLineSink, follow bool) {
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
tailerID := managedProcessTailerID(identity, stream)
|
||||
if tailer, exists := supervisor.tailers[tailerID]; exists {
|
||||
if !follow {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed.output status=tail_reuse job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset)
|
||||
return
|
||||
}
|
||||
path := filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(ref))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
tailer := &managedProcessTailer{cancel: cancel, drain: make(chan struct{})}
|
||||
if !follow {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
supervisor.tailers[tailerID] = tailer
|
||||
log.Printf("RUN phase=process.managed.output status=tail_start job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset)
|
||||
go supervisor.tailOutput(ctx, tailerID, tailer, identity, stream, path, offset, sink)
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) tailOutput(ctx context.Context, tailerID string, tailer *managedProcessTailer, identity ProcessIdentity, stream string, path string, offset int64, sink ManagedProcessLineSink) {
|
||||
defer supervisor.removeTailer(tailerID, tailer, identity)
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_open_failed job=%s pid=%d stream=%s path=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
if offset > 0 {
|
||||
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_seek_failed job=%s pid=%d stream=%s path=%s offset=%d error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), offset, RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
reader := bufio.NewReader(file)
|
||||
defer func() {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_stop job=%s pid=%d stream=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, offset)
|
||||
}()
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
startOffset := offset
|
||||
endOffset := offset + int64(len(line))
|
||||
text := strings.TrimSpace(line)
|
||||
if text != "" {
|
||||
for {
|
||||
if sinkErr := sink(identity, ManagedProcessLine{Text: text, StartOffset: startOffset, EndOffset: endOffset}); sinkErr == nil {
|
||||
log.Printf("RUN phase=process.managed.output status=line job=%s pid=%d stream=%s line=%q", safeOptional(identity.JobID), identity.PID, stream, RedactText(text))
|
||||
break
|
||||
} else {
|
||||
log.Printf("RUN phase=process.managed.output status=spool_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(sinkErr.Error()))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(managedProcessOutputRetryDelay):
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
if offsetErr := supervisor.updateOutputOffset(identity, stream, endOffset); offsetErr == nil {
|
||||
break
|
||||
} else {
|
||||
log.Printf("RUN phase=process.managed.output status=offset_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(offsetErr.Error()))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(managedProcessOutputRetryDelay):
|
||||
}
|
||||
}
|
||||
offset = endOffset
|
||||
}
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if err != io.EOF {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tailer.drain:
|
||||
return
|
||||
case <-time.After(managedProcessOutputPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) removeTailer(tailerID string, tailer *managedProcessTailer, identity ProcessIdentity) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
if current, ok := supervisor.tailers[tailerID]; ok && current == tailer {
|
||||
delete(supervisor.tailers, tailerID)
|
||||
}
|
||||
key := managedProcessGenerationKey(identity)
|
||||
if retired, ok := supervisor.retired[key]; ok && !supervisor.hasPendingOutput(retired) {
|
||||
delete(supervisor.retired, key)
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
supervisor.retired[key] = retired
|
||||
log.Printf("RUN phase=process.managed.output status=retired_prune_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) updateOutputOffset(identity ProcessIdentity, stream string, offset int64) error {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
item, ok := supervisor.items[identity.Scope]
|
||||
retiredKey := ""
|
||||
if !ok || !sameManagedProcessGeneration(item, identity) {
|
||||
retiredKey = managedProcessGenerationKey(identity)
|
||||
item, ok = supervisor.retired[retiredKey]
|
||||
if !ok || !sameManagedProcessGeneration(item, identity) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if stream == "stdout" {
|
||||
item.StdoutOffset = offset
|
||||
} else {
|
||||
item.StderrOffset = offset
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
if retiredKey == "" {
|
||||
supervisor.items[identity.Scope] = item
|
||||
} else {
|
||||
supervisor.retired[retiredKey] = item
|
||||
}
|
||||
return supervisor.persistLocked()
|
||||
}
|
||||
|
||||
func sameManagedProcessGeneration(current ProcessIdentity, expected ProcessIdentity) bool {
|
||||
if current.LogSessionID != "" || expected.LogSessionID != "" {
|
||||
if current.LogSessionID == "" || current.LogSessionID != expected.LogSessionID {
|
||||
return false
|
||||
}
|
||||
return sameManagedProcessOutputFiles(current, expected)
|
||||
}
|
||||
return current.PID == expected.PID && current.StdoutLogRef == expected.StdoutLogRef && current.StderrLogRef == expected.StderrLogRef
|
||||
}
|
||||
|
||||
func sameManagedProcessOutputFiles(current ProcessIdentity, expected ProcessIdentity) bool {
|
||||
if current.StdoutLogRef != "" && expected.StdoutLogRef != "" && current.StdoutLogRef != expected.StdoutLogRef {
|
||||
return false
|
||||
}
|
||||
if current.StderrLogRef != "" && expected.StderrLogRef != "" && current.StderrLogRef != expected.StderrLogRef {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) drainTailersAfter(identity ProcessIdentity, delay time.Duration) {
|
||||
go func() {
|
||||
time.Sleep(delay)
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
supervisor.drainTailersLocked(identity)
|
||||
}()
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) drainTailersLocked(identity ProcessIdentity) {
|
||||
for _, stream := range []string{"stdout", "stderr"} {
|
||||
if tailer, ok := supervisor.tailers[managedProcessTailerID(identity, stream)]; ok {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func beginManagedProcessTailerDrain(tailer *managedProcessTailer) {
|
||||
select {
|
||||
case <-tailer.drain:
|
||||
default:
|
||||
close(tailer.drain)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) stopTailersLocked(identity ProcessIdentity) {
|
||||
for _, stream := range []string{"stdout", "stderr"} {
|
||||
tailerID := managedProcessTailerID(identity, stream)
|
||||
if tailer, ok := supervisor.tailers[tailerID]; ok {
|
||||
tailer.cancel()
|
||||
delete(supervisor.tailers, tailerID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func managedProcessTailerID(identity ProcessIdentity, stream string) string {
|
||||
generation := identity.LogSessionID
|
||||
if generation == "" {
|
||||
generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef)
|
||||
} else {
|
||||
generation += "\x00" + identity.StdoutLogRef + "\x00" + identity.StderrLogRef
|
||||
}
|
||||
return identity.Scope + "\x00" + generation + "\x00" + stream
|
||||
}
|
||||
|
||||
func managedProcessGenerationKey(identity ProcessIdentity) string {
|
||||
generation := identity.LogSessionID
|
||||
if generation == "" {
|
||||
generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef)
|
||||
}
|
||||
return identity.Scope + "\x00" + generation
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) hasPendingOutput(identity ProcessIdentity) bool {
|
||||
for _, item := range []struct {
|
||||
ref string
|
||||
offset int64
|
||||
}{{identity.StdoutLogRef, identity.StdoutOffset}, {identity.StderrLogRef, identity.StderrOffset}} {
|
||||
if item.ref == "" {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(item.ref)))
|
||||
if err == nil && info.Size() > item.offset {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) isAlive(item ProcessIdentity) bool {
|
||||
return processAlivePID(item.PID)
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) load() error {
|
||||
body, err := os.ReadFile(supervisor.path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var file processJournal
|
||||
if err := json.Unmarshal(body, &file); err != nil {
|
||||
return err
|
||||
}
|
||||
for key, item := range file.Items {
|
||||
supervisor.items[key] = item
|
||||
}
|
||||
for key, item := range file.Retired {
|
||||
supervisor.retired[key] = item
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) migrateLegacySessions() error {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
changed := false
|
||||
for key, item := range supervisor.items {
|
||||
if item.State != "running" || !supervisor.isAlive(item) {
|
||||
continue
|
||||
}
|
||||
if item.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate legacy managed process log session: %w", err)
|
||||
}
|
||||
item.LogSessionID = logSessionID
|
||||
changed = true
|
||||
}
|
||||
if item.StartedAt.IsZero() {
|
||||
item.StartedAt = time.Now().UTC()
|
||||
changed = true
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[key] = item
|
||||
}
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
return fmt.Errorf("persist legacy managed process log session: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) persistLocked() error {
|
||||
body, err := json.Marshal(processJournal{Version: managedProcessJournalVersion, Items: supervisor.items, Retired: supervisor.retired})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporary := supervisor.path + ".tmp"
|
||||
if err := os.WriteFile(temporary, body, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, supervisor.path); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processOutputBase(scope string, startedAt time.Time) string {
|
||||
sum := sha256.Sum256([]byte(scope + "\x00" + startedAt.Format(time.RFC3339Nano)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func managedProcessStopEventName(identity ProcessIdentity) string {
|
||||
seed := strings.Join([]string{
|
||||
"run-managed-stop-v1",
|
||||
identity.RunEndpointID,
|
||||
identity.ServerInstanceID,
|
||||
identity.Scope,
|
||||
identity.LogSessionID,
|
||||
}, "\x00")
|
||||
sum := sha256.Sum256([]byte(seed))
|
||||
return "Local\\run-managed-stop-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func newManagedProcessLogSessionID() (string, error) {
|
||||
var value [16]byte
|
||||
if _, err := rand.Read(value[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(value[:]), nil
|
||||
}
|
||||
|
||||
func fingerprintArgs(args []string) string {
|
||||
sum := sha256.Sum256([]byte(strings.Join(args, "\x00")))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build !windows
|
||||
|
||||
package runtime
|
||||
|
||||
import "os/exec"
|
||||
|
||||
func configureManagedProcessCommand(_ *exec.Cmd) {}
|
||||
|
||||
func RunManagedProcessHelper(_ []string) (bool, int) { return false, 0 }
|
||||
@@ -0,0 +1,745 @@
|
||||
//go:build windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
const managedProcessHelperFlag = "--run-managed-process-helper"
|
||||
|
||||
func configureManagedProcessCommand(cmd *exec.Cmd) {
|
||||
// Run may itself be launched by a service or task scheduler job that
|
||||
// terminates its process tree on shutdown. The helper is the durable
|
||||
// owner of the game process, so ask Windows to keep it outside that job.
|
||||
// The helper itself does not need a console: its stdout/stderr are already
|
||||
// durable files. CREATE_NEW_CONSOLE creates a second hidden conhost in a
|
||||
// non-interactive Task Scheduler session and prevents the child pseudo
|
||||
// console from initializing on Windows Server.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_NO_WINDOW | windows.CREATE_BREAKAWAY_FROM_JOB, HideWindow: true}
|
||||
}
|
||||
|
||||
// Windows console output is collected by a child helper that owns the
|
||||
// pseudo-console. The helper inherits the durable stdout/stderr files, so it
|
||||
// remains attached to the supervised process when Run itself is updated or
|
||||
// restarted. The new Run instance resumes tailing those files by offset.
|
||||
type windowsFileManagedProcess struct {
|
||||
cmd *exec.Cmd
|
||||
helperExecutable string
|
||||
targetPID int
|
||||
}
|
||||
|
||||
type managedProcessHelperSpec struct {
|
||||
Command ProcessCommand `json:"command"`
|
||||
StopEventName string `json:"stopEventName,omitempty"`
|
||||
StdoutPath string `json:"stdoutPath,omitempty"`
|
||||
StderrPath string `json:"stderrPath,omitempty"`
|
||||
PIDPath string `json:"pidPath,omitempty"`
|
||||
}
|
||||
|
||||
func startManagedProcess(command ProcessCommand, files managedProcessFiles, stopEventName string) (managedProcess, error) {
|
||||
pidPath := files.stdout.Name() + ".pid"
|
||||
_ = os.Remove(pidPath)
|
||||
body, err := json.Marshal(managedProcessHelperSpec{Command: command, StopEventName: stopEventName, StdoutPath: files.stdout.Name(), StderrPath: files.stderr.Name(), PIDPath: pidPath})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode managed process helper spec: %w", err)
|
||||
}
|
||||
payload := base64.RawURLEncoding.EncodeToString(body)
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve Run executable for managed process helper: %w", err)
|
||||
}
|
||||
helperExecutable, err := prepareManagedProcessHelperExecutable(executable, files.stdout.Name())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare managed process helper executable: %w", err)
|
||||
}
|
||||
cmd := exec.Command(helperExecutable, managedProcessHelperFlag, payload)
|
||||
cmd.Stdout = files.stdout
|
||||
cmd.Stderr = files.stderr
|
||||
configureManagedProcessCommand(cmd)
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = os.Remove(helperExecutable)
|
||||
return nil, err
|
||||
}
|
||||
targetPID := cmd.Process.Pid
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if body, readErr := os.ReadFile(pidPath); readErr == nil {
|
||||
if parsed, parseErr := strconv.Atoi(strings.TrimSpace(string(body))); parseErr == nil && parsed > 0 {
|
||||
targetPID = parsed
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return &windowsFileManagedProcess{cmd: cmd, helperExecutable: helperExecutable, targetPID: targetPID}, nil
|
||||
}
|
||||
|
||||
func prepareManagedProcessHelperExecutable(executable, outputPath string) (string, error) {
|
||||
directory := filepath.Dir(outputPath)
|
||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
temporary, err := os.CreateTemp(directory, ".run-managed-helper-*.exe")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
helper := temporary.Name()
|
||||
if err := temporary.Close(); err != nil {
|
||||
_ = os.Remove(helper)
|
||||
return "", err
|
||||
}
|
||||
if err := os.Remove(helper); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := copyExecutable(executable, helper); err != nil {
|
||||
_ = os.Remove(helper)
|
||||
return "", err
|
||||
}
|
||||
return helper, nil
|
||||
}
|
||||
|
||||
func (process *windowsFileManagedProcess) PID() int {
|
||||
return process.cmd.Process.Pid
|
||||
}
|
||||
|
||||
func (process *windowsFileManagedProcess) TargetPID() int {
|
||||
if process.targetPID > 0 {
|
||||
return process.targetPID
|
||||
}
|
||||
return process.PID()
|
||||
}
|
||||
|
||||
func (process *windowsFileManagedProcess) Wait() (int, error) {
|
||||
err := process.cmd.Wait()
|
||||
if process.helperExecutable != "" {
|
||||
_ = os.Remove(process.helperExecutable)
|
||||
}
|
||||
if process.cmd.ProcessState == nil {
|
||||
return -1, err
|
||||
}
|
||||
return process.cmd.ProcessState.ExitCode(), err
|
||||
}
|
||||
|
||||
func (process *windowsFileManagedProcess) Kill() error {
|
||||
return process.cmd.Process.Kill()
|
||||
}
|
||||
|
||||
// RunManagedProcessHelper is invoked by the same executable in a detached
|
||||
// child process. It is deliberately not a worker mode and does not register
|
||||
// with Platform.
|
||||
func RunManagedProcessHelper(args []string) (bool, int) {
|
||||
if len(args) < 3 || args[1] != managedProcessHelperFlag {
|
||||
return false, 0
|
||||
}
|
||||
body, err := base64.RawURLEncoding.DecodeString(args[2])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid managed process helper payload: %v\n", err)
|
||||
return true, 2
|
||||
}
|
||||
var spec managedProcessHelperSpec
|
||||
if err := json.Unmarshal(body, &spec); err != nil || len(spec.Command.Args) == 0 {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("managed process command is empty")
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "invalid managed process helper spec: %v\n", err)
|
||||
return true, 2
|
||||
}
|
||||
code, err := runManagedProcessHelper(spec)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "managed process helper failed: %v\n", err)
|
||||
if code == 0 {
|
||||
code = 1
|
||||
}
|
||||
}
|
||||
return true, code
|
||||
}
|
||||
|
||||
func runManagedProcessHelper(spec managedProcessHelperSpec) (int, error) {
|
||||
defer scheduleManagedProcessHelperCleanup()
|
||||
stdout, closeStdout, err := openManagedProcessOutput(spec.StdoutPath, os.Stdout)
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("open managed process stdout: %w", err)
|
||||
}
|
||||
defer closeStdout()
|
||||
stderr, closeStderr, err := openManagedProcessOutput(spec.StderrPath, os.Stderr)
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("open managed process stderr: %w", err)
|
||||
}
|
||||
defer closeStderr()
|
||||
command := spec.Command
|
||||
stopEventName := spec.StopEventName
|
||||
// The plugin-declared pipes mode uses ordinary inherited handles. The
|
||||
// durable output files are attached directly to the child process, so this
|
||||
// path remains valid when Run itself is updated or restarted. Console mode
|
||||
// is deliberately kept separate for applications that switch from
|
||||
// redirected handles to a Windows console after startup.
|
||||
if command.OutputMode != "console" {
|
||||
return runManagedProcessWithPipes(command, stopEventName, spec.PIDPath, stdout, stderr)
|
||||
}
|
||||
return runManagedProcessWithPseudoConsole(command, stopEventName, spec.PIDPath, stdout, stderr)
|
||||
}
|
||||
|
||||
func scheduleManagedProcessHelperCleanup() {
|
||||
executable, err := os.Executable()
|
||||
if err != nil || strings.TrimSpace(executable) == "" {
|
||||
return
|
||||
}
|
||||
commandLine := "ping 127.0.0.1 -n 2 >nul & del /f /q " + quoteWindowsCommandArg(executable)
|
||||
cleanup := exec.Command("cmd.exe", "/d", "/c", commandLine)
|
||||
cleanup.Stdout = io.Discard
|
||||
cleanup.Stderr = io.Discard
|
||||
configureManagedProcessCommand(cleanup)
|
||||
_ = cleanup.Start()
|
||||
}
|
||||
|
||||
func openManagedProcessOutput(path string, fallback *os.File) (io.Writer, func(), error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fallback, func() {}, nil
|
||||
}
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return file, func() { _ = file.Close() }, nil
|
||||
}
|
||||
|
||||
func runManagedProcessWithPipes(command ProcessCommand, stopEventName string, pidPath string, stdout io.Writer, stderr io.Writer) (int, error) {
|
||||
if len(command.Args) == 0 {
|
||||
return 1, fmt.Errorf("managed process command is empty")
|
||||
}
|
||||
application := command.Args[0]
|
||||
if strings.EqualFold(application, "cmd.exe") {
|
||||
if comspec := os.Getenv("ComSpec"); comspec != "" {
|
||||
application = comspec
|
||||
}
|
||||
}
|
||||
cmd := exec.Command(application, command.Args[1:]...)
|
||||
cmd.Dir = command.WorkDir
|
||||
cmd.Env = os.Environ()
|
||||
for key, value := range command.Env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Stdout = stdout
|
||||
cmd.Stderr = stderr
|
||||
configureManagedProcessCommand(cmd)
|
||||
if err := cmd.Start(); err != nil {
|
||||
return 1, fmt.Errorf("start managed process with pipes: %w", err)
|
||||
}
|
||||
if err := writeManagedProcessPID(pidPath, cmd.Process.Pid); err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return 1, fmt.Errorf("persist managed process pid: %w", err)
|
||||
}
|
||||
|
||||
processHandle, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(cmd.Process.Pid))
|
||||
if err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return 1, fmt.Errorf("open managed process handle: %w", err)
|
||||
}
|
||||
job, err := createManagedProcessJob(processHandle)
|
||||
_ = windows.CloseHandle(processHandle)
|
||||
if err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return 1, err
|
||||
}
|
||||
defer windows.CloseHandle(job)
|
||||
stopCleanup, err := watchManagedProcessStop(job, stopEventName)
|
||||
if err != nil {
|
||||
_ = windows.TerminateJobObject(job, 1)
|
||||
_ = cmd.Wait()
|
||||
return 1, err
|
||||
}
|
||||
defer stopCleanup()
|
||||
|
||||
waitErr := cmd.Wait()
|
||||
if cmd.ProcessState == nil {
|
||||
if waitErr != nil {
|
||||
return 1, waitErr
|
||||
}
|
||||
return 1, fmt.Errorf("managed process has no exit state")
|
||||
}
|
||||
exitCode := cmd.ProcessState.ExitCode()
|
||||
if waitErr != nil {
|
||||
// exec.Cmd returns *exec.ExitError for a normal non-zero exit. Return
|
||||
// the actual code without treating it as an internal supervisor error;
|
||||
// the outer helper will propagate the code to the durable supervisor.
|
||||
if _, ok := waitErr.(*exec.ExitError); !ok {
|
||||
return 1, waitErr
|
||||
}
|
||||
}
|
||||
if exitCode > 255 {
|
||||
return 1, fmt.Errorf("managed process exited with code %d", exitCode)
|
||||
}
|
||||
return exitCode, nil
|
||||
}
|
||||
|
||||
func writeManagedProcessPID(path string, pid int) error {
|
||||
if strings.TrimSpace(path) == "" || pid <= 0 {
|
||||
return nil
|
||||
}
|
||||
temporary := path + ".tmp"
|
||||
if err := os.WriteFile(temporary, []byte(strconv.Itoa(pid)+"\n"), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, path); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type synchronizedOutputWriter struct {
|
||||
mu sync.Mutex
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (writer *synchronizedOutputWriter) Write(body []byte) (int, error) {
|
||||
writer.mu.Lock()
|
||||
defer writer.mu.Unlock()
|
||||
return writer.w.Write(body)
|
||||
}
|
||||
|
||||
func runManagedProcessWithPseudoConsole(command ProcessCommand, stopEventName string, pidPath string, stdout io.Writer, stderr io.Writer) (int, error) {
|
||||
// Direct console executables use a pseudo-console so programs that require
|
||||
// a console handle still have a bounded, hidden console surface. This is
|
||||
// also used for plugin-declared console capture around a Windows shell.
|
||||
inputRead, inputWrite, err := createPseudoConsolePipe()
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("create pseudo-console input pipe: %w", err)
|
||||
}
|
||||
outputRead, outputWrite, err := createPseudoConsolePipe()
|
||||
if err != nil {
|
||||
closePseudoConsoleHandles(inputRead, inputWrite)
|
||||
return 1, fmt.Errorf("create pseudo-console output pipe: %w", err)
|
||||
}
|
||||
|
||||
var console windows.Handle
|
||||
if err := windows.CreatePseudoConsole(windows.Coord{X: 160, Y: 50}, inputRead, outputWrite, 0, &console); err != nil {
|
||||
closePseudoConsoleHandles(inputRead, inputWrite, outputRead, outputWrite)
|
||||
return 1, fmt.Errorf("create pseudo-console: %w", err)
|
||||
}
|
||||
_ = windows.CloseHandle(inputRead)
|
||||
_ = windows.CloseHandle(outputWrite)
|
||||
|
||||
attributes, err := windows.NewProcThreadAttributeList(1)
|
||||
if err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("create pseudo-console process attributes: %w", err)
|
||||
}
|
||||
defer attributes.Delete()
|
||||
// PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE expects the HPCON handle value as
|
||||
// lpValue, not the address of the local variable that stores the handle.
|
||||
if err := attributes.Update(windows.PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE, unsafe.Pointer(uintptr(console)), unsafe.Sizeof(console)); err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("configure pseudo-console process attributes: %w", err)
|
||||
}
|
||||
|
||||
applicationPath := command.Args[0]
|
||||
commandIsCmd := strings.EqualFold(applicationPath, "cmd.exe")
|
||||
if commandIsCmd {
|
||||
applicationPath = os.Getenv("ComSpec")
|
||||
if applicationPath == "" {
|
||||
applicationPath = `C:\Windows\System32\cmd.exe`
|
||||
}
|
||||
}
|
||||
// CreateProcess receives both an application name and a command line.
|
||||
// Keep the command-line program name identical to the resolved application
|
||||
// name; cmd.exe can fail with STATUS_DLL_INIT_FAILED when the former is
|
||||
// left as the short name while the latter is an absolute path.
|
||||
commandArgs := append([]string(nil), command.Args...)
|
||||
commandArgs[0] = applicationPath
|
||||
commandLine, err := windows.UTF16FromString(windows.ComposeCommandLine(commandArgs))
|
||||
if err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("encode managed process command: %w", err)
|
||||
}
|
||||
var applicationName *uint16
|
||||
if !commandIsCmd && strings.ContainsAny(applicationPath, `:\`) {
|
||||
applicationName, err = windows.UTF16PtrFromString(applicationPath)
|
||||
if err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("encode managed process executable: %w", err)
|
||||
}
|
||||
}
|
||||
environment, err := managedProcessEnvironment(command.Env)
|
||||
if err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("encode managed process environment: %w", err)
|
||||
}
|
||||
var workDir *uint16
|
||||
if command.WorkDir != "" {
|
||||
workDir, err = windows.UTF16PtrFromString(command.WorkDir)
|
||||
if err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("encode managed process working directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A pseudo-console supplies the child's console surface itself; do not
|
||||
// combine it with STARTF_USESHOWWINDOW, which makes cmd.exe fail during
|
||||
// initialization on some non-interactive Windows Server sessions.
|
||||
startup := &windows.StartupInfoEx{StartupInfo: windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfoEx{}))}, ProcThreadAttributeList: attributes.List()}
|
||||
var processInfo windows.ProcessInformation
|
||||
// The helper has already detached from any parent job. A second
|
||||
// CREATE_BREAKAWAY_FROM_JOB on the pseudo-console client is rejected by
|
||||
// some Windows Server builds during console initialization and surfaces as
|
||||
// STATUS_DLL_INIT_FAILED from the otherwise valid child process.
|
||||
if err := windows.CreateProcess(applicationName, &commandLine[0], nil, nil, false, windows.CREATE_UNICODE_ENVIRONMENT|windows.EXTENDED_STARTUPINFO_PRESENT, environment, workDir, &startup.StartupInfo, &processInfo); err != nil {
|
||||
windows.ClosePseudoConsole(console)
|
||||
closePseudoConsoleHandles(inputWrite, outputRead)
|
||||
return 1, fmt.Errorf("start managed process in pseudo-console: %w", err)
|
||||
}
|
||||
_ = windows.CloseHandle(processInfo.Thread)
|
||||
_ = windows.CloseHandle(inputWrite)
|
||||
if err := writeManagedProcessPID(pidPath, int(processInfo.ProcessId)); err != nil {
|
||||
_ = windows.TerminateProcess(processInfo.Process, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
windows.ClosePseudoConsole(console)
|
||||
_ = windows.CloseHandle(outputRead)
|
||||
return 1, fmt.Errorf("persist managed process pid: %w", err)
|
||||
}
|
||||
|
||||
job, err := createManagedProcessJob(processInfo.Process)
|
||||
if err != nil {
|
||||
_ = windows.TerminateProcess(processInfo.Process, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
windows.ClosePseudoConsole(console)
|
||||
_ = windows.CloseHandle(outputRead)
|
||||
return 1, err
|
||||
}
|
||||
defer windows.CloseHandle(job)
|
||||
stopCleanup, err := watchManagedProcessStop(job, stopEventName)
|
||||
if err != nil {
|
||||
_ = windows.TerminateJobObject(job, 1)
|
||||
windows.ClosePseudoConsole(console)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
return 1, err
|
||||
}
|
||||
defer stopCleanup()
|
||||
|
||||
outputFile := os.NewFile(uintptr(outputRead), "run-pseudo-console-output")
|
||||
if outputFile == nil {
|
||||
_ = windows.TerminateProcess(processInfo.Process, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
windows.ClosePseudoConsole(console)
|
||||
return 1, fmt.Errorf("open pseudo-console output")
|
||||
}
|
||||
outputDone := make(chan struct{})
|
||||
outputWriter := &synchronizedOutputWriter{w: stdout}
|
||||
go func() {
|
||||
_, _ = io.Copy(outputWriter, outputFile)
|
||||
close(outputDone)
|
||||
}()
|
||||
|
||||
_, waitErr := windows.WaitForSingleObject(processInfo.Process, windows.INFINITE)
|
||||
var exitCode uint32
|
||||
if waitErr == nil {
|
||||
waitErr = windows.GetExitCodeProcess(processInfo.Process, &exitCode)
|
||||
}
|
||||
windows.ClosePseudoConsole(console)
|
||||
<-outputDone
|
||||
_ = outputFile.Close()
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
if waitErr != nil {
|
||||
return 1, waitErr
|
||||
}
|
||||
if exitCode > 255 {
|
||||
return 1, fmt.Errorf("managed process exited with code %d", exitCode)
|
||||
}
|
||||
return int(exitCode), nil
|
||||
}
|
||||
|
||||
func runManagedShellWithPipes(command ProcessCommand, stopEventName string, stdout io.Writer, stderr io.Writer) (int, error) {
|
||||
inputRead, inputWrite, err := createPseudoConsolePipe()
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("create managed shell input pipe: %w", err)
|
||||
}
|
||||
stdoutRead, stdoutWrite, err := createPseudoConsolePipe()
|
||||
if err != nil {
|
||||
closePseudoConsoleHandles(inputRead, inputWrite)
|
||||
return 1, fmt.Errorf("create managed shell stdout pipe: %w", err)
|
||||
}
|
||||
stderrRead, stderrWrite, err := createPseudoConsolePipe()
|
||||
if err != nil {
|
||||
closePseudoConsoleHandles(inputRead, inputWrite, stdoutRead, stdoutWrite)
|
||||
return 1, fmt.Errorf("create managed shell stderr pipe: %w", err)
|
||||
}
|
||||
closeOnError := func() {
|
||||
closePseudoConsoleHandles(inputRead, inputWrite, stdoutRead, stdoutWrite, stderrRead, stderrWrite)
|
||||
}
|
||||
if err := windows.SetHandleInformation(stdoutRead, windows.HANDLE_FLAG_INHERIT, 0); err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("make managed shell stdout pipe private: %w", err)
|
||||
}
|
||||
if err := windows.SetHandleInformation(stderrRead, windows.HANDLE_FLAG_INHERIT, 0); err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("make managed shell stderr pipe private: %w", err)
|
||||
}
|
||||
|
||||
applicationPath := os.Getenv("ComSpec")
|
||||
if applicationPath == "" {
|
||||
applicationPath = `C:\Windows\System32\cmd.exe`
|
||||
}
|
||||
commandArgs := append([]string(nil), command.Args...)
|
||||
commandArgs[0] = applicationPath
|
||||
commandLine, err := windows.UTF16FromString(windows.ComposeCommandLine(commandArgs))
|
||||
if err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("encode managed shell command: %w", err)
|
||||
}
|
||||
applicationName, err := windows.UTF16PtrFromString(applicationPath)
|
||||
if err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("encode managed shell executable: %w", err)
|
||||
}
|
||||
environment, err := managedProcessEnvironment(command.Env)
|
||||
if err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("encode managed shell environment: %w", err)
|
||||
}
|
||||
var workDir *uint16
|
||||
if command.WorkDir != "" {
|
||||
workDir, err = windows.UTF16PtrFromString(command.WorkDir)
|
||||
if err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("encode managed shell working directory: %w", err)
|
||||
}
|
||||
}
|
||||
startup := &windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfo{})), Flags: windows.STARTF_USESTDHANDLES | windows.STARTF_USESHOWWINDOW, ShowWindow: windows.SW_HIDE, StdInput: inputRead, StdOutput: stdoutWrite, StdErr: stderrWrite}
|
||||
var processInfo windows.ProcessInformation
|
||||
if err := windows.CreateProcess(applicationName, &commandLine[0], nil, nil, true, windows.CREATE_BREAKAWAY_FROM_JOB|windows.CREATE_NO_WINDOW|windows.CREATE_UNICODE_ENVIRONMENT, environment, workDir, startup, &processInfo); err != nil {
|
||||
closeOnError()
|
||||
return 1, fmt.Errorf("start managed shell: %w", err)
|
||||
}
|
||||
_ = windows.CloseHandle(processInfo.Thread)
|
||||
closePseudoConsoleHandles(inputRead, inputWrite, stdoutWrite, stderrWrite)
|
||||
|
||||
job, err := createManagedProcessJob(processInfo.Process)
|
||||
if err != nil {
|
||||
_ = windows.TerminateProcess(processInfo.Process, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
closePseudoConsoleHandles(stdoutRead, stderrRead)
|
||||
return 1, err
|
||||
}
|
||||
defer windows.CloseHandle(job)
|
||||
stopCleanup, err := watchManagedProcessStop(job, stopEventName)
|
||||
if err != nil {
|
||||
_ = windows.TerminateJobObject(job, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
return 1, err
|
||||
}
|
||||
defer stopCleanup()
|
||||
stdoutFile := os.NewFile(uintptr(stdoutRead), "run-managed-shell-stdout")
|
||||
stderrFile := os.NewFile(uintptr(stderrRead), "run-managed-shell-stderr")
|
||||
if stdoutFile == nil || stderrFile == nil {
|
||||
if stdoutFile != nil {
|
||||
_ = stdoutFile.Close()
|
||||
}
|
||||
if stderrFile != nil {
|
||||
_ = stderrFile.Close()
|
||||
}
|
||||
_ = windows.TerminateProcess(processInfo.Process, 1)
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
return 1, fmt.Errorf("open managed shell output pipes")
|
||||
}
|
||||
outputDone := make(chan struct{})
|
||||
outputWriter := &synchronizedOutputWriter{w: stdout}
|
||||
go func() {
|
||||
_, _ = io.Copy(outputWriter, stdoutFile)
|
||||
_ = stdoutFile.Close()
|
||||
close(outputDone)
|
||||
}()
|
||||
errorDone := make(chan struct{})
|
||||
errorWriter := &synchronizedOutputWriter{w: stderr}
|
||||
go func() {
|
||||
_, _ = io.Copy(errorWriter, stderrFile)
|
||||
_ = stderrFile.Close()
|
||||
close(errorDone)
|
||||
}()
|
||||
|
||||
_, waitErr := windows.WaitForSingleObject(processInfo.Process, windows.INFINITE)
|
||||
var exitCode uint32
|
||||
if waitErr == nil {
|
||||
waitErr = windows.GetExitCodeProcess(processInfo.Process, &exitCode)
|
||||
}
|
||||
<-outputDone
|
||||
<-errorDone
|
||||
_ = windows.CloseHandle(processInfo.Process)
|
||||
if waitErr != nil {
|
||||
return 1, waitErr
|
||||
}
|
||||
if exitCode > 255 {
|
||||
return 1, fmt.Errorf("managed shell exited with code %d", exitCode)
|
||||
}
|
||||
return int(exitCode), nil
|
||||
}
|
||||
|
||||
func createManagedProcessJob(process windows.Handle) (windows.Handle, error) {
|
||||
job, err := windows.CreateJobObject(nil, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("create managed process job: %w", err)
|
||||
}
|
||||
if err := windows.AssignProcessToJobObject(job, process); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return 0, fmt.Errorf("assign managed process to job: %w", err)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func watchManagedProcessStop(job windows.Handle, name string) (func(), error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return func() {}, nil
|
||||
}
|
||||
eventName, err := windows.UTF16PtrFromString(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode managed process stop event: %w", err)
|
||||
}
|
||||
event, eventErr := windows.CreateEvent(nil, 1, 0, eventName)
|
||||
if eventErr != nil && eventErr != windows.ERROR_ALREADY_EXISTS {
|
||||
if event != 0 {
|
||||
_ = windows.CloseHandle(event)
|
||||
}
|
||||
return nil, fmt.Errorf("create managed process stop event: %w", eventErr)
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
result, waitErr := windows.WaitForSingleObject(event, 100)
|
||||
if waitErr != nil {
|
||||
return
|
||||
}
|
||||
if result == windows.WAIT_OBJECT_0 {
|
||||
_ = windows.TerminateJobObject(job, 1)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(stop)
|
||||
<-done
|
||||
_ = windows.CloseHandle(event)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func requestManagedProcessStop(identity ProcessIdentity) error {
|
||||
if strings.TrimSpace(identity.StopEventName) != "" {
|
||||
name, err := windows.UTF16PtrFromString(identity.StopEventName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
event, openErr := windows.OpenEvent(windows.EVENT_MODIFY_STATE|windows.SYNCHRONIZE, false, name)
|
||||
if openErr == nil {
|
||||
setErr := windows.SetEvent(event)
|
||||
_ = windows.CloseHandle(event)
|
||||
return setErr
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("managed process stop event is unavailable")
|
||||
}
|
||||
|
||||
func forceManagedProcessStop(identity ProcessIdentity) error {
|
||||
if err := requestManagedProcessStop(identity); err == nil {
|
||||
return nil
|
||||
}
|
||||
if identity.PID <= 0 {
|
||||
return fmt.Errorf("managed process pid is invalid")
|
||||
}
|
||||
// A recovered process can outlive its helper. In that case no stop-event
|
||||
// watcher remains. Target only the persisted process PID here: /T would
|
||||
// recursively terminate descendants and can take down a game process that
|
||||
// was deliberately kept alive while Run is being restarted or updated.
|
||||
command := exec.Command("taskkill.exe", "/PID", strconv.Itoa(identity.PID), "/F")
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
return command.Run()
|
||||
}
|
||||
|
||||
func createPseudoConsolePipe() (windows.Handle, windows.Handle, error) {
|
||||
var readHandle, writeHandle windows.Handle
|
||||
// ConPTY owns these handles through its internal duplication. Keeping the
|
||||
// pipe ends non-inheritable matches the Windows ConPTY contract and avoids
|
||||
// leaking the pseudo-console handles into the client process.
|
||||
if err := windows.CreatePipe(&readHandle, &writeHandle, nil, 0); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return readHandle, writeHandle, nil
|
||||
}
|
||||
|
||||
func closePseudoConsoleHandles(handles ...windows.Handle) {
|
||||
for _, handle := range handles {
|
||||
if handle != 0 && handle != windows.InvalidHandle {
|
||||
_ = windows.CloseHandle(handle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func managedProcessEnvironment(values map[string]string) (*uint16, error) {
|
||||
if len(values) == 0 {
|
||||
// A nil environment tells CreateProcess to inherit the helper's
|
||||
// environment. This preserves Windows' special drive-current-directory
|
||||
// entries and avoids rebuilding a potentially incomplete environment
|
||||
// block for the common case.
|
||||
return nil, nil
|
||||
}
|
||||
environment := make(map[string]string, len(values))
|
||||
for _, entry := range os.Environ() {
|
||||
keyEnd := strings.IndexByte(entry, '=')
|
||||
if strings.HasPrefix(entry, "=") {
|
||||
// Windows stores drive current directories as =C:=C:\\...;
|
||||
// the first equals sign is part of that variable's name.
|
||||
if next := strings.IndexByte(entry[1:], '='); next >= 0 {
|
||||
keyEnd = next + 1
|
||||
}
|
||||
}
|
||||
if keyEnd > 0 {
|
||||
key := entry[:keyEnd]
|
||||
environment[strings.ToUpper(key)] = entry
|
||||
}
|
||||
}
|
||||
for key, value := range values {
|
||||
environment[strings.ToUpper(key)] = key + "=" + value
|
||||
}
|
||||
entries := make([]string, 0, len(environment))
|
||||
for _, entry := range environment {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return strings.ToUpper(entries[i]) < strings.ToUpper(entries[j]) })
|
||||
encoded := utf16.Encode([]rune(strings.Join(entries, "\x00") + "\x00\x00"))
|
||||
return &encoded[0], nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//go:build windows
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func TestConfigureManagedProcessCommandPreservesInheritedConsole(t *testing.T) {
|
||||
cmd := exec.Command("cmd.exe")
|
||||
configureManagedProcessCommand(cmd)
|
||||
if cmd.SysProcAttr == nil {
|
||||
t.Fatal("expected Windows process attributes")
|
||||
}
|
||||
if !cmd.SysProcAttr.HideWindow {
|
||||
t.Fatal("expected managed console window to stay hidden")
|
||||
}
|
||||
expected := uint32(windows.CREATE_NO_WINDOW | windows.CREATE_BREAKAWAY_FROM_JOB)
|
||||
if cmd.SysProcAttr.CreationFlags != expected {
|
||||
t.Fatalf("expected a hidden breakaway helper without a console, got %#x", cmd.SysProcAttr.CreationFlags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteManagedProcessPIDUsesAtomicSidecar(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "process.stdout.log.pid")
|
||||
if err := writeManagedProcessPID(path, 321); err != nil {
|
||||
t.Fatalf("write managed process pid: %v", err)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read managed process pid: %v", err)
|
||||
}
|
||||
if got, err := strconv.Atoi(string(body[:len(body)-1])); err != nil || got != 321 {
|
||||
t.Fatalf("expected pid sidecar to contain 321, got %q err=%v", body, err)
|
||||
}
|
||||
if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Fatalf("pid sidecar temp file should not remain, err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareManagedProcessHelperExecutableCopiesRunOutsideCurrentPath(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
current := filepath.Join(root, "run.exe")
|
||||
output := filepath.Join(root, "state", "process-output", "stdout.log")
|
||||
if err := os.WriteFile(current, []byte("run-binary"), 0o700); err != nil {
|
||||
t.Fatalf("write current executable: %v", err)
|
||||
}
|
||||
helper, err := prepareManagedProcessHelperExecutable(current, output)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare helper executable: %v", err)
|
||||
}
|
||||
defer os.Remove(helper)
|
||||
if helper == current || filepath.Dir(helper) != filepath.Dir(output) {
|
||||
t.Fatalf("expected helper beside durable process output, helper=%q current=%q", helper, current)
|
||||
}
|
||||
body, err := os.ReadFile(helper)
|
||||
if err != nil {
|
||||
t.Fatalf("read helper executable: %v", err)
|
||||
}
|
||||
if string(body) != "run-binary" {
|
||||
t.Fatalf("expected helper to copy current executable, got %q", body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
ProtectedRequestStatusSucceeded = "succeeded"
|
||||
ProtectedRequestStatusFailed = "failed"
|
||||
ProtectedRequestStatusUnknown = "unknown"
|
||||
)
|
||||
|
||||
// ErrProtectedRequestUnknown lets a Run-owned transport report that a request
|
||||
// is syntactically safe but not one of its declared operations. It is terminal
|
||||
// and affects only this request.
|
||||
var ErrProtectedRequestUnknown = errors.New("protected request outcome unknown")
|
||||
|
||||
// ProtectedRequest contains only Platform-authorized text and logical binding.
|
||||
// Handlers resolve their own private transport configuration locally; they must
|
||||
// not return it in an outcome, error, log line, or result.
|
||||
type ProtectedRequest struct {
|
||||
JobID string
|
||||
ServerInstanceID string
|
||||
FencingToken uint64
|
||||
Kind string
|
||||
TransportKey string
|
||||
TargetKey string
|
||||
RequestText string
|
||||
}
|
||||
|
||||
type ProtectedRequestOutcome struct {
|
||||
Status string
|
||||
Stdout string
|
||||
Stderr string
|
||||
}
|
||||
|
||||
type ProtectedRequestHandler interface {
|
||||
ExecuteProtectedRequest(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error)
|
||||
}
|
||||
|
||||
type ProtectedRequestHandlerFunc func(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error)
|
||||
|
||||
func (fn ProtectedRequestHandlerFunc) ExecuteProtectedRequest(ctx context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) {
|
||||
return fn(ctx, request)
|
||||
}
|
||||
|
||||
// ProtectedRequestRegistry is configured by the local Run package owner. It
|
||||
// uses only logical lookup keys, so plugin and Platform payloads cannot select
|
||||
// an arbitrary program, DSN, socket, or host path.
|
||||
type ProtectedRequestRegistry struct {
|
||||
mu sync.RWMutex
|
||||
handlers map[string]ProtectedRequestHandler
|
||||
}
|
||||
|
||||
func NewProtectedRequestRegistry() *ProtectedRequestRegistry {
|
||||
return &ProtectedRequestRegistry{handlers: map[string]ProtectedRequestHandler{}}
|
||||
}
|
||||
|
||||
func (registry *ProtectedRequestRegistry) Register(kind string, transportKey string, handler ProtectedRequestHandler) error {
|
||||
if !validProtectedRequestKind(kind) || !protocol.ValidLogicalFileKey(transportKey) || handler == nil {
|
||||
return fmt.Errorf("protected request kind, transport key, and handler are required")
|
||||
}
|
||||
registry.mu.Lock()
|
||||
defer registry.mu.Unlock()
|
||||
if registry.handlers == nil {
|
||||
registry.handlers = map[string]ProtectedRequestHandler{}
|
||||
}
|
||||
registry.handlers[protectedRequestRegistryKey(kind, transportKey)] = handler
|
||||
return nil
|
||||
}
|
||||
|
||||
func (registry *ProtectedRequestRegistry) handler(kind string, transportKey string) (ProtectedRequestHandler, bool) {
|
||||
if registry == nil {
|
||||
return nil, false
|
||||
}
|
||||
registry.mu.RLock()
|
||||
defer registry.mu.RUnlock()
|
||||
handler, exists := registry.handlers[protectedRequestRegistryKey(kind, transportKey)]
|
||||
return handler, exists
|
||||
}
|
||||
|
||||
func protectedRequestRegistryKey(kind string, transportKey string) string {
|
||||
return kind + "\x00" + transportKey
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ExecuteProtectedRequest(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.ProtectedRequestExecutionInputResponse) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil || !protectedRequestInputMatchesExecutor(input, assignment) {
|
||||
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_binding_invalid")
|
||||
}
|
||||
handler, exists := executor.protectedRequests.handler(input.Kind, input.TransportKey)
|
||||
if !exists {
|
||||
if input.Kind == "rcon" && assignment.ExecutionInput.SourceRCON != nil {
|
||||
return executor.executeProtectedSourceRCON(ctx, assignment, input)
|
||||
}
|
||||
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusUnknown, "protected_request_transport_unknown")
|
||||
}
|
||||
executionCtx, cancel := context.WithTimeout(ctx, time.Duration(assignment.ExecutionInput.TimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
outcome, err := handler.ExecuteProtectedRequest(executionCtx, ProtectedRequest{JobID: input.JobID, ServerInstanceID: input.ServerInstanceID, FencingToken: input.FencingToken, Kind: input.Kind, TransportKey: input.TransportKey, TargetKey: input.TargetKey, RequestText: input.RequestText})
|
||||
if input.Kind == "program" {
|
||||
executor.writeProtectedProgramLogs(ctx, assignment, outcome)
|
||||
}
|
||||
if errors.Is(executionCtx.Err(), context.Canceled) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request cancelled"}, Message: "protected request cancelled", ErrorCode: "protected_request_cancelled"}
|
||||
}
|
||||
if errors.Is(executionCtx.Err(), context.DeadlineExceeded) {
|
||||
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_timeout")
|
||||
}
|
||||
if errors.Is(err, ErrProtectedRequestUnknown) || outcome.Status == ProtectedRequestStatusUnknown {
|
||||
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusUnknown, "protected_request_unknown")
|
||||
}
|
||||
if err != nil || outcome.Status == ProtectedRequestStatusFailed || outcome.Status != "" && outcome.Status != ProtectedRequestStatusSucceeded {
|
||||
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_failed")
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request completed"}, Message: "protected request completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + input.Kind, Summary: "approved protected request executed through logical Run transport"}}
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) executeProtectedSourceRCON(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.ProtectedRequestExecutionInputResponse) LifecycleExecutionResult {
|
||||
result := executor.ExecuteSourceRCON(ctx, assignment, input.RequestText)
|
||||
if result.State != lifecycleResultStateSucceeded {
|
||||
return result
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected RCON request delivered"}, Message: "protected RCON request delivered", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected.rcon", Summary: "approved protected RCON request delivered through Source RCON"}}
|
||||
}
|
||||
|
||||
func protectedRequestInputMatchesExecutor(input protocol.ProtectedRequestExecutionInputResponse, assignment protocol.RunJobAssignment) bool {
|
||||
return protocol.ValidProtectedRequestExecutionInput(input) && input.JobID == assignment.JobID && input.ServerInstanceID == assignment.ServerInstanceID && input.FencingToken == assignment.FencingToken && input.TargetKey == assignment.TargetKey && input.TransportKey == assignment.ExecutionInput.RemoteAdapterKey && input.Kind == protectedRequestKindForCapability(assignment.Capability)
|
||||
}
|
||||
|
||||
func protectedRequestKindForCapability(capability string) string {
|
||||
switch capability {
|
||||
case protocol.RunCapabilityRemoteRunProtectedSQL:
|
||||
return "sql"
|
||||
case protocol.RunCapabilityRemoteRunProtectedRCON:
|
||||
return "rcon"
|
||||
case protocol.RunCapabilityRemoteRunProgram:
|
||||
return "program"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func validProtectedRequestKind(kind string) bool {
|
||||
return kind == "sql" || kind == "rcon" || kind == "program"
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) writeProtectedProgramLogs(ctx context.Context, assignment protocol.RunJobAssignment, outcome ProtectedRequestOutcome) {
|
||||
for _, item := range []struct{ stream, body string }{{"management-program.stdout", outcome.Stdout}, {"management-program.stderr", outcome.Stderr}} {
|
||||
for _, line := range splitProtectedProgramLines(item.body) {
|
||||
_ = executor.logSink.Append(ctx, assignment, item.stream, sanitizeProtectedProgramLogLine(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func splitProtectedProgramLines(value string) []string {
|
||||
value = strings.ToValidUTF8(strings.TrimSpace(value), "")
|
||||
if len(value) > maxLifecycleOutputBytes {
|
||||
value = strings.ToValidUTF8(value[:maxLifecycleOutputBytes], "")
|
||||
}
|
||||
lines := strings.Split(value, "\n")
|
||||
bounded := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
if strings.TrimSpace(line) != "" {
|
||||
bounded = append(bounded, line)
|
||||
}
|
||||
}
|
||||
return bounded
|
||||
}
|
||||
|
||||
func sanitizeProtectedProgramLogLine(line string) string {
|
||||
if containsProtectedProgramPrivateText(line) {
|
||||
return "[redacted protected management-program output]"
|
||||
}
|
||||
return RedactText(line)
|
||||
}
|
||||
|
||||
func containsProtectedProgramPrivateText(value string) bool {
|
||||
lower := strings.ToLower(value)
|
||||
for _, marker := range []string{"password=", "password:", "secret=", "secret:", "token=", "token:", "credential", "bearer ", "api_key", "apikey", "dsn=", "path=", "file=", "database=", "://", "mysql:", "postgres:", "sqlite:", "file:", "socket", "named pipe"} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, field := range strings.Fields(value) {
|
||||
field = strings.Trim(field, "\"'()[]{}<>,;")
|
||||
if strings.HasPrefix(field, "/") || strings.HasPrefix(field, "./") || strings.HasPrefix(field, "../") || strings.HasPrefix(field, `\\`) || len(field) >= 3 && ((field[0] >= 'a' && field[0] <= 'z') || (field[0] >= 'A' && field[0] <= 'Z')) && field[1] == ':' && (field[2] == '/' || field[2] == '\\') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func protectedRequestFailure(capability string, status string, code string) LifecycleExecutionResult {
|
||||
kind := protectedRequestKindForCapability(capability)
|
||||
if kind == "" {
|
||||
kind = "unknown"
|
||||
}
|
||||
if status == ProtectedRequestStatusUnknown {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request outcome is unknown"}, Message: "protected request outcome is unknown", ErrorCode: "protected_request_unknown", ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + kind + ".unknown", Summary: "protected request outcome is unknown"}}
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "protected request failed"}, Message: "protected request failed", ErrorCode: code, ExecutionResult: protocol.RunJobExecutionResult{Kind: "protected." + kind + ".failed", Summary: "protected request failed safely"}}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
func TestWorkerExecutesFencedProtectedProgramAndSpoolsDedicatedLogs(t *testing.T) {
|
||||
registry := NewProtectedRequestRegistry()
|
||||
requestText := `{"operation":"status"}`
|
||||
handlerCalled := false
|
||||
if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) {
|
||||
handlerCalled = true
|
||||
if request.ServerInstanceID != "server-worker" || request.FencingToken != 12 || request.TargetKey != "scum-program" || request.RequestText != requestText {
|
||||
t.Fatalf("unexpected protected request: %+v", request)
|
||||
}
|
||||
return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded, Stdout: "SCUM ready\npassword=hidden\nconfig /Users/private/scum.ini\ndsn mysql://private", Stderr: "bounded warning"}, nil
|
||||
})); err != nil {
|
||||
t.Fatalf("register protected handler: %v", err)
|
||||
}
|
||||
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 12)
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = assignment
|
||||
client.protectedInput = protectedWorkerInput(assignment, "program", requestText)
|
||||
logSpool, err := spool.NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("log spool: %v", err)
|
||||
}
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithProtectedRequestRegistry(registry), WithProcessLogSink(&SpoolLogSink{Spool: logSpool}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register worker: %v", err)
|
||||
}
|
||||
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
||||
t.Fatalf("claim protected request handled=%v err=%v", handled, err)
|
||||
}
|
||||
if !handlerCalled || len(client.protectedRequests) != 1 || client.protectedRequests[0].FencingToken != assignment.FencingToken {
|
||||
t.Fatalf("expected one fenced protected input read: %+v", client.protectedRequests)
|
||||
}
|
||||
if len(client.resultRequests) != 1 || client.resultRequests[0].State != lifecycleResultStateSucceeded || client.resultRequests[0].ExecutionResult.Kind != "protected.program" {
|
||||
t.Fatalf("unexpected protected result: %+v", client.resultRequests)
|
||||
}
|
||||
batches, err := logSpool.Pending()
|
||||
entryCount := 0
|
||||
for _, batch := range batches {
|
||||
entryCount += len(batch.Entries)
|
||||
}
|
||||
if err != nil || entryCount != 5 {
|
||||
t.Fatalf("expected five program log lines: batches=%+v err=%v", batches, err)
|
||||
}
|
||||
redactedEntries := 0
|
||||
for _, batch := range batches {
|
||||
if batch.Source != "management-program" || batch.Source == "file" || batch.Source == "process" {
|
||||
t.Fatalf("program output used wrong log source: %+v", batch)
|
||||
}
|
||||
for _, entry := range batch.Entries {
|
||||
if entry.Redacted {
|
||||
redactedEntries++
|
||||
}
|
||||
if strings.Contains(entry.Line, "password=hidden") || strings.Contains(entry.Line, "/Users/") || strings.Contains(entry.Line, "://") {
|
||||
t.Fatalf("program log leaked protected output: %+v", entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
if redactedEntries != 3 {
|
||||
t.Fatalf("expected three explicitly redacted private lines, got %d", redactedEntries)
|
||||
}
|
||||
serialized, err := json.Marshal([]any{client.resultRequests, client.protectedRequests, worker.journal.ActiveJobs()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, private := range []string{requestText, "password=hidden", "/Users/private/scum.ini", "mysql://private", "bounded warning"} {
|
||||
if strings.Contains(string(serialized), private) {
|
||||
t.Fatalf("protected text or output leaked into control projection %q: %s", private, serialized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedRequestUnknownAndBindingFailureAreIsolated(t *testing.T) {
|
||||
registry := NewProtectedRequestRegistry()
|
||||
called := 0
|
||||
if err := registry.Register("rcon", "scum-management", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) {
|
||||
called++
|
||||
if request.RequestText == "unknown.command" {
|
||||
return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown}, ErrProtectedRequestUnknown
|
||||
}
|
||||
return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded}, nil
|
||||
})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry))
|
||||
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 21)
|
||||
unknown := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "unknown.command"))
|
||||
if unknown.State != lifecycleResultStateFailed || unknown.ErrorCode != "protected_request_unknown" || unknown.Retryable {
|
||||
t.Fatalf("unexpected unknown outcome: %+v", unknown)
|
||||
}
|
||||
succeeded := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "status"))
|
||||
if succeeded.State != lifecycleResultStateSucceeded || called != 2 {
|
||||
t.Fatalf("unknown request affected later request: result=%+v called=%d", succeeded, called)
|
||||
}
|
||||
mismatched := protectedWorkerInput(assignment, "rcon", "status")
|
||||
mismatched.FencingToken++
|
||||
failed := executor.ExecuteProtectedRequest(context.Background(), assignment, mismatched)
|
||||
if failed.ErrorCode != "protected_request_binding_invalid" || called != 2 {
|
||||
t.Fatalf("binding failure reached handler: result=%+v called=%d", failed, called)
|
||||
}
|
||||
encoded, _ := json.Marshal([]LifecycleExecutionResult{unknown, failed})
|
||||
if strings.Contains(string(encoded), "unknown.command") || strings.Contains(string(encoded), "status") {
|
||||
t.Fatalf("safe failure leaked request text: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedRCONFallsBackToSourceRCONPlan(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
defer listener.Close()
|
||||
password := strings.Repeat("f", 64)
|
||||
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 41)
|
||||
assignment.ExecutionInput.WorkspaceScope = "run-local"
|
||||
assignment.ExecutionInput.TimeoutSeconds = 5
|
||||
assignment.ExecutionInput.SourceRCON = &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port}
|
||||
root := t.TempDir()
|
||||
writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password)
|
||||
commands := make(chan string, 1)
|
||||
serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, packet sourceRCONPacket) error {
|
||||
commands <- packet.body
|
||||
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue})
|
||||
})
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "#ListPlayers"))
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "protected.rcon" {
|
||||
t.Fatalf("expected protected RCON delivery through Source RCON, got %+v", result)
|
||||
}
|
||||
if got := <-commands; got != "#ListPlayers" {
|
||||
t.Fatalf("expected SCUM command delivery, got %q", got)
|
||||
}
|
||||
awaitSourceRCONServer(t, serverDone)
|
||||
serialized, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(serialized), "#ListPlayers") || strings.Contains(string(serialized), password) {
|
||||
t.Fatalf("protected Source RCON result leaked private input: %s", serialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownProtectedProgramKeepsSafeDiagnosticInProgramLogOnly(t *testing.T) {
|
||||
registry := NewProtectedRequestRegistry()
|
||||
if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error) {
|
||||
return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown, Stderr: "unknown field database=/private/scum.db"}, ErrProtectedRequestUnknown
|
||||
})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordingLogSink{}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry), WithProcessLogSink(sink))
|
||||
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 31)
|
||||
result := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "program", `{"unexpected":true}`))
|
||||
if result.ErrorCode != "protected_request_unknown" || result.State != lifecycleResultStateFailed {
|
||||
t.Fatalf("unexpected unknown program result: %+v", result)
|
||||
}
|
||||
if len(sink.lines) != 1 || !strings.HasPrefix(sink.lines[0], "management-program.stderr:") || strings.Contains(sink.lines[0], "/private/") || !strings.Contains(sink.lines[0], "[redacted protected") {
|
||||
t.Fatalf("unknown program diagnostic was not safely channelized: %+v", sink.lines)
|
||||
}
|
||||
}
|
||||
|
||||
func protectedWorkerAssignment(capability string, kind string, key string, fence uint64) protocol.RunJobAssignment {
|
||||
assignment := workerJobAssignment(capability)
|
||||
assignment.TargetKey = key
|
||||
assignment.InputRef = "input://protected-request/" + assignment.JobID
|
||||
assignment.FencingToken = fence
|
||||
assignment.MaxAttempts = 1
|
||||
assignment.ExecutionInput = protocol.RunJobExecutionInput{RemoteAdapterKey: key, RemoteAdapterKind: "protected-" + kind, TimeoutSeconds: 5}
|
||||
return assignment
|
||||
}
|
||||
|
||||
func protectedWorkerInput(assignment protocol.RunJobAssignment, kind string, text string) protocol.ProtectedRequestExecutionInputResponse {
|
||||
return protocol.ProtectedRequestExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, FencingToken: assignment.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: time.Now().UTC().Add(time.Minute), Kind: kind, TransportKey: assignment.ExecutionInput.RemoteAdapterKey, TargetKey: assignment.TargetKey, RequestText: text}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type RemoteAdapterRequest struct {
|
||||
JobID string
|
||||
ServerInstanceID string
|
||||
AdapterKey string
|
||||
AdapterKind string
|
||||
TargetKey string
|
||||
Capability string
|
||||
InputRef string
|
||||
}
|
||||
|
||||
type RemoteAdapterOutcome struct {
|
||||
Message string
|
||||
ResultRef string
|
||||
Retryable bool
|
||||
}
|
||||
|
||||
type RemoteAdapter interface {
|
||||
Execute(context.Context, RemoteAdapterRequest) (RemoteAdapterOutcome, error)
|
||||
}
|
||||
|
||||
type RemoteAdapterFunc func(context.Context, RemoteAdapterRequest) (RemoteAdapterOutcome, error)
|
||||
|
||||
func (fn RemoteAdapterFunc) Execute(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) {
|
||||
return fn(ctx, request)
|
||||
}
|
||||
|
||||
type RemoteAdapterRegistry struct {
|
||||
mu sync.RWMutex
|
||||
adapters map[string]RemoteAdapter
|
||||
}
|
||||
|
||||
func NewRemoteAdapterRegistry() *RemoteAdapterRegistry {
|
||||
registry := &RemoteAdapterRegistry{adapters: map[string]RemoteAdapter{}}
|
||||
for _, kind := range []string{"ftp", "rsync", "run-file", "run-process", "database", "log-transfer"} {
|
||||
registry.adapters[kind] = declaredRemoteAdapter{kind: kind}
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
func (registry *RemoteAdapterRegistry) Register(kind string, adapter RemoteAdapter) error {
|
||||
kind = strings.TrimSpace(kind)
|
||||
if kind == "" || adapter == nil {
|
||||
return fmt.Errorf("remote adapter kind and implementation are required")
|
||||
}
|
||||
registry.mu.Lock()
|
||||
defer registry.mu.Unlock()
|
||||
registry.adapters[kind] = adapter
|
||||
return nil
|
||||
}
|
||||
|
||||
func (registry *RemoteAdapterRegistry) adapter(kind string) (RemoteAdapter, bool) {
|
||||
registry.mu.RLock()
|
||||
defer registry.mu.RUnlock()
|
||||
adapter, exists := registry.adapters[kind]
|
||||
return adapter, exists
|
||||
}
|
||||
|
||||
func ExecuteRemoteAccessJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
return ExecuteRemoteAccessJobWithRegistry(ctx, assignment, NewRemoteAdapterRegistry())
|
||||
}
|
||||
|
||||
func ExecuteRemoteAccessJobWithRegistry(ctx context.Context, assignment protocol.RunJobAssignment, registry *RemoteAdapterRegistry) LifecycleExecutionResult {
|
||||
if assignment.ExecutionInput.SourceRCON != nil {
|
||||
return lifecycleFailure("source_rcon_requires_worker_transport", "Source RCON commands require the one-time worker transport")
|
||||
}
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
if strings.Contains(err.Error(), "remoteAdapterKey") {
|
||||
return lifecycleFailure("unsafe_remote_adapter_target", "remote adapter key must be an approved logical key")
|
||||
}
|
||||
return lifecycleFailure("unsafe_remote_access_job", err.Error())
|
||||
}
|
||||
if !isSupportedRemoteCapability(assignment.Capability) {
|
||||
return lifecycleFailure("unsupported_remote_access_capability", "unsupported remote access capability")
|
||||
}
|
||||
if registry == nil {
|
||||
return lifecycleFailure("remote_adapter_unavailable", "remote adapter registry is unavailable")
|
||||
}
|
||||
adapterKind := strings.TrimSpace(assignment.ExecutionInput.RemoteAdapterKind)
|
||||
if adapterKind == "" {
|
||||
adapterKind = adapterKindForCapability(assignment.Capability)
|
||||
}
|
||||
adapterKey := strings.TrimSpace(assignment.ExecutionInput.RemoteAdapterKey)
|
||||
if adapterKey == "" {
|
||||
adapterKey = assignment.TargetKey
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(adapterKey) || !protocol.ValidLogicalFileKey(assignment.TargetKey) {
|
||||
return lifecycleFailure("unsafe_remote_adapter_target", "remote adapter and target must use approved logical keys")
|
||||
}
|
||||
if !adapterKindAllowsCapability(adapterKind, assignment.Capability) {
|
||||
return lifecycleFailure("remote_adapter_capability_mismatch", "remote adapter kind does not allow requested capability")
|
||||
}
|
||||
adapter, exists := registry.adapter(adapterKind)
|
||||
if !exists {
|
||||
return lifecycleFailure("remote_adapter_unavailable", "declared remote adapter is unavailable")
|
||||
}
|
||||
|
||||
executionCtx := ctx
|
||||
cancel := func() {}
|
||||
if timeout := assignment.ExecutionInput.TimeoutSeconds; timeout > 0 {
|
||||
if timeout > 300 {
|
||||
return lifecycleFailure("unsafe_remote_adapter_timeout", "remote adapter timeout exceeds bound")
|
||||
}
|
||||
executionCtx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
request := RemoteAdapterRequest{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, AdapterKey: adapterKey, AdapterKind: adapterKind, TargetKey: assignment.TargetKey, Capability: assignment.Capability, InputRef: assignment.InputRef}
|
||||
outcome, err := adapter.Execute(executionCtx, request)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(executionCtx.Err(), context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter timed out"}, Message: "remote adapter timed out", ErrorCode: "remote_adapter_timeout", Retryable: true}
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(executionCtx.Err(), context.Canceled) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter cancelled"}, Message: "remote adapter cancelled", ErrorCode: "remote_adapter_cancelled"}
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter failed"}, Message: "remote adapter failed", ErrorCode: "remote_adapter_failed", Retryable: outcome.Retryable}
|
||||
}
|
||||
if err := executionCtx.Err(); err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter timed out"}, Message: "remote adapter timed out", ErrorCode: "remote_adapter_timeout", Retryable: true}
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter cancelled"}, Message: "remote adapter cancelled", ErrorCode: "remote_adapter_cancelled"}
|
||||
}
|
||||
resultRef := outcome.ResultRef
|
||||
if resultRef == "" {
|
||||
resultRef = fmt.Sprintf("artifact://jobs/%s/remote-access-result", url.PathEscape(assignment.JobID))
|
||||
}
|
||||
message := strings.TrimSpace(outcome.Message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("%s completed through declared %s adapter", assignment.Capability, adapterKind)
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "remote adapter completed"}, ResultRef: resultRef, Message: message}
|
||||
}
|
||||
|
||||
type declaredRemoteAdapter struct {
|
||||
kind string
|
||||
}
|
||||
|
||||
func (adapter declaredRemoteAdapter) Execute(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return RemoteAdapterOutcome{}, ctx.Err()
|
||||
default:
|
||||
}
|
||||
if request.AdapterKind != adapter.kind || request.ServerInstanceID == "" || request.JobID == "" {
|
||||
return RemoteAdapterOutcome{}, fmt.Errorf("remote adapter request identity mismatch")
|
||||
}
|
||||
return RemoteAdapterOutcome{Message: fmt.Sprintf("%s completed through bounded remote access envelope", request.Capability), ResultRef: fmt.Sprintf("artifact://jobs/%s/remote-access-result", url.PathEscape(request.JobID))}, nil
|
||||
}
|
||||
|
||||
func adapterKindForCapability(capability string) string {
|
||||
switch capability {
|
||||
case protocol.RunCapabilityRemoteFTPRead, protocol.RunCapabilityRemoteFTPWrite:
|
||||
return "ftp"
|
||||
case protocol.RunCapabilityRemoteRsyncRead, protocol.RunCapabilityRemoteRsyncWrite:
|
||||
return "rsync"
|
||||
case protocol.RunCapabilityRemoteRunFilesRead, protocol.RunCapabilityRemoteRunFilesWrite:
|
||||
return "run-file"
|
||||
case protocol.RunCapabilityRemoteRunProcessStart, protocol.RunCapabilityRemoteRunProcessStop:
|
||||
return "run-process"
|
||||
case protocol.RunCapabilityRemoteRunDBMySQLQuery, protocol.RunCapabilityRemoteRunDBSQLiteQuery:
|
||||
return "database"
|
||||
case protocol.RunCapabilityRemoteRunRCONCommand:
|
||||
return "rcon"
|
||||
case protocol.RunCapabilityRemoteRunLogsTransfer:
|
||||
return "log-transfer"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func adapterKindAllowsCapability(kind string, capability string) bool {
|
||||
return kind != "" && kind == adapterKindForCapability(capability)
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestRemoteAdapterRegistryHonorsTimeoutAndCancellation(t *testing.T) {
|
||||
registry := NewRemoteAdapterRegistry()
|
||||
if err := registry.Register("database", RemoteAdapterFunc(func(ctx context.Context, request RemoteAdapterRequest) (RemoteAdapterOutcome, error) {
|
||||
<-ctx.Done()
|
||||
return RemoteAdapterOutcome{}, ctx.Err()
|
||||
})); err != nil {
|
||||
t.Fatalf("register blocking adapter: %v", err)
|
||||
}
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
|
||||
assignment.TargetKey = "db/sqlite/query"
|
||||
assignment.InputRef = "input://server-1/db/sqlite/query/1"
|
||||
assignment.ExecutionInput.RemoteAdapterKind = "database"
|
||||
assignment.ExecutionInput.RemoteAdapterKey = "db-sqlite"
|
||||
assignment.ExecutionInput.TimeoutSeconds = 1
|
||||
started := time.Now()
|
||||
result := ExecuteRemoteAccessJobWithRegistry(context.Background(), assignment, registry)
|
||||
if result.ErrorCode != "remote_adapter_timeout" || !result.Retryable || time.Since(started) > 3*time.Second {
|
||||
t.Fatalf("expected bounded remote timeout, got %+v", result)
|
||||
}
|
||||
|
||||
cancelCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
result = ExecuteRemoteAccessJobWithRegistry(cancelCtx, assignment, NewRemoteAdapterRegistry())
|
||||
if result.ErrorCode != "remote_adapter_cancelled" || result.State != "cancelled" {
|
||||
t.Fatalf("expected remote cancellation, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteAdapterRejectsKindMismatchAndUnsafeProjection(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
|
||||
assignment.TargetKey = "db/sqlite/query"
|
||||
assignment.InputRef = "input://server-1/db/sqlite/query/1"
|
||||
assignment.ExecutionInput.RemoteAdapterKind = "rcon"
|
||||
result := ExecuteRemoteAccessJob(context.Background(), assignment)
|
||||
if result.ErrorCode != "remote_adapter_capability_mismatch" {
|
||||
t.Fatalf("expected kind mismatch rejection, got %+v", result)
|
||||
}
|
||||
assignment.ExecutionInput.RemoteAdapterKind = "database"
|
||||
assignment.ExecutionInput.RemoteAdapterKey = "tcp://unapproved"
|
||||
result = ExecuteRemoteAccessJob(context.Background(), assignment)
|
||||
if result.ErrorCode != "unsafe_remote_adapter_target" || strings.Contains(result.Message, "tcp://") {
|
||||
t.Fatalf("expected unsafe adapter target rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
RuntimeModeLocalProcess = "local-process"
|
||||
RuntimeModeHostedFTPRCON = "hosted-ftp-rcon"
|
||||
RuntimeModeFTPOnly = "ftp-only"
|
||||
RuntimeModeCustomClient = "custom-client"
|
||||
)
|
||||
|
||||
type RuntimeProfiles struct {
|
||||
Discovery []RuntimeDiscoveryProbe `json:"discovery,omitempty"`
|
||||
LifecycleProfiles []RuntimeLifecycleProfile `json:"lifecycleProfiles,omitempty"`
|
||||
DependencyProbes []RuntimeDependencyProbe `json:"dependencyProbes,omitempty"`
|
||||
InstallPlans []RuntimeInstallPlan `json:"installPlans,omitempty"`
|
||||
LogSources []RuntimeLogSource `json:"logSources,omitempty"`
|
||||
TransportProfiles []RuntimeTransportProfile `json:"transportProfiles,omitempty"`
|
||||
ClientManagers []RuntimeClientManagerSpec `json:"clientManagers,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeDiscoveryProbe struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
TargetKey string `json:"targetKey"`
|
||||
Required bool `json:"required,omitempty"`
|
||||
Platforms []string `json:"platforms,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeLifecycleProfile struct {
|
||||
Key string `json:"key"`
|
||||
Mode string `json:"mode"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
ActionRefs map[string]string `json:"actionRefs,omitempty"`
|
||||
TransportKeys []string `json:"transportKeys,omitempty"`
|
||||
ClientManagerRef string `json:"clientManagerRef,omitempty"`
|
||||
Platforms []string `json:"platforms,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeDependencyProbe struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
TargetKey string `json:"targetKey"`
|
||||
Required bool `json:"required,omitempty"`
|
||||
Platforms []string `json:"platforms,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeInstallPlan struct {
|
||||
Key string `json:"key"`
|
||||
Title string `json:"title"`
|
||||
Platforms []string `json:"platforms,omitempty"`
|
||||
Steps []RuntimeInstallStep `json:"steps"`
|
||||
}
|
||||
|
||||
type RuntimeInstallStep struct {
|
||||
Type string `json:"type"`
|
||||
TargetKey string `json:"targetKey"`
|
||||
PackageManager string `json:"packageManager,omitempty"`
|
||||
PackageName string `json:"packageName,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
DownloadRef string `json:"downloadRef,omitempty"`
|
||||
Checksum string `json:"checksum,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeLogSource struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
TargetKey string `json:"targetKey,omitempty"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
CursorKind string `json:"cursorKind,omitempty"`
|
||||
RetentionDays int `json:"retentionDays,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeTransportProfile struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
TargetKey string `json:"targetKey,omitempty"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
}
|
||||
|
||||
type RuntimeClientManagerSpec struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type RuntimeBindingSet struct {
|
||||
ProfileKey string `json:"profileKey"`
|
||||
Mode string `json:"mode"`
|
||||
Bindings map[string]string `json:"bindings,omitempty"`
|
||||
MissingKeys []string `json:"missingKeys,omitempty"`
|
||||
}
|
||||
|
||||
type RuntimeResolution struct {
|
||||
ProfileKey string `json:"profileKey"`
|
||||
Mode string `json:"mode"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
ActionRefs map[string]string `json:"actionRefs,omitempty"`
|
||||
TransportKeys []string `json:"transportKeys,omitempty"`
|
||||
Transports []RuntimeTransportProfile `json:"transports,omitempty"`
|
||||
LogSources []RuntimeLogSource `json:"logSources,omitempty"`
|
||||
Discovery []RuntimeDiscoveryProbe `json:"discovery,omitempty"`
|
||||
ClientManagerRef string `json:"clientManagerRef,omitempty"`
|
||||
MissingKeys []string `json:"missingKeys,omitempty"`
|
||||
Available bool `json:"available"`
|
||||
}
|
||||
|
||||
func ResolveRuntimeProfile(profiles RuntimeProfiles, profileKey string, targetOS string, binding RuntimeBindingSet) (RuntimeResolution, error) {
|
||||
profile, ok := findLifecycleProfile(profiles.LifecycleProfiles, profileKey)
|
||||
if !ok {
|
||||
return RuntimeResolution{}, fmt.Errorf("runtime profile is not declared")
|
||||
}
|
||||
if !supportedRuntimeMode(profile.Mode) {
|
||||
return RuntimeResolution{}, fmt.Errorf("runtime mode is unsupported")
|
||||
}
|
||||
if targetOS != "" && !supportsPlatform(profile.Platforms, targetOS) {
|
||||
return RuntimeResolution{}, fmt.Errorf("runtime profile does not support target platform")
|
||||
}
|
||||
if binding.ProfileKey != "" && binding.ProfileKey != profile.Key {
|
||||
return RuntimeResolution{}, fmt.Errorf("runtime binding profile does not match")
|
||||
}
|
||||
if binding.Mode != "" && binding.Mode != profile.Mode {
|
||||
return RuntimeResolution{}, fmt.Errorf("runtime binding mode does not match")
|
||||
}
|
||||
if err := validateRuntimeProfile(profile); err != nil {
|
||||
return RuntimeResolution{}, err
|
||||
}
|
||||
|
||||
transports, err := resolveTransports(profile.TransportKeys, profiles.TransportProfiles)
|
||||
if err != nil {
|
||||
return RuntimeResolution{}, err
|
||||
}
|
||||
missing := missingRuntimeBindingKeys(profile, transports, profiles.Discovery, profiles.LogSources, binding)
|
||||
return RuntimeResolution{
|
||||
ProfileKey: profile.Key,
|
||||
Mode: profile.Mode,
|
||||
Capabilities: append([]string(nil), profile.Capabilities...),
|
||||
ActionRefs: copyStringMap(profile.ActionRefs),
|
||||
TransportKeys: append([]string(nil), profile.TransportKeys...),
|
||||
Transports: transports,
|
||||
LogSources: safeLogSources(profiles.LogSources, targetOS),
|
||||
Discovery: safeDiscovery(profiles.Discovery, targetOS),
|
||||
ClientManagerRef: profile.ClientManagerRef,
|
||||
MissingKeys: missing,
|
||||
Available: len(missing) == 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func findLifecycleProfile(profiles []RuntimeLifecycleProfile, key string) (RuntimeLifecycleProfile, bool) {
|
||||
for _, profile := range profiles {
|
||||
if profile.Key == key {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
return RuntimeLifecycleProfile{}, false
|
||||
}
|
||||
|
||||
func validateRuntimeProfile(profile RuntimeLifecycleProfile) error {
|
||||
if !protocol.ValidLogicalFileKey(profile.Key) {
|
||||
return fmt.Errorf("runtime profile key is unsafe")
|
||||
}
|
||||
for _, capability := range profile.Capabilities {
|
||||
if strings.TrimSpace(capability) == "" || containsUnsafeRuntimeText(capability) {
|
||||
return fmt.Errorf("runtime capability is unsafe")
|
||||
}
|
||||
}
|
||||
for action, ref := range profile.ActionRefs {
|
||||
if !protocol.ValidLogicalFileKey(action) || !protocol.ValidLogicalFileKey(ref) {
|
||||
return fmt.Errorf("runtime action ref is unsafe")
|
||||
}
|
||||
}
|
||||
if profile.ClientManagerRef != "" && !protocol.ValidLogicalFileKey(profile.ClientManagerRef) {
|
||||
return fmt.Errorf("client manager ref is unsafe")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveTransports(keys []string, profiles []RuntimeTransportProfile) ([]RuntimeTransportProfile, error) {
|
||||
out := make([]RuntimeTransportProfile, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if !protocol.ValidLogicalFileKey(key) {
|
||||
return nil, fmt.Errorf("transport key is unsafe")
|
||||
}
|
||||
found := false
|
||||
for _, profile := range profiles {
|
||||
if profile.Key != key {
|
||||
continue
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(profile.Key) || (profile.TargetKey != "" && !protocol.ValidLogicalFileKey(profile.TargetKey)) {
|
||||
return nil, fmt.Errorf("transport profile is unsafe")
|
||||
}
|
||||
out = append(out, profile)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("transport profile %q is not declared", key)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func missingRuntimeBindingKeys(profile RuntimeLifecycleProfile, transports []RuntimeTransportProfile, discovery []RuntimeDiscoveryProbe, logs []RuntimeLogSource, binding RuntimeBindingSet) []string {
|
||||
required := map[string]struct{}{}
|
||||
for _, transport := range transports {
|
||||
if transport.TargetKey != "" {
|
||||
required[transport.TargetKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, probe := range discovery {
|
||||
if probe.Required && probe.TargetKey != "" {
|
||||
required[probe.TargetKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
logTargets := map[string]struct{}{}
|
||||
for _, source := range logs {
|
||||
if source.TargetKey != "" {
|
||||
logTargets[source.TargetKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
if profile.ClientManagerRef != "" {
|
||||
required[profile.ClientManagerRef] = struct{}{}
|
||||
}
|
||||
for _, key := range binding.MissingKeys {
|
||||
if _, logTarget := logTargets[key]; logTarget {
|
||||
continue
|
||||
}
|
||||
if protocol.ValidLogicalFileKey(key) {
|
||||
required[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
missing := make([]string, 0, len(required))
|
||||
for key := range required {
|
||||
if _, ok := binding.Bindings[key]; !ok {
|
||||
missing = append(missing, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(missing)
|
||||
return missing
|
||||
}
|
||||
|
||||
func safeDiscovery(probes []RuntimeDiscoveryProbe, targetOS string) []RuntimeDiscoveryProbe {
|
||||
out := []RuntimeDiscoveryProbe{}
|
||||
for _, probe := range probes {
|
||||
if supportsPlatform(probe.Platforms, targetOS) && protocol.ValidLogicalFileKey(probe.Key) && protocol.ValidLogicalFileKey(probe.TargetKey) {
|
||||
out = append(out, probe)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func safeLogSources(sources []RuntimeLogSource, targetOS string) []RuntimeLogSource {
|
||||
_ = targetOS
|
||||
out := []RuntimeLogSource{}
|
||||
for _, source := range sources {
|
||||
if protocol.ValidLogicalFileKey(source.Key) && protocol.ValidLogicalFileKey(source.StreamKey) && (source.TargetKey == "" || protocol.ValidLogicalFileKey(source.TargetKey)) {
|
||||
out = append(out, source)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func supportedRuntimeMode(mode string) bool {
|
||||
switch mode {
|
||||
case RuntimeModeLocalProcess, RuntimeModeHostedFTPRCON, RuntimeModeFTPOnly, RuntimeModeCustomClient:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func supportsPlatform(platforms []string, targetOS string) bool {
|
||||
if targetOS == "" || len(platforms) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, platform := range platforms {
|
||||
if platform == targetOS {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func copyStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,611 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
selfUpdateManifestVersion = 1
|
||||
maxSelfUpdateBytes = int64(512 * 1024 * 1024)
|
||||
maxSelfUpdateEntries = 8
|
||||
defaultUpdateHealthWait = 30 * time.Second
|
||||
)
|
||||
|
||||
var ErrSelfUpdateRestartRequested = errors.New("Run self-update restart requested")
|
||||
|
||||
type SelfUpdateManifest struct {
|
||||
Version int `json:"version"`
|
||||
JobID string `json:"jobId"`
|
||||
Attempt int `json:"attempt"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
ArtifactChecksum string `json:"artifactChecksum"`
|
||||
ArtifactSizeBytes int64 `json:"artifactSizeBytes"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
TargetRelease string `json:"targetRelease"`
|
||||
CurrentExecutable string `json:"currentExecutable"`
|
||||
StagedExecutable string `json:"stagedExecutable"`
|
||||
BackupExecutable string `json:"backupExecutable"`
|
||||
HealthFile string `json:"healthFile"`
|
||||
WorkingDirectory string `json:"workingDirectory"`
|
||||
Phase string `json:"phase"`
|
||||
DownloadedBytes int64 `json:"downloadedBytes"`
|
||||
BinaryChecksum string `json:"binaryChecksum,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type SelfUpdateActivator interface {
|
||||
Activate(string) error
|
||||
}
|
||||
|
||||
type ProcessSelfUpdateActivator struct{}
|
||||
|
||||
func (ProcessSelfUpdateActivator) Activate(manifestPath string) error {
|
||||
manifest, err := loadSelfUpdateManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
command := exec.Command(manifest.StagedExecutable)
|
||||
command.Dir = manifest.WorkingDirectory
|
||||
command.Env = append(cleanUpdateEnvironment(os.Environ()), "RUN_MODE=self-update-helper", "RUN_UPDATE_MANIFEST="+manifestPath)
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
return command.Start()
|
||||
}
|
||||
|
||||
func (worker *Worker) executeRunSelfUpdate(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_self_update_job", err.Error())
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_unregistered", "Run worker is not registered")
|
||||
}
|
||||
input, err := worker.client.GetRunUpdateInput(ctx, protocol.RunUpdateInputRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt})
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_input_failed", "could not load fenced Run update input")
|
||||
}
|
||||
if err := validateRunUpdateInput(assignment, input); err != nil {
|
||||
return lifecycleFailure("unsafe_self_update_input", err.Error())
|
||||
}
|
||||
transactionRoot := filepath.Join(worker.cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID))
|
||||
if err := os.MkdirAll(transactionRoot, 0o700); err != nil {
|
||||
return lifecycleFailure("self_update_workspace_failed", "could not create update transaction workspace")
|
||||
}
|
||||
manifestPath := filepath.Join(transactionRoot, "manifest.json")
|
||||
archivePath := filepath.Join(transactionRoot, "update.archive")
|
||||
manifest, err := prepareSelfUpdateManifest(manifestPath, assignment, input, transactionRoot)
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
if manifest.Phase != "staged" {
|
||||
manifest.Phase = "downloading"
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
if err := worker.downloadRunUpdate(ctx, assignment, input, archivePath, manifestPath, &manifest); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update download cancelled"}, Message: "Run update download cancelled", ErrorCode: "run_self_update_cancelled"}
|
||||
}
|
||||
return lifecycleFailure("self_update_download_failed", err.Error())
|
||||
}
|
||||
stagedPath := filepath.Join(transactionRoot, input.ExecutableName+".staged")
|
||||
binaryChecksum, err := stageRunUpdateBinary(archivePath, input.PackageFormat, input.ExecutableName, stagedPath)
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_extract_failed", err.Error())
|
||||
}
|
||||
manifest.StagedExecutable = stagedPath
|
||||
manifest.BinaryChecksum = binaryChecksum
|
||||
manifest.Phase = "staged"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
}
|
||||
evidence, _ := json.Marshal(protocol.RunUpdateExecutionEvidence{TargetRelease: input.TargetRelease, Phase: "staged"})
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", safeWorkspaceName(assignment.JobID)), Message: "Run update verified and staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: input.Checksum, SizeBytes: input.SizeBytes, Summary: "verified update staged", Content: string(evidence)}, ActivationManifest: manifestPath}
|
||||
}
|
||||
|
||||
func validateRunUpdateInput(assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse) error {
|
||||
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID || assignment.InputRef != "artifact://"+input.ArtifactID {
|
||||
return fmt.Errorf("Run update input scope does not match job")
|
||||
}
|
||||
if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH {
|
||||
return fmt.Errorf("Run update target does not match this executable")
|
||||
}
|
||||
if input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" && input.PackageFormat != "raw-executable" {
|
||||
return fmt.Errorf("Run update package format is unsupported")
|
||||
}
|
||||
if input.SizeBytes <= 0 || input.SizeBytes > maxSelfUpdateBytes || input.ChunkSizeBytes <= 0 || input.ChunkSizeBytes > 1024*1024 || !validSHA256(input.Checksum) {
|
||||
return fmt.Errorf("Run update artifact bounds are invalid")
|
||||
}
|
||||
expectedName := "run"
|
||||
if runtime.GOOS == "windows" {
|
||||
expectedName = "run.exe"
|
||||
}
|
||||
if input.ExecutableName != expectedName || !protocol.ValidLogicalFileKey(input.TargetRelease) {
|
||||
return fmt.Errorf("Run update executable or release identity is unsafe")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareSelfUpdateManifest(path string, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, root string) (SelfUpdateManifest, error) {
|
||||
if existing, err := loadSelfUpdateManifest(path); err == nil {
|
||||
if existing.JobID != assignment.JobID || existing.ArtifactID != input.ArtifactID || existing.ArtifactChecksum != input.Checksum || existing.TargetRelease != input.TargetRelease || existing.Attempt > assignment.Attempt {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("existing update transaction does not match active attempt")
|
||||
}
|
||||
if existing.Phase == "staged" {
|
||||
if existing.StagedExecutable == "" || !pathWithinRoot(root, existing.StagedExecutable) || existing.BinaryChecksum == "" {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("staged update manifest is outside the transaction workspace")
|
||||
}
|
||||
checksum, _, checksumErr := checksumFile(existing.StagedExecutable)
|
||||
if checksumErr != nil || checksum != existing.BinaryChecksum {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("staged Run binary checksum changed")
|
||||
}
|
||||
}
|
||||
existing.Attempt = assignment.Attempt
|
||||
existing.LeaseToken = assignment.LeaseToken
|
||||
return existing, nil
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
current, err := os.Executable()
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
current, err = filepath.Abs(current)
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
info, err := os.Lstat(current)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("current Run executable is not a regular file")
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, CurrentExecutable: current, BackupExecutable: filepath.Join(root, "previous-run.backup"), HealthFile: filepath.Join(root, "healthy"), WorkingDirectory: workingDirectory, Phase: "downloading", CreatedAt: now, UpdatedAt: now}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func pathWithinRoot(root, path string) bool {
|
||||
rootAbs, rootErr := filepath.Abs(root)
|
||||
pathAbs, pathErr := filepath.Abs(path)
|
||||
if rootErr != nil || pathErr != nil {
|
||||
return false
|
||||
}
|
||||
relative, err := filepath.Rel(rootAbs, pathAbs)
|
||||
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && relative != "."
|
||||
}
|
||||
|
||||
func (worker *Worker) downloadRunUpdate(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, archivePath, manifestPath string, manifest *SelfUpdateManifest) error {
|
||||
file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
offset := info.Size()
|
||||
if offset < 0 || offset > input.SizeBytes {
|
||||
return fmt.Errorf("partial update artifact has invalid size")
|
||||
}
|
||||
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
for offset < input.SizeBytes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
length := input.ChunkSizeBytes
|
||||
if remaining := input.SizeBytes - offset; int64(length) > remaining {
|
||||
length = int(remaining)
|
||||
}
|
||||
chunk, err := worker.client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Offset: offset, Length: length})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if chunk.JobID != assignment.JobID || chunk.ArtifactID != input.ArtifactID || chunk.Offset != offset || chunk.TotalBytes != input.SizeBytes || chunk.Checksum != input.Checksum || len(chunk.Payload) == 0 || len(chunk.Payload) > length {
|
||||
return fmt.Errorf("Run update chunk acknowledgement does not match request")
|
||||
}
|
||||
if _, err := file.Write(chunk.Payload); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
offset += int64(len(chunk.Payload))
|
||||
manifest.DownloadedBytes = offset
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, *manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
checksum, size, err := checksumFile(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size != input.SizeBytes || checksum != input.Checksum {
|
||||
_ = os.Remove(archivePath)
|
||||
return fmt.Errorf("Run update artifact checksum mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stageRunUpdateBinary(artifactPath, format, executableName, destination string) (string, error) {
|
||||
if format != "raw-executable" {
|
||||
return extractRunUpdateBinary(artifactPath, format, executableName, destination)
|
||||
}
|
||||
info, err := os.Stat(artifactPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxSelfUpdateBytes {
|
||||
return "", fmt.Errorf("Run update executable exceeds bounds")
|
||||
}
|
||||
input, err := os.Open(artifactPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer input.Close()
|
||||
temporary := destination + ".tmp"
|
||||
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
written, copyErr := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1))
|
||||
if copyErr == nil && written != info.Size() {
|
||||
copyErr = fmt.Errorf("Run update executable size does not match artifact")
|
||||
}
|
||||
if syncErr := output.Sync(); copyErr == nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := output.Close(); copyErr == nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return "", copyErr
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return "", err
|
||||
}
|
||||
if err := os.Chmod(destination, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
checksum, _, err := checksumFile(destination)
|
||||
return checksum, err
|
||||
}
|
||||
|
||||
func extractRunUpdateBinary(archivePath, format, executableName, destination string) (string, error) {
|
||||
found := false
|
||||
entries := 0
|
||||
writeEntry := func(name string, mode os.FileMode, reader io.Reader, size int64) error {
|
||||
entries++
|
||||
if entries > maxSelfUpdateEntries || size < 0 || size > maxSelfUpdateBytes {
|
||||
return fmt.Errorf("Run update archive exceeds bounds")
|
||||
}
|
||||
clean := filepath.ToSlash(filepath.Clean(name))
|
||||
if clean != name || strings.Contains(clean, "../") || strings.HasPrefix(clean, "/") || strings.Contains(clean, `\`) {
|
||||
return fmt.Errorf("Run update archive entry is unsafe")
|
||||
}
|
||||
if clean == "config.json" {
|
||||
_, err := io.Copy(io.Discard, io.LimitReader(reader, size+1))
|
||||
return err
|
||||
}
|
||||
if clean != executableName || found || mode&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("Run update archive contains unexpected entry")
|
||||
}
|
||||
found = true
|
||||
temporary := destination + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
written, copyErr := io.Copy(file, io.LimitReader(reader, maxSelfUpdateBytes+1))
|
||||
if copyErr == nil && written != size {
|
||||
copyErr = fmt.Errorf("Run update binary size does not match archive")
|
||||
}
|
||||
if syncErr := file.Sync(); copyErr == nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := file.Close(); copyErr == nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return copyErr
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return os.Chmod(destination, 0o700)
|
||||
}
|
||||
|
||||
if format == "zip" {
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reader, err := zip.OpenReader(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
if info.Size() > maxSelfUpdateBytes {
|
||||
return "", fmt.Errorf("Run update archive exceeds size limit")
|
||||
}
|
||||
for _, entry := range reader.File {
|
||||
if entry.FileInfo().IsDir() || entry.Mode()&os.ModeType != 0 {
|
||||
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
||||
}
|
||||
stream, err := entry.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = writeEntry(entry.Name, entry.Mode(), stream, int64(entry.UncompressedSize64))
|
||||
_ = stream.Close()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA {
|
||||
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
||||
}
|
||||
if err := writeEntry(header.Name, os.FileMode(header.Mode), tarReader, header.Size); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("Run update archive does not contain expected executable")
|
||||
}
|
||||
checksum, _, err := checksumFile(destination)
|
||||
return checksum, err
|
||||
}
|
||||
|
||||
func ApplySelfUpdateManifest(manifestPath string) error {
|
||||
manifest, err := loadSelfUpdateManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
helper, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
helper, _ = filepath.Abs(helper)
|
||||
staged, _ := filepath.Abs(manifest.StagedExecutable)
|
||||
if helper != staged || manifest.TargetOS != runtime.GOOS || manifest.TargetArch != runtime.GOARCH || manifest.Phase != "staged" {
|
||||
return fmt.Errorf("self-update helper scope does not match staged transaction")
|
||||
}
|
||||
manifest.Phase = "activating"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := replaceRunExecutable(manifest); err != nil {
|
||||
manifest.Phase = "rolled-back"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(manifest.HealthFile)
|
||||
command, err := startRunAfterUpdate(manifest, "succeeded")
|
||||
if err != nil {
|
||||
_ = rollbackRunExecutable(manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return err
|
||||
}
|
||||
wait := defaultUpdateHealthWait
|
||||
if value, parseErr := strconv.Atoi(os.Getenv("RUN_UPDATE_HEALTH_TIMEOUT_MS")); parseErr == nil && value > 0 && value <= 300000 {
|
||||
wait = time.Duration(value) * time.Millisecond
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(manifest.HealthFile); err == nil {
|
||||
manifest.Phase = "succeeded"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
return persistSelfUpdateManifest(manifestPath, manifest)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
_ = command.Process.Kill()
|
||||
if err := rollbackRunExecutable(manifest); err != nil {
|
||||
return fmt.Errorf("updated Run health timed out and rollback failed: %w", err)
|
||||
}
|
||||
manifest.Phase = "rolled-back"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return fmt.Errorf("updated Run did not become healthy before timeout")
|
||||
}
|
||||
|
||||
func replaceRunExecutable(manifest SelfUpdateManifest) error {
|
||||
if checksum, _, err := checksumFile(manifest.StagedExecutable); err != nil || checksum != manifest.BinaryChecksum {
|
||||
return fmt.Errorf("staged Run binary checksum changed")
|
||||
}
|
||||
_ = os.Remove(manifest.BackupExecutable)
|
||||
var lastErr error
|
||||
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if err := os.Rename(manifest.CurrentExecutable, manifest.BackupExecutable); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err := copyExecutable(manifest.StagedExecutable, manifest.CurrentExecutable); err != nil {
|
||||
_ = os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("could not back up current Run executable: %w", lastErr)
|
||||
}
|
||||
|
||||
func rollbackRunExecutable(manifest SelfUpdateManifest) error {
|
||||
if _, err := os.Stat(manifest.BackupExecutable); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(manifest.CurrentExecutable)
|
||||
return os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
||||
}
|
||||
|
||||
func startRunAfterUpdate(manifest SelfUpdateManifest, outcome string) (*exec.Cmd, error) {
|
||||
command := exec.Command(manifest.CurrentExecutable)
|
||||
command.Dir = manifest.WorkingDirectory
|
||||
environment := cleanUpdateEnvironment(os.Environ())
|
||||
environment = append(environment, "RUN_MODE=worker", "RUN_UPDATE_JOB_ID="+manifest.JobID, "RUN_UPDATE_OUTCOME="+outcome, "RUN_UPDATE_ATTEMPT="+strconv.Itoa(manifest.Attempt), "RUN_UPDATE_LEASE_TOKEN="+manifest.LeaseToken)
|
||||
if outcome == "succeeded" {
|
||||
environment = append(environment, "RUN_VERSION="+manifest.TargetRelease, "RUN_UPDATE_HEALTH_FILE="+manifest.HealthFile)
|
||||
}
|
||||
command.Env = environment
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
if err := command.Start(); err != nil {
|
||||
return command, err
|
||||
}
|
||||
return command, nil
|
||||
}
|
||||
|
||||
func MarkSelfUpdateHealthy(path string) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, []byte("healthy\n"), 0o600)
|
||||
}
|
||||
|
||||
func loadSelfUpdateManifest(path string) (SelfUpdateManifest, error) {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
var manifest SelfUpdateManifest
|
||||
if err := json.Unmarshal(body, &manifest); err != nil {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("decode self-update manifest: %w", err)
|
||||
}
|
||||
if manifest.Version != selfUpdateManifestVersion || manifest.JobID == "" || manifest.Attempt <= 0 || manifest.LeaseToken == "" || manifest.ArtifactID == "" || !validSHA256(manifest.ArtifactChecksum) || manifest.ArtifactSizeBytes <= 0 || manifest.ArtifactSizeBytes > maxSelfUpdateBytes || !protocol.ValidLogicalFileKey(manifest.TargetRelease) {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("self-update manifest is invalid")
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func persistSelfUpdateManifest(path string, manifest SelfUpdateManifest) error {
|
||||
body, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, body, 0o600)
|
||||
}
|
||||
|
||||
func checksumFile(path string) (string, int64, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, io.LimitReader(file, maxSelfUpdateBytes+1))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if size > maxSelfUpdateBytes {
|
||||
return "", size, fmt.Errorf("file exceeds self-update size limit")
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil
|
||||
}
|
||||
|
||||
func copyExecutable(source, destination string) error {
|
||||
input, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer input.Close()
|
||||
temporary := destination + ".update-tmp"
|
||||
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)); err != nil {
|
||||
_ = output.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := output.Sync(); err != nil {
|
||||
_ = output.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return os.Chmod(destination, 0o700)
|
||||
}
|
||||
|
||||
func cleanUpdateEnvironment(environment []string) []string {
|
||||
blocked := map[string]bool{"RUN_UPDATE_MANIFEST": true, "RUN_UPDATE_HEALTH_FILE": true, "RUN_UPDATE_HEALTH_TIMEOUT_MS": true, "RUN_UPDATE_JOB_ID": true, "RUN_UPDATE_OUTCOME": true, "RUN_UPDATE_ATTEMPT": true, "RUN_UPDATE_LEASE_TOKEN": true, "RUN_MODE": true, "RUN_VERSION": true}
|
||||
out := make([]string, 0, len(environment))
|
||||
for _, entry := range environment {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if !blocked[key] {
|
||||
out = append(out, entry)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestRunSelfUpdateResumesPartialDownloadAndRejectsChecksum(t *testing.T) {
|
||||
assignment, input, payload := selfUpdateTestFixture(t)
|
||||
client := newFakeWorkerClient()
|
||||
client.updateInput = input
|
||||
client.updatePayload = payload
|
||||
cfg := workerTestConfig(t)
|
||||
transactionRoot := filepath.Join(cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID))
|
||||
if err := os.MkdirAll(transactionRoot, 0o700); err != nil {
|
||||
t.Fatalf("create transaction root: %v", err)
|
||||
}
|
||||
partial := len(payload) / 3
|
||||
if err := os.WriteFile(filepath.Join(transactionRoot, "update.archive"), payload[:partial], 0o600); err != nil {
|
||||
t.Fatalf("write partial update: %v", err)
|
||||
}
|
||||
worker, err := NewWorker(cfg, client, WithSelfUpdateActivator(&recordingSelfUpdateActivator{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
result := worker.executeRunSelfUpdate(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "run.update.staged" || result.ActivationManifest == "" {
|
||||
t.Fatalf("expected staged self-update, got %+v", result)
|
||||
}
|
||||
if len(client.updateChunkOffsets) == 0 || client.updateChunkOffsets[0] != int64(partial) {
|
||||
t.Fatalf("expected resumable range from %d, got %+v", partial, client.updateChunkOffsets)
|
||||
}
|
||||
|
||||
badClient := newFakeWorkerClient()
|
||||
badInput := input
|
||||
badInput.Checksum = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
badClient.updateInput = badInput
|
||||
badClient.updatePayload = payload
|
||||
badWorker, err := NewWorker(workerTestConfig(t), badClient, WithSelfUpdateActivator(&recordingSelfUpdateActivator{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new bad checksum worker: %v", err)
|
||||
}
|
||||
badWorker.state.SessionToken = "session-token"
|
||||
bad := badWorker.executeRunSelfUpdate(context.Background(), assignment)
|
||||
if bad.State != lifecycleResultStateFailed || bad.ErrorCode != "self_update_download_failed" {
|
||||
t.Fatalf("expected final checksum rejection, got %+v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelfUpdateActivationWaitsForAcceptedTerminalResult(t *testing.T) {
|
||||
assignment, input, payload := selfUpdateTestFixture(t)
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = assignment
|
||||
client.updateInput = input
|
||||
client.updatePayload = payload
|
||||
client.resultErr = errors.New("stale lease")
|
||||
activator := &recordingSelfUpdateActivator{}
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithSelfUpdateActivator(activator))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
handled, err := worker.ClaimAndRunOnce(context.Background())
|
||||
if !handled || err == nil {
|
||||
t.Fatalf("expected rejected result error, handled=%v err=%v", handled, err)
|
||||
}
|
||||
if activator.manifestPath != "" {
|
||||
t.Fatalf("stale terminal result must not activate update: %s", activator.manifestPath)
|
||||
}
|
||||
if worker.journal.ActiveCount() != 1 {
|
||||
t.Fatal("staged result must remain recoverable until Platform accepts it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelfUpdateReplacementPreservesRollbackAndRejectsTraversal(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
current := filepath.Join(root, "run")
|
||||
staged := filepath.Join(root, "staged-run")
|
||||
backup := filepath.Join(root, "previous-run")
|
||||
if err := os.WriteFile(current, []byte("old-run"), 0o700); err != nil {
|
||||
t.Fatalf("write current: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(staged, []byte("new-run"), 0o700); err != nil {
|
||||
t.Fatalf("write staged: %v", err)
|
||||
}
|
||||
checksum, _, err := checksumFile(staged)
|
||||
if err != nil {
|
||||
t.Fatalf("checksum staged: %v", err)
|
||||
}
|
||||
manifest := SelfUpdateManifest{StagedExecutable: staged, CurrentExecutable: current, BackupExecutable: backup, BinaryChecksum: checksum}
|
||||
if err := replaceRunExecutable(manifest); err != nil {
|
||||
t.Fatalf("replace executable: %v", err)
|
||||
}
|
||||
if body, _ := os.ReadFile(current); string(body) != "new-run" {
|
||||
t.Fatalf("expected new executable, got %q", body)
|
||||
}
|
||||
if err := rollbackRunExecutable(manifest); err != nil {
|
||||
t.Fatalf("rollback executable: %v", err)
|
||||
}
|
||||
if body, _ := os.ReadFile(current); string(body) != "old-run" {
|
||||
t.Fatalf("expected previous executable after rollback, got %q", body)
|
||||
}
|
||||
|
||||
archivePath := filepath.Join(root, "unsafe.tar.gz")
|
||||
var archive bytes.Buffer
|
||||
gzipWriter := gzip.NewWriter(&archive)
|
||||
tarWriter := tar.NewWriter(gzipWriter)
|
||||
body := []byte("escape")
|
||||
if err := tarWriter.WriteHeader(&tar.Header{Name: "../run", Mode: 0o700, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatalf("write unsafe header: %v", err)
|
||||
}
|
||||
_, _ = tarWriter.Write(body)
|
||||
_ = tarWriter.Close()
|
||||
_ = gzipWriter.Close()
|
||||
if err := os.WriteFile(archivePath, archive.Bytes(), 0o600); err != nil {
|
||||
t.Fatalf("write unsafe archive: %v", err)
|
||||
}
|
||||
if _, err := extractRunUpdateBinary(archivePath, "tar.gz", "run", filepath.Join(root, "escaped")); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||
t.Fatalf("expected traversal rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareSelfUpdateManifestRevalidatesStagedBinaryAfterRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate)
|
||||
assignment.TargetKey = "run/update"
|
||||
assignment.InputRef = "artifact://artifact-run-staged"
|
||||
input := protocol.RunUpdateInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, ArtifactID: "artifact-run-staged", Checksum: "sha256:" + strings.Repeat("a", 64), SizeBytes: 16, TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, PackageFormat: "tar.gz", ExecutableName: "run", TargetRelease: "release-staged", ChunkSizeBytes: 8}
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
stagedPath := filepath.Join(root, "run.staged")
|
||||
if err := os.WriteFile(stagedPath, []byte("staged-binary"), 0o700); err != nil {
|
||||
t.Fatalf("write staged binary: %v", err)
|
||||
}
|
||||
checksum, _, err := checksumFile(stagedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("checksum staged binary: %v", err)
|
||||
}
|
||||
manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, StagedExecutable: stagedPath, BinaryChecksum: checksum, Phase: "staged", CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC()}
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
t.Fatalf("persist staged manifest: %v", err)
|
||||
}
|
||||
if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err != nil {
|
||||
t.Fatalf("revalidate staged manifest: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(stagedPath, []byte("tampered-binary"), 0o700); err != nil {
|
||||
t.Fatalf("tamper staged binary: %v", err)
|
||||
}
|
||||
if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err == nil || !strings.Contains(err.Error(), "checksum changed") {
|
||||
t.Fatalf("expected staged checksum rejection, got %v", err)
|
||||
}
|
||||
manifest.StagedExecutable = filepath.Join(root, "..", "outside")
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
t.Fatalf("persist unsafe staged manifest: %v", err)
|
||||
}
|
||||
if _, err := prepareSelfUpdateManifest(manifestPath, assignment, input, root); err == nil || !strings.Contains(err.Error(), "outside") {
|
||||
t.Fatalf("expected staged path rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func selfUpdateTestFixture(t *testing.T) (protocol.RunJobAssignment, protocol.RunUpdateInputResponse, []byte) {
|
||||
t.Helper()
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityRunSelfUpdate)
|
||||
assignment.TargetKey = "run/update"
|
||||
assignment.InputRef = "artifact://artifact-run-latest"
|
||||
assignment.State = "running"
|
||||
assignment.LeaseToken = "lease-update"
|
||||
assignment.Attempt = 1
|
||||
executableName := "run"
|
||||
if runtime.GOOS == "windows" {
|
||||
executableName = "run.exe"
|
||||
}
|
||||
payload := []byte("self-update-test-binary")
|
||||
input := protocol.RunUpdateInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, ArtifactID: "artifact-run-latest", Checksum: bytesChecksum(payload), SizeBytes: int64(len(payload)), TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, PackageFormat: "raw-executable", ExecutableName: executableName, TargetRelease: "run-release-test", ChunkSizeBytes: 64}
|
||||
return assignment, input, payload
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/domain"
|
||||
)
|
||||
|
||||
func SmokeSummary(cfg config.Config) domain.ExecutorStatus {
|
||||
return domain.ExecutorStatus{
|
||||
Mode: cfg.Mode,
|
||||
PlatformURL: cfg.PlatformURL,
|
||||
Status: "ok",
|
||||
ExposedHostPath: false,
|
||||
Capabilities: append([]string{
|
||||
"control.hello",
|
||||
"control.heartbeat",
|
||||
}, SupportedRunCapabilitiesForComponent(cfg.ComponentKind)...),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
)
|
||||
|
||||
func TestSmokeSummaryDoesNotExposeHostPaths(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{
|
||||
Mode: "smoke",
|
||||
PlatformURL: "http://platform.test",
|
||||
})
|
||||
|
||||
if summary.Status != "ok" {
|
||||
t.Fatalf("expected ok status, got %q", summary.Status)
|
||||
}
|
||||
if summary.ExposedHostPath {
|
||||
t.Fatal("smoke summary must not expose host paths")
|
||||
}
|
||||
if len(summary.Capabilities) == 0 {
|
||||
t.Fatal("expected baseline capabilities")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
sourceRCONAuthRequestID int32 = 1
|
||||
sourceRCONCommandRequestID int32 = 2
|
||||
sourceRCONResponseValue int32 = 0
|
||||
sourceRCONAuthResponse int32 = 2
|
||||
sourceRCONExecuteCommand int32 = 2
|
||||
sourceRCONAuthenticate int32 = 3
|
||||
sourceRCONMaxPacketSize = 4096
|
||||
sourceRCONMaxCommandBytes = 4000
|
||||
sourceRCONMaxResponsePackets = 32
|
||||
sourceRCONMaxResponseBytes = 64 * 1024
|
||||
sourceRCONIOTimeout = 10 * time.Second
|
||||
sourceRCONMaxExecutionTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
type sourceRCONPacket struct {
|
||||
id int32
|
||||
typeCode int32
|
||||
body string
|
||||
}
|
||||
|
||||
type sourceRCONConfig struct {
|
||||
password string
|
||||
}
|
||||
|
||||
type sourceRCONError struct {
|
||||
code string
|
||||
}
|
||||
|
||||
func (err sourceRCONError) Error() string { return err.code }
|
||||
|
||||
// ExecuteSourceRCON connects only to the local UE4SS listener described by a
|
||||
// frozen plan. The command, config, password, and response body are transient.
|
||||
func (executor LifecycleExecutor) ExecuteSourceRCON(ctx context.Context, assignment protocol.RunJobAssignment, command string) LifecycleExecutionResult {
|
||||
if assignment.ExecutionInput.SourceRCON == nil || protocol.ValidateRunJobAssignment(assignment) != nil {
|
||||
return lifecycleFailure("unsafe_source_rcon_plan", "Source RCON plan is invalid")
|
||||
}
|
||||
if executor.runtimeTargetOS != "windows" || executor.runtimeTargetArch != "amd64" {
|
||||
return lifecycleFailure("unsupported_extension_platform", "Source RCON requires Windows amd64")
|
||||
}
|
||||
if !validSourceRCONCommand(command) {
|
||||
return lifecycleFailure("source_rcon_input_invalid", "Source RCON command input is invalid")
|
||||
}
|
||||
|
||||
executionCtx, cancel := sourceRCONExecutionContext(ctx, assignment.ExecutionInput.TimeoutSeconds)
|
||||
defer cancel()
|
||||
resolver := NewWorkspaceResolver(executor.workspaceRoot)
|
||||
scope, err := resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
return lifecycleFailure("source_rcon_workspace_unavailable", "Source RCON workspace is unavailable")
|
||||
}
|
||||
configPath, err := sourceRCONConfigPath(resolver, scope, *assignment.ExecutionInput.SourceRCON)
|
||||
if err != nil {
|
||||
return lifecycleFailure("source_rcon_config_unavailable", "Source RCON configuration is unavailable")
|
||||
}
|
||||
config, err := loadSourceRCONConfig(configPath, assignment.ExecutionInput.SourceRCON.Port)
|
||||
if err != nil {
|
||||
return lifecycleFailure("source_rcon_config_invalid", "Source RCON configuration is invalid")
|
||||
}
|
||||
if err := executeSourceRCONWire(executionCtx, assignment.ExecutionInput.SourceRCON.Port, config.password, command); err != nil {
|
||||
if errors.Is(executionCtx.Err(), context.Canceled) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Source RCON command cancelled"}, Message: "Source RCON command cancelled", ErrorCode: "source_rcon_cancelled"}
|
||||
}
|
||||
if errors.Is(executionCtx.Err(), context.DeadlineExceeded) {
|
||||
return lifecycleFailure("source_rcon_timeout", "Source RCON command timed out")
|
||||
}
|
||||
var sourceErr sourceRCONError
|
||||
if errors.As(err, &sourceErr) {
|
||||
return lifecycleFailure(sourceErr.code, sourceRCONSafeMessage(sourceErr.code))
|
||||
}
|
||||
return lifecycleFailure("source_rcon_execution_failed", "Source RCON command failed")
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Source RCON command delivered"},
|
||||
Message: "Source RCON command delivered",
|
||||
ExecutionResult: protocol.RunJobExecutionResult{
|
||||
Kind: "source-rcon",
|
||||
Summary: "one-time loopback Source RCON command delivered",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sourceRCONExecutionContext(ctx context.Context, timeoutSeconds int) (context.Context, context.CancelFunc) {
|
||||
timeout := time.Duration(timeoutSeconds) * time.Second
|
||||
if timeout <= 0 || timeout > sourceRCONMaxExecutionTimeout {
|
||||
timeout = sourceRCONMaxExecutionTimeout
|
||||
}
|
||||
return context.WithTimeout(ctx, timeout)
|
||||
}
|
||||
|
||||
func validSourceRCONCommand(command string) bool {
|
||||
return strings.TrimSpace(command) != "" && utf8.ValidString(command) && len([]byte(command)) <= sourceRCONMaxCommandBytes && !strings.ContainsAny(command, "\x00\r\n")
|
||||
}
|
||||
|
||||
func loadSourceRCONConfig(path string, expectedPort int) (sourceRCONConfig, error) {
|
||||
body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes)
|
||||
if err != nil || !found || !utf8.Valid(body) {
|
||||
return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"}
|
||||
}
|
||||
content := strings.ReplaceAll(string(body), "\r\n", "\n")
|
||||
if !strings.Contains(content, managedRCONConfigMarker) {
|
||||
return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"}
|
||||
}
|
||||
values := map[string]string{}
|
||||
inRCON := false
|
||||
for _, rawLine := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
if line == "[rcon]" {
|
||||
inRCON = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "[") {
|
||||
inRCON = false
|
||||
continue
|
||||
}
|
||||
if !inRCON || line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key != "bind_address" && key != "port" && key != "password" {
|
||||
continue
|
||||
}
|
||||
if _, duplicate := values[key]; duplicate {
|
||||
return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"}
|
||||
}
|
||||
values[key] = strings.TrimSpace(value)
|
||||
}
|
||||
configuredPort, err := strconv.Atoi(values["port"])
|
||||
if err != nil || values["bind_address"] != "127.0.0.1" || configuredPort != expectedPort || len(values["password"]) != 64 {
|
||||
return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"}
|
||||
}
|
||||
if _, err := hex.DecodeString(values["password"]); err != nil {
|
||||
return sourceRCONConfig{}, sourceRCONError{code: "source_rcon_config_invalid"}
|
||||
}
|
||||
return sourceRCONConfig{password: values["password"]}, nil
|
||||
}
|
||||
|
||||
func sourceRCONConfigPath(resolver WorkspaceResolver, scope string, plan protocol.RuntimeSourceRCONPlan) (string, error) {
|
||||
markerPath, err := resolver.ExistingTarget(scope, plan.DeploymentStateRef)
|
||||
if err != nil {
|
||||
return "", sourceRCONError{code: "source_rcon_config_unavailable"}
|
||||
}
|
||||
marker, found, err := loadManagedDLLExtensionMarker(markerPath)
|
||||
if err != nil || !found || !sourceRCONMarkerMatchesPlan(marker, plan) {
|
||||
return "", sourceRCONError{code: "source_rcon_config_unavailable"}
|
||||
}
|
||||
configPath, err := resolver.ExistingTarget(scope, marker.ConfigRef)
|
||||
if err != nil {
|
||||
return "", sourceRCONError{code: "source_rcon_config_unavailable"}
|
||||
}
|
||||
return configPath, nil
|
||||
}
|
||||
|
||||
func sourceRCONMarkerMatchesPlan(marker managedDLLExtensionMarker, plan protocol.RuntimeSourceRCONPlan) bool {
|
||||
if marker.ExtensionKey != plan.ExtensionKey || marker.ModKey != plan.ModKey || marker.RCONPort != plan.Port || !managedRCONConfigRefForMod(marker.ConfigRef, plan.ModKey) {
|
||||
return false
|
||||
}
|
||||
return marker.ConfigRef == plan.ConfigRef || strings.HasSuffix(marker.ConfigRef, "/"+plan.ConfigRef)
|
||||
}
|
||||
|
||||
func executeSourceRCONWire(ctx context.Context, port int, password string, command string) error {
|
||||
dialer := net.Dialer{Timeout: sourceRCONIOTimeout}
|
||||
connection, err := dialer.DialContext(ctx, "tcp4", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
|
||||
if err != nil {
|
||||
return sourceRCONError{code: "source_rcon_connection_failed"}
|
||||
}
|
||||
defer connection.Close()
|
||||
if err := writeSourceRCONPacket(ctx, connection, sourceRCONPacket{id: sourceRCONAuthRequestID, typeCode: sourceRCONAuthenticate, body: password}); err != nil {
|
||||
return sourceRCONError{code: "source_rcon_connection_failed"}
|
||||
}
|
||||
auth, err := readSourceRCONPacket(ctx, connection)
|
||||
if err != nil {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
if auth.typeCode != sourceRCONAuthResponse || auth.id == -1 {
|
||||
return sourceRCONError{code: "source_rcon_authentication_failed"}
|
||||
}
|
||||
if auth.id != sourceRCONAuthRequestID || auth.body != "" {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
if err := writeSourceRCONPacket(ctx, connection, sourceRCONPacket{id: sourceRCONCommandRequestID, typeCode: sourceRCONExecuteCommand, body: command}); err != nil {
|
||||
return sourceRCONError{code: "source_rcon_connection_failed"}
|
||||
}
|
||||
responseBytes := 0
|
||||
sourceError := false
|
||||
responsePrefix := make([]byte, 0, len("error:"))
|
||||
for packetIndex := 0; packetIndex < sourceRCONMaxResponsePackets; packetIndex++ {
|
||||
response, err := readSourceRCONPacket(ctx, connection)
|
||||
if err != nil {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
if response.id != sourceRCONCommandRequestID || response.typeCode != sourceRCONResponseValue {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
responseBytes += len([]byte(response.body))
|
||||
if responseBytes > sourceRCONMaxResponseBytes {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
if len(responsePrefix) < cap(responsePrefix) {
|
||||
remaining := cap(responsePrefix) - len(responsePrefix)
|
||||
chunk := []byte(response.body)
|
||||
if len(chunk) > remaining {
|
||||
chunk = chunk[:remaining]
|
||||
}
|
||||
responsePrefix = append(responsePrefix, chunk...)
|
||||
}
|
||||
if len(responsePrefix) == len("error:") && strings.EqualFold(string(responsePrefix), "error:") {
|
||||
sourceError = true
|
||||
}
|
||||
if response.body == "" {
|
||||
if sourceError {
|
||||
return sourceRCONError{code: "source_rcon_command_failed"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
|
||||
func writeSourceRCONPacket(ctx context.Context, connection net.Conn, packet sourceRCONPacket) error {
|
||||
if !utf8.ValidString(packet.body) || len([]byte(packet.body)) > sourceRCONMaxCommandBytes {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
size := 8 + len(packet.body) + 2
|
||||
if size < 10 || size > sourceRCONMaxPacketSize {
|
||||
return sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
buffer := make([]byte, 4+size)
|
||||
binary.LittleEndian.PutUint32(buffer[0:4], uint32(size))
|
||||
binary.LittleEndian.PutUint32(buffer[4:8], uint32(packet.id))
|
||||
binary.LittleEndian.PutUint32(buffer[8:12], uint32(packet.typeCode))
|
||||
copy(buffer[12:], packet.body)
|
||||
if err := setSourceRCONDeadline(ctx, connection); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := connection.Write(buffer)
|
||||
return err
|
||||
}
|
||||
|
||||
func readSourceRCONPacket(ctx context.Context, connection net.Conn) (sourceRCONPacket, error) {
|
||||
if err := setSourceRCONDeadline(ctx, connection); err != nil {
|
||||
return sourceRCONPacket{}, err
|
||||
}
|
||||
var sizeBuffer [4]byte
|
||||
if _, err := io.ReadFull(connection, sizeBuffer[:]); err != nil {
|
||||
return sourceRCONPacket{}, err
|
||||
}
|
||||
size := int(int32(binary.LittleEndian.Uint32(sizeBuffer[:])))
|
||||
if size < 10 || size > sourceRCONMaxPacketSize {
|
||||
return sourceRCONPacket{}, sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
payload := make([]byte, size)
|
||||
if _, err := io.ReadFull(connection, payload); err != nil {
|
||||
return sourceRCONPacket{}, err
|
||||
}
|
||||
if payload[size-2] != 0 || payload[size-1] != 0 || !utf8.Valid(payload[8:size-2]) {
|
||||
return sourceRCONPacket{}, sourceRCONError{code: "source_rcon_protocol_failed"}
|
||||
}
|
||||
return sourceRCONPacket{id: int32(binary.LittleEndian.Uint32(payload[0:4])), typeCode: int32(binary.LittleEndian.Uint32(payload[4:8])), body: string(payload[8 : size-2])}, nil
|
||||
}
|
||||
|
||||
func setSourceRCONDeadline(ctx context.Context, connection net.Conn) error {
|
||||
deadline := time.Now().Add(sourceRCONIOTimeout)
|
||||
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
|
||||
deadline = contextDeadline
|
||||
}
|
||||
return connection.SetDeadline(deadline)
|
||||
}
|
||||
|
||||
func sourceRCONSafeMessage(code string) string {
|
||||
switch code {
|
||||
case "source_rcon_connection_failed":
|
||||
return "Source RCON listener is unavailable"
|
||||
case "source_rcon_authentication_failed":
|
||||
return "Source RCON authentication failed"
|
||||
case "source_rcon_command_failed":
|
||||
return "Source RCON command was rejected"
|
||||
case "source_rcon_protocol_failed":
|
||||
return "Source RCON protocol exchange failed"
|
||||
default:
|
||||
return "Source RCON command failed"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestExecuteSourceRCONAuthenticatesRunsAndRedacts(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
defer listener.Close()
|
||||
password := strings.Repeat("a", 64)
|
||||
assignment := sourceRCONAssignment(port)
|
||||
root := t.TempDir()
|
||||
writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password)
|
||||
commands := make(chan string, 1)
|
||||
serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, command sourceRCONPacket) error {
|
||||
commands <- command.body
|
||||
if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue, body: "queued"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue})
|
||||
})
|
||||
|
||||
command := "SetTime 12"
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, command)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ErrorCode != "" || result.ExecutionResult.Kind != "source-rcon" {
|
||||
t.Fatalf("expected successful Source RCON delivery, got %+v", result)
|
||||
}
|
||||
if got := <-commands; got != command {
|
||||
t.Fatalf("expected transient command delivery, got %q", got)
|
||||
}
|
||||
awaitSourceRCONServer(t, serverDone)
|
||||
serialized, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal result: %v", err)
|
||||
}
|
||||
for _, private := range []string{command, password, "queued"} {
|
||||
if strings.Contains(string(serialized), private) {
|
||||
t.Fatalf("Source RCON result exposed private wire data %q: %s", private, serialized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSourceRCONReadsConfigFromManagedNestedDeployment(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
defer listener.Close()
|
||||
password := strings.Repeat("e", 64)
|
||||
assignment := sourceRCONAssignment(port)
|
||||
root := t.TempDir()
|
||||
writeSourceRCONConfigAt(t, root, assignment, "bin/"+assignment.ExecutionInput.SourceRCON.ConfigRef, "127.0.0.1", password)
|
||||
serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, command sourceRCONPacket) error {
|
||||
if command.body != "rcon.status" {
|
||||
return fmt.Errorf("unexpected nested deployment command %q", command.body)
|
||||
}
|
||||
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: command.id, typeCode: sourceRCONResponseValue})
|
||||
})
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status")
|
||||
if result.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected nested managed config delivery, got %+v", result)
|
||||
}
|
||||
awaitSourceRCONServer(t, serverDone)
|
||||
}
|
||||
|
||||
func TestExecuteSourceRCONRedactsSourceErrorsAndMalformedPackets(t *testing.T) {
|
||||
password := strings.Repeat("b", 64)
|
||||
command := "SpawnItem secret-item"
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
respond func(net.Conn, sourceRCONPacket) error
|
||||
wantCode string
|
||||
private string
|
||||
}{
|
||||
{
|
||||
name: "source error response",
|
||||
respond: func(connection net.Conn, packet sourceRCONPacket) error {
|
||||
if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "err"}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "or: denied secret-item"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue})
|
||||
},
|
||||
wantCode: "source_rcon_command_failed",
|
||||
private: "denied secret-item",
|
||||
},
|
||||
{
|
||||
name: "malformed response packet",
|
||||
respond: func(connection net.Conn, _ sourceRCONPacket) error {
|
||||
var size [4]byte
|
||||
binary.LittleEndian.PutUint32(size[:], sourceRCONMaxPacketSize+1)
|
||||
_, err := connection.Write(size[:])
|
||||
return err
|
||||
},
|
||||
wantCode: "source_rcon_protocol_failed",
|
||||
private: "source response body",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
defer listener.Close()
|
||||
assignment := sourceRCONAssignment(port)
|
||||
root := t.TempDir()
|
||||
writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password)
|
||||
serverDone := serveSourceRCONSession(listener, password, testCase.respond)
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, command)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != testCase.wantCode || result.Retryable {
|
||||
t.Fatalf("expected safe non-retryable %s failure, got %+v", testCase.wantCode, result)
|
||||
}
|
||||
awaitSourceRCONServer(t, serverDone)
|
||||
serialized, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal result: %v", err)
|
||||
}
|
||||
for _, private := range []string{command, password, testCase.private} {
|
||||
if strings.Contains(string(serialized), private) {
|
||||
t.Fatalf("Source RCON failure exposed private wire data %q: %s", private, serialized)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSourceRCONRejectsUnsafeConfigAndNonWindowsBeforeDial(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
acceptResult := make(chan error, 1)
|
||||
go func() {
|
||||
connection, err := listener.Accept()
|
||||
if err == nil {
|
||||
_ = connection.Close()
|
||||
}
|
||||
acceptResult <- err
|
||||
}()
|
||||
password := strings.Repeat("c", 64)
|
||||
assignment := sourceRCONAssignment(port)
|
||||
root := t.TempDir()
|
||||
writeSourceRCONConfig(t, root, assignment, "0.0.0.0", password)
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status")
|
||||
if result.ErrorCode != "source_rcon_config_invalid" {
|
||||
t.Fatalf("expected unsafe local config rejection, got %+v", result)
|
||||
}
|
||||
acceptReturned := false
|
||||
acceptedConnection := false
|
||||
select {
|
||||
case err := <-acceptResult:
|
||||
acceptReturned = true
|
||||
if err == nil {
|
||||
acceptedConnection = true
|
||||
}
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
// No connection is expected before the listener is closed below.
|
||||
}
|
||||
if acceptedConnection {
|
||||
t.Fatal("unsafe config opened a socket")
|
||||
}
|
||||
if err := listener.Close(); err != nil {
|
||||
t.Fatalf("close listener: %v", err)
|
||||
}
|
||||
if !acceptReturned {
|
||||
if err := <-acceptResult; err == nil {
|
||||
t.Fatal("unsafe config opened a socket")
|
||||
}
|
||||
}
|
||||
|
||||
linuxResult := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithDLLExtensionRuntimeTarget("linux", "amd64")).ExecuteSourceRCON(context.Background(), assignment, "rcon.status")
|
||||
if linuxResult.ErrorCode != "unsupported_extension_platform" {
|
||||
t.Fatalf("expected non-Windows rejection, got %+v", linuxResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSourceRCONConsumesInputOnceWithoutJournalOrResultLeakage(t *testing.T) {
|
||||
listener, port := newSourceRCONListener(t)
|
||||
defer listener.Close()
|
||||
password := strings.Repeat("d", 64)
|
||||
command := "SendChat 4 \"maintenance complete\""
|
||||
client := newFakeWorkerClient()
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityRemoteRunRCONCommand)
|
||||
assignment.TargetKey = "rcon.password"
|
||||
assignment.InputRef = "input://source-rcon/job-worker"
|
||||
assignment.MaxAttempts = 1
|
||||
assignment.ExecutionInput = protocol.RunJobExecutionInput{
|
||||
WorkspaceScope: "run-local",
|
||||
RemoteAdapterKey: "rcon",
|
||||
RemoteAdapterKind: "rcon",
|
||||
TimeoutSeconds: 5,
|
||||
SourceRCON: &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port},
|
||||
}
|
||||
client.claimJob = assignment
|
||||
client.sourceRCONInput = protocol.SourceRCONExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, Command: command}
|
||||
serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, packet sourceRCONPacket) error {
|
||||
if err := writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue, body: "accepted"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue})
|
||||
})
|
||||
config := workerTestConfig(t)
|
||||
writeSourceRCONConfig(t, config.WorkspaceRoot, assignment, "127.0.0.1", password)
|
||||
worker, err := NewWorker(config, client, WithDLLExtensionRuntimeTarget("windows", "amd64"))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register worker: %v", err)
|
||||
}
|
||||
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
||||
t.Fatalf("claim Source RCON job handled=%v err=%v", handled, err)
|
||||
}
|
||||
awaitSourceRCONServer(t, serverDone)
|
||||
if len(client.sourceRCONRequests) != 1 || client.sourceRCONRequests[0].JobID != assignment.JobID {
|
||||
t.Fatalf("expected one active-lease Source RCON input read, got %+v", client.sourceRCONRequests)
|
||||
}
|
||||
if len(client.resultRequests) != 1 || client.resultRequests[0].Retryable || client.resultRequests[0].State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected one non-retryable safe result, got %+v", client.resultRequests)
|
||||
}
|
||||
for _, projection := range []any{worker.journal.ActiveJobs(), client.resultRequests, client.sourceRCONRequests} {
|
||||
body, marshalErr := json.Marshal(projection)
|
||||
if marshalErr != nil {
|
||||
t.Fatalf("marshal safe projection: %v", marshalErr)
|
||||
}
|
||||
for _, private := range []string{command, password, "accepted"} {
|
||||
if strings.Contains(string(body), private) {
|
||||
t.Fatalf("journal or result projection exposed %q: %s", private, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newSourceRCONListener(t *testing.T) (net.Listener, int) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen Source RCON fixture: %v", err)
|
||||
}
|
||||
address, ok := listener.Addr().(*net.TCPAddr)
|
||||
if !ok || address.Port < 1024 {
|
||||
_ = listener.Close()
|
||||
t.Fatal("invalid Source RCON fixture address")
|
||||
}
|
||||
return listener, address.Port
|
||||
}
|
||||
|
||||
func sourceRCONAssignment(port int) protocol.RunJobAssignment {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunRCONCommand)
|
||||
assignment.TargetKey = "rcon.password"
|
||||
assignment.InputRef = "input://source-rcon/job-1"
|
||||
assignment.MaxAttempts = 1
|
||||
assignment.ExecutionInput = protocol.RunJobExecutionInput{
|
||||
WorkspaceScope: "run-local",
|
||||
RemoteAdapterKey: "rcon",
|
||||
RemoteAdapterKind: "rcon",
|
||||
TimeoutSeconds: 5,
|
||||
SourceRCON: &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port},
|
||||
}
|
||||
return assignment
|
||||
}
|
||||
|
||||
func writeSourceRCONConfig(t *testing.T, root string, assignment protocol.RunJobAssignment, bindAddress string, password string) {
|
||||
writeSourceRCONConfigAt(t, root, assignment, assignment.ExecutionInput.SourceRCON.ConfigRef, bindAddress, password)
|
||||
}
|
||||
|
||||
func writeSourceRCONConfigAt(t *testing.T, root string, assignment protocol.RunJobAssignment, configRef string, bindAddress string, password string) {
|
||||
t.Helper()
|
||||
resolver := NewWorkspaceResolver(root)
|
||||
scope, err := resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("create Source RCON scope: %v", err)
|
||||
}
|
||||
path, _, err := resolver.WritableTarget(scope, configRef)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve Source RCON config: %v", err)
|
||||
}
|
||||
body := fmt.Sprintf("%s\n[rcon]\nbind_address=%s\nport=%d\npassword=%s\n", managedRCONConfigMarker, bindAddress, assignment.ExecutionInput.SourceRCON.Port, password)
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatalf("write Source RCON config: %v", err)
|
||||
}
|
||||
markerPath, _, err := resolver.WritableTarget(scope, assignment.ExecutionInput.SourceRCON.DeploymentStateRef)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve Source RCON deployment marker: %v", err)
|
||||
}
|
||||
markerBody, err := json.Marshal(managedDLLExtensionMarker{Version: ue4ssExtensionMarkerVersion, ReleaseVersion: "1.0.0", Checksum: "sha256:" + strings.Repeat("a", 64), SizeBytes: 1, ExtensionKey: assignment.ExecutionInput.SourceRCON.ExtensionKey, ModKey: assignment.ExecutionInput.SourceRCON.ModKey, ConfigRef: configRef, RCONPort: assignment.ExecutionInput.SourceRCON.Port})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal Source RCON deployment marker: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(markerPath, markerBody, 0o600); err != nil {
|
||||
t.Fatalf("write Source RCON deployment marker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func serveSourceRCONSession(listener net.Listener, password string, respond func(net.Conn, sourceRCONPacket) error) <-chan error {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
connection, err := listener.Accept()
|
||||
if err != nil {
|
||||
done <- err
|
||||
return
|
||||
}
|
||||
defer connection.Close()
|
||||
context, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
auth, err := readSourceRCONPacket(context, connection)
|
||||
if err != nil {
|
||||
done <- err
|
||||
return
|
||||
}
|
||||
if auth.id != sourceRCONAuthRequestID || auth.typeCode != sourceRCONAuthenticate || auth.body != password {
|
||||
done <- fmt.Errorf("unexpected auth packet")
|
||||
return
|
||||
}
|
||||
if err := writeSourceRCONPacket(context, connection, sourceRCONPacket{id: auth.id, typeCode: sourceRCONAuthResponse}); err != nil {
|
||||
done <- err
|
||||
return
|
||||
}
|
||||
command, err := readSourceRCONPacket(context, connection)
|
||||
if err != nil {
|
||||
done <- err
|
||||
return
|
||||
}
|
||||
if command.id != sourceRCONCommandRequestID || command.typeCode != sourceRCONExecuteCommand {
|
||||
done <- fmt.Errorf("unexpected command packet")
|
||||
return
|
||||
}
|
||||
done <- respond(connection, command)
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
func awaitSourceRCONServer(t *testing.T, done <-chan error) {
|
||||
t.Helper()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Source RCON fixture: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Source RCON fixture did not finish")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const sqliteSchemaProbeDriver = "sqlite"
|
||||
|
||||
// SQLiteSchemaProbeExecutor performs only fixed SQLite introspection queries.
|
||||
// The assignment carries no SQL and the only local target is a package-scoped,
|
||||
// logical database key.
|
||||
type SQLiteSchemaProbeExecutor struct{ resolver WorkspaceResolver }
|
||||
|
||||
func NewSQLiteSchemaProbeExecutor(workspaceRoot string) *SQLiteSchemaProbeExecutor {
|
||||
return &SQLiteSchemaProbeExecutor{resolver: NewWorkspaceResolver(workspaceRoot)}
|
||||
}
|
||||
|
||||
func (executor *SQLiteSchemaProbeExecutor) Execute(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return sqliteSchemaProbeFailure(assignment, "invalid_request", false)
|
||||
}
|
||||
request := *assignment.ExecutionInput.SQLiteSchemaProbe
|
||||
probe := protocol.SQLiteSchemaProbeResult{RequestID: request.RequestID, JobID: assignment.JobID, Binding: request.Binding, Status: "failed", Limits: request.Limits}
|
||||
scope, err := executor.resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "target_unavailable", false)
|
||||
}
|
||||
path, err := executor.resolver.ExistingTarget(scope, assignment.TargetKey)
|
||||
if err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "target_unavailable", false)
|
||||
}
|
||||
sourceFingerprint, err := fingerprintSQLiteSource(path)
|
||||
if err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "source_unavailable", true)
|
||||
}
|
||||
probe.SourceFingerprint = sourceFingerprint
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, time.Duration(request.Limits.TimeoutMS)*time.Millisecond)
|
||||
defer cancel()
|
||||
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+path+"?mode=ro")
|
||||
if err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "sqlite_open_failed", true)
|
||||
}
|
||||
defer database.Close()
|
||||
database.SetMaxOpenConns(1)
|
||||
database.SetConnMaxLifetime(time.Minute)
|
||||
if _, err := database.ExecContext(probeCtx, "PRAGMA query_only = ON"); err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, sqliteProbeErrorCode(probeCtx, err), true)
|
||||
}
|
||||
objects, err := inspectSQLiteSchema(probeCtx, database, request.Limits)
|
||||
if err != nil {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, sqliteProbeErrorCode(probeCtx, err), true)
|
||||
}
|
||||
probe.Objects = objects
|
||||
if sourceFingerprintAfter, err := fingerprintSQLiteSource(path); err != nil || sourceFingerprintAfter != probe.SourceFingerprint {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "source_changed", true)
|
||||
}
|
||||
probe.SchemaFingerprint = digestValue(schemaFingerprintInput(objects))
|
||||
probe.ObservedAt = time.Now().UTC()
|
||||
probe.Status = "succeeded"
|
||||
if !finalizeSQLiteSchemaProbe(&probe) || sqliteSchemaProbeSize(probe) > request.Limits.MaxResultBytes {
|
||||
return sqliteSchemaProbeTerminalFailure(probe, "result_limit_exceeded", false)
|
||||
}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "SQLite schema probe completed"}, Message: "SQLite schema probe completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", Checksum: probe.ResultDigest, SizeBytes: int64(sqliteSchemaProbeSize(probe)), Summary: "bounded query-only SQLite schema metadata", SQLiteSchemaProbe: &probe}}
|
||||
}
|
||||
|
||||
func sqliteSchemaProbeFailure(assignment protocol.RunJobAssignment, code string, retryable bool) LifecycleExecutionResult {
|
||||
probe := protocol.SQLiteSchemaProbeResult{JobID: assignment.JobID, Status: "failed", SafeError: protocol.SQLiteSchemaProbeSafeError{Code: code, Retryable: retryable}}
|
||||
if assignment.ExecutionInput.SQLiteSchemaProbe != nil {
|
||||
probe.RequestID, probe.Binding, probe.Limits = assignment.ExecutionInput.SQLiteSchemaProbe.RequestID, assignment.ExecutionInput.SQLiteSchemaProbe.Binding, assignment.ExecutionInput.SQLiteSchemaProbe.Limits
|
||||
}
|
||||
return sqliteSchemaProbeTerminalFailure(probe, code, retryable)
|
||||
}
|
||||
|
||||
func sqliteSchemaProbeTerminalFailure(probe protocol.SQLiteSchemaProbeResult, code string, retryable bool) LifecycleExecutionResult {
|
||||
probe.Status, probe.ObservedAt, probe.SafeError = "failed", time.Now().UTC(), protocol.SQLiteSchemaProbeSafeError{Code: code, Retryable: retryable}
|
||||
_ = finalizeSQLiteSchemaProbe(&probe)
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "SQLite schema probe failed"}, Message: "SQLite schema probe failed", ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "sqlite.schema-probe", Checksum: probe.ResultDigest, SizeBytes: int64(sqliteSchemaProbeSize(probe)), Summary: "bounded query-only SQLite schema probe failed", SQLiteSchemaProbe: &probe}}
|
||||
}
|
||||
|
||||
func inspectSQLiteSchema(ctx context.Context, database *sql.DB, limits protocol.SQLiteSchemaProbeLimits) ([]protocol.SQLiteSchemaProbeObject, error) {
|
||||
rows, err := database.QueryContext(ctx, "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY type, name LIMIT ?", limits.MaxObjects)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type sqliteObjectIdentity struct{ name, kind string }
|
||||
identities := make([]sqliteObjectIdentity, 0, limits.MaxObjects)
|
||||
for rows.Next() {
|
||||
var name, kind string
|
||||
if err := rows.Scan(&name, &kind); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
identities = append(identities, sqliteObjectIdentity{name: name, kind: kind})
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects := make([]protocol.SQLiteSchemaProbeObject, 0, len(identities))
|
||||
for _, identity := range identities {
|
||||
object, err := inspectSQLiteObject(ctx, database, identity.name, identity.kind, limits, len(objects) < limits.MaxCardinalityReads)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects = append(objects, object)
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func inspectSQLiteObject(ctx context.Context, database *sql.DB, name, kind string, limits protocol.SQLiteSchemaProbeLimits, includeCardinality bool) (protocol.SQLiteSchemaProbeObject, error) {
|
||||
object := protocol.SQLiteSchemaProbeObject{ObjectHash: digestValue(kind + "\x00" + name), Kind: kind, NameFingerprint: digestValue(name)}
|
||||
columns, err := database.QueryContext(ctx, "SELECT cid, name, type, [notnull], pk FROM pragma_table_info(?) ORDER BY cid LIMIT ?", name, limits.MaxColumnsPerObject)
|
||||
if err != nil {
|
||||
return object, err
|
||||
}
|
||||
for columns.Next() {
|
||||
var ordinal, notNull, primaryKey int
|
||||
var columnName, declaredType string
|
||||
if err := columns.Scan(&ordinal, &columnName, &declaredType, ¬Null, &primaryKey); err != nil {
|
||||
columns.Close()
|
||||
return object, err
|
||||
}
|
||||
nullable := notNull == 0
|
||||
object.DeclaredColumns = append(object.DeclaredColumns, protocol.SQLiteSchemaProbeColumn{NameFingerprint: digestValue(columnName), DeclaredType: safeSQLiteDeclaredType(declaredType), Nullable: &nullable, PrimaryKey: primaryKey != 0, Ordinal: ordinal})
|
||||
}
|
||||
if err := columns.Close(); err != nil {
|
||||
return object, err
|
||||
}
|
||||
indexes, err := database.QueryContext(ctx, "SELECT name, [unique] FROM pragma_index_list(?) ORDER BY seq LIMIT ?", name, limits.MaxIndexesPerObject)
|
||||
if err != nil {
|
||||
return object, err
|
||||
}
|
||||
type sqliteIndexIdentity struct {
|
||||
name string
|
||||
unique bool
|
||||
}
|
||||
indexIdentities := make([]sqliteIndexIdentity, 0, limits.MaxIndexesPerObject)
|
||||
for indexes.Next() {
|
||||
var indexName string
|
||||
var unique int
|
||||
if err := indexes.Scan(&indexName, &unique); err != nil {
|
||||
indexes.Close()
|
||||
return object, err
|
||||
}
|
||||
indexIdentities = append(indexIdentities, sqliteIndexIdentity{name: indexName, unique: unique != 0})
|
||||
}
|
||||
if err := indexes.Close(); err != nil {
|
||||
return object, err
|
||||
}
|
||||
for _, identity := range indexIdentities {
|
||||
item, err := inspectSQLiteIndex(ctx, database, identity.name, identity.unique, limits.MaxColumnsPerObject)
|
||||
if err != nil {
|
||||
return object, err
|
||||
}
|
||||
object.Indexes = append(object.Indexes, item)
|
||||
}
|
||||
foreignKeys, err := database.QueryContext(ctx, "SELECT [table], [from], [to] FROM pragma_foreign_key_list(?) ORDER BY id, seq LIMIT ?", name, limits.MaxForeignKeys)
|
||||
if err != nil {
|
||||
return object, err
|
||||
}
|
||||
for foreignKeys.Next() {
|
||||
var destination, from, to string
|
||||
if err := foreignKeys.Scan(&destination, &from, &to); err != nil {
|
||||
foreignKeys.Close()
|
||||
return object, err
|
||||
}
|
||||
object.ForeignKeys = append(object.ForeignKeys, protocol.SQLiteSchemaProbeForeignKey{FromColumnHash: digestValue(from), ToObjectHash: digestValue("table\x00" + destination), ToColumnHash: digestValue(to)})
|
||||
}
|
||||
if err := foreignKeys.Close(); err != nil {
|
||||
return object, err
|
||||
}
|
||||
if includeCardinality {
|
||||
var count int64
|
||||
if err := database.QueryRowContext(ctx, "SELECT count(*) FROM "+quoteSQLiteIdentifier(name)).Scan(&count); err != nil {
|
||||
return object, err
|
||||
}
|
||||
object.ApproximateRows = &count
|
||||
}
|
||||
if limits.MaxSampleRows > 0 {
|
||||
rows, err := database.QueryContext(ctx, "SELECT * FROM "+quoteSQLiteIdentifier(name)+" LIMIT ?", limits.MaxSampleRows)
|
||||
if err != nil {
|
||||
return object, err
|
||||
}
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
return object, err
|
||||
}
|
||||
for rows.Next() {
|
||||
values := make([]any, len(columns))
|
||||
pointers := make([]any, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
rows.Close()
|
||||
return object, err
|
||||
}
|
||||
object.SampleFingerprints = append(object.SampleFingerprints, digestValue(canonicalSQLiteRow(columns, values)))
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return object, err
|
||||
}
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func inspectSQLiteIndex(ctx context.Context, database *sql.DB, name string, unique bool, maxColumns int) (protocol.SQLiteSchemaProbeIndex, error) {
|
||||
index := protocol.SQLiteSchemaProbeIndex{NameFingerprint: digestValue(name), Unique: unique}
|
||||
rows, err := database.QueryContext(ctx, "SELECT name FROM pragma_index_info(?) ORDER BY seqno LIMIT ?", name, maxColumns)
|
||||
if err != nil {
|
||||
return index, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var column string
|
||||
if err := rows.Scan(&column); err != nil {
|
||||
return index, err
|
||||
}
|
||||
index.ColumnHashes = append(index.ColumnHashes, digestValue(column))
|
||||
}
|
||||
return index, rows.Err()
|
||||
}
|
||||
|
||||
func quoteSQLiteIdentifier(value string) string {
|
||||
return `"` + strings.ReplaceAll(value, `"`, `""`) + `"`
|
||||
}
|
||||
func digestBytes(value []byte) string {
|
||||
sum := sha256.Sum256(value)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
func digestValue(value string) string { return digestBytes([]byte(value)) }
|
||||
func fingerprintSQLiteSource(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
func schemaFingerprintInput(objects []protocol.SQLiteSchemaProbeObject) string {
|
||||
body, _ := json.Marshal(objects)
|
||||
return string(body)
|
||||
}
|
||||
func finalizeSQLiteSchemaProbe(probe *protocol.SQLiteSchemaProbeResult) bool {
|
||||
probe.ResultDigest = ""
|
||||
body, err := json.Marshal(probe)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
probe.ResultDigest = digestBytes(body)
|
||||
return true
|
||||
}
|
||||
func sqliteSchemaProbeSize(probe protocol.SQLiteSchemaProbeResult) int {
|
||||
body, _ := json.Marshal(probe)
|
||||
return len(body)
|
||||
}
|
||||
func canonicalSQLiteRow(columns []string, values []any) string {
|
||||
body, _ := json.Marshal(struct {
|
||||
Columns []string `json:"columns"`
|
||||
Values []any `json:"values"`
|
||||
}{columns, values})
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func safeSQLiteDeclaredType(value string) string {
|
||||
value = strings.ToUpper(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if len(value) > 80 {
|
||||
return "OTHER"
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || strings.ContainsRune("_(), ", char) {
|
||||
continue
|
||||
}
|
||||
return "OTHER"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func sqliteProbeErrorCode(ctx context.Context, err error) string {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return "cancelled"
|
||||
}
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return "timeout"
|
||||
}
|
||||
lower := strings.ToLower(fmt.Sprint(err))
|
||||
if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") {
|
||||
return "database_busy"
|
||||
}
|
||||
return "sqlite_read_failed"
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestSQLiteSchemaProbeExecutesOnlyAgainstScopedLogicalTarget(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("create workspace scope: %v", err)
|
||||
}
|
||||
databasePath := filepath.Join(scope, "databases", "current.db")
|
||||
createSQLiteProbeFixture(t, databasePath)
|
||||
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+databasePath+"?mode=ro")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inspectSQLiteSchema(context.Background(), database, assignment.ExecutionInput.SQLiteSchemaProbe.Limits); err != nil {
|
||||
t.Fatalf("inspect fixture directly: %v", err)
|
||||
}
|
||||
database.Close()
|
||||
|
||||
result := NewSQLiteSchemaProbeExecutor(root).Execute(context.Background(), assignment)
|
||||
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.SQLiteSchemaProbe == nil {
|
||||
t.Fatalf("expected successful probe envelope, got %+v", result)
|
||||
}
|
||||
probe := result.ExecutionResult.SQLiteSchemaProbe
|
||||
if probe.JobID != assignment.JobID || probe.Binding != assignment.ExecutionInput.SQLiteSchemaProbe.Binding || probe.Status != "succeeded" || probe.ObservedAt.IsZero() || probe.ResultDigest == "" || probe.SourceFingerprint == "" || probe.SchemaFingerprint == "" {
|
||||
t.Fatalf("unexpected probe identity envelope: %+v", probe)
|
||||
}
|
||||
if len(probe.Objects) != 2 || probe.Objects[0].ApproximateRows == nil || len(probe.Objects[1].SampleFingerprints) == 0 {
|
||||
t.Fatalf("expected bounded table evidence, got %+v", probe.Objects)
|
||||
}
|
||||
serialized := mustJSON(t, probe)
|
||||
for _, value := range []string{"current.db", "members", "alpha", root, "SELECT", "sqlite:"} {
|
||||
if strings.Contains(serialized, value) {
|
||||
t.Fatalf("probe leaked protected source material %q: %s", value, serialized)
|
||||
}
|
||||
}
|
||||
if result.ExecutionResult.Content != "" {
|
||||
t.Fatalf("probe must not return content: %+v", result.ExecutionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSchemaProbeRejectsUnscopedOrUnsafeRequests(t *testing.T) {
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
assignment.TargetKey = "/tmp/current.db"
|
||||
result := NewSQLiteSchemaProbeExecutor(t.TempDir()).Execute(context.Background(), assignment)
|
||||
if result.ErrorCode != "invalid_request" || result.ExecutionResult.SQLiteSchemaProbe == nil {
|
||||
t.Fatalf("expected safe invalid probe failure, got %+v", result)
|
||||
}
|
||||
assignment = sqliteSchemaProbeAssignment()
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Binding.DatabaseIdentity = "C:/host/path"
|
||||
result = NewSQLiteSchemaProbeExecutor(t.TempDir()).Execute(context.Background(), assignment)
|
||||
if result.ErrorCode != "invalid_request" || strings.Contains(result.Message, "C:/") {
|
||||
t.Fatalf("expected redacted binding rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSchemaProbeEnforcesResultLimit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := sqliteSchemaProbeAssignment()
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
createSQLiteProbeFixture(t, filepath.Join(scope, "databases", "current.db"))
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe.Limits.MaxResultBytes = 1
|
||||
result := NewSQLiteSchemaProbeExecutor(root).Execute(context.Background(), assignment)
|
||||
if result.ErrorCode != "result_limit_exceeded" || result.State != lifecycleResultStateFailed {
|
||||
t.Fatalf("expected result bound failure, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func sqliteSchemaProbeAssignment() protocol.RunJobAssignment {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteProbe)
|
||||
assignment.TargetKey, assignment.MaxAttempts, assignment.FencingToken = "databases/current.db", 1, 7
|
||||
assignment.ExecutionInput.WorkspaceScope = "profile-default"
|
||||
assignment.ExecutionInput.SQLiteSchemaProbe = &protocol.SQLiteSchemaProbeRequest{RequestID: "probe-1", Binding: protocol.SQLiteSchemaProbeBinding{ServerInstanceID: assignment.ServerInstanceID, RunBindingID: "binding-1", RunEndpointID: assignment.RunEndpointID, PluginID: "game.example", PluginVersion: "1.0.0", AdapterVersion: "adapter-1", GameVersion: "1.0", DatabaseIdentity: "database-1"}, Limits: protocol.SQLiteSchemaProbeLimits{MaxObjects: 8, MaxColumnsPerObject: 8, MaxIndexesPerObject: 8, MaxForeignKeys: 8, MaxCardinalityReads: 8, MaxSampleRows: 2, TimeoutMS: 1000, MaxResultBytes: 128 * 1024}}
|
||||
return assignment
|
||||
}
|
||||
|
||||
func createSQLiteProbeFixture(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database, err := sql.Open(sqliteSchemaProbeDriver, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec("CREATE TABLE members (id INTEGER PRIMARY KEY, name TEXT NOT NULL); CREATE TABLE groups (id INTEGER PRIMARY KEY, member_id INTEGER REFERENCES members(id)); CREATE UNIQUE INDEX members_name ON members(name); INSERT INTO members(name) VALUES ('alpha'), ('beta');"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, value any) string {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
ue4ssExtensionStateRoot = "runtime/ue4ss-dll"
|
||||
ue4ssExtensionMarkerVersion = 2
|
||||
maxUE4SSMetadataBytes int64 = 16 * 1024
|
||||
maxUE4SSDLLBytes int64 = 128 * 1024 * 1024
|
||||
maxSCUMExecutableBytes int64 = 2 * 1024 * 1024 * 1024
|
||||
managedRCONConfigMarker = "; managed by Run UE4SS DLL extension"
|
||||
)
|
||||
|
||||
type dllExtensionError struct {
|
||||
code string
|
||||
message string
|
||||
}
|
||||
|
||||
func (err dllExtensionError) Error() string { return err.message }
|
||||
|
||||
type managedDLLExtensionMarker struct {
|
||||
Version int `json:"version"`
|
||||
ReleaseVersion string `json:"releaseVersion"`
|
||||
Checksum string `json:"checksum"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
ExtensionKey string `json:"extensionKey"`
|
||||
ModKey string `json:"modKey"`
|
||||
ConfigRef string `json:"configRef"`
|
||||
RCONPort int `json:"rconPort"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) synchronizeUE4SSDLLExtensions(ctx context.Context, assignment protocol.RunJobAssignment, template LifecycleActionTemplate, scope string) error {
|
||||
if executor.runtimeTargetOS != "windows" || executor.runtimeTargetArch != "amd64" {
|
||||
return dllExtensionError{code: "unsupported_extension_platform", message: "UE4SS DLL extensions require Windows amd64"}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
executableKey := template.TargetExecutableKey
|
||||
// Older generated plugin packages do not have targetExecutableKey yet,
|
||||
// but SCUM's existing start action already carries the same logical path
|
||||
// in SERVER_EXECUTABLE_REF. Keep those packages forward-compatible.
|
||||
if executableKey == "" && template.Environment != nil {
|
||||
executableKey = template.Environment["SERVER_EXECUTABLE_REF"]
|
||||
}
|
||||
if executableKey == "" && template.Env != nil {
|
||||
executableKey = template.Env["SERVER_EXECUTABLE_REF"]
|
||||
}
|
||||
if executableKey == "" && strings.HasSuffix(strings.ToLower(template.ExecutableKey), ".exe") {
|
||||
executableKey = template.ExecutableKey
|
||||
}
|
||||
if executableKey == "" || !strings.HasSuffix(strings.ToLower(executableKey), ".exe") {
|
||||
return dllExtensionError{code: "extension_scum_executable_invalid", message: "UE4SS DLL extensions require a declared SCUM executable"}
|
||||
}
|
||||
|
||||
resolver := NewWorkspaceResolver(executor.workspaceRoot)
|
||||
targetResolver, targetScope := resolver, scope
|
||||
if deployment := assignment.ExecutionInput.Deployment; deployment != nil && deployment.ServerRoot != "" {
|
||||
root := filepath.Clean(deployment.ServerRoot)
|
||||
if root == "." || !filepath.IsAbs(root) {
|
||||
return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared executable root is unsafe"}
|
||||
}
|
||||
targetResolver = NewWorkspaceResolver(filepath.Dir(root))
|
||||
targetScope = root
|
||||
}
|
||||
executable, err := declaredLifecycleExecutable(targetResolver, targetScope, executableKey)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable is unavailable"}
|
||||
}
|
||||
executableChecksum, _, err := checksumRegularFile(executable, maxSCUMExecutableBytes)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable cannot be verified"}
|
||||
}
|
||||
for _, plan := range assignment.ExecutionInput.DLLExtensions {
|
||||
if !strings.EqualFold(executableChecksum, plan.SCUMExecutableChecksum) {
|
||||
return dllExtensionError{code: "extension_scum_checksum_mismatch", message: "declared SCUM executable does not match the extension release"}
|
||||
}
|
||||
}
|
||||
|
||||
gameRootKey, err := gameRootKeyForExecutable(template.ExecutableKey)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "extension_scum_executable_invalid", message: "declared SCUM executable location is unsafe"}
|
||||
}
|
||||
if err := verifyUE4SSBootstrap(targetResolver, targetScope, gameRootKey); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, plan := range assignment.ExecutionInput.DLLExtensions {
|
||||
if err := executor.synchronizeUE4SSDLLExtension(ctx, targetResolver, targetScope, gameRootKey, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func declaredLifecycleExecutable(resolver WorkspaceResolver, scope string, executableKey string) (string, error) {
|
||||
if !protocol.ValidLogicalFileKey(filepath.ToSlash(executableKey)) {
|
||||
return "", fmt.Errorf("declared executable key is unsafe")
|
||||
}
|
||||
return resolver.ExistingTarget(scope, executableKey)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) synchronizeUE4SSDLLExtension(ctx context.Context, resolver WorkspaceResolver, scope string, gameRootKey string, plan protocol.RuntimeDLLExtensionPlan) error {
|
||||
activeKey := gameRelativeKey(gameRootKey, plan.DLLRef)
|
||||
configRef := managedRCONConfigRef(gameRootKey, plan.ModKey)
|
||||
activePath, _, err := resolver.WritableTarget(scope, activeKey)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_workspace_failed", message: "extension deployment workspace is unavailable"}
|
||||
}
|
||||
markerPath, stagePath, previousPath, err := extensionStatePaths(resolver, scope, plan)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_workspace_failed", message: "extension state workspace is unavailable"}
|
||||
}
|
||||
marker, markerFound, err := loadManagedDLLExtensionMarker(markerPath)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_state_failed", message: "extension release state cannot be read"}
|
||||
}
|
||||
unchanged := markerFound && markerMatchesDeployment(marker, plan, configRef) && managedDLLMatchesPlan(activePath, plan)
|
||||
if !unchanged {
|
||||
_ = os.Remove(stagePath)
|
||||
defer os.Remove(stagePath)
|
||||
downloadedSize, downloadedChecksum, downloadErr := executor.dependencyDownloader.Download(ctx, plan.ReleaseURL, stagePath, plan.SizeBytes)
|
||||
if downloadErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return dllExtensionError{code: "dll_extension_download_failed", message: "declared DLL download failed"}
|
||||
}
|
||||
verifiedChecksum, verifiedSize, verifyErr := checksumRegularFile(stagePath, plan.SizeBytes)
|
||||
if verifyErr != nil || downloadedSize != plan.SizeBytes || verifiedSize != plan.SizeBytes || !strings.EqualFold(downloadedChecksum, plan.Checksum) || !strings.EqualFold(verifiedChecksum, plan.Checksum) {
|
||||
return dllExtensionError{code: "dll_extension_verify_failed", message: "declared DLL did not match its fixed release checksum"}
|
||||
}
|
||||
}
|
||||
if err := executor.ensureLoopbackRCONConfig(resolver, scope, gameRootKey, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := executor.ensureUE4SSModsIndex(resolver, scope, gameRootKey, plan.ModKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if unchanged {
|
||||
return nil
|
||||
}
|
||||
if err := executor.activateManagedDLLExtension(activePath, stagePath, previousPath, markerPath, plan, configRef); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyUE4SSBootstrap(resolver WorkspaceResolver, scope string, gameRootKey string) error {
|
||||
for _, filename := range []string{"dwmapi.dll", "UE4SS.dll"} {
|
||||
if _, err := resolver.ExistingTarget(scope, gameRelativeKey(gameRootKey, filename)); err != nil {
|
||||
return dllExtensionError{code: "ue4ss_bootstrap_missing", message: "required UE4SS bootstrap files are not installed"}
|
||||
}
|
||||
}
|
||||
if err := existingRuntimeDirectory(scope, gameRelativeKey(gameRootKey, "ue4ss")); err != nil {
|
||||
return dllExtensionError{code: "ue4ss_bootstrap_missing", message: "required UE4SS bootstrap files are not installed"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gameRootKeyForExecutable(executableKey string) (string, error) {
|
||||
normalized := filepath.ToSlash(executableKey)
|
||||
if !protocol.ValidLogicalFileKey(normalized) || strings.HasPrefix(normalized, "/") || strings.Contains(normalized, `\`) {
|
||||
return "", fmt.Errorf("executable key is unsafe")
|
||||
}
|
||||
parent := pathpkg.Dir(normalized)
|
||||
if parent == "." {
|
||||
return "", nil
|
||||
}
|
||||
return parent, nil
|
||||
}
|
||||
|
||||
func gameRelativeKey(gameRootKey string, relativeKey string) string {
|
||||
if gameRootKey == "" {
|
||||
return relativeKey
|
||||
}
|
||||
return gameRootKey + "/" + relativeKey
|
||||
}
|
||||
|
||||
func extensionStatePaths(resolver WorkspaceResolver, scope string, plan protocol.RuntimeDLLExtensionPlan) (string, string, string, error) {
|
||||
baseKey := ue4ssExtensionStateRoot + "/" + plan.TargetKey
|
||||
markerPath, _, err := resolver.WritableTarget(scope, baseKey+"/release.json")
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
stagePath, _, err := resolver.WritableTarget(scope, baseKey+"/download.staged")
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
previousPath, _, err := resolver.WritableTarget(scope, baseKey+"/previous.dll")
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
return markerPath, stagePath, previousPath, nil
|
||||
}
|
||||
|
||||
func loadManagedDLLExtensionMarker(path string) (managedDLLExtensionMarker, bool, error) {
|
||||
body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes)
|
||||
if err != nil || !found {
|
||||
return managedDLLExtensionMarker{}, found, err
|
||||
}
|
||||
var marker managedDLLExtensionMarker
|
||||
if err := json.Unmarshal(body, &marker); err != nil {
|
||||
return managedDLLExtensionMarker{}, false, nil
|
||||
}
|
||||
if marker.Version != ue4ssExtensionMarkerVersion || !protocol.ValidLogicalFileKey(marker.ExtensionKey) || !protocol.ValidLogicalFileKey(marker.ModKey) || !managedRCONConfigRefForMod(marker.ConfigRef, marker.ModKey) || !protocolValidSHA256(marker.Checksum) || marker.SizeBytes < 1 || marker.ReleaseVersion == "" || marker.RCONPort < 1024 || marker.RCONPort > 65535 {
|
||||
return managedDLLExtensionMarker{}, false, nil
|
||||
}
|
||||
return marker, true, nil
|
||||
}
|
||||
|
||||
func markerMatchesPlan(marker managedDLLExtensionMarker, plan protocol.RuntimeDLLExtensionPlan) bool {
|
||||
return marker.Version == ue4ssExtensionMarkerVersion && marker.ReleaseVersion == plan.Version && strings.EqualFold(marker.Checksum, plan.Checksum) && marker.SizeBytes == plan.SizeBytes && marker.ExtensionKey == plan.Key && marker.ModKey == plan.ModKey && marker.RCONPort == plan.RCONPort
|
||||
}
|
||||
|
||||
func markerMatchesDeployment(marker managedDLLExtensionMarker, plan protocol.RuntimeDLLExtensionPlan, configRef string) bool {
|
||||
return markerMatchesPlan(marker, plan) && marker.ConfigRef == configRef
|
||||
}
|
||||
|
||||
func managedDLLMatchesPlan(path string, plan protocol.RuntimeDLLExtensionPlan) bool {
|
||||
checksum, size, err := checksumRegularFile(path, plan.SizeBytes)
|
||||
return err == nil && size == plan.SizeBytes && strings.EqualFold(checksum, plan.Checksum)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) activateManagedDLLExtension(activePath string, stagePath string, previousPath string, markerPath string, plan protocol.RuntimeDLLExtensionPlan, configRef string) error {
|
||||
if _, _, err := checksumRegularFile(stagePath, plan.SizeBytes); err != nil {
|
||||
return dllExtensionError{code: "dll_extension_verify_failed", message: "staged DLL cannot be verified"}
|
||||
}
|
||||
previousMarker, previousMarkerFound, markerErr := readBoundedRegularFile(markerPath, maxUE4SSMetadataBytes)
|
||||
if markerErr != nil {
|
||||
return dllExtensionError{code: "dll_extension_state_failed", message: "extension release state cannot be read"}
|
||||
}
|
||||
activeExists := false
|
||||
if _, _, err := checksumRegularFile(activePath, maxUE4SSDLLBytes); err == nil {
|
||||
activeExists = true
|
||||
if err := copyRegularFileAtomic(activePath, previousPath, maxUE4SSDLLBytes, 0o600); err != nil {
|
||||
return dllExtensionError{code: "dll_extension_activation_failed", message: "previous DLL could not be retained"}
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return dllExtensionError{code: "dll_extension_activation_failed", message: "current DLL cannot be safely replaced"}
|
||||
}
|
||||
if err := os.Rename(stagePath, activePath); err != nil {
|
||||
return dllExtensionError{code: "dll_extension_activation_failed", message: "verified DLL could not be activated"}
|
||||
}
|
||||
if err := os.Chmod(activePath, 0o600); err != nil {
|
||||
rollbackManagedDLLExtension(activePath, previousPath, activeExists)
|
||||
return dllExtensionError{code: "dll_extension_activation_failed", message: "activated DLL permissions could not be secured"}
|
||||
}
|
||||
marker := managedDLLExtensionMarker{Version: ue4ssExtensionMarkerVersion, ReleaseVersion: plan.Version, Checksum: strings.ToLower(plan.Checksum), SizeBytes: plan.SizeBytes, ExtensionKey: plan.Key, ModKey: plan.ModKey, ConfigRef: configRef, RCONPort: plan.RCONPort, UpdatedAt: time.Now().UTC()}
|
||||
body, err := json.Marshal(marker)
|
||||
if err != nil || executor.writeRuntimeFile(markerPath, body, 0o600) != nil {
|
||||
rollbackManagedDLLExtension(activePath, previousPath, activeExists)
|
||||
if previousMarkerFound {
|
||||
_ = executor.writeRuntimeFile(markerPath, previousMarker, 0o600)
|
||||
} else {
|
||||
_ = os.Remove(markerPath)
|
||||
}
|
||||
return dllExtensionError{code: "dll_extension_activation_failed", message: "extension release state could not be activated"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rollbackManagedDLLExtension(activePath string, previousPath string, activeExists bool) {
|
||||
if activeExists {
|
||||
_ = copyRegularFileAtomic(previousPath, activePath, maxUE4SSDLLBytes, 0o600)
|
||||
return
|
||||
}
|
||||
_ = os.Remove(activePath)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ensureLoopbackRCONConfig(resolver WorkspaceResolver, scope string, gameRootKey string, plan protocol.RuntimeDLLExtensionPlan) error {
|
||||
configKey := managedRCONConfigRef(gameRootKey, plan.ModKey)
|
||||
configPath, _, err := resolver.WritableTarget(scope, configKey)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be prepared"}
|
||||
}
|
||||
if managedLoopbackRCONConfigMatches(configPath, plan.RCONPort) {
|
||||
return nil
|
||||
}
|
||||
password, err := randomRCONPassword()
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be secured"}
|
||||
}
|
||||
body := fmt.Sprintf("%s\n[rcon]\nbind_address=127.0.0.1\nport=%d\npassword=%s\n", managedRCONConfigMarker, plan.RCONPort, password)
|
||||
if err := executor.writeRuntimeFile(configPath, []byte(body), 0o600); err != nil {
|
||||
return dllExtensionError{code: "dll_extension_config_failed", message: "loopback RCON configuration cannot be written"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func managedRCONConfigRef(gameRootKey string, modKey string) string {
|
||||
return gameRelativeKey(gameRootKey, "ue4ss/Mods/"+modKey+"/config.ini")
|
||||
}
|
||||
|
||||
func managedRCONConfigRefForMod(configRef string, modKey string) bool {
|
||||
baseRef := "ue4ss/Mods/" + modKey + "/config.ini"
|
||||
return protocol.ValidLogicalFileKey(configRef) && (configRef == baseRef || strings.HasSuffix(configRef, "/"+baseRef))
|
||||
}
|
||||
|
||||
func managedLoopbackRCONConfigMatches(path string, port int) bool {
|
||||
body, found, err := readBoundedRegularFile(path, maxUE4SSMetadataBytes)
|
||||
if err != nil || !found {
|
||||
return false
|
||||
}
|
||||
content := strings.ReplaceAll(string(body), "\r\n", "\n")
|
||||
if !strings.Contains(content, managedRCONConfigMarker) {
|
||||
return false
|
||||
}
|
||||
values := map[string]string{}
|
||||
inRCON := false
|
||||
for _, rawLine := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
if line == "[rcon]" {
|
||||
inRCON = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "[") {
|
||||
inRCON = false
|
||||
continue
|
||||
}
|
||||
if !inRCON || line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key != "bind_address" && key != "port" && key != "password" {
|
||||
continue
|
||||
}
|
||||
if _, duplicate := values[key]; duplicate {
|
||||
return false
|
||||
}
|
||||
values[key] = strings.TrimSpace(value)
|
||||
}
|
||||
configuredPort, err := strconv.Atoi(values["port"])
|
||||
if err != nil || values["bind_address"] != "127.0.0.1" || configuredPort != port || len(values["password"]) != 64 {
|
||||
return false
|
||||
}
|
||||
_, err = hex.DecodeString(values["password"])
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func randomRCONPassword() (string, error) {
|
||||
bytes := make([]byte, 32)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(bytes), nil
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ensureUE4SSModsIndex(resolver WorkspaceResolver, scope string, gameRootKey string, modKey string) error {
|
||||
modsKey := gameRelativeKey(gameRootKey, "ue4ss/Mods/mods.txt")
|
||||
modsPath, _, err := resolver.WritableTarget(scope, modsKey)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be prepared"}
|
||||
}
|
||||
body, found, err := readBoundedRegularFile(modsPath, maxUE4SSMetadataBytes)
|
||||
if err != nil {
|
||||
return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be read"}
|
||||
}
|
||||
content := ""
|
||||
if found {
|
||||
content = strings.ReplaceAll(string(body), "\r\n", "\n")
|
||||
}
|
||||
lines := strings.Split(content, "\n")
|
||||
if content == "" {
|
||||
lines = nil
|
||||
}
|
||||
updated := make([]string, 0, len(lines)+1)
|
||||
declared := false
|
||||
for _, line := range lines {
|
||||
if modsIndexLineKey(line) == modKey {
|
||||
if !declared {
|
||||
updated = append(updated, modKey+" : 1")
|
||||
declared = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
updated = append(updated, line)
|
||||
}
|
||||
if !declared {
|
||||
updated = append(updated, modKey+" : 1")
|
||||
}
|
||||
next := strings.Join(updated, "\n")
|
||||
if !strings.HasSuffix(next, "\n") {
|
||||
next += "\n"
|
||||
}
|
||||
if content == next {
|
||||
return nil
|
||||
}
|
||||
if err := executor.writeRuntimeFile(modsPath, []byte(next), 0o600); err != nil {
|
||||
return dllExtensionError{code: "dll_extension_mods_failed", message: "UE4SS mods index cannot be updated"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func modsIndexLineKey(line string) string {
|
||||
withoutComment := strings.SplitN(line, "#", 2)[0]
|
||||
parts := strings.SplitN(strings.TrimSpace(withoutComment), ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
|
||||
func existingRuntimeDirectory(scope string, key string) error {
|
||||
if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) {
|
||||
return fmt.Errorf("directory key is unsafe")
|
||||
}
|
||||
cleanScope, err := filepath.Abs(scope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current := cleanScope
|
||||
for _, part := range strings.Split(filepath.ToSlash(key), "/") {
|
||||
current = filepath.Join(current, part)
|
||||
info, err := os.Lstat(current)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("required directory is unavailable")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checksumRegularFile(path string, maxBytes int64) (string, int64, error) {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", 0, fmt.Errorf("file is not regular")
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, io.LimitReader(file, maxBytes+1))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if size > maxBytes {
|
||||
return "", size, fmt.Errorf("file exceeds maximum size")
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil
|
||||
}
|
||||
|
||||
func copyRegularFileAtomic(source string, destination string, maxBytes int64, mode os.FileMode) error {
|
||||
checksum, size, err := checksumRegularFile(source, maxBytes)
|
||||
if err != nil || checksum == "" || size < 1 {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("source file is empty")
|
||||
}
|
||||
input, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer input.Close()
|
||||
temporary := destination + ".copying"
|
||||
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
remove := true
|
||||
defer func() {
|
||||
_ = output.Close()
|
||||
if remove {
|
||||
_ = os.Remove(temporary)
|
||||
}
|
||||
}()
|
||||
written, err := io.Copy(output, io.LimitReader(input, maxBytes+1))
|
||||
if err != nil || written != size || written > maxBytes {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("source file changed during copy")
|
||||
}
|
||||
if err := output.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
return err
|
||||
}
|
||||
remove = false
|
||||
return os.Chmod(destination, mode)
|
||||
}
|
||||
|
||||
func readBoundedRegularFile(path string, maxBytes int64) ([]byte, bool, error) {
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() || info.Size() > maxBytes {
|
||||
return nil, false, fmt.Errorf("file is not a bounded regular file")
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer file.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(file, maxBytes+1))
|
||||
if err != nil || int64(len(body)) > maxBytes {
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, false, fmt.Errorf("file exceeds maximum size")
|
||||
}
|
||||
return body, true, nil
|
||||
}
|
||||
|
||||
func protocolValidSHA256(value string) bool {
|
||||
if len(value) != len("sha256:")+64 || !strings.HasPrefix(value, "sha256:") {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:"))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func dllExtensionLifecycleFailure(err error) LifecycleExecutionResult {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "DLL extension synchronization cancelled"}, Message: "DLL extension synchronization cancelled", ErrorCode: "dll_extension_cancelled"}
|
||||
}
|
||||
var extensionErr dllExtensionError
|
||||
if errors.As(err, &extensionErr) {
|
||||
return lifecycleFailure(extensionErr.code, extensionErr.message)
|
||||
}
|
||||
return lifecycleFailure("dll_extension_sync_failed", "DLL extension synchronization failed")
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type ue4ssTestDownloader struct {
|
||||
mu sync.Mutex
|
||||
payload []byte
|
||||
calls int
|
||||
lastURL string
|
||||
started chan struct{}
|
||||
unblock <-chan struct{}
|
||||
startOnce sync.Once
|
||||
}
|
||||
|
||||
func (downloader *ue4ssTestDownloader) Download(ctx context.Context, sourceURL string, destination string, _ int64) (int64, string, error) {
|
||||
downloader.mu.Lock()
|
||||
downloader.calls++
|
||||
downloader.lastURL = sourceURL
|
||||
payload := append([]byte(nil), downloader.payload...)
|
||||
started := downloader.started
|
||||
unblock := downloader.unblock
|
||||
downloader.mu.Unlock()
|
||||
if started != nil {
|
||||
downloader.startOnce.Do(func() { close(started) })
|
||||
}
|
||||
if unblock != nil {
|
||||
select {
|
||||
case <-unblock:
|
||||
case <-ctx.Done():
|
||||
return 0, "", ctx.Err()
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
if err := os.WriteFile(destination, payload, 0o600); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return int64(len(payload)), bytesChecksum(payload), nil
|
||||
}
|
||||
|
||||
func (downloader *ue4ssTestDownloader) SetPayload(payload []byte) {
|
||||
downloader.mu.Lock()
|
||||
defer downloader.mu.Unlock()
|
||||
downloader.payload = append([]byte(nil), payload...)
|
||||
}
|
||||
|
||||
func (downloader *ue4ssTestDownloader) Count() int {
|
||||
downloader.mu.Lock()
|
||||
defer downloader.mu.Unlock()
|
||||
return downloader.calls
|
||||
}
|
||||
|
||||
func (downloader *ue4ssTestDownloader) LastURL() string {
|
||||
downloader.mu.Lock()
|
||||
defer downloader.mu.Unlock()
|
||||
return downloader.lastURL
|
||||
}
|
||||
|
||||
type ue4ssManagedProcessSupervisor struct {
|
||||
mu sync.Mutex
|
||||
starts int
|
||||
command ProcessCommand
|
||||
}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) Start(_ context.Context, command ProcessCommand, identity ProcessIdentity, output ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
supervisor.mu.Lock()
|
||||
supervisor.starts++
|
||||
supervisor.command = ProcessCommand{WorkDir: command.WorkDir, Args: append([]string(nil), command.Args...), Env: command.Env, Timeout: command.Timeout}
|
||||
supervisor.mu.Unlock()
|
||||
if output.Stdout != nil {
|
||||
_ = output.Stdout(identity, ManagedProcessLine{Text: "ue4ss managed process started", EndOffset: int64(len("ue4ss managed process started"))})
|
||||
}
|
||||
identity.State = "running"
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) Stop(_ context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
||||
identity.State = "stopped"
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
||||
if identity.State == "" {
|
||||
identity.State = "stopped"
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) ResumeOutput(ManagedProcessOutput) {}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) Starts() int {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
return supervisor.starts
|
||||
}
|
||||
|
||||
func (supervisor *ue4ssManagedProcessSupervisor) Command() ProcessCommand {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
return ProcessCommand{WorkDir: supervisor.command.WorkDir, Args: append([]string(nil), supervisor.command.Args...), Env: supervisor.command.Env, Timeout: supervisor.command.Timeout}
|
||||
}
|
||||
|
||||
type ue4ssExtensionFixture struct {
|
||||
assignment protocol.RunJobAssignment
|
||||
plan protocol.RuntimeDLLExtensionPlan
|
||||
activePath string
|
||||
markerPath string
|
||||
previous string
|
||||
stagePath string
|
||||
configPath string
|
||||
modsPath string
|
||||
}
|
||||
|
||||
func TestUE4SSDLLExtensionSynchronizesNoOpsUpdatesAndKeepsPriorRelease(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fixture := newUE4SSExtensionFixture(t, root, true)
|
||||
payloadV1 := []byte("scum-simple-rcon DLL release one")
|
||||
downloader := &ue4ssTestDownloader{payload: payloadV1}
|
||||
supervisor := &ue4ssManagedProcessSupervisor{}
|
||||
executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64")
|
||||
|
||||
first := executor.Execute(fixture.assignment)
|
||||
if first.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected first DLL sync and start to succeed, got %+v", first)
|
||||
}
|
||||
if downloader.Count() != 1 || downloader.LastURL() != fixture.plan.ReleaseURL {
|
||||
t.Fatalf("expected one frozen release download, count=%d url=%q", downloader.Count(), downloader.LastURL())
|
||||
}
|
||||
assertUE4SSFileEquals(t, fixture.activePath, payloadV1)
|
||||
marker, found, err := loadManagedDLLExtensionMarker(fixture.markerPath)
|
||||
if err != nil || !found || !markerMatchesPlan(marker, fixture.plan) || marker.ConfigRef != "bin/ue4ss/Mods/scum_simple_rcon/config.ini" {
|
||||
t.Fatalf("expected managed release marker for first DLL, marker=%+v found=%v err=%v", marker, found, err)
|
||||
}
|
||||
config := readUE4SSFile(t, fixture.configPath)
|
||||
if !managedLoopbackRCONConfigMatches(fixture.configPath, fixture.plan.RCONPort) {
|
||||
t.Fatal("expected protected loopback RCON configuration")
|
||||
}
|
||||
password := rconPasswordFromConfig(string(config))
|
||||
if len(password) != 64 {
|
||||
t.Fatal("expected a 32-byte generated RCON password")
|
||||
}
|
||||
if _, err := hex.DecodeString(password); err != nil {
|
||||
t.Fatalf("expected hexadecimal generated RCON password: %v", err)
|
||||
}
|
||||
if info, err := os.Stat(fixture.configPath); err != nil || info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("expected protected RCON configuration permissions, info=%v err=%v", info, err)
|
||||
}
|
||||
mods := string(readUE4SSFile(t, fixture.modsPath))
|
||||
if strings.Count(mods, "scum_simple_rcon : 1\n") != 1 || strings.Contains(mods, "scum_simple_rcon : 0") {
|
||||
t.Fatalf("expected exactly one enabled managed mod entry, mods=%q", mods)
|
||||
}
|
||||
command := supervisor.Command()
|
||||
if len(command.Args) != 1 || filepath.Base(command.Args[0]) != "SCUMServer.exe" || strings.HasSuffix(strings.ToLower(command.Args[0]), ".dll") {
|
||||
t.Fatalf("expected normal SCUM executable start without a DLL loader, command=%+v", command)
|
||||
}
|
||||
|
||||
second := executor.Execute(fixture.assignment)
|
||||
if second.State != lifecycleResultStateSucceeded || downloader.Count() != 1 {
|
||||
t.Fatalf("expected unchanged release to skip download, result=%+v downloads=%d", second, downloader.Count())
|
||||
}
|
||||
if nextConfig := readUE4SSFile(t, fixture.configPath); !bytes.Equal(config, nextConfig) {
|
||||
t.Fatal("expected unchanged release to retain its protected RCON configuration")
|
||||
}
|
||||
|
||||
payloadV2 := []byte("scum-simple-rcon DLL release two")
|
||||
planV2 := fixture.plan
|
||||
planV2.Version = "1.1.0"
|
||||
planV2.Checksum = bytesChecksum(payloadV2)
|
||||
planV2.SizeBytes = int64(len(payloadV2))
|
||||
downloader.SetPayload(payloadV2)
|
||||
updatedAssignment := fixture.assignment
|
||||
updatedAssignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{planV2}
|
||||
updated := executor.Execute(updatedAssignment)
|
||||
if updated.State != lifecycleResultStateSucceeded || downloader.Count() != 2 {
|
||||
t.Fatalf("expected changed release to update before start, result=%+v downloads=%d", updated, downloader.Count())
|
||||
}
|
||||
assertUE4SSFileEquals(t, fixture.activePath, payloadV2)
|
||||
assertUE4SSFileEquals(t, fixture.previous, payloadV1)
|
||||
marker, found, err = loadManagedDLLExtensionMarker(fixture.markerPath)
|
||||
if err != nil || !found || !markerMatchesPlan(marker, planV2) {
|
||||
t.Fatalf("expected managed release marker for updated DLL, marker=%+v found=%v err=%v", marker, found, err)
|
||||
}
|
||||
if mods := string(readUE4SSFile(t, fixture.modsPath)); strings.Count(mods, "scum_simple_rcon : 1\n") != 1 {
|
||||
t.Fatalf("expected deterministic mod index after update, mods=%q", mods)
|
||||
}
|
||||
|
||||
badPlan := planV2
|
||||
badPlan.Version = "1.2.0"
|
||||
badPlan.Checksum = bytesChecksum([]byte("expected-but-not-delivered"))
|
||||
badPlan.SizeBytes = int64(len([]byte("expected-but-not-delivered")))
|
||||
downloader.SetPayload([]byte("tampered release payload"))
|
||||
badAssignment := fixture.assignment
|
||||
badAssignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{badPlan}
|
||||
failed := executor.Execute(badAssignment)
|
||||
if failed.State != lifecycleResultStateFailed || failed.ErrorCode != "dll_extension_verify_failed" {
|
||||
t.Fatalf("expected checksum mismatch to fail before process start, got %+v", failed)
|
||||
}
|
||||
if supervisor.Starts() != 3 {
|
||||
t.Fatalf("expected failed update not to start SCUM, starts=%d", supervisor.Starts())
|
||||
}
|
||||
assertUE4SSFileEquals(t, fixture.activePath, payloadV2)
|
||||
assertUE4SSFileEquals(t, fixture.previous, payloadV1)
|
||||
marker, found, err = loadManagedDLLExtensionMarker(fixture.markerPath)
|
||||
if err != nil || !found || !markerMatchesPlan(marker, planV2) {
|
||||
t.Fatalf("expected failed update to retain prior managed release, marker=%+v found=%v err=%v", marker, found, err)
|
||||
}
|
||||
if _, err := os.Stat(fixture.stagePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expected failed staged DLL to be removed, err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUE4SSDLLExtensionRollsBackWhenReleaseMarkerCannotActivate(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fixture := newUE4SSExtensionFixture(t, root, true)
|
||||
payloadV1 := []byte("scum-simple-rcon DLL release one")
|
||||
downloader := &ue4ssTestDownloader{payload: payloadV1}
|
||||
supervisor := &ue4ssManagedProcessSupervisor{}
|
||||
executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64")
|
||||
if result := executor.Execute(fixture.assignment); result.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("install initial release: %+v", result)
|
||||
}
|
||||
|
||||
payloadV2 := []byte("scum-simple-rcon DLL release two")
|
||||
planV2 := fixture.plan
|
||||
planV2.Version = "1.1.0"
|
||||
planV2.Checksum = bytesChecksum(payloadV2)
|
||||
planV2.SizeBytes = int64(len(payloadV2))
|
||||
downloader.SetPayload(payloadV2)
|
||||
failMarkerWrite := true
|
||||
executor.runtimeFileWriter = func(path string, body []byte, mode os.FileMode) error {
|
||||
if path == fixture.markerPath && failMarkerWrite {
|
||||
failMarkerWrite = false
|
||||
return errors.New("injected marker write failure")
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, body, mode)
|
||||
}
|
||||
assignment := fixture.assignment
|
||||
assignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{planV2}
|
||||
result := executor.Execute(assignment)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != "dll_extension_activation_failed" {
|
||||
t.Fatalf("expected failed marker activation to fail closed, got %+v", result)
|
||||
}
|
||||
if supervisor.Starts() != 1 {
|
||||
t.Fatalf("expected marker activation failure not to start SCUM, starts=%d", supervisor.Starts())
|
||||
}
|
||||
assertUE4SSFileEquals(t, fixture.activePath, payloadV1)
|
||||
assertUE4SSFileEquals(t, fixture.previous, payloadV1)
|
||||
marker, found, err := loadManagedDLLExtensionMarker(fixture.markerPath)
|
||||
if err != nil || !found || !markerMatchesPlan(marker, fixture.plan) {
|
||||
t.Fatalf("expected previous release marker after rollback, marker=%+v found=%v err=%v", marker, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUE4SSDLLExtensionRejectsLinuxBeforeDownloadOrStart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fixture := newUE4SSExtensionFixture(t, root, false)
|
||||
downloader := &ue4ssTestDownloader{payload: []byte("must not download")}
|
||||
supervisor := &ue4ssManagedProcessSupervisor{}
|
||||
executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "linux", "amd64")
|
||||
|
||||
result := executor.Execute(fixture.assignment)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != "unsupported_extension_platform" {
|
||||
t.Fatalf("expected Linux DLL extension rejection, got %+v", result)
|
||||
}
|
||||
if downloader.Count() != 0 || supervisor.Starts() != 0 {
|
||||
t.Fatalf("expected Linux rejection before download or start, downloads=%d starts=%d", downloader.Count(), supervisor.Starts())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUE4SSDLLExtensionRequiresInstalledBootstrap(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fixture := newUE4SSExtensionFixture(t, root, false)
|
||||
downloader := &ue4ssTestDownloader{payload: []byte("must not download")}
|
||||
supervisor := &ue4ssManagedProcessSupervisor{}
|
||||
executor := newUE4SSExtensionExecutor(root, downloader, supervisor, "windows", "amd64")
|
||||
|
||||
result := executor.Execute(fixture.assignment)
|
||||
if result.State != lifecycleResultStateFailed || result.ErrorCode != "ue4ss_bootstrap_missing" {
|
||||
t.Fatalf("expected bootstrap precondition failure, got %+v", result)
|
||||
}
|
||||
if downloader.Count() != 0 || supervisor.Starts() != 0 {
|
||||
t.Fatalf("expected missing bootstrap to prevent download and start, downloads=%d starts=%d", downloader.Count(), supervisor.Starts())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUE4SSDLLExtensionSlowDownloadPreservesHeartbeat(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
cfg := workerTestConfig(t)
|
||||
fixture := newUE4SSExtensionFixture(t, cfg.WorkspaceRoot, true)
|
||||
releaseDownload := make(chan struct{})
|
||||
downloader := &ue4ssTestDownloader{payload: []byte("scum-simple-rcon DLL release one"), started: make(chan struct{}), unblock: releaseDownload}
|
||||
supervisor := &ue4ssManagedProcessSupervisor{}
|
||||
worker, err := NewWorker(cfg, client,
|
||||
WithDependencyDownloader(downloader),
|
||||
WithManagedProcessSupervisor(supervisor),
|
||||
WithDLLExtensionRuntimeTarget("windows", "amd64"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
worker.state.SessionToken = "session-token"
|
||||
|
||||
executionDone := make(chan LifecycleExecutionResult, 1)
|
||||
go func() { executionDone <- worker.executeAssignment(context.Background(), fixture.assignment) }()
|
||||
select {
|
||||
case <-downloader.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
close(releaseDownload)
|
||||
t.Fatal("expected lifecycle job to begin its bounded DLL download")
|
||||
}
|
||||
|
||||
heartbeatDone := make(chan error, 1)
|
||||
go func() { heartbeatDone <- worker.HeartbeatOnce(context.Background()) }()
|
||||
select {
|
||||
case err := <-heartbeatDone:
|
||||
if err != nil {
|
||||
close(releaseDownload)
|
||||
t.Fatalf("heartbeat during DLL download: %v", err)
|
||||
}
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
close(releaseDownload)
|
||||
t.Fatal("slow DLL download blocked the control heartbeat")
|
||||
}
|
||||
if len(client.heartbeatRequests) != 1 {
|
||||
close(releaseDownload)
|
||||
t.Fatalf("expected heartbeat request during DLL download, got %d", len(client.heartbeatRequests))
|
||||
}
|
||||
close(releaseDownload)
|
||||
select {
|
||||
case result := <-executionDone:
|
||||
if result.State != lifecycleResultStateSucceeded {
|
||||
t.Fatalf("expected lifecycle job to finish after download release, got %+v", result)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("lifecycle job did not finish after slow download was released")
|
||||
}
|
||||
}
|
||||
|
||||
func newUE4SSExtensionExecutor(root string, downloader DependencyDownloader, supervisor ManagedProcessSupervisor, targetOS string, targetArch string) LifecycleExecutor {
|
||||
return NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithDependencyDownloader(downloader),
|
||||
WithManagedProcessSupervisor(supervisor),
|
||||
WithDLLExtensionRuntimeTarget(targetOS, targetArch),
|
||||
)
|
||||
}
|
||||
|
||||
func newUE4SSExtensionFixture(t *testing.T, root string, bootstrap bool) ue4ssExtensionFixture {
|
||||
t.Helper()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.TargetKey = "actions/start.json"
|
||||
assignment.ExecutionInput.WorkspaceScope = "run-local"
|
||||
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
||||
if err != nil {
|
||||
t.Fatalf("create workspace scope: %v", err)
|
||||
}
|
||||
writeUE4SSFixtureFile(t, filepath.Join(scope, "actions", "start.json"), []byte(`{"version":1,"action":"start","mode":"supervised","executableKey":"bin/SCUMServer.exe"}`), 0o600)
|
||||
executable := []byte("SCUM server executable fixture")
|
||||
writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "SCUMServer.exe"), executable, 0o700)
|
||||
if bootstrap {
|
||||
writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "dwmapi.dll"), []byte("UE4SS proxy fixture"), 0o600)
|
||||
writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "UE4SS.dll"), []byte("UE4SS loader fixture"), 0o600)
|
||||
writeUE4SSFixtureFile(t, filepath.Join(scope, "bin", "ue4ss", "Mods", "mods.txt"), []byte("OtherMod : 1\nscum_simple_rcon : 0\nscum_simple_rcon : 1\n"), 0o600)
|
||||
}
|
||||
plan := protocol.RuntimeDLLExtensionPlan{
|
||||
Key: "scum-simple-rcon",
|
||||
Version: "1.0.0",
|
||||
ReleaseURL: "https://cdn.npc0.com/scum_simple_rcon_ue4s.dll",
|
||||
Checksum: bytesChecksum([]byte("scum-simple-rcon DLL release one")),
|
||||
SizeBytes: int64(len([]byte("scum-simple-rcon DLL release one"))),
|
||||
TargetKey: "ue4ss/scum-simple-rcon",
|
||||
ModKey: "scum_simple_rcon",
|
||||
DLLRef: "ue4ss/Mods/scum_simple_rcon/dlls/main.dll",
|
||||
SCUMExecutableChecksum: bytesChecksum(executable),
|
||||
UE4SSABI: "ue4ss-3.0",
|
||||
RCONPort: 27015,
|
||||
}
|
||||
assignment.ExecutionInput.DLLExtensions = []protocol.RuntimeDLLExtensionPlan{plan}
|
||||
return ue4ssExtensionFixture{
|
||||
assignment: assignment,
|
||||
plan: plan,
|
||||
activePath: filepath.Join(scope, "bin", "ue4ss", "Mods", plan.ModKey, "dlls", "main.dll"),
|
||||
markerPath: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "release.json"),
|
||||
previous: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "previous.dll"),
|
||||
stagePath: filepath.Join(scope, "runtime", "ue4ss-dll", "ue4ss", "scum-simple-rcon", "download.staged"),
|
||||
configPath: filepath.Join(scope, "bin", "ue4ss", "Mods", plan.ModKey, "config.ini"),
|
||||
modsPath: filepath.Join(scope, "bin", "ue4ss", "Mods", "mods.txt"),
|
||||
}
|
||||
}
|
||||
|
||||
func writeUE4SSFixtureFile(t *testing.T, path string, body []byte, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatalf("create fixture directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, body, mode); err != nil {
|
||||
t.Fatalf("write fixture file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertUE4SSFileEquals(t *testing.T, path string, expected []byte) {
|
||||
t.Helper()
|
||||
if actual := readUE4SSFile(t, path); !bytes.Equal(actual, expected) {
|
||||
t.Fatalf("unexpected managed file contents at %s", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func readUE4SSFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture file %s: %v", filepath.Base(path), err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func rconPasswordFromConfig(content string) string {
|
||||
inRCON := false
|
||||
for _, rawLine := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
if line == "[rcon]" {
|
||||
inRCON = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "[") {
|
||||
inRCON = false
|
||||
continue
|
||||
}
|
||||
if inRCON && strings.HasPrefix(line, "password=") {
|
||||
return strings.TrimPrefix(line, "password=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
+1364
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,197 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const maxExecutionContentBytes = 64 * 1024
|
||||
|
||||
type WorkspaceResolver struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewWorkspaceResolver(root string) WorkspaceResolver {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
root = filepath.Join(".", ".run-workspace")
|
||||
}
|
||||
return WorkspaceResolver{root: root}
|
||||
}
|
||||
|
||||
func (resolver WorkspaceResolver) Scope(serverInstanceID string, profileKey string) (string, error) {
|
||||
if err := validateWorkspaceComponent(serverInstanceID, "serverInstanceId"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateWorkspaceComponent(profileKey, "profileKey"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
root, err := filepath.Abs(resolver.root)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace root: %w", err)
|
||||
}
|
||||
if err := ensureDirectory(root); err != nil {
|
||||
return "", fmt.Errorf("secure workspace root: %w", err)
|
||||
}
|
||||
instances := filepath.Join(root, "instances")
|
||||
if err := ensureDirectory(instances); err != nil {
|
||||
return "", fmt.Errorf("secure workspace instances: %w", err)
|
||||
}
|
||||
serverDir := filepath.Join(instances, serverInstanceID)
|
||||
if err := ensureDirectory(serverDir); err != nil {
|
||||
return "", fmt.Errorf("secure server workspace: %w", err)
|
||||
}
|
||||
scope := filepath.Join(serverDir, profileKey)
|
||||
if err := ensureDirectory(scope); err != nil {
|
||||
return "", fmt.Errorf("secure profile workspace: %w", err)
|
||||
}
|
||||
return scope, nil
|
||||
}
|
||||
|
||||
func (resolver WorkspaceResolver) ExistingTarget(scope string, key string) (string, error) {
|
||||
path, err := resolver.target(scope, key, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("target must be a regular file")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (resolver WorkspaceResolver) ExistingDirectory(scope string, key string) (string, error) {
|
||||
if strings.TrimSpace(scope) == "" || !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, string(rune(92))) {
|
||||
return "", fmt.Errorf("logical directory is unsafe")
|
||||
}
|
||||
cleanScope, err := filepath.Abs(scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
root, err := filepath.Abs(resolver.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(root, cleanScope)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("workspace scope escapes root")
|
||||
}
|
||||
current := cleanScope
|
||||
for _, part := range strings.Split(filepath.ToSlash(key), "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return "", fmt.Errorf("logical directory contains unsafe component")
|
||||
}
|
||||
current = filepath.Join(current, part)
|
||||
info, statErr := os.Lstat(current)
|
||||
if statErr != nil {
|
||||
return "", statErr
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return "", fmt.Errorf("logical directory is not a real directory")
|
||||
}
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
|
||||
func (resolver WorkspaceResolver) WritableTarget(scope string, key string) (string, string, error) {
|
||||
if strings.HasPrefix(key, "actions/") || strings.HasPrefix(key, "state/") || key == "actions" || key == "state" {
|
||||
return "", "", fmt.Errorf("target is reserved")
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(key), "/")
|
||||
parent := scope
|
||||
for _, part := range parts[:len(parts)-1] {
|
||||
parent = filepath.Join(parent, part)
|
||||
if err := ensureDirectory(parent); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
path, err := resolver.target(scope, key, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return path, filepath.Dir(path), nil
|
||||
}
|
||||
|
||||
func (resolver WorkspaceResolver) target(scope string, key string, allowMissingFinal bool) (string, error) {
|
||||
if strings.TrimSpace(scope) == "" {
|
||||
return "", fmt.Errorf("workspace scope is invalid")
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) {
|
||||
return "", fmt.Errorf("logical key is unsafe")
|
||||
}
|
||||
cleanScope, err := filepath.Abs(scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
root, err := filepath.Abs(resolver.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(root, cleanScope)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("workspace scope escapes root")
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(key), "/")
|
||||
current := cleanScope
|
||||
for index, part := range parts {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return "", fmt.Errorf("logical key contains unsafe component")
|
||||
}
|
||||
current = filepath.Join(current, part)
|
||||
info, statErr := os.Lstat(current)
|
||||
if statErr != nil {
|
||||
if allowMissingFinal && index == len(parts)-1 && os.IsNotExist(statErr) {
|
||||
return current, nil
|
||||
}
|
||||
return "", statErr
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", fmt.Errorf("logical key contains a symlink")
|
||||
}
|
||||
if index < len(parts)-1 && !info.IsDir() {
|
||||
return "", fmt.Errorf("logical key parent is not a directory")
|
||||
}
|
||||
if index == len(parts)-1 && info.Mode()&os.ModeType != 0 {
|
||||
return "", fmt.Errorf("target is a special file")
|
||||
}
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
|
||||
func ensureDirectory(path string) error {
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("path is not a real directory")
|
||||
}
|
||||
return os.Chmod(path, 0o700)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
parent := filepath.Dir(path)
|
||||
if parent != path {
|
||||
if err := ensureDirectory(parent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Mkdir(path, 0o700); err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("created path is not a real directory")
|
||||
}
|
||||
return os.Chmod(path, 0o700)
|
||||
}
|
||||
|
||||
func validateWorkspaceComponent(value string, field string) error {
|
||||
if strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) || value == "." || value == ".." || strings.ContainsAny(value, `/\`) || !protocol.ValidLogicalFileKey(value) {
|
||||
return fmt.Errorf("%s is unsafe", field)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
type workspaceSeedFile struct {
|
||||
Path string `json:"path"`
|
||||
Content string `json:"content"`
|
||||
Encoding string `json:"encoding,omitempty"`
|
||||
Mode int `json:"mode,omitempty"`
|
||||
}
|
||||
|
||||
// MaterializeWorkspaceSeed writes platform-packaged plugin assets into the
|
||||
// scoped run workspace. The seed contains plugin-owned files only; run treats
|
||||
// them as opaque generic lifecycle assets.
|
||||
func MaterializeWorkspaceSeed(cfg config.Config) error {
|
||||
startedAt := time.Now()
|
||||
encoded := strings.TrimSpace(cfg.WorkspaceSeed)
|
||||
if encoded == "" {
|
||||
log.Printf("RUN phase=workspace_seed status=skipped reason=empty workspace=%s", safeOptional(cfg.WorkspaceRoot))
|
||||
return nil
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed status=decoding workspace=%s encodedBytes=%d componentKey=%s", safeOptional(cfg.WorkspaceRoot), len(encoded), safeOptional(cfg.ComponentKey))
|
||||
payload, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=workspace_seed status=decode_failed workspace=%s error=%s", safeOptional(cfg.WorkspaceRoot), RedactText(err.Error()))
|
||||
return fmt.Errorf("decode workspace seed: %w", err)
|
||||
}
|
||||
var files []workspaceSeedFile
|
||||
if err := json.Unmarshal(payload, &files); err != nil {
|
||||
log.Printf("RUN phase=workspace_seed status=manifest_failed workspace=%s payloadBytes=%d error=%s", safeOptional(cfg.WorkspaceRoot), len(payload), RedactText(err.Error()))
|
||||
return fmt.Errorf("decode workspace seed manifest: %w", err)
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed status=decoded workspace=%s payloadBytes=%d files=%d", safeOptional(cfg.WorkspaceRoot), len(payload), len(files))
|
||||
if len(files) == 0 {
|
||||
log.Printf("RUN phase=workspace_seed status=skipped reason=no_files workspace=%s durationMs=%d", safeOptional(cfg.WorkspaceRoot), time.Since(startedAt).Milliseconds())
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.ServerInstanceID) == "" {
|
||||
log.Printf("RUN phase=workspace_seed status=failed reason=missing_server workspace=%s", safeOptional(cfg.WorkspaceRoot))
|
||||
return fmt.Errorf("workspace seed requires a server instance id")
|
||||
}
|
||||
scope, err := seededWorkspaceScope(cfg)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=workspace_seed status=scope_failed workspace=%s server=%s componentKey=%s error=%s", safeOptional(cfg.WorkspaceRoot), safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKey), RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed status=scope_ready workspace=%s scope=%s files=%d", safeOptional(cfg.WorkspaceRoot), safeOptional(scope), len(files))
|
||||
totalBytes := 0
|
||||
for index, file := range files {
|
||||
written, err := writeWorkspaceSeedFile(scope, file, index+1, len(files))
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=workspace_seed.file status=failed index=%d total=%d path=%s error=%s", index+1, len(files), safeOptional(file.Path), RedactText(err.Error()))
|
||||
return err
|
||||
}
|
||||
totalBytes += written
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed status=complete workspace=%s scope=%s files=%d bytes=%d durationMs=%d", safeOptional(cfg.WorkspaceRoot), safeOptional(scope), len(files), totalBytes, time.Since(startedAt).Milliseconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
func seededWorkspaceScope(cfg config.Config) (string, error) {
|
||||
if strings.TrimSpace(cfg.ComponentKey) != "" {
|
||||
return NewWorkspaceResolver(cfg.WorkspaceRoot).Scope(cfg.ServerInstanceID, cfg.ComponentKey)
|
||||
}
|
||||
return scopedServerWorkspace(cfg.WorkspaceRoot, cfg.ServerInstanceID)
|
||||
}
|
||||
|
||||
func writeWorkspaceSeedFile(scope string, file workspaceSeedFile, index int, total int) (int, error) {
|
||||
target, err := workspaceSeedTarget(scope, file.Path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
mode := os.FileMode(file.Mode)
|
||||
if mode == 0 {
|
||||
mode = 0o600
|
||||
}
|
||||
if mode&0o777 != mode || mode&0o022 != 0 {
|
||||
return 0, fmt.Errorf("workspace seed file mode is unsafe")
|
||||
}
|
||||
body, err := workspaceSeedFileContent(file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed.file status=writing index=%d total=%d path=%s target=%s bytes=%d mode=%#o", index, total, safeOptional(file.Path), safeOptional(target), len(body), mode)
|
||||
if err := ensureDirectory(filepath.Dir(target)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed.file status=directory_ready index=%d total=%d dir=%s", index, total, safeOptional(filepath.Dir(target)))
|
||||
if err := os.WriteFile(target, body, mode); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed.file status=written index=%d total=%d path=%s target=%s bytes=%d mode=%#o", index, total, safeOptional(file.Path), safeOptional(target), len(body), mode)
|
||||
return len(body), nil
|
||||
}
|
||||
|
||||
func workspaceSeedFileContent(file workspaceSeedFile) ([]byte, error) {
|
||||
switch strings.TrimSpace(file.Encoding) {
|
||||
case "":
|
||||
return []byte(file.Content), nil
|
||||
case "base64":
|
||||
body, err := base64.StdEncoding.DecodeString(strings.TrimSpace(file.Content))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("workspace seed file content is not valid base64")
|
||||
}
|
||||
return body, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("workspace seed file encoding is unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func workspaceSeedTarget(scope string, key string) (string, error) {
|
||||
if strings.TrimSpace(scope) == "" {
|
||||
return "", fmt.Errorf("workspace scope is invalid")
|
||||
}
|
||||
if !protocol.ValidLogicalFileKey(key) || filepath.IsAbs(key) || strings.Contains(key, `\`) {
|
||||
return "", fmt.Errorf("workspace seed path is unsafe")
|
||||
}
|
||||
cleanScope, err := filepath.Abs(scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(key), "/")
|
||||
current := cleanScope
|
||||
for _, part := range parts {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return "", fmt.Errorf("workspace seed path contains unsafe component")
|
||||
}
|
||||
current = filepath.Join(current, part)
|
||||
}
|
||||
rel, err := filepath.Rel(cleanScope, current)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("workspace seed path escapes scope")
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
Reference in New Issue
Block a user