package runtime import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log" "net/http" "os" "path/filepath" "runtime" "strings" "sync" "time" "browser.local/run/config" "browser.local/run/protocol" "browser.local/run/spool" ) type WorkerClient interface { Hello(context.Context, protocol.RunHelloRequest) (protocol.RunHelloResponse, error) Heartbeat(context.Context, protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) ReportLifecycle(context.Context, protocol.RunLifecycleReportRequest) (protocol.RunLifecycleReportResponse, error) GetRunLogStreamProgress(context.Context, protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) ClaimJob(context.Context, protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) AckJob(context.Context, protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) UpdateJobProgress(context.Context, protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) CompleteJob(context.Context, protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) GetDistributionBuildInput(context.Context, protocol.DistributionBuildInputRequest) (protocol.DistributionBuildInputResponse, error) GetDependencyExecutionInput(context.Context, protocol.DependencyExecutionInputRequest) (protocol.DependencyExecutionInputResponse, error) GetSourceRCONExecutionInput(context.Context, protocol.SourceRCONExecutionInputRequest) (protocol.SourceRCONExecutionInputResponse, error) GetProtectedRequestExecutionInput(context.Context, protocol.ProtectedRequestExecutionInputRequest) (protocol.ProtectedRequestExecutionInputResponse, error) GetRunUpdateInput(context.Context, protocol.RunUpdateInputRequest) (protocol.RunUpdateInputResponse, error) ReadRunUpdateChunk(context.Context, protocol.RunUpdateChunkRequest) (protocol.RunUpdateChunkResponse, error) ReportRunUpdateHealth(context.Context, protocol.RunUpdateHealthRequest) (protocol.RunUpdateHealthResponse, error) IngestMetricBatch(context.Context, protocol.MetricBatchIngestRequest) (protocol.MetricBatchIngestResponse, error) OpenArtifactTransfer(context.Context, protocol.ArtifactTransferOpenRequest) (protocol.ArtifactTransferOpenResponse, error) UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) CompleteArtifactTransfer(context.Context, protocol.ArtifactTransferCompleteRequest) (protocol.ArtifactTransferCompleteResponse, error) PollJobCancel(context.Context, protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) ReconcileJobs(context.Context, protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) } const ( jobActivePollInterval = 10 * time.Second jobClaimWaitSeconds = 25 durableUploaderFlushTimeout = 5 * time.Second fileArtifactChunkSize = 1024 * 1024 maxFileArtifactBytes = int64(512 * 1024 * 1024) ) type Worker struct { cfg config.Config client WorkerClient executor LifecycleExecutor state WorkerState journal *JobJournal stateMu sync.RWMutex sessionRefreshMu sync.Mutex sequenceMu sync.Mutex restartMu sync.Mutex restartRequested bool observationMu sync.Mutex reportedObservations map[string]uint64 metricCollector MetricCollector } type WorkerState struct { RunEndpointID string SessionToken string SessionExpiresAt time.Time Capabilities []string Capacity protocol.RunCapacityReport LastHeartbeat time.Time Sequence uint64 } func NewWorker(cfg config.Config, client WorkerClient, options ...LifecycleExecutorOption) (*Worker, error) { if client == nil { return nil, fmt.Errorf("worker client is required") } if cfg.RunEndpointID == "" { cfg.RunEndpointID = config.DefaultEndpointID } if cfg.DisplayName == "" { cfg.DisplayName = config.DefaultDisplayName } if cfg.Version == "" { cfg.Version = config.DefaultVersion } if cfg.MaxJobs <= 0 { cfg.MaxJobs = 1 } executorOptions := append([]LifecycleExecutorOption{ WithLifecycleWorkspaceRoot(cfg.WorkspaceRoot), WithManagedProcessStateRoot(managedProcessStateRoot(cfg)), WithManagedProcessOutputRoot(cfg.WorkspaceRoot), WithLocalStartupDiagnostics(cfg.LocalStartupDiagnostics), }, options...) if err := migrateManagedProcessState(cfg, managedProcessStateRoot(cfg)); err != nil { return nil, err } journal, err := NewPersistentJobJournal(cfg.WorkspaceRoot) if err != nil { return nil, err } worker := &Worker{ cfg: cfg, client: client, executor: NewLifecycleExecutor(executorOptions...), state: WorkerState{ RunEndpointID: cfg.RunEndpointID, Capabilities: SupportedRunCapabilitiesForComponent(cfg.ComponentKind), Capacity: protocol.RunCapacityReport{MaxJobs: cfg.MaxJobs}, }, journal: journal, reportedObservations: map[string]uint64{}, metricCollector: defaultMetricCollector{}, } if worker.executor.metricCollector != nil { worker.metricCollector = worker.executor.metricCollector } return worker, nil } func migrateLegacyManagedProcessState(cfg config.Config, stateRoot string) error { workspaceRoot := cfg.WorkspaceRoot if strings.TrimSpace(workspaceRoot) == "" { workspaceRoot = filepath.Join(".", ".run-workspace") } legacyPath := filepath.Join(workspaceRoot, "state", "processes.json") newPath := filepath.Join(stateRoot, "state", "processes.json") if filepath.Clean(legacyPath) == filepath.Clean(newPath) { return nil } if _, err := os.Stat(newPath); err == nil { return nil } else if !os.IsNotExist(err) { return err } body, err := os.ReadFile(legacyPath) if os.IsNotExist(err) { return nil } if err != nil { return fmt.Errorf("read legacy managed process state: %w", err) } var legacy processJournal if err := json.Unmarshal(body, &legacy); err != nil { return fmt.Errorf("decode legacy managed process state: %w", err) } filtered := processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{}, Retired: map[string]ProcessIdentity{}} for key, item := range legacy.Items { if managedProcessBelongsToRun(item, cfg) { filtered.Items[key] = item } } for key, item := range legacy.Retired { if managedProcessBelongsToRun(item, cfg) { filtered.Retired[key] = item } } if len(filtered.Items) == 0 && len(filtered.Retired) == 0 { return nil } if err := ensureDirectory(filepath.Dir(newPath)); err != nil { return fmt.Errorf("create isolated managed process state: %w", err) } encoded, err := json.Marshal(filtered) if err != nil { return fmt.Errorf("encode isolated managed process state: %w", err) } if err := os.WriteFile(newPath, encoded, 0o600); err != nil { return fmt.Errorf("write isolated managed process state: %w", err) } log.Printf("RUN phase=process.managed status=legacy_state_migrated items=%d retired=%d", len(filtered.Items), len(filtered.Retired)) return nil } // migrateManagedProcessState also imports matching journals from prior Run // state namespaces. A generated package can change its component metadata // while retaining the same endpoint/server/profile; that must not create a // second autonomous process for the same server during a Run update. func migrateManagedProcessState(cfg config.Config, stateRoot string) error { if err := migrateLegacyManagedProcessState(cfg, stateRoot); err != nil { return err } workspaceRoot := cfg.WorkspaceRoot if strings.TrimSpace(workspaceRoot) == "" { workspaceRoot = filepath.Join(".", ".run-workspace") } targetPath := filepath.Join(stateRoot, "state", "processes.json") target := processJournal{Version: managedProcessJournalVersion, Items: map[string]ProcessIdentity{}, Retired: map[string]ProcessIdentity{}} if body, err := os.ReadFile(targetPath); err == nil { if err := json.Unmarshal(body, &target); err != nil { return fmt.Errorf("decode managed process state: %w", err) } if target.Items == nil { target.Items = map[string]ProcessIdentity{} } if target.Retired == nil { target.Retired = map[string]ProcessIdentity{} } } else if !os.IsNotExist(err) { return fmt.Errorf("read managed process state: %w", err) } changed := false err := filepath.WalkDir(workspaceRoot, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if entry.IsDir() || entry.Name() != "processes.json" { return nil } if filepath.Clean(path) == filepath.Clean(targetPath) { return nil } body, err := os.ReadFile(path) if err != nil { return err } var source processJournal if err := json.Unmarshal(body, &source); err != nil { return fmt.Errorf("decode managed process state candidate: %w", err) } for key, item := range source.Items { if !managedProcessBelongsToRun(item, cfg) { continue } existing, exists := target.Items[key] if !exists || (!processAlivePID(existing.PID) && processAlivePID(item.PID)) { target.Items[key] = item changed = true } } for key, item := range source.Retired { if managedProcessBelongsToRun(item, cfg) { if _, exists := target.Retired[key]; !exists { target.Retired[key] = item changed = true } } } return nil }) if err != nil { return fmt.Errorf("scan managed process state: %w", err) } if !changed { return nil } if err := ensureDirectory(filepath.Dir(targetPath)); err != nil { return fmt.Errorf("create managed process state directory: %w", err) } body, err := json.Marshal(target) if err != nil { return fmt.Errorf("encode managed process state: %w", err) } temporary := targetPath + ".tmp" if err := os.WriteFile(temporary, body, 0o600); err != nil { return fmt.Errorf("write managed process state: %w", err) } if err := os.Rename(temporary, targetPath); err != nil { _ = os.Remove(temporary) return fmt.Errorf("replace managed process state: %w", err) } log.Printf("RUN phase=process.managed status=state_namespaces_merged items=%d retired=%d", len(target.Items), len(target.Retired)) return nil } func managedProcessBelongsToRun(item ProcessIdentity, cfg config.Config) bool { if item.RunEndpointID == "" || item.ServerInstanceID == "" || item.RunEndpointID != cfg.RunEndpointID || item.ServerInstanceID != cfg.ServerInstanceID { return false } return cfg.ComponentKey == "" || item.ProfileKey == "" || item.ProfileKey == cfg.ComponentKey } func managedProcessStateRoot(cfg config.Config) string { workspaceRoot := cfg.WorkspaceRoot if strings.TrimSpace(workspaceRoot) == "" { workspaceRoot = filepath.Join(".", ".run-workspace") } identity := strings.Join([]string{ "run-process-state-v1", cfg.RunEndpointID, cfg.ServerInstanceID, cfg.PluginID, cfg.ComponentKind, cfg.ComponentKey, }, "\x00") digest := sha256.Sum256([]byte(identity)) return filepath.Join(workspaceRoot, "run-services", hex.EncodeToString(digest[:])) } func (worker *Worker) Register(ctx context.Context) error { worker.sessionRefreshMu.Lock() defer worker.sessionRefreshMu.Unlock() return worker.registerUnlocked(ctx) } func (worker *Worker) registerUnlocked(ctx context.Context) error { state := worker.State() log.Printf("RUN phase=register status=starting endpoint=%s version=%s server=%s plugin=%s component=%s componentKey=%s capabilities=%d maxJobs=%d", worker.cfg.RunEndpointID, worker.cfg.Version, worker.cfg.ServerInstanceID, worker.cfg.PluginID, worker.cfg.ComponentKind, safeOptional(worker.cfg.ComponentKey), len(state.Capabilities), worker.cfg.MaxJobs) response, err := worker.client.Hello(ctx, protocol.RunHelloRequest{ RegistrationToken: worker.cfg.RegistrationToken, RunEndpointID: worker.cfg.RunEndpointID, ServerInstanceID: worker.cfg.ServerInstanceID, PluginID: worker.cfg.PluginID, ComponentKind: worker.cfg.ComponentKind, ComponentKey: worker.cfg.ComponentKey, KeyGeneration: worker.cfg.KeyGeneration, DisplayName: worker.cfg.DisplayName, Version: worker.cfg.Version, Status: "online", Platform: runtime.GOOS, Architecture: runtime.GOARCH, CapabilityReport: protocol.RunCapabilityReport{ Capabilities: state.Capabilities, Fingerprint: capabilityFingerprint(state.Capabilities), }, Capacity: worker.capacityReportFor(state), }) if err != nil { log.Printf("RUN phase=register status=failed endpoint=%s error=%s", worker.cfg.RunEndpointID, RedactText(err.Error())) return err } if !response.Accepted || response.SessionToken == "" { log.Printf("RUN phase=register status=rejected endpoint=%s accepted=%t sessionTokenPresent=%t", worker.cfg.RunEndpointID, response.Accepted, response.SessionToken != "") return fmt.Errorf("run hello was not accepted") } worker.stateMu.Lock() worker.state.SessionToken = response.SessionToken worker.state.SessionExpiresAt = response.SessionExpiresAt state = worker.state state.Capabilities = append([]string(nil), state.Capabilities...) worker.stateMu.Unlock() if sink, ok := worker.executor.logSink.(*SpoolLogSink); ok { sink.mu.Lock() sink.RunEndpointID = state.RunEndpointID sink.SessionToken = state.SessionToken sink.Progress = func(ctx context.Context, serverInstanceID string, streamID string) (uint64, error) { current, err := worker.registeredState() if err != nil { return 0, err } response, err := worker.client.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{RunEndpointID: current.RunEndpointID, SessionToken: current.SessionToken, ServerInstanceID: serverInstanceID, LogStreamID: streamID}) if err != nil { return 0, err } if !response.Accepted || response.LogStreamID != streamID { return 0, fmt.Errorf("run log stream progress was not accepted") } return response.LatestSeq, nil } sink.mu.Unlock() } if hook, ok := worker.executor.artifactHook.(*QueueArtifactHook); ok { hook.RunEndpointID = state.RunEndpointID hook.SessionToken = state.SessionToken } worker.executor.ResumeManagedProcessLogs(ctx) log.Printf("RUN phase=register status=accepted endpoint=%s sessionExpiresAt=%s heartbeatSeconds=%d", state.RunEndpointID, response.SessionExpiresAt.Format(time.RFC3339), response.HeartbeatIntervalSeconds) return nil } func (worker *Worker) registeredState() (WorkerState, error) { state := worker.State() if state.SessionToken == "" { return state, fmt.Errorf("worker is not registered") } return state, nil } func (worker *Worker) HeartbeatOnce(ctx context.Context) error { state, err := worker.registeredState() if err != nil { return err } if !state.SessionExpiresAt.IsZero() && !time.Now().UTC().Add(time.Minute).Before(state.SessionExpiresAt) { log.Printf("RUN phase=heartbeat status=session_expiring endpoint=%s sessionExpiresAt=%s", state.RunEndpointID, state.SessionExpiresAt.Format(time.RFC3339)) return worker.reregisterAndReconcile(ctx, "heartbeat_session_expiring", state.SessionToken) } log.Printf("RUN phase=heartbeat status=starting endpoint=%s activeJobs=%d", state.RunEndpointID, worker.journal.ActiveCount()) response, err := worker.client.Heartbeat(ctx, protocol.RunHeartbeatRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, Version: worker.cfg.Version, Status: "online", CapabilityFingerprint: capabilityFingerprint(state.Capabilities), Capacity: worker.capacityReportFor(state), }) if err != nil { if sessionInvalidError(err) { log.Printf("RUN phase=heartbeat status=session_invalid endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) return worker.reregisterAndReconcile(ctx, "heartbeat_session_invalid", state.SessionToken) } log.Printf("RUN phase=heartbeat status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) return err } if !response.Accepted { log.Printf("RUN phase=heartbeat status=rejected endpoint=%s", state.RunEndpointID) return fmt.Errorf("heartbeat was not accepted") } worker.stateMu.Lock() if worker.state.SessionToken == state.SessionToken { worker.state.LastHeartbeat = response.ServerTime } worker.stateMu.Unlock() log.Printf("RUN phase=heartbeat status=accepted endpoint=%s serverTime=%s nextSeconds=%d", state.RunEndpointID, response.ServerTime.Format(time.RFC3339), response.NextHeartbeatSeconds) return nil } func (worker *Worker) reregisterAndReconcile(ctx context.Context, reason string, observedToken string) error { worker.sessionRefreshMu.Lock() defer worker.sessionRefreshMu.Unlock() state := worker.State() if observedToken != "" && state.SessionToken != "" && state.SessionToken != observedToken { log.Printf("RUN phase=register status=already_refreshed reason=%s endpoint=%s", safeOptional(reason), state.RunEndpointID) return worker.ReconcileOnce(ctx) } log.Printf("RUN phase=register status=refreshing reason=%s endpoint=%s", safeOptional(reason), state.RunEndpointID) if err := worker.registerUnlocked(ctx); err != nil { return err } return worker.ReconcileOnce(ctx) } func (worker *Worker) ClaimAndRunOnce(ctx context.Context) (bool, error) { state, err := worker.registeredState() if err != nil { return false, err } log.Printf("RUN phase=claim status=polling endpoint=%s activeJobs=%d", state.RunEndpointID, worker.journal.ActiveCount()) claim, err := worker.client.ClaimJob(ctx, protocol.RunJobClaimRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, Capabilities: state.Capabilities, Capacity: worker.capacityReportFor(state), WaitSeconds: jobClaimWaitSeconds, }) if err != nil { if sessionInvalidError(err) { log.Printf("RUN phase=claim status=session_invalid endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) return false, worker.reregisterAndReconcile(ctx, "claim_session_invalid", state.SessionToken) } log.Printf("RUN phase=claim status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) return false, err } if !claim.Accepted || !claim.HasJob || claim.Job == nil { log.Printf("RUN phase=claim status=idle endpoint=%s accepted=%t hasJob=%t", state.RunEndpointID, claim.Accepted, claim.HasJob) return false, nil } assignment := *claim.Job log.Printf("RUN phase=claim status=assigned job=%s capability=%s target=%s attempt=%d server=%s workspaceScope=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), assignment.Attempt, assignment.ServerInstanceID, safeOptional(assignment.ExecutionInput.WorkspaceScope)) return true, worker.runAssignment(ctx, assignment) } func sessionInvalidError(err error) bool { var sessionError interface{ SessionInvalid() bool } return errors.As(err, &sessionError) && sessionError.SessionInvalid() } func (worker *Worker) runAssignment(ctx context.Context, assignment protocol.RunJobAssignment) error { state, err := worker.registeredState() if err != nil { return err } if assignment.RunEndpointID != state.RunEndpointID { return fmt.Errorf("job assignment endpoint does not match registered Run endpoint") } log.Printf("RUN phase=job status=journal_store job=%s capability=%s attempt=%d", assignment.JobID, assignment.Capability, assignment.Attempt) if err := worker.journal.Store(assignment); err != nil { log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } log.Printf("RUN phase=job status=ack_start job=%s capability=%s", assignment.JobID, assignment.Capability) ack, err := worker.client.AckJob(ctx, protocol.RunJobAckRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Message: "job accepted by run worker", }) if err != nil { log.Printf("RUN phase=job status=ack_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } assignment = ack.Job if !ack.Accepted { log.Printf("RUN phase=job status=ack_rejected job=%s", assignment.JobID) return fmt.Errorf("job acknowledgement was not accepted") } log.Printf("RUN phase=job status=ack_accepted job=%s attempt=%d", assignment.JobID, assignment.Attempt) if err := worker.journal.Store(assignment); err != nil { log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } progressSequence := worker.nextProgressSequence(assignment.ProgressSequence) state, err = worker.registeredState() if err != nil { return err } log.Printf("RUN phase=job status=progress_start job=%s sequence=%d", assignment.JobID, progressSequence) progress, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Progress: protocol.RunJobProgressReport{Percent: 10, Message: "lifecycle execution started"}, Sequence: progressSequence, }) if err != nil { log.Printf("RUN phase=job status=progress_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } if !progress.Accepted { log.Printf("RUN phase=job status=progress_rejected job=%s", assignment.JobID) return fmt.Errorf("job progress was not accepted") } assignment = progress.Job log.Printf("RUN phase=job status=progress_accepted job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) if err := worker.journal.Store(assignment); err != nil { log.Printf("RUN phase=job status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } log.Printf("RUN phase=job status=execute_start job=%s capability=%s", assignment.JobID, assignment.Capability) execution, assignment, cancelledByPlatform, err := worker.executeWithJobPolling(ctx, assignment) if err != nil { log.Printf("RUN phase=job status=execute_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } log.Printf("RUN phase=job status=execute_done job=%s state=%s errorCode=%s message=%s cancelledByPlatform=%t", assignment.JobID, execution.State, safeOptional(execution.ErrorCode), safeOptional(execution.Message), cancelledByPlatform) if cancelledByPlatform && execution.State == lifecycleResultStateSucceeded { execution = LifecycleExecutionResult{ State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "cancelled by platform"}, Message: "cancelled by platform", ErrorCode: "lifecycle_cancelled", } } state, err = worker.registeredState() if err != nil { return err } resultRequest := LifecycleResultRequest(assignment, state.SessionToken, execution) log.Printf("RUN phase=job status=result_store job=%s state=%s", assignment.JobID, resultRequest.State) if err := worker.journal.StorePendingResult(resultRequest, execution.ActivationManifest); err != nil { log.Printf("RUN phase=job status=result_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } log.Printf("RUN phase=job status=result_submit job=%s state=%s", assignment.JobID, resultRequest.State) result, err := worker.client.CompleteJob(ctx, resultRequest) if err != nil { log.Printf("RUN phase=job status=result_submit_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } if !result.Accepted { log.Printf("RUN phase=job status=result_rejected job=%s", assignment.JobID) return fmt.Errorf("job result was not accepted") } log.Printf("RUN phase=job status=result_accepted job=%s state=%s", assignment.JobID, resultRequest.State) if err := worker.journal.Delete(assignment.JobID); err != nil { log.Printf("RUN phase=job status=journal_delete_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } log.Printf("RUN phase=job status=complete job=%s state=%s", assignment.JobID, resultRequest.State) if execution.ActivationManifest != "" { log.Printf("RUN phase=self_update status=activate_start job=%s", assignment.JobID) if err := worker.executor.selfUpdateActivator.Activate(execution.ActivationManifest); err != nil { log.Printf("RUN phase=self_update status=activate_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return fmt.Errorf("launch self-update helper: %w", err) } worker.restartMu.Lock() worker.restartRequested = true worker.restartMu.Unlock() } return nil } func (worker *Worker) executeWithJobPolling(ctx context.Context, assignment protocol.RunJobAssignment) (LifecycleExecutionResult, protocol.RunJobAssignment, bool, error) { jobCtx, cancel := context.WithCancel(ctx) defer cancel() cancelledByPlatform := false pollCancel := func() { state, err := worker.registeredState() if err != nil { log.Printf("RUN phase=job.cancel_poll status=skipped_unregistered job=%s error=%s", assignment.JobID, RedactText(err.Error())) return } log.Printf("RUN phase=job.cancel_poll status=starting job=%s", assignment.JobID) response, err := worker.client.PollJobCancel(ctx, protocol.RunJobCancelPollRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, }) if err != nil { log.Printf("RUN phase=job.cancel_poll status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return } if err == nil && response.HasCancel { log.Printf("RUN phase=job.cancel_poll status=cancel_requested job=%s", assignment.JobID) cancelledByPlatform = true cancel() return } log.Printf("RUN phase=job.cancel_poll status=clear job=%s", assignment.JobID) } pollCancel() executionCh := make(chan LifecycleExecutionResult, 1) executionAssignment := assignment go func() { executionCh <- worker.executeAssignment(jobCtx, executionAssignment) }() log.Printf("RUN phase=job.execute status=worker_started job=%s pollSeconds=%d", assignment.JobID, int(jobActivePollInterval/time.Second)) ticker := time.NewTicker(jobActivePollInterval) defer ticker.Stop() for { select { case execution := <-executionCh: log.Printf("RUN phase=job.execute status=worker_finished job=%s state=%s", assignment.JobID, execution.State) return execution, assignment, cancelledByPlatform, nil case <-ctx.Done(): log.Printf("RUN phase=job.execute status=context_done job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error())) cancel() execution := <-executionCh return execution, assignment, cancelledByPlatform, ctx.Err() case <-ticker.C: log.Printf("RUN phase=job.execute status=active job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) if !cancelledByPlatform { pollCancel() } if cancelledByPlatform { continue } state, err := worker.registeredState() if err != nil { cancel() log.Printf("RUN phase=job.execute status=lease_renew_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err } progress, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Progress: protocol.RunJobProgressReport{Percent: assignment.Progress.Percent, Message: "lifecycle execution active"}, Sequence: worker.nextProgressSequence(assignment.ProgressSequence), }) if err != nil || !progress.Accepted { cancel() if err == nil { err = fmt.Errorf("job lease renewal was not accepted") } log.Printf("RUN phase=job.execute status=lease_renew_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err } assignment = progress.Job log.Printf("RUN phase=job.execute status=lease_renewed job=%s percent=%d", assignment.JobID, assignment.Progress.Percent) if err := worker.journal.Store(assignment); err != nil { cancel() log.Printf("RUN phase=job.execute status=journal_store_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return LifecycleExecutionResult{}, assignment, cancelledByPlatform, err } } } } func (worker *Worker) nextProgressSequence(minimum uint64) uint64 { worker.sequenceMu.Lock() defer worker.sequenceMu.Unlock() worker.stateMu.Lock() defer worker.stateMu.Unlock() if worker.state.Sequence < minimum { worker.state.Sequence = minimum } worker.state.Sequence++ return worker.state.Sequence } func (worker *Worker) executeAssignment(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { log.Printf("RUN phase=job.dispatch status=select job=%s capability=%s inputRef=%s target=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.InputRef), safeOptional(assignment.TargetKey)) if protocol.IsProtectedRequestCapability(assignment.Capability) { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=protected_request", assignment.JobID) return worker.executeProtectedRequestJob(ctx, assignment) } if assignment.ExecutionInput.SourceRCON != nil { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=source_rcon", assignment.JobID) return worker.executeSourceRCONJob(ctx, assignment) } if assignment.Capability == protocol.RunCapabilityDistributionBuild { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=distribution_build", assignment.JobID) return worker.executeDistributionBuild(ctx, assignment) } if assignment.Capability == protocol.RunCapabilityDependenciesCheck || assignment.Capability == protocol.RunCapabilityDependenciesInstall { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=dependencies", assignment.JobID) return worker.executeDependencyJob(ctx, assignment) } if assignment.Capability == protocol.RunCapabilityRunSelfUpdate { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=self_update", assignment.JobID) return worker.executeRunSelfUpdate(ctx, assignment) } if assignment.Capability == protocol.RunCapabilityLogsBackfill { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=logs_backfill", assignment.JobID) return worker.executor.ExecuteLogBackfill(ctx, assignment) } if assignment.Capability == protocol.RunCapabilityRemoteRunDBSQLiteProbe { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=sqlite_schema_probe", assignment.JobID) if worker.executor.sqliteSchemaProbe == nil { return lifecycleFailure("sqlite_probe_unavailable", "SQLite schema probe executor is unavailable") } if assignment.ExecutionInput.SQLiteSchemaProbe != nil { targetKey, err := worker.materializeSQLiteProbeDataTarget(ctx, assignment) if err != nil { return sqliteProbeFailureForDataTarget(assignment, err) } assignment.TargetKey = targetKey } return worker.executor.sqliteSchemaProbe.Execute(ctx, assignment) } if isSupportedLifecycleCapability(assignment.Capability) { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=lifecycle", assignment.JobID) return worker.executor.ExecuteContext(ctx, assignment) } if supportedCapability(SupportedFileCapabilities(), assignment.Capability) { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=file", assignment.JobID) return worker.executeFileJob(ctx, assignment) } if isSupportedDistributionCapability(assignment.Capability) { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=distribution", assignment.JobID) return ExecuteDistributionJob(ctx, assignment) } if isSupportedRemoteCapability(assignment.Capability) { log.Printf("RUN phase=job.dispatch status=selected job=%s executor=remote", assignment.JobID) return ExecuteRemoteAccessJob(ctx, assignment) } log.Printf("RUN phase=job.dispatch status=unsupported job=%s capability=%s", assignment.JobID, assignment.Capability) return lifecycleFailure("unsupported_run_capability", "unsupported run capability") } func (worker *Worker) executeFileJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { if assignment.Capability != protocol.RunCapabilityFilesRead { return worker.executor.ExecuteContext(ctx, assignment) } fileExecutor := worker.executor.fileExecutor if fileExecutor == nil { return lifecycleExecutionFailure("file_executor_unavailable", "file executor is unavailable", false) } scope, targetPath, deploymentRoot, err := fileExecutor.existingReadTargetForAssignment(assignment) if err != nil { return lifecycleExecutionFailure("file_read_failed", err.Error(), false) } info, err := os.Stat(targetPath) if err != nil { return lifecycleExecutionFailure("file_read_failed", err.Error(), false) } limit := assignment.ExecutionInput.MaxReadBytes if limit <= 0 || limit > maxExecutionContentBytes { limit = maxExecutionContentBytes } if info.Size() <= int64(limit) { return fileExecutor.read(ctx, scope, deploymentRoot, assignment) } if info.Size() > maxFileArtifactBytes { return lifecycleExecutionFailure("file_read_too_large", "file exceeds artifact transfer limit", false) } checksum, err := checksumServerFileForArtifact(ctx, targetPath) if err != nil { if errors.Is(err, context.Canceled) { return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false) } return lifecycleExecutionFailure("file_read_failed", err.Error(), false) } artifactID := "artifact-" + safeWorkspaceName(assignment.JobID) + "-file-read" if err := worker.uploadFileArtifact(ctx, assignment, artifactID, targetPath, info.Size(), checksum); err != nil { if errors.Is(err, context.Canceled) { return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false) } return lifecycleExecutionFailure("file_artifact_upload_failed", "file artifact upload failed", false) } metadata := fileExecutor.metadata(scope, assignment.TargetKey, checksum, info.Size()) return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file read completed"}, ResultRef: "artifact://" + artifactID, Message: "file read completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.read", Version: metadata.Version, Checksum: checksum, SizeBytes: info.Size(), Summary: "large file transferred as artifact"}} } func (worker *Worker) executeProtectedRequestJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { if err := protocol.ValidateRunJobAssignment(assignment); err != nil { return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_assignment_invalid") } state, err := worker.registeredState() if err != nil { return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_unregistered") } input, err := worker.client.GetProtectedRequestExecutionInput(ctx, protocol.ProtectedRequestExecutionInputRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, FencingToken: assignment.FencingToken, }) if err != nil || !protectedRequestInputMatchesAssignment(input, assignment, state.RunEndpointID) { return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_input_unavailable") } return worker.executor.ExecuteProtectedRequest(ctx, assignment, input) } func protectedRequestInputMatchesAssignment(input protocol.ProtectedRequestExecutionInputResponse, assignment protocol.RunJobAssignment, endpointID string) bool { return protocol.ValidProtectedRequestExecutionInput(input) && input.JobID == assignment.JobID && input.ServerInstanceID == assignment.ServerInstanceID && input.RunEndpointID == endpointID && input.FencingToken == assignment.FencingToken && input.TargetKey == assignment.TargetKey && input.TransportKey == assignment.ExecutionInput.RemoteAdapterKey && input.Kind == protectedRequestKindForCapability(assignment.Capability) } func (worker *Worker) executeSourceRCONJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { if err := protocol.ValidateRunJobAssignment(assignment); err != nil { return lifecycleFailure("unsafe_source_rcon_plan", "Source RCON plan is invalid") } state, err := worker.registeredState() if err != nil { return lifecycleFailure("source_rcon_unregistered", "Run worker is not registered") } input, err := worker.client.GetSourceRCONExecutionInput(ctx, protocol.SourceRCONExecutionInputRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, }) if err != nil || input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != state.RunEndpointID { return lifecycleFailure("source_rcon_input_unavailable", "Source RCON command input is unavailable") } return worker.executor.ExecuteSourceRCON(ctx, assignment, input.Command) } func supportedCapability(capabilities []string, target string) bool { for _, capability := range capabilities { if capability == target { return true } } return false } func (worker *Worker) ReconcileOnce(ctx context.Context) error { state, err := worker.registeredState() if err != nil { return err } activeBefore := worker.journal.ActiveCount() log.Printf("RUN phase=reconcile status=starting endpoint=%s activeJobs=%d", state.RunEndpointID, activeBefore) response, err := worker.client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, ActiveJobs: worker.journal.ReconcileEntries(), }) if err != nil { log.Printf("RUN phase=reconcile status=failed endpoint=%s error=%s", state.RunEndpointID, RedactText(err.Error())) return err } if !response.Accepted { log.Printf("RUN phase=reconcile status=rejected endpoint=%s", state.RunEndpointID) return fmt.Errorf("job reconciliation was not accepted") } confirmed := map[string]struct{}{} for _, job := range response.ConfirmedJobs { if job.RunEndpointID != state.RunEndpointID { return fmt.Errorf("reconciled job endpoint does not match registered Run endpoint") } if err := worker.journal.Store(job); err != nil { return err } confirmed[job.JobID] = struct{}{} } for _, jobID := range response.DiscardJobIDs { if err := worker.journal.Delete(jobID); err != nil { return err } confirmed[jobID] = struct{}{} } for _, job := range worker.journal.ActiveJobs() { if _, accounted := confirmed[job.JobID]; !accounted { if err := worker.journal.Delete(job.JobID); err != nil { return err } } } log.Printf("RUN phase=reconcile status=accepted endpoint=%s confirmed=%d discarded=%d activeBefore=%d activeAfter=%d", state.RunEndpointID, len(response.ConfirmedJobs), len(response.DiscardJobIDs), activeBefore, worker.journal.ActiveCount()) return nil } func (worker *Worker) RecoverActiveJobs(ctx context.Context) error { activeJobs := worker.journal.ActiveJobs() log.Printf("RUN phase=recover status=starting activeJobs=%d", len(activeJobs)) for _, assignment := range activeJobs { if pending, ok := worker.journal.PendingResult(assignment.JobID); ok { log.Printf("RUN phase=recover status=pending_result job=%s", assignment.JobID) activationManifest := worker.journal.PendingActivation(assignment.JobID) state, err := worker.registeredState() if err != nil { return err } pending.SessionToken = state.SessionToken result, err := worker.client.CompleteJob(ctx, pending) if err != nil { log.Printf("RUN phase=recover status=result_submit_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } if !result.Accepted { log.Printf("RUN phase=recover status=result_rejected job=%s", assignment.JobID) return fmt.Errorf("recovered job result was not accepted") } if err := worker.journal.Delete(assignment.JobID); err != nil { return err } if activationManifest != "" { if err := worker.executor.selfUpdateActivator.Activate(activationManifest); err != nil { return fmt.Errorf("launch recovered self-update helper: %w", err) } worker.restartMu.Lock() worker.restartRequested = true worker.restartMu.Unlock() } continue } log.Printf("RUN phase=recover status=rerun_active_job job=%s capability=%s", assignment.JobID, assignment.Capability) if err := worker.runAssignment(ctx, assignment); err != nil { log.Printf("RUN phase=recover status=rerun_failed job=%s error=%s", assignment.JobID, RedactText(err.Error())) return err } } log.Printf("RUN phase=recover status=complete activeJobs=%d", worker.journal.ActiveCount()) return nil } func (worker *Worker) Run(ctx context.Context) error { log.Printf("RUN phase=run status=starting endpoint=%s", worker.cfg.RunEndpointID) if err := worker.Register(ctx); err != nil { return err } if err := worker.ReconcileOnce(ctx); err != nil { return err } if err := worker.RecoverActiveJobs(ctx); err != nil { return err } if err := worker.RunAutonomousLifecycleOnce(ctx); err != nil { return err } if err := worker.reportAutonomousProcessObservations(ctx); err != nil { log.Printf("RUN phase=autonomous_lifecycle.observation status=degraded error=%s", RedactText(err.Error())) } worker.reportMetricsDegraded(ctx, "startup") if err := MarkSelfUpdateHealthy(worker.cfg.UpdateHealthFile); err != nil { log.Printf("RUN phase=self_update_health status=mark_failed error=%s", RedactText(err.Error())) return err } if err := worker.reportRunUpdateHealth(ctx); err != nil { return err } heartbeatInterval := durationOrDefault(worker.cfg.HeartbeatInterval, 15*time.Second) jobInterval := durationOrDefault(worker.cfg.PollInterval, 2*time.Second) state := worker.State() log.Printf("RUN phase=run status=ready endpoint=%s heartbeatSeconds=%d jobPollSeconds=%d", state.RunEndpointID, int(heartbeatInterval/time.Second), int(jobInterval/time.Second)) heartbeatTicker := time.NewTicker(heartbeatInterval) defer heartbeatTicker.Stop() workerCtx, cancelWorker := context.WithCancel(ctx) uploaderDone := make(chan struct{}) go worker.runDurableUploaders(workerCtx, uploaderDone) jobDone := make(chan error, 1) go func() { jobDone <- worker.runJobLoop(workerCtx, jobInterval) }() defer func() { cancelWorker() <-uploaderDone }() for { select { case <-ctx.Done(): log.Printf("RUN phase=run status=context_done error=%s", RedactText(ctx.Err().Error())) return ctx.Err() case err := <-jobDone: log.Printf("RUN phase=run status=job_loop_done error=%s", errorSummary(err)) return err case <-heartbeatTicker.C: if err := worker.HeartbeatOnce(ctx); err != nil { log.Printf("RUN phase=heartbeat status=retry_scheduled backoffMs=%d", boundedRetryBackoff(worker.cfg.RetryBackoff).Milliseconds()) heartbeatTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff)) continue } if err := worker.reportAutonomousProcessObservations(ctx); err != nil { log.Printf("RUN phase=autonomous_lifecycle.observation status=degraded error=%s", RedactText(err.Error())) } worker.reportMetricsDegraded(ctx, "heartbeat") heartbeatTicker.Reset(heartbeatInterval) } } } func (worker *Worker) reportRunUpdateHealth(ctx context.Context) error { if worker.cfg.UpdateJobID == "" && worker.cfg.UpdateOutcome == "" && worker.cfg.UpdateAttempt == 0 && worker.cfg.UpdateLeaseToken == "" { log.Printf("RUN phase=self_update_health status=skipped") return nil } if worker.cfg.UpdateJobID == "" || (worker.cfg.UpdateOutcome != "succeeded" && worker.cfg.UpdateOutcome != "rolled-back") || worker.cfg.UpdateAttempt <= 0 || worker.cfg.UpdateLeaseToken == "" { log.Printf("RUN phase=self_update_health status=invalid_config") return fmt.Errorf("self-update health report configuration is incomplete") } log.Printf("RUN phase=self_update_health status=reporting job=%s outcome=%s attempt=%d", worker.cfg.UpdateJobID, worker.cfg.UpdateOutcome, worker.cfg.UpdateAttempt) state, err := worker.registeredState() if err != nil { return err } response, err := worker.client.ReportRunUpdateHealth(ctx, protocol.RunUpdateHealthRequest{ RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: worker.cfg.UpdateJobID, LeaseToken: worker.cfg.UpdateLeaseToken, Attempt: worker.cfg.UpdateAttempt, Outcome: worker.cfg.UpdateOutcome, Version: worker.cfg.Version, }) if err != nil { log.Printf("RUN phase=self_update_health status=failed job=%s error=%s", worker.cfg.UpdateJobID, RedactText(err.Error())) return err } if !response.Accepted || response.JobID != worker.cfg.UpdateJobID { log.Printf("RUN phase=self_update_health status=rejected job=%s", worker.cfg.UpdateJobID) return fmt.Errorf("self-update health report was not accepted") } log.Printf("RUN phase=self_update_health status=accepted job=%s", worker.cfg.UpdateJobID) return nil } func (worker *Worker) runJobLoop(ctx context.Context, interval time.Duration) error { log.Printf("RUN phase=job_loop status=starting fallbackPollMs=%d claimWaitSeconds=%d", interval.Milliseconds(), jobClaimWaitSeconds) for { select { case <-ctx.Done(): log.Printf("RUN phase=job_loop status=context_done error=%s", RedactText(ctx.Err().Error())) return ctx.Err() default: } if worker.journal.ActiveCount() > 0 { log.Printf("RUN phase=job_loop status=active_jobs activeJobs=%d", worker.journal.ActiveCount()) if err := worker.ReconcileOnce(ctx); err != nil { log.Printf("RUN phase=job_loop status=reconcile_failed error=%s", RedactText(err.Error())) if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil { return err } continue } if err := worker.RecoverActiveJobs(ctx); err != nil { log.Printf("RUN phase=job_loop status=recover_failed error=%s", RedactText(err.Error())) if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil { return err } continue } } claimStartedAt := time.Now() handled, err := worker.ClaimAndRunOnce(ctx) if err != nil { log.Printf("RUN phase=job_loop status=claim_failed error=%s", RedactText(err.Error())) if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil { return err } continue } worker.restartMu.Lock() restartRequested := worker.restartRequested worker.restartMu.Unlock() if restartRequested { log.Printf("RUN phase=job_loop status=restart_requested") return ErrSelfUpdateRestartRequested } if !handled && time.Since(claimStartedAt) < interval { if err := waitWorkerLoop(ctx, interval-time.Since(claimStartedAt)); err != nil { return err } } } } func waitWorkerLoop(ctx context.Context, delay time.Duration) error { if delay <= 0 { return nil } timer := time.NewTimer(delay) defer timer.Stop() select { case <-ctx.Done(): return ctx.Err() case <-timer.C: return nil } } type durableLogClient interface { IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) } type durableArtifactClient interface { UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) } type sessionLogBatchClient struct { client durableLogClient runEndpointID string sessionToken string } type sessionLogStreamProgressClient struct { client interface { GetRunLogStreamProgress(context.Context, protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) } runEndpointID string sessionToken string serverID string } func (client sessionLogStreamProgressClient) GetRunLogStreamProgress(ctx context.Context, streamID string) (uint64, error) { response, err := client.client.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{ RunEndpointID: client.runEndpointID, SessionToken: client.sessionToken, ServerInstanceID: client.serverID, LogStreamID: streamID, }) if err != nil { return 0, err } if !response.Accepted || response.LogStreamID != streamID { return 0, fmt.Errorf("platform log stream progress response is invalid") } return response.LatestSeq, nil } func (client sessionLogBatchClient) IngestLogBatch(ctx context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) { batch.RunEndpointID = client.runEndpointID batch.SessionToken = client.sessionToken if checksum, err := checksumForLogEntries(batch.Entries); err == nil { batch.Checksum = checksum } response, err := client.client.IngestLogBatch(ctx, batch) if err != nil && (logBatchSequenceGapError(err) || logBatchAcknowledgedRangeConflict(err)) { reason := "platform_sequence_gap" if logBatchAcknowledgedRangeConflict(err) { reason = "platform_acknowledged_range_conflict" } log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=%s error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, reason, RedactText(err.Error())) return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection(reason, err) } if err != nil && logBatchLegacySessionMetadataError(err) && strings.TrimSpace(batch.LogSessionID) == "" && !batch.SessionStartedAt.IsZero() { log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=legacy_session_metadata error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, RedactText(err.Error())) return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection("legacy_session_metadata", err) } if err != nil && logBatchSessionMetadataMismatchError(err) { log.Printf("RUN phase=durable_uploaders.logs status=spool_quarantine stream=%s firstSeq=%d lastSeq=%d reason=session_metadata_mismatch error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, batch.LastSeq, RedactText(err.Error())) return protocol.LogBatchIngestResponse{}, spool.PermanentLogBatchRejection("session_metadata_mismatch", err) } return response, err } func logBatchSequenceGapError(err error) bool { var gapError interface{ LogBatchSequenceGap() bool } return errors.As(err, &gapError) && gapError.LogBatchSequenceGap() } func logBatchAcknowledgedRangeConflict(err error) bool { var conflict interface{ LogBatchAcknowledgedRangeConflict() bool } return errors.As(err, &conflict) && conflict.LogBatchAcknowledgedRangeConflict() } func logBatchLegacySessionMetadataError(err error) bool { var legacyMetadata interface{ LogBatchLegacySessionMetadata() bool } return errors.As(err, &legacyMetadata) && legacyMetadata.LogBatchLegacySessionMetadata() } func logBatchSessionMetadataMismatchError(err error) bool { var mismatch interface{ LogBatchSessionMetadataMismatch() bool } return errors.As(err, &mismatch) && mismatch.LogBatchSessionMetadataMismatch() } func artifactChunkMissingOnPlatform(err error) bool { var httpErr interface{ HTTPStatus() int } return errors.As(err, &httpErr) && httpErr.HTTPStatus() == http.StatusNotFound } type sessionArtifactChunkClient struct { client durableArtifactClient runEndpointID string sessionToken string } func (client sessionArtifactChunkClient) UploadArtifactChunk(ctx context.Context, chunk protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) { chunk.RunEndpointID = client.runEndpointID chunk.SessionToken = client.sessionToken response, err := client.client.UploadArtifactChunk(ctx, chunk) if err != nil { if artifactChunkMissingOnPlatform(err) { log.Printf("RUN phase=durable_uploaders.artifacts status=drop_stale transfer=%s artifact=%s chunk=%d reason=platform_transfer_missing", safeOptional(chunk.TransferID), safeOptional(chunk.ArtifactID), chunk.ChunkIndex) return protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: chunk.TransferID, ArtifactID: chunk.ArtifactID, ChunkIndex: chunk.ChunkIndex}, nil } return response, err } return response, nil } func (worker *Worker) runDurableUploaders(ctx context.Context, done chan<- struct{}) { defer close(done) logSink, hasLogSink := worker.executor.logSink.(*SpoolLogSink) artifactHook, hasArtifactHook := worker.executor.artifactHook.(*QueueArtifactHook) logClient, hasLogClient := worker.client.(durableLogClient) artifactClient, hasArtifactClient := worker.client.(durableArtifactClient) if (!hasLogSink || !hasLogClient) && (!hasArtifactHook || !hasArtifactClient) { log.Printf("RUN phase=durable_uploaders status=disabled logs=%t artifacts=%t", hasLogSink && hasLogClient, hasArtifactHook && hasArtifactClient) return } log.Printf("RUN phase=durable_uploaders status=starting logs=%t artifacts=%t", hasLogSink && hasLogClient, hasArtifactHook && hasArtifactClient) ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): log.Printf("RUN phase=durable_uploaders status=stopping") return case <-ticker.C: state, err := worker.registeredState() if err != nil { log.Printf("RUN phase=durable_uploaders status=skipped_unregistered error=%s", RedactText(err.Error())) continue } if hasLogSink && hasLogClient { startedAt := time.Now() log.Printf("RUN phase=durable_uploaders.logs status=flush_start timeoutMs=%d", durableUploaderFlushTimeout.Milliseconds()) flushCtx, cancel := context.WithTimeout(ctx, durableUploaderFlushTimeout) client := sessionLogBatchClient{client: logClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken} flushed, err := logSink.Spool.Flush(flushCtx, client) if err != nil { log.Printf("RUN phase=durable_uploaders.logs status=flush_failed flushed=%d durationMs=%d error=%s", flushed, time.Since(startedAt).Milliseconds(), RedactText(err.Error())) } else if flushed > 0 { log.Printf("RUN phase=durable_uploaders.logs status=flushed count=%d durationMs=%d", flushed, time.Since(startedAt).Milliseconds()) } cancel() } if hasArtifactHook && hasArtifactClient { startedAt := time.Now() log.Printf("RUN phase=durable_uploaders.artifacts status=flush_start timeoutMs=%d", durableUploaderFlushTimeout.Milliseconds()) flushCtx, cancel := context.WithTimeout(ctx, durableUploaderFlushTimeout) client := sessionArtifactChunkClient{client: artifactClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken} flushed, err := artifactHook.Queue.Flush(flushCtx, client) if err != nil { log.Printf("RUN phase=durable_uploaders.artifacts status=flush_failed flushed=%d durationMs=%d error=%s", flushed, time.Since(startedAt).Milliseconds(), RedactText(err.Error())) } else if flushed > 0 { log.Printf("RUN phase=durable_uploaders.artifacts status=flushed count=%d durationMs=%d", flushed, time.Since(startedAt).Milliseconds()) } cancel() } } } } func checksumForLogEntries(entries []protocol.LogEntry) (string, error) { stable := make([]logEntryChecksumBody, len(entries)) for i, entry := range entries { stable[i] = logEntryChecksumBody{ Seq: entry.Seq, Timestamp: entry.Timestamp.UTC().Format("2006-01-02T15:04:05.000000000Z07:00"), Level: entry.Level, Line: entry.Line, Fields: entry.Fields, Redacted: entry.Redacted, } } encoded, err := json.Marshal(stable) if err != nil { return "", err } sum := sha256.Sum256(encoded) return "sha256:" + hex.EncodeToString(sum[:]), nil } type logEntryChecksumBody struct { Seq uint64 `json:"seq"` Timestamp string `json:"timestamp"` Level string `json:"level,omitempty"` Line string `json:"line"` Fields map[string]string `json:"fields,omitempty"` Redacted bool `json:"redacted"` } func (worker *Worker) capacityReport() protocol.RunCapacityReport { return worker.capacityReportFor(worker.State()) } func (worker *Worker) capacityReportFor(state WorkerState) protocol.RunCapacityReport { return protocol.RunCapacityReport{ MaxJobs: state.Capacity.MaxJobs, RunningJobs: worker.journal.ActiveCount(), QueuedJobs: 0, Summary: "worker control active; job capacity reported separately", } } func (worker *Worker) State() WorkerState { worker.stateMu.RLock() defer worker.stateMu.RUnlock() state := worker.state state.Capabilities = append([]string(nil), state.Capabilities...) return state } type SpoolLogSink struct { RunEndpointID string SessionToken string Spool spool.LogSpool mu sync.Mutex Progress func(context.Context, string, string) (uint64, error) } func (sink *SpoolLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error { return sink.append(ctx, assignment, stream, line, nil) } func (sink *SpoolLogSink) AppendWithCursor(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, cursor ProcessLogCursor) error { return sink.append(ctx, assignment, stream, line, &spool.LogSourceCursor{StartOffset: cursor.StartOffset, EndOffset: cursor.EndOffset}) } func (sink *SpoolLogSink) append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, cursor *spool.LogSourceCursor) error { sink.mu.Lock() defer sink.mu.Unlock() redactedLine := RedactText(line) redacted := line != redactedLine || strings.HasPrefix(redactedLine, "[redacted protected ") streamKey := declaredProcessStreamKey(assignment, stream) logStreamID := logStreamIDForAssignment(assignment, streamKey) entry := protocol.LogEntry{Timestamp: time.Now().UTC(), Level: "info", Line: redactedLine, Redacted: redacted} source := "process" if strings.HasPrefix(stream, "management-program.") { source = "management-program" } else if assignment.Capability == protocol.RunCapabilityLogsBackfill { source = "file" } var recoverProgress func(context.Context, string) (uint64, error) runStreamPrefix := "run." + assignment.RunEndpointID + "." + assignment.ServerInstanceID + "." if sink.Progress != nil && strings.HasPrefix(logStreamID, runStreamPrefix) { recoverProgress = func(ctx context.Context, streamID string) (uint64, error) { return sink.Progress(ctx, assignment.ServerInstanceID, streamID) } } if strings.TrimSpace(assignment.LogSessionID) != "" { // A generation-scoped stream is globally fresh, so its first durable // append must not depend on platform availability or the worker ctx. recoverProgress = nil } _, _, err := sink.Spool.EnqueueNextAggregated(ctx, protocol.LogBatchIngestRequest{ RunEndpointID: sink.RunEndpointID, SessionToken: sink.SessionToken, LogStreamID: logStreamID, ServerInstanceID: assignment.ServerInstanceID, StreamKey: streamKey, Source: source, LogSessionID: assignment.LogSessionID, SessionStartedAt: assignment.SessionStartedAt, Entries: []protocol.LogEntry{entry}, }, cursor, recoverProgress, checksumForLogEntries) return err } func logStreamIDForAssignment(assignment protocol.RunJobAssignment, streamKey string) string { if strings.TrimSpace(assignment.LogSessionID) != "" { return fmt.Sprintf("run.%s.%s.%s.%s", assignment.RunEndpointID, assignment.ServerInstanceID, assignment.LogSessionID, streamKey) } if autonomousLifecycleLogAssignment(assignment) { return fmt.Sprintf("run.%s.%s.%s", assignment.RunEndpointID, assignment.ServerInstanceID, streamKey) } return fmt.Sprintf("job.%s.%s", assignment.JobID, streamKey) } func autonomousLifecycleLogAssignment(assignment protocol.RunJobAssignment) bool { return strings.HasPrefix(assignment.IdempotencyKey, "autonomous:") || strings.HasPrefix(assignment.LeaseToken, "local-autonomous-") || strings.HasPrefix(assignment.JobID, "autonomous-") } func declaredProcessStreamKey(assignment protocol.RunJobAssignment, stream string) string { kind := "" if stream == "stdout" { kind = "process.stdout" } else if stream == "stderr" { kind = "process.stderr" } if kind != "" { for _, source := range assignment.ExecutionInput.LogSources { if source.Kind == kind && strings.TrimSpace(source.StreamKey) != "" { return source.StreamKey } } } return stream } type QueueArtifactHook struct { RunEndpointID string SessionToken string Queue spool.ArtifactQueue } func (hook QueueArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, result ProcessResult) (string, error) { ref := fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID) payload := []byte(RedactText(result.Stdout + result.Stderr)) if len(payload) == 0 { payload = []byte("lifecycle result metadata") } artifactID := "artifact-" + assignment.JobID + "-lifecycle" if err := hook.Queue.Enqueue(protocol.ArtifactChunkUploadRequest{ RunEndpointID: hook.RunEndpointID, SessionToken: hook.SessionToken, TransferID: "transfer-" + assignment.JobID, ArtifactID: artifactID, ChunkIndex: 0, Offset: 0, SizeBytes: len(payload), Checksum: checksumForText(string(payload)), Payload: payload, }); err != nil { return "", err } return ref, nil } func capabilityFingerprint(capabilities []string) string { return checksumForText(strings.Join(capabilities, ",")) } func durationOrDefault(value time.Duration, fallback time.Duration) time.Duration { if value <= 0 { return fallback } return value } func boundedRetryBackoff(value time.Duration) time.Duration { value = durationOrDefault(value, time.Second) if value > 30*time.Second { return 30 * time.Second } return value }