package runtime import ( "archive/tar" "archive/zip" "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "time" "browser.local/run/protocol" ) const ( selfUpdateManifestVersion = 1 maxSelfUpdateBytes = int64(512 * 1024 * 1024) maxSelfUpdateEntries = 8 defaultUpdateHealthWait = 30 * time.Second ) var ErrSelfUpdateRestartRequested = errors.New("Run self-update restart requested") type SelfUpdateManifest struct { Version int `json:"version"` JobID string `json:"jobId"` Attempt int `json:"attempt"` LeaseToken string `json:"leaseToken"` ArtifactID string `json:"artifactId"` ArtifactChecksum string `json:"artifactChecksum"` ArtifactSizeBytes int64 `json:"artifactSizeBytes"` TargetOS string `json:"targetOs"` TargetArch string `json:"targetArch"` TargetRelease string `json:"targetRelease"` CurrentExecutable string `json:"currentExecutable"` StagedExecutable string `json:"stagedExecutable"` BackupExecutable string `json:"backupExecutable"` HealthFile string `json:"healthFile"` WorkingDirectory string `json:"workingDirectory"` Phase string `json:"phase"` DownloadedBytes int64 `json:"downloadedBytes"` BinaryChecksum string `json:"binaryChecksum,omitempty"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } type SelfUpdateActivator interface { Activate(string) error } type ProcessSelfUpdateActivator struct{} func (ProcessSelfUpdateActivator) Activate(manifestPath string) error { manifest, err := loadSelfUpdateManifest(manifestPath) if err != nil { return err } command := exec.Command(manifest.StagedExecutable) command.Dir = manifest.WorkingDirectory command.Env = append(cleanUpdateEnvironment(os.Environ()), "RUN_MODE=self-update-helper", "RUN_UPDATE_MANIFEST="+manifestPath) command.Stdout = io.Discard command.Stderr = io.Discard return command.Start() } func (worker *Worker) executeRunSelfUpdate(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult { if err := protocol.ValidateRunJobAssignment(assignment); err != nil { return lifecycleFailure("unsafe_self_update_job", err.Error()) } state, err := worker.registeredState() if err != nil { return lifecycleFailure("self_update_unregistered", "Run worker is not registered") } input, err := worker.client.GetRunUpdateInput(ctx, protocol.RunUpdateInputRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt}) if err != nil { return lifecycleFailure("self_update_input_failed", "could not load fenced Run update input") } if err := validateRunUpdateInput(assignment, input); err != nil { return lifecycleFailure("unsafe_self_update_input", err.Error()) } transactionRoot := filepath.Join(worker.cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID)) if err := os.MkdirAll(transactionRoot, 0o700); err != nil { return lifecycleFailure("self_update_workspace_failed", "could not create update transaction workspace") } manifestPath := filepath.Join(transactionRoot, "manifest.json") archivePath := filepath.Join(transactionRoot, "update.archive") manifest, err := prepareSelfUpdateManifest(manifestPath, assignment, input, transactionRoot) if err != nil { return lifecycleFailure("self_update_manifest_failed", err.Error()) } if manifest.Phase != "staged" { manifest.Phase = "downloading" if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { return lifecycleFailure("self_update_manifest_failed", err.Error()) } if err := worker.downloadRunUpdate(ctx, assignment, input, archivePath, manifestPath, &manifest); err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update download cancelled"}, Message: "Run update download cancelled", ErrorCode: "run_self_update_cancelled"} } return lifecycleFailure("self_update_download_failed", err.Error()) } stagedPath := filepath.Join(transactionRoot, input.ExecutableName+".staged") binaryChecksum, err := stageRunUpdateBinary(archivePath, input.PackageFormat, input.ExecutableName, stagedPath) if err != nil { return lifecycleFailure("self_update_extract_failed", err.Error()) } manifest.StagedExecutable = stagedPath manifest.BinaryChecksum = binaryChecksum manifest.Phase = "staged" manifest.UpdatedAt = time.Now().UTC() if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { return lifecycleFailure("self_update_manifest_failed", err.Error()) } } evidence, _ := json.Marshal(protocol.RunUpdateExecutionEvidence{TargetRelease: input.TargetRelease, Phase: "staged"}) return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", safeWorkspaceName(assignment.JobID)), Message: "Run update verified and staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: input.Checksum, SizeBytes: input.SizeBytes, Summary: "verified update staged", Content: string(evidence)}, ActivationManifest: manifestPath} } func validateRunUpdateInput(assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse) error { if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID || assignment.InputRef != "artifact://"+input.ArtifactID { return fmt.Errorf("Run update input scope does not match job") } if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH { return fmt.Errorf("Run update target does not match this executable") } if input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" && input.PackageFormat != "raw-executable" { return fmt.Errorf("Run update package format is unsupported") } if input.SizeBytes <= 0 || input.SizeBytes > maxSelfUpdateBytes || input.ChunkSizeBytes <= 0 || input.ChunkSizeBytes > 1024*1024 || !validSHA256(input.Checksum) { return fmt.Errorf("Run update artifact bounds are invalid") } expectedName := "run" if runtime.GOOS == "windows" { expectedName = "run.exe" } if input.ExecutableName != expectedName || !protocol.ValidLogicalFileKey(input.TargetRelease) { return fmt.Errorf("Run update executable or release identity is unsafe") } return nil } func prepareSelfUpdateManifest(path string, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, root string) (SelfUpdateManifest, error) { if existing, err := loadSelfUpdateManifest(path); err == nil { if existing.JobID != assignment.JobID || existing.ArtifactID != input.ArtifactID || existing.ArtifactChecksum != input.Checksum || existing.TargetRelease != input.TargetRelease || existing.Attempt > assignment.Attempt { return SelfUpdateManifest{}, fmt.Errorf("existing update transaction does not match active attempt") } if existing.Phase == "staged" { if existing.StagedExecutable == "" || !pathWithinRoot(root, existing.StagedExecutable) || existing.BinaryChecksum == "" { return SelfUpdateManifest{}, fmt.Errorf("staged update manifest is outside the transaction workspace") } checksum, _, checksumErr := checksumFile(existing.StagedExecutable) if checksumErr != nil || checksum != existing.BinaryChecksum { return SelfUpdateManifest{}, fmt.Errorf("staged Run binary checksum changed") } } existing.Attempt = assignment.Attempt existing.LeaseToken = assignment.LeaseToken return existing, nil } else if !errors.Is(err, os.ErrNotExist) { return SelfUpdateManifest{}, err } current, err := os.Executable() if err != nil { return SelfUpdateManifest{}, err } current, err = filepath.Abs(current) if err != nil { return SelfUpdateManifest{}, err } info, err := os.Lstat(current) if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { return SelfUpdateManifest{}, fmt.Errorf("current Run executable is not a regular file") } workingDirectory, err := os.Getwd() if err != nil { return SelfUpdateManifest{}, err } now := time.Now().UTC() manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, CurrentExecutable: current, BackupExecutable: filepath.Join(root, "previous-run.backup"), HealthFile: filepath.Join(root, "healthy"), WorkingDirectory: workingDirectory, Phase: "downloading", CreatedAt: now, UpdatedAt: now} return manifest, nil } func pathWithinRoot(root, path string) bool { rootAbs, rootErr := filepath.Abs(root) pathAbs, pathErr := filepath.Abs(path) if rootErr != nil || pathErr != nil { return false } relative, err := filepath.Rel(rootAbs, pathAbs) return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && relative != "." } func (worker *Worker) downloadRunUpdate(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, archivePath, manifestPath string, manifest *SelfUpdateManifest) error { file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return err } defer file.Close() info, err := file.Stat() if err != nil { return err } offset := info.Size() if offset < 0 || offset > input.SizeBytes { return fmt.Errorf("partial update artifact has invalid size") } if _, err := file.Seek(offset, io.SeekStart); err != nil { return err } for offset < input.SizeBytes { if ctx.Err() != nil { return ctx.Err() } state, err := worker.registeredState() if err != nil { return err } length := input.ChunkSizeBytes if remaining := input.SizeBytes - offset; int64(length) > remaining { length = int(remaining) } chunk, err := worker.client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Offset: offset, Length: length}) if err != nil { return err } if chunk.JobID != assignment.JobID || chunk.ArtifactID != input.ArtifactID || chunk.Offset != offset || chunk.TotalBytes != input.SizeBytes || chunk.Checksum != input.Checksum || len(chunk.Payload) == 0 || len(chunk.Payload) > length { return fmt.Errorf("Run update chunk acknowledgement does not match request") } if _, err := file.Write(chunk.Payload); err != nil { return err } if err := file.Sync(); err != nil { return err } offset += int64(len(chunk.Payload)) manifest.DownloadedBytes = offset manifest.UpdatedAt = time.Now().UTC() if err := persistSelfUpdateManifest(manifestPath, *manifest); err != nil { return err } } if err := file.Close(); err != nil { return err } checksum, size, err := checksumFile(archivePath) if err != nil { return err } if size != input.SizeBytes || checksum != input.Checksum { _ = os.Remove(archivePath) return fmt.Errorf("Run update artifact checksum mismatch") } return nil } func stageRunUpdateBinary(artifactPath, format, executableName, destination string) (string, error) { if format != "raw-executable" { return extractRunUpdateBinary(artifactPath, format, executableName, destination) } info, err := os.Stat(artifactPath) if err != nil { return "", err } if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxSelfUpdateBytes { return "", fmt.Errorf("Run update executable exceeds bounds") } input, err := os.Open(artifactPath) if err != nil { return "", err } defer input.Close() temporary := destination + ".tmp" output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) if err != nil { return "", err } written, copyErr := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)) if copyErr == nil && written != info.Size() { copyErr = fmt.Errorf("Run update executable size does not match artifact") } if syncErr := output.Sync(); copyErr == nil { copyErr = syncErr } if closeErr := output.Close(); copyErr == nil { copyErr = closeErr } if copyErr != nil { _ = os.Remove(temporary) return "", copyErr } if err := os.Rename(temporary, destination); err != nil { _ = os.Remove(temporary) return "", err } if err := os.Chmod(destination, 0o700); err != nil { return "", err } checksum, _, err := checksumFile(destination) return checksum, err } func extractRunUpdateBinary(archivePath, format, executableName, destination string) (string, error) { found := false entries := 0 writeEntry := func(name string, mode os.FileMode, reader io.Reader, size int64) error { entries++ if entries > maxSelfUpdateEntries || size < 0 || size > maxSelfUpdateBytes { return fmt.Errorf("Run update archive exceeds bounds") } clean := filepath.ToSlash(filepath.Clean(name)) if clean != name || strings.Contains(clean, "../") || strings.HasPrefix(clean, "/") || strings.Contains(clean, `\`) { return fmt.Errorf("Run update archive entry is unsafe") } if clean == "config.json" { _, err := io.Copy(io.Discard, io.LimitReader(reader, size+1)) return err } if clean != executableName || found || mode&os.ModeSymlink != 0 { return fmt.Errorf("Run update archive contains unexpected entry") } found = true temporary := destination + ".tmp" file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) if err != nil { return err } written, copyErr := io.Copy(file, io.LimitReader(reader, maxSelfUpdateBytes+1)) if copyErr == nil && written != size { copyErr = fmt.Errorf("Run update binary size does not match archive") } if syncErr := file.Sync(); copyErr == nil { copyErr = syncErr } if closeErr := file.Close(); copyErr == nil { copyErr = closeErr } if copyErr != nil { _ = os.Remove(temporary) return copyErr } if err := os.Rename(temporary, destination); err != nil { _ = os.Remove(temporary) return err } return os.Chmod(destination, 0o700) } if format == "zip" { info, err := os.Stat(archivePath) if err != nil { return "", err } reader, err := zip.OpenReader(archivePath) if err != nil { return "", err } defer reader.Close() if info.Size() > maxSelfUpdateBytes { return "", fmt.Errorf("Run update archive exceeds size limit") } for _, entry := range reader.File { if entry.FileInfo().IsDir() || entry.Mode()&os.ModeType != 0 { return "", fmt.Errorf("Run update archive contains non-regular entry") } stream, err := entry.Open() if err != nil { return "", err } err = writeEntry(entry.Name, entry.Mode(), stream, int64(entry.UncompressedSize64)) _ = stream.Close() if err != nil { return "", err } } } else { file, err := os.Open(archivePath) if err != nil { return "", err } defer file.Close() gzipReader, err := gzip.NewReader(file) if err != nil { return "", err } defer gzipReader.Close() tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() if errors.Is(err, io.EOF) { break } if err != nil { return "", err } if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { return "", fmt.Errorf("Run update archive contains non-regular entry") } if err := writeEntry(header.Name, os.FileMode(header.Mode), tarReader, header.Size); err != nil { return "", err } } } if !found { return "", fmt.Errorf("Run update archive does not contain expected executable") } checksum, _, err := checksumFile(destination) return checksum, err } func ApplySelfUpdateManifest(manifestPath string) error { manifest, err := loadSelfUpdateManifest(manifestPath) if err != nil { return err } helper, err := os.Executable() if err != nil { return err } helper, _ = filepath.Abs(helper) staged, _ := filepath.Abs(manifest.StagedExecutable) if helper != staged || manifest.TargetOS != runtime.GOOS || manifest.TargetArch != runtime.GOARCH || manifest.Phase != "staged" { return fmt.Errorf("self-update helper scope does not match staged transaction") } manifest.Phase = "activating" manifest.UpdatedAt = time.Now().UTC() if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil { return err } if err := replaceRunExecutable(manifest); err != nil { manifest.Phase = "rolled-back" manifest.UpdatedAt = time.Now().UTC() _ = persistSelfUpdateManifest(manifestPath, manifest) _, _ = startRunAfterUpdate(manifest, "rolled-back") return err } _ = os.Remove(manifest.HealthFile) command, err := startRunAfterUpdate(manifest, "succeeded") if err != nil { _ = rollbackRunExecutable(manifest) _, _ = startRunAfterUpdate(manifest, "rolled-back") return err } wait := defaultUpdateHealthWait if value, parseErr := strconv.Atoi(os.Getenv("RUN_UPDATE_HEALTH_TIMEOUT_MS")); parseErr == nil && value > 0 && value <= 300000 { wait = time.Duration(value) * time.Millisecond } deadline := time.Now().Add(wait) for time.Now().Before(deadline) { if _, err := os.Stat(manifest.HealthFile); err == nil { manifest.Phase = "succeeded" manifest.UpdatedAt = time.Now().UTC() return persistSelfUpdateManifest(manifestPath, manifest) } time.Sleep(100 * time.Millisecond) } _ = command.Process.Kill() if err := rollbackRunExecutable(manifest); err != nil { return fmt.Errorf("updated Run health timed out and rollback failed: %w", err) } manifest.Phase = "rolled-back" manifest.UpdatedAt = time.Now().UTC() _ = persistSelfUpdateManifest(manifestPath, manifest) _, _ = startRunAfterUpdate(manifest, "rolled-back") return fmt.Errorf("updated Run did not become healthy before timeout") } func replaceRunExecutable(manifest SelfUpdateManifest) error { if checksum, _, err := checksumFile(manifest.StagedExecutable); err != nil || checksum != manifest.BinaryChecksum { return fmt.Errorf("staged Run binary checksum changed") } _ = os.Remove(manifest.BackupExecutable) var lastErr error for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { if err := os.Rename(manifest.CurrentExecutable, manifest.BackupExecutable); err != nil { lastErr = err continue } if err := copyExecutable(manifest.StagedExecutable, manifest.CurrentExecutable); err != nil { _ = os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable) return err } return nil } return fmt.Errorf("could not back up current Run executable: %w", lastErr) } func rollbackRunExecutable(manifest SelfUpdateManifest) error { if _, err := os.Stat(manifest.BackupExecutable); err != nil { return err } _ = os.Remove(manifest.CurrentExecutable) return os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable) } func startRunAfterUpdate(manifest SelfUpdateManifest, outcome string) (*exec.Cmd, error) { command := exec.Command(manifest.CurrentExecutable) command.Dir = manifest.WorkingDirectory environment := cleanUpdateEnvironment(os.Environ()) environment = append(environment, "RUN_MODE=worker", "RUN_UPDATE_JOB_ID="+manifest.JobID, "RUN_UPDATE_OUTCOME="+outcome, "RUN_UPDATE_ATTEMPT="+strconv.Itoa(manifest.Attempt), "RUN_UPDATE_LEASE_TOKEN="+manifest.LeaseToken) if outcome == "succeeded" { environment = append(environment, "RUN_VERSION="+manifest.TargetRelease, "RUN_UPDATE_HEALTH_FILE="+manifest.HealthFile) } command.Env = environment command.Stdout = io.Discard command.Stderr = io.Discard if err := command.Start(); err != nil { return command, err } return command, nil } func MarkSelfUpdateHealthy(path string) error { if strings.TrimSpace(path) == "" { return nil } return writeRuntimeAtomicFile(path, []byte("healthy\n"), 0o600) } func loadSelfUpdateManifest(path string) (SelfUpdateManifest, error) { body, err := os.ReadFile(path) if err != nil { return SelfUpdateManifest{}, err } var manifest SelfUpdateManifest if err := json.Unmarshal(body, &manifest); err != nil { return SelfUpdateManifest{}, fmt.Errorf("decode self-update manifest: %w", err) } if manifest.Version != selfUpdateManifestVersion || manifest.JobID == "" || manifest.Attempt <= 0 || manifest.LeaseToken == "" || manifest.ArtifactID == "" || !validSHA256(manifest.ArtifactChecksum) || manifest.ArtifactSizeBytes <= 0 || manifest.ArtifactSizeBytes > maxSelfUpdateBytes || !protocol.ValidLogicalFileKey(manifest.TargetRelease) { return SelfUpdateManifest{}, fmt.Errorf("self-update manifest is invalid") } return manifest, nil } func persistSelfUpdateManifest(path string, manifest SelfUpdateManifest) error { body, err := json.MarshalIndent(manifest, "", " ") if err != nil { return err } return writeRuntimeAtomicFile(path, body, 0o600) } func checksumFile(path string) (string, int64, error) { file, err := os.Open(path) if err != nil { return "", 0, err } defer file.Close() hash := sha256.New() size, err := io.Copy(hash, io.LimitReader(file, maxSelfUpdateBytes+1)) if err != nil { return "", 0, err } if size > maxSelfUpdateBytes { return "", size, fmt.Errorf("file exceeds self-update size limit") } return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil } func copyExecutable(source, destination string) error { input, err := os.Open(source) if err != nil { return err } defer input.Close() temporary := destination + ".update-tmp" output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) if err != nil { return err } if _, err := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)); err != nil { _ = output.Close() _ = os.Remove(temporary) return err } if err := output.Sync(); err != nil { _ = output.Close() _ = os.Remove(temporary) return err } if err := output.Close(); err != nil { _ = os.Remove(temporary) return err } if err := os.Rename(temporary, destination); err != nil { _ = os.Remove(temporary) return err } return os.Chmod(destination, 0o700) } func cleanUpdateEnvironment(environment []string) []string { blocked := map[string]bool{"RUN_UPDATE_MANIFEST": true, "RUN_UPDATE_HEALTH_FILE": true, "RUN_UPDATE_HEALTH_TIMEOUT_MS": true, "RUN_UPDATE_JOB_ID": true, "RUN_UPDATE_OUTCOME": true, "RUN_UPDATE_ATTEMPT": true, "RUN_UPDATE_LEASE_TOKEN": true, "RUN_MODE": true, "RUN_VERSION": true} out := make([]string, 0, len(environment)) for _, entry := range environment { key, _, _ := strings.Cut(entry, "=") if !blocked[key] { out = append(out, entry) } } return out }