init
This commit is contained in:
npc0-hue
@@ -0,0 +1,611 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
selfUpdateManifestVersion = 1
|
||||
maxSelfUpdateBytes = int64(512 * 1024 * 1024)
|
||||
maxSelfUpdateEntries = 8
|
||||
defaultUpdateHealthWait = 30 * time.Second
|
||||
)
|
||||
|
||||
var ErrSelfUpdateRestartRequested = errors.New("Run self-update restart requested")
|
||||
|
||||
type SelfUpdateManifest struct {
|
||||
Version int `json:"version"`
|
||||
JobID string `json:"jobId"`
|
||||
Attempt int `json:"attempt"`
|
||||
LeaseToken string `json:"leaseToken"`
|
||||
ArtifactID string `json:"artifactId"`
|
||||
ArtifactChecksum string `json:"artifactChecksum"`
|
||||
ArtifactSizeBytes int64 `json:"artifactSizeBytes"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
TargetRelease string `json:"targetRelease"`
|
||||
CurrentExecutable string `json:"currentExecutable"`
|
||||
StagedExecutable string `json:"stagedExecutable"`
|
||||
BackupExecutable string `json:"backupExecutable"`
|
||||
HealthFile string `json:"healthFile"`
|
||||
WorkingDirectory string `json:"workingDirectory"`
|
||||
Phase string `json:"phase"`
|
||||
DownloadedBytes int64 `json:"downloadedBytes"`
|
||||
BinaryChecksum string `json:"binaryChecksum,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type SelfUpdateActivator interface {
|
||||
Activate(string) error
|
||||
}
|
||||
|
||||
type ProcessSelfUpdateActivator struct{}
|
||||
|
||||
func (ProcessSelfUpdateActivator) Activate(manifestPath string) error {
|
||||
manifest, err := loadSelfUpdateManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
command := exec.Command(manifest.StagedExecutable)
|
||||
command.Dir = manifest.WorkingDirectory
|
||||
command.Env = append(cleanUpdateEnvironment(os.Environ()), "RUN_MODE=self-update-helper", "RUN_UPDATE_MANIFEST="+manifestPath)
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
return command.Start()
|
||||
}
|
||||
|
||||
func (worker *Worker) executeRunSelfUpdate(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
return lifecycleFailure("unsafe_self_update_job", err.Error())
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_unregistered", "Run worker is not registered")
|
||||
}
|
||||
input, err := worker.client.GetRunUpdateInput(ctx, protocol.RunUpdateInputRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt})
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_input_failed", "could not load fenced Run update input")
|
||||
}
|
||||
if err := validateRunUpdateInput(assignment, input); err != nil {
|
||||
return lifecycleFailure("unsafe_self_update_input", err.Error())
|
||||
}
|
||||
transactionRoot := filepath.Join(worker.cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID))
|
||||
if err := os.MkdirAll(transactionRoot, 0o700); err != nil {
|
||||
return lifecycleFailure("self_update_workspace_failed", "could not create update transaction workspace")
|
||||
}
|
||||
manifestPath := filepath.Join(transactionRoot, "manifest.json")
|
||||
archivePath := filepath.Join(transactionRoot, "update.archive")
|
||||
manifest, err := prepareSelfUpdateManifest(manifestPath, assignment, input, transactionRoot)
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
if manifest.Phase != "staged" {
|
||||
manifest.Phase = "downloading"
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
if err := worker.downloadRunUpdate(ctx, assignment, input, archivePath, manifestPath, &manifest); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update download cancelled"}, Message: "Run update download cancelled", ErrorCode: "run_self_update_cancelled"}
|
||||
}
|
||||
return lifecycleFailure("self_update_download_failed", err.Error())
|
||||
}
|
||||
stagedPath := filepath.Join(transactionRoot, input.ExecutableName+".staged")
|
||||
binaryChecksum, err := stageRunUpdateBinary(archivePath, input.PackageFormat, input.ExecutableName, stagedPath)
|
||||
if err != nil {
|
||||
return lifecycleFailure("self_update_extract_failed", err.Error())
|
||||
}
|
||||
manifest.StagedExecutable = stagedPath
|
||||
manifest.BinaryChecksum = binaryChecksum
|
||||
manifest.Phase = "staged"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
||||
}
|
||||
}
|
||||
evidence, _ := json.Marshal(protocol.RunUpdateExecutionEvidence{TargetRelease: input.TargetRelease, Phase: "staged"})
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", safeWorkspaceName(assignment.JobID)), Message: "Run update verified and staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: input.Checksum, SizeBytes: input.SizeBytes, Summary: "verified update staged", Content: string(evidence)}, ActivationManifest: manifestPath}
|
||||
}
|
||||
|
||||
func validateRunUpdateInput(assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse) error {
|
||||
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID || assignment.InputRef != "artifact://"+input.ArtifactID {
|
||||
return fmt.Errorf("Run update input scope does not match job")
|
||||
}
|
||||
if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH {
|
||||
return fmt.Errorf("Run update target does not match this executable")
|
||||
}
|
||||
if input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" && input.PackageFormat != "raw-executable" {
|
||||
return fmt.Errorf("Run update package format is unsupported")
|
||||
}
|
||||
if input.SizeBytes <= 0 || input.SizeBytes > maxSelfUpdateBytes || input.ChunkSizeBytes <= 0 || input.ChunkSizeBytes > 1024*1024 || !validSHA256(input.Checksum) {
|
||||
return fmt.Errorf("Run update artifact bounds are invalid")
|
||||
}
|
||||
expectedName := "run"
|
||||
if runtime.GOOS == "windows" {
|
||||
expectedName = "run.exe"
|
||||
}
|
||||
if input.ExecutableName != expectedName || !protocol.ValidLogicalFileKey(input.TargetRelease) {
|
||||
return fmt.Errorf("Run update executable or release identity is unsafe")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareSelfUpdateManifest(path string, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, root string) (SelfUpdateManifest, error) {
|
||||
if existing, err := loadSelfUpdateManifest(path); err == nil {
|
||||
if existing.JobID != assignment.JobID || existing.ArtifactID != input.ArtifactID || existing.ArtifactChecksum != input.Checksum || existing.TargetRelease != input.TargetRelease || existing.Attempt > assignment.Attempt {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("existing update transaction does not match active attempt")
|
||||
}
|
||||
if existing.Phase == "staged" {
|
||||
if existing.StagedExecutable == "" || !pathWithinRoot(root, existing.StagedExecutable) || existing.BinaryChecksum == "" {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("staged update manifest is outside the transaction workspace")
|
||||
}
|
||||
checksum, _, checksumErr := checksumFile(existing.StagedExecutable)
|
||||
if checksumErr != nil || checksum != existing.BinaryChecksum {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("staged Run binary checksum changed")
|
||||
}
|
||||
}
|
||||
existing.Attempt = assignment.Attempt
|
||||
existing.LeaseToken = assignment.LeaseToken
|
||||
return existing, nil
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
current, err := os.Executable()
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
current, err = filepath.Abs(current)
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
info, err := os.Lstat(current)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("current Run executable is not a regular file")
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, CurrentExecutable: current, BackupExecutable: filepath.Join(root, "previous-run.backup"), HealthFile: filepath.Join(root, "healthy"), WorkingDirectory: workingDirectory, Phase: "downloading", CreatedAt: now, UpdatedAt: now}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func pathWithinRoot(root, path string) bool {
|
||||
rootAbs, rootErr := filepath.Abs(root)
|
||||
pathAbs, pathErr := filepath.Abs(path)
|
||||
if rootErr != nil || pathErr != nil {
|
||||
return false
|
||||
}
|
||||
relative, err := filepath.Rel(rootAbs, pathAbs)
|
||||
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && relative != "."
|
||||
}
|
||||
|
||||
func (worker *Worker) downloadRunUpdate(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, archivePath, manifestPath string, manifest *SelfUpdateManifest) error {
|
||||
file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
offset := info.Size()
|
||||
if offset < 0 || offset > input.SizeBytes {
|
||||
return fmt.Errorf("partial update artifact has invalid size")
|
||||
}
|
||||
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
for offset < input.SizeBytes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
state, err := worker.registeredState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
length := input.ChunkSizeBytes
|
||||
if remaining := input.SizeBytes - offset; int64(length) > remaining {
|
||||
length = int(remaining)
|
||||
}
|
||||
chunk, err := worker.client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Offset: offset, Length: length})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if chunk.JobID != assignment.JobID || chunk.ArtifactID != input.ArtifactID || chunk.Offset != offset || chunk.TotalBytes != input.SizeBytes || chunk.Checksum != input.Checksum || len(chunk.Payload) == 0 || len(chunk.Payload) > length {
|
||||
return fmt.Errorf("Run update chunk acknowledgement does not match request")
|
||||
}
|
||||
if _, err := file.Write(chunk.Payload); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
offset += int64(len(chunk.Payload))
|
||||
manifest.DownloadedBytes = offset
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, *manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
checksum, size, err := checksumFile(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size != input.SizeBytes || checksum != input.Checksum {
|
||||
_ = os.Remove(archivePath)
|
||||
return fmt.Errorf("Run update artifact checksum mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stageRunUpdateBinary(artifactPath, format, executableName, destination string) (string, error) {
|
||||
if format != "raw-executable" {
|
||||
return extractRunUpdateBinary(artifactPath, format, executableName, destination)
|
||||
}
|
||||
info, err := os.Stat(artifactPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxSelfUpdateBytes {
|
||||
return "", fmt.Errorf("Run update executable exceeds bounds")
|
||||
}
|
||||
input, err := os.Open(artifactPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer input.Close()
|
||||
temporary := destination + ".tmp"
|
||||
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
written, copyErr := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1))
|
||||
if copyErr == nil && written != info.Size() {
|
||||
copyErr = fmt.Errorf("Run update executable size does not match artifact")
|
||||
}
|
||||
if syncErr := output.Sync(); copyErr == nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := output.Close(); copyErr == nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return "", copyErr
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return "", err
|
||||
}
|
||||
if err := os.Chmod(destination, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
checksum, _, err := checksumFile(destination)
|
||||
return checksum, err
|
||||
}
|
||||
|
||||
func extractRunUpdateBinary(archivePath, format, executableName, destination string) (string, error) {
|
||||
found := false
|
||||
entries := 0
|
||||
writeEntry := func(name string, mode os.FileMode, reader io.Reader, size int64) error {
|
||||
entries++
|
||||
if entries > maxSelfUpdateEntries || size < 0 || size > maxSelfUpdateBytes {
|
||||
return fmt.Errorf("Run update archive exceeds bounds")
|
||||
}
|
||||
clean := filepath.ToSlash(filepath.Clean(name))
|
||||
if clean != name || strings.Contains(clean, "../") || strings.HasPrefix(clean, "/") || strings.Contains(clean, `\`) {
|
||||
return fmt.Errorf("Run update archive entry is unsafe")
|
||||
}
|
||||
if clean == "config.json" {
|
||||
_, err := io.Copy(io.Discard, io.LimitReader(reader, size+1))
|
||||
return err
|
||||
}
|
||||
if clean != executableName || found || mode&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("Run update archive contains unexpected entry")
|
||||
}
|
||||
found = true
|
||||
temporary := destination + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
written, copyErr := io.Copy(file, io.LimitReader(reader, maxSelfUpdateBytes+1))
|
||||
if copyErr == nil && written != size {
|
||||
copyErr = fmt.Errorf("Run update binary size does not match archive")
|
||||
}
|
||||
if syncErr := file.Sync(); copyErr == nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := file.Close(); copyErr == nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return copyErr
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return os.Chmod(destination, 0o700)
|
||||
}
|
||||
|
||||
if format == "zip" {
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reader, err := zip.OpenReader(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
if info.Size() > maxSelfUpdateBytes {
|
||||
return "", fmt.Errorf("Run update archive exceeds size limit")
|
||||
}
|
||||
for _, entry := range reader.File {
|
||||
if entry.FileInfo().IsDir() || entry.Mode()&os.ModeType != 0 {
|
||||
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
||||
}
|
||||
stream, err := entry.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = writeEntry(entry.Name, entry.Mode(), stream, int64(entry.UncompressedSize64))
|
||||
_ = stream.Close()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA {
|
||||
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
||||
}
|
||||
if err := writeEntry(header.Name, os.FileMode(header.Mode), tarReader, header.Size); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("Run update archive does not contain expected executable")
|
||||
}
|
||||
checksum, _, err := checksumFile(destination)
|
||||
return checksum, err
|
||||
}
|
||||
|
||||
func ApplySelfUpdateManifest(manifestPath string) error {
|
||||
manifest, err := loadSelfUpdateManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
helper, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
helper, _ = filepath.Abs(helper)
|
||||
staged, _ := filepath.Abs(manifest.StagedExecutable)
|
||||
if helper != staged || manifest.TargetOS != runtime.GOOS || manifest.TargetArch != runtime.GOARCH || manifest.Phase != "staged" {
|
||||
return fmt.Errorf("self-update helper scope does not match staged transaction")
|
||||
}
|
||||
manifest.Phase = "activating"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := replaceRunExecutable(manifest); err != nil {
|
||||
manifest.Phase = "rolled-back"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(manifest.HealthFile)
|
||||
command, err := startRunAfterUpdate(manifest, "succeeded")
|
||||
if err != nil {
|
||||
_ = rollbackRunExecutable(manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return err
|
||||
}
|
||||
wait := defaultUpdateHealthWait
|
||||
if value, parseErr := strconv.Atoi(os.Getenv("RUN_UPDATE_HEALTH_TIMEOUT_MS")); parseErr == nil && value > 0 && value <= 300000 {
|
||||
wait = time.Duration(value) * time.Millisecond
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(manifest.HealthFile); err == nil {
|
||||
manifest.Phase = "succeeded"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
return persistSelfUpdateManifest(manifestPath, manifest)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
_ = command.Process.Kill()
|
||||
if err := rollbackRunExecutable(manifest); err != nil {
|
||||
return fmt.Errorf("updated Run health timed out and rollback failed: %w", err)
|
||||
}
|
||||
manifest.Phase = "rolled-back"
|
||||
manifest.UpdatedAt = time.Now().UTC()
|
||||
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
||||
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
||||
return fmt.Errorf("updated Run did not become healthy before timeout")
|
||||
}
|
||||
|
||||
func replaceRunExecutable(manifest SelfUpdateManifest) error {
|
||||
if checksum, _, err := checksumFile(manifest.StagedExecutable); err != nil || checksum != manifest.BinaryChecksum {
|
||||
return fmt.Errorf("staged Run binary checksum changed")
|
||||
}
|
||||
_ = os.Remove(manifest.BackupExecutable)
|
||||
var lastErr error
|
||||
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if err := os.Rename(manifest.CurrentExecutable, manifest.BackupExecutable); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err := copyExecutable(manifest.StagedExecutable, manifest.CurrentExecutable); err != nil {
|
||||
_ = os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("could not back up current Run executable: %w", lastErr)
|
||||
}
|
||||
|
||||
func rollbackRunExecutable(manifest SelfUpdateManifest) error {
|
||||
if _, err := os.Stat(manifest.BackupExecutable); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(manifest.CurrentExecutable)
|
||||
return os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
||||
}
|
||||
|
||||
func startRunAfterUpdate(manifest SelfUpdateManifest, outcome string) (*exec.Cmd, error) {
|
||||
command := exec.Command(manifest.CurrentExecutable)
|
||||
command.Dir = manifest.WorkingDirectory
|
||||
environment := cleanUpdateEnvironment(os.Environ())
|
||||
environment = append(environment, "RUN_MODE=worker", "RUN_UPDATE_JOB_ID="+manifest.JobID, "RUN_UPDATE_OUTCOME="+outcome, "RUN_UPDATE_ATTEMPT="+strconv.Itoa(manifest.Attempt), "RUN_UPDATE_LEASE_TOKEN="+manifest.LeaseToken)
|
||||
if outcome == "succeeded" {
|
||||
environment = append(environment, "RUN_VERSION="+manifest.TargetRelease, "RUN_UPDATE_HEALTH_FILE="+manifest.HealthFile)
|
||||
}
|
||||
command.Env = environment
|
||||
command.Stdout = io.Discard
|
||||
command.Stderr = io.Discard
|
||||
if err := command.Start(); err != nil {
|
||||
return command, err
|
||||
}
|
||||
return command, nil
|
||||
}
|
||||
|
||||
func MarkSelfUpdateHealthy(path string) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, []byte("healthy\n"), 0o600)
|
||||
}
|
||||
|
||||
func loadSelfUpdateManifest(path string) (SelfUpdateManifest, error) {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return SelfUpdateManifest{}, err
|
||||
}
|
||||
var manifest SelfUpdateManifest
|
||||
if err := json.Unmarshal(body, &manifest); err != nil {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("decode self-update manifest: %w", err)
|
||||
}
|
||||
if manifest.Version != selfUpdateManifestVersion || manifest.JobID == "" || manifest.Attempt <= 0 || manifest.LeaseToken == "" || manifest.ArtifactID == "" || !validSHA256(manifest.ArtifactChecksum) || manifest.ArtifactSizeBytes <= 0 || manifest.ArtifactSizeBytes > maxSelfUpdateBytes || !protocol.ValidLogicalFileKey(manifest.TargetRelease) {
|
||||
return SelfUpdateManifest{}, fmt.Errorf("self-update manifest is invalid")
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func persistSelfUpdateManifest(path string, manifest SelfUpdateManifest) error {
|
||||
body, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeRuntimeAtomicFile(path, body, 0o600)
|
||||
}
|
||||
|
||||
func checksumFile(path string) (string, int64, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, io.LimitReader(file, maxSelfUpdateBytes+1))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if size > maxSelfUpdateBytes {
|
||||
return "", size, fmt.Errorf("file exceeds self-update size limit")
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil
|
||||
}
|
||||
|
||||
func copyExecutable(source, destination string) error {
|
||||
input, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer input.Close()
|
||||
temporary := destination + ".update-tmp"
|
||||
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)); err != nil {
|
||||
_ = output.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := output.Sync(); err != nil {
|
||||
_ = output.Close()
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return os.Chmod(destination, 0o700)
|
||||
}
|
||||
|
||||
func cleanUpdateEnvironment(environment []string) []string {
|
||||
blocked := map[string]bool{"RUN_UPDATE_MANIFEST": true, "RUN_UPDATE_HEALTH_FILE": true, "RUN_UPDATE_HEALTH_TIMEOUT_MS": true, "RUN_UPDATE_JOB_ID": true, "RUN_UPDATE_OUTCOME": true, "RUN_UPDATE_ATTEMPT": true, "RUN_UPDATE_LEASE_TOKEN": true, "RUN_MODE": true, "RUN_VERSION": true}
|
||||
out := make([]string, 0, len(environment))
|
||||
for _, entry := range environment {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if !blocked[key] {
|
||||
out = append(out, entry)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user