628 lines
23 KiB
Go
628 lines
23 KiB
Go
package runtime
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
const (
|
|
selfUpdateManifestVersion = 1
|
|
maxSelfUpdateBytes = int64(512 * 1024 * 1024)
|
|
maxSelfUpdateEntries = 8
|
|
defaultUpdateHealthWait = 30 * time.Second
|
|
selfUpdateProgressEvery = int64(8 * 1024 * 1024)
|
|
)
|
|
|
|
var ErrSelfUpdateRestartRequested = errors.New("Run self-update restart requested")
|
|
|
|
type SelfUpdateManifest struct {
|
|
Version int `json:"version"`
|
|
JobID string `json:"jobId"`
|
|
Attempt int `json:"attempt"`
|
|
LeaseToken string `json:"leaseToken"`
|
|
ArtifactID string `json:"artifactId"`
|
|
ArtifactChecksum string `json:"artifactChecksum"`
|
|
ArtifactSizeBytes int64 `json:"artifactSizeBytes"`
|
|
TargetOS string `json:"targetOs"`
|
|
TargetArch string `json:"targetArch"`
|
|
TargetRelease string `json:"targetRelease"`
|
|
CurrentExecutable string `json:"currentExecutable"`
|
|
StagedExecutable string `json:"stagedExecutable"`
|
|
BackupExecutable string `json:"backupExecutable"`
|
|
HealthFile string `json:"healthFile"`
|
|
WorkingDirectory string `json:"workingDirectory"`
|
|
Phase string `json:"phase"`
|
|
DownloadedBytes int64 `json:"downloadedBytes"`
|
|
BinaryChecksum string `json:"binaryChecksum,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type SelfUpdateActivator interface {
|
|
Activate(string) error
|
|
}
|
|
|
|
type ProcessSelfUpdateActivator struct{}
|
|
|
|
func (ProcessSelfUpdateActivator) Activate(manifestPath string) error {
|
|
manifest, err := loadSelfUpdateManifest(manifestPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
command := exec.Command(manifest.StagedExecutable)
|
|
command.Dir = manifest.WorkingDirectory
|
|
command.Env = append(cleanUpdateEnvironment(os.Environ()), "RUN_MODE=self-update-helper", "RUN_UPDATE_MANIFEST="+manifestPath)
|
|
command.Stdout = io.Discard
|
|
command.Stderr = io.Discard
|
|
return command.Start()
|
|
}
|
|
|
|
func (worker *Worker) executeRunSelfUpdate(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
|
return lifecycleFailure("unsafe_self_update_job", err.Error())
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return lifecycleFailure("self_update_unregistered", "Run worker is not registered")
|
|
}
|
|
input, err := worker.client.GetRunUpdateInput(ctx, protocol.RunUpdateInputRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt})
|
|
if err != nil {
|
|
return lifecycleFailure("self_update_input_failed", "could not load fenced Run update input")
|
|
}
|
|
if err := validateRunUpdateInput(assignment, input); err != nil {
|
|
return lifecycleFailure("unsafe_self_update_input", err.Error())
|
|
}
|
|
transactionRoot := filepath.Join(worker.cfg.WorkspaceRoot, "self-updates", safeWorkspaceName(assignment.JobID))
|
|
if err := os.MkdirAll(transactionRoot, 0o700); err != nil {
|
|
return lifecycleFailure("self_update_workspace_failed", "could not create update transaction workspace")
|
|
}
|
|
manifestPath := filepath.Join(transactionRoot, "manifest.json")
|
|
archivePath := filepath.Join(transactionRoot, "update.archive")
|
|
manifest, err := prepareSelfUpdateManifest(manifestPath, assignment, input, transactionRoot)
|
|
if err != nil {
|
|
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
|
}
|
|
if manifest.Phase != "staged" {
|
|
manifest.Phase = "downloading"
|
|
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
|
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
|
}
|
|
if err := worker.downloadRunUpdate(ctx, assignment, input, archivePath, manifestPath, &manifest); err != nil {
|
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
return LifecycleExecutionResult{State: lifecycleResultStateCancelled, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update download cancelled"}, Message: "Run update download cancelled", ErrorCode: "run_self_update_cancelled"}
|
|
}
|
|
return lifecycleFailure("self_update_download_failed", err.Error())
|
|
}
|
|
stagedPath := filepath.Join(transactionRoot, input.ExecutableName+".staged")
|
|
binaryChecksum, err := stageRunUpdateBinary(archivePath, input.PackageFormat, input.ExecutableName, stagedPath)
|
|
if err != nil {
|
|
return lifecycleFailure("self_update_extract_failed", err.Error())
|
|
}
|
|
manifest.StagedExecutable = stagedPath
|
|
manifest.BinaryChecksum = binaryChecksum
|
|
manifest.Phase = "staged"
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
|
return lifecycleFailure("self_update_manifest_failed", err.Error())
|
|
}
|
|
}
|
|
evidence, _ := json.Marshal(protocol.RunUpdateExecutionEvidence{TargetRelease: input.TargetRelease, Phase: "staged"})
|
|
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: fmt.Sprintf("artifact://jobs/%s/run-update-staged", safeWorkspaceName(assignment.JobID)), Message: "Run update verified and staged", ExecutionResult: protocol.RunJobExecutionResult{Kind: "run.update.staged", Checksum: input.Checksum, SizeBytes: input.SizeBytes, Summary: "verified update staged", Content: string(evidence)}, ActivationManifest: manifestPath}
|
|
}
|
|
|
|
func validateRunUpdateInput(assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse) error {
|
|
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID || input.RunEndpointID != assignment.RunEndpointID || assignment.InputRef != "artifact://"+input.ArtifactID {
|
|
return fmt.Errorf("Run update input scope does not match job")
|
|
}
|
|
if input.TargetOS != runtime.GOOS || input.TargetArch != runtime.GOARCH {
|
|
return fmt.Errorf("Run update target does not match this executable")
|
|
}
|
|
if input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" && input.PackageFormat != "raw-executable" {
|
|
return fmt.Errorf("Run update package format is unsupported")
|
|
}
|
|
if input.SizeBytes <= 0 || input.SizeBytes > maxSelfUpdateBytes || input.ChunkSizeBytes <= 0 || input.ChunkSizeBytes > 1024*1024 || !validSHA256(input.Checksum) {
|
|
return fmt.Errorf("Run update artifact bounds are invalid")
|
|
}
|
|
expectedName := "run"
|
|
if runtime.GOOS == "windows" {
|
|
expectedName = "run.exe"
|
|
}
|
|
if input.ExecutableName != expectedName || !protocol.ValidLogicalFileKey(input.TargetRelease) {
|
|
return fmt.Errorf("Run update executable or release identity is unsafe")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func prepareSelfUpdateManifest(path string, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, root string) (SelfUpdateManifest, error) {
|
|
if existing, err := loadSelfUpdateManifest(path); err == nil {
|
|
if existing.JobID != assignment.JobID || existing.ArtifactID != input.ArtifactID || existing.ArtifactChecksum != input.Checksum || existing.TargetRelease != input.TargetRelease || existing.Attempt > assignment.Attempt {
|
|
return SelfUpdateManifest{}, fmt.Errorf("existing update transaction does not match active attempt")
|
|
}
|
|
if existing.Phase == "staged" {
|
|
if existing.StagedExecutable == "" || !pathWithinRoot(root, existing.StagedExecutable) || existing.BinaryChecksum == "" {
|
|
return SelfUpdateManifest{}, fmt.Errorf("staged update manifest is outside the transaction workspace")
|
|
}
|
|
checksum, _, checksumErr := checksumFile(existing.StagedExecutable)
|
|
if checksumErr != nil || checksum != existing.BinaryChecksum {
|
|
return SelfUpdateManifest{}, fmt.Errorf("staged Run binary checksum changed")
|
|
}
|
|
}
|
|
existing.Attempt = assignment.Attempt
|
|
existing.LeaseToken = assignment.LeaseToken
|
|
return existing, nil
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return SelfUpdateManifest{}, err
|
|
}
|
|
current, err := os.Executable()
|
|
if err != nil {
|
|
return SelfUpdateManifest{}, err
|
|
}
|
|
current, err = filepath.Abs(current)
|
|
if err != nil {
|
|
return SelfUpdateManifest{}, err
|
|
}
|
|
info, err := os.Lstat(current)
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
return SelfUpdateManifest{}, fmt.Errorf("current Run executable is not a regular file")
|
|
}
|
|
workingDirectory, err := os.Getwd()
|
|
if err != nil {
|
|
return SelfUpdateManifest{}, err
|
|
}
|
|
now := time.Now().UTC()
|
|
manifest := SelfUpdateManifest{Version: selfUpdateManifestVersion, JobID: assignment.JobID, Attempt: assignment.Attempt, LeaseToken: assignment.LeaseToken, ArtifactID: input.ArtifactID, ArtifactChecksum: input.Checksum, ArtifactSizeBytes: input.SizeBytes, TargetOS: input.TargetOS, TargetArch: input.TargetArch, TargetRelease: input.TargetRelease, CurrentExecutable: current, BackupExecutable: filepath.Join(root, "previous-run.backup"), HealthFile: filepath.Join(root, "healthy"), WorkingDirectory: workingDirectory, Phase: "downloading", CreatedAt: now, UpdatedAt: now}
|
|
return manifest, nil
|
|
}
|
|
|
|
func pathWithinRoot(root, path string) bool {
|
|
rootAbs, rootErr := filepath.Abs(root)
|
|
pathAbs, pathErr := filepath.Abs(path)
|
|
if rootErr != nil || pathErr != nil {
|
|
return false
|
|
}
|
|
relative, err := filepath.Rel(rootAbs, pathAbs)
|
|
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && relative != "."
|
|
}
|
|
|
|
func (worker *Worker) downloadRunUpdate(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.RunUpdateInputResponse, archivePath, manifestPath string, manifest *SelfUpdateManifest) error {
|
|
file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
offset := info.Size()
|
|
if offset < 0 || offset > input.SizeBytes {
|
|
return fmt.Errorf("partial update artifact has invalid size")
|
|
}
|
|
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
|
return err
|
|
}
|
|
manifest.DownloadedBytes = offset
|
|
lastPersistedOffset := offset
|
|
persistProgress := func(force bool) error {
|
|
if !force && offset < input.SizeBytes && offset-lastPersistedOffset < selfUpdateProgressEvery {
|
|
return nil
|
|
}
|
|
manifest.DownloadedBytes = offset
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
if err := persistSelfUpdateManifest(manifestPath, *manifest); err != nil {
|
|
return err
|
|
}
|
|
lastPersistedOffset = offset
|
|
return nil
|
|
}
|
|
for offset < input.SizeBytes {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
length := input.ChunkSizeBytes
|
|
if remaining := input.SizeBytes - offset; int64(length) > remaining {
|
|
length = int(remaining)
|
|
}
|
|
chunk, err := worker.client.ReadRunUpdateChunk(ctx, protocol.RunUpdateChunkRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, JobID: assignment.JobID, LeaseToken: assignment.LeaseToken, Attempt: assignment.Attempt, Offset: offset, Length: length})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if chunk.JobID != assignment.JobID || chunk.ArtifactID != input.ArtifactID || chunk.Offset != offset || chunk.TotalBytes != input.SizeBytes || chunk.Checksum != input.Checksum || len(chunk.Payload) == 0 || len(chunk.Payload) > length {
|
|
return fmt.Errorf("Run update chunk acknowledgement does not match request")
|
|
}
|
|
if _, err := file.Write(chunk.Payload); err != nil {
|
|
return err
|
|
}
|
|
offset += int64(len(chunk.Payload))
|
|
if err := persistProgress(false); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := file.Sync(); err != nil {
|
|
return err
|
|
}
|
|
if err := persistProgress(true); err != nil {
|
|
return err
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
return err
|
|
}
|
|
checksum, size, err := checksumFile(archivePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if size != input.SizeBytes || checksum != input.Checksum {
|
|
_ = os.Remove(archivePath)
|
|
return fmt.Errorf("Run update artifact checksum mismatch")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func stageRunUpdateBinary(artifactPath, format, executableName, destination string) (string, error) {
|
|
if format != "raw-executable" {
|
|
return extractRunUpdateBinary(artifactPath, format, executableName, destination)
|
|
}
|
|
info, err := os.Stat(artifactPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxSelfUpdateBytes {
|
|
return "", fmt.Errorf("Run update executable exceeds bounds")
|
|
}
|
|
input, err := os.Open(artifactPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer input.Close()
|
|
temporary := destination + ".tmp"
|
|
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
written, copyErr := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1))
|
|
if copyErr == nil && written != info.Size() {
|
|
copyErr = fmt.Errorf("Run update executable size does not match artifact")
|
|
}
|
|
if syncErr := output.Sync(); copyErr == nil {
|
|
copyErr = syncErr
|
|
}
|
|
if closeErr := output.Close(); copyErr == nil {
|
|
copyErr = closeErr
|
|
}
|
|
if copyErr != nil {
|
|
_ = os.Remove(temporary)
|
|
return "", copyErr
|
|
}
|
|
if err := os.Rename(temporary, destination); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return "", err
|
|
}
|
|
if err := os.Chmod(destination, 0o700); err != nil {
|
|
return "", err
|
|
}
|
|
checksum, _, err := checksumFile(destination)
|
|
return checksum, err
|
|
}
|
|
|
|
func extractRunUpdateBinary(archivePath, format, executableName, destination string) (string, error) {
|
|
found := false
|
|
entries := 0
|
|
writeEntry := func(name string, mode os.FileMode, reader io.Reader, size int64) error {
|
|
entries++
|
|
if entries > maxSelfUpdateEntries || size < 0 || size > maxSelfUpdateBytes {
|
|
return fmt.Errorf("Run update archive exceeds bounds")
|
|
}
|
|
clean := filepath.ToSlash(filepath.Clean(name))
|
|
if clean != name || strings.Contains(clean, "../") || strings.HasPrefix(clean, "/") || strings.Contains(clean, `\`) {
|
|
return fmt.Errorf("Run update archive entry is unsafe")
|
|
}
|
|
if clean == "config.json" {
|
|
_, err := io.Copy(io.Discard, io.LimitReader(reader, size+1))
|
|
return err
|
|
}
|
|
if clean != executableName || found || mode&os.ModeSymlink != 0 {
|
|
return fmt.Errorf("Run update archive contains unexpected entry")
|
|
}
|
|
found = true
|
|
temporary := destination + ".tmp"
|
|
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
written, copyErr := io.Copy(file, io.LimitReader(reader, maxSelfUpdateBytes+1))
|
|
if copyErr == nil && written != size {
|
|
copyErr = fmt.Errorf("Run update binary size does not match archive")
|
|
}
|
|
if syncErr := file.Sync(); copyErr == nil {
|
|
copyErr = syncErr
|
|
}
|
|
if closeErr := file.Close(); copyErr == nil {
|
|
copyErr = closeErr
|
|
}
|
|
if copyErr != nil {
|
|
_ = os.Remove(temporary)
|
|
return copyErr
|
|
}
|
|
if err := os.Rename(temporary, destination); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return err
|
|
}
|
|
return os.Chmod(destination, 0o700)
|
|
}
|
|
|
|
if format == "zip" {
|
|
info, err := os.Stat(archivePath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
reader, err := zip.OpenReader(archivePath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer reader.Close()
|
|
if info.Size() > maxSelfUpdateBytes {
|
|
return "", fmt.Errorf("Run update archive exceeds size limit")
|
|
}
|
|
for _, entry := range reader.File {
|
|
if entry.FileInfo().IsDir() || entry.Mode()&os.ModeType != 0 {
|
|
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
|
}
|
|
stream, err := entry.Open()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
err = writeEntry(entry.Name, entry.Mode(), stream, int64(entry.UncompressedSize64))
|
|
_ = stream.Close()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
} else {
|
|
file, err := os.Open(archivePath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
gzipReader, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer gzipReader.Close()
|
|
tarReader := tar.NewReader(gzipReader)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA {
|
|
return "", fmt.Errorf("Run update archive contains non-regular entry")
|
|
}
|
|
if err := writeEntry(header.Name, os.FileMode(header.Mode), tarReader, header.Size); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
return "", fmt.Errorf("Run update archive does not contain expected executable")
|
|
}
|
|
checksum, _, err := checksumFile(destination)
|
|
return checksum, err
|
|
}
|
|
|
|
func ApplySelfUpdateManifest(manifestPath string) error {
|
|
manifest, err := loadSelfUpdateManifest(manifestPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
helper, err := os.Executable()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
helper, _ = filepath.Abs(helper)
|
|
staged, _ := filepath.Abs(manifest.StagedExecutable)
|
|
if helper != staged || manifest.TargetOS != runtime.GOOS || manifest.TargetArch != runtime.GOARCH || manifest.Phase != "staged" {
|
|
return fmt.Errorf("self-update helper scope does not match staged transaction")
|
|
}
|
|
manifest.Phase = "activating"
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
if err := persistSelfUpdateManifest(manifestPath, manifest); err != nil {
|
|
return err
|
|
}
|
|
if err := replaceRunExecutable(manifest); err != nil {
|
|
manifest.Phase = "rolled-back"
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
|
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
|
return err
|
|
}
|
|
_ = os.Remove(manifest.HealthFile)
|
|
command, err := startRunAfterUpdate(manifest, "succeeded")
|
|
if err != nil {
|
|
_ = rollbackRunExecutable(manifest)
|
|
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
|
return err
|
|
}
|
|
wait := defaultUpdateHealthWait
|
|
if value, parseErr := strconv.Atoi(os.Getenv("RUN_UPDATE_HEALTH_TIMEOUT_MS")); parseErr == nil && value > 0 && value <= 300000 {
|
|
wait = time.Duration(value) * time.Millisecond
|
|
}
|
|
deadline := time.Now().Add(wait)
|
|
for time.Now().Before(deadline) {
|
|
if _, err := os.Stat(manifest.HealthFile); err == nil {
|
|
manifest.Phase = "succeeded"
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
return persistSelfUpdateManifest(manifestPath, manifest)
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
_ = command.Process.Kill()
|
|
if err := rollbackRunExecutable(manifest); err != nil {
|
|
return fmt.Errorf("updated Run health timed out and rollback failed: %w", err)
|
|
}
|
|
manifest.Phase = "rolled-back"
|
|
manifest.UpdatedAt = time.Now().UTC()
|
|
_ = persistSelfUpdateManifest(manifestPath, manifest)
|
|
_, _ = startRunAfterUpdate(manifest, "rolled-back")
|
|
return fmt.Errorf("updated Run did not become healthy before timeout")
|
|
}
|
|
|
|
func replaceRunExecutable(manifest SelfUpdateManifest) error {
|
|
if checksum, _, err := checksumFile(manifest.StagedExecutable); err != nil || checksum != manifest.BinaryChecksum {
|
|
return fmt.Errorf("staged Run binary checksum changed")
|
|
}
|
|
_ = os.Remove(manifest.BackupExecutable)
|
|
var lastErr error
|
|
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if err := os.Rename(manifest.CurrentExecutable, manifest.BackupExecutable); err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
if err := copyExecutable(manifest.StagedExecutable, manifest.CurrentExecutable); err != nil {
|
|
_ = os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("could not back up current Run executable: %w", lastErr)
|
|
}
|
|
|
|
func rollbackRunExecutable(manifest SelfUpdateManifest) error {
|
|
if _, err := os.Stat(manifest.BackupExecutable); err != nil {
|
|
return err
|
|
}
|
|
_ = os.Remove(manifest.CurrentExecutable)
|
|
return os.Rename(manifest.BackupExecutable, manifest.CurrentExecutable)
|
|
}
|
|
|
|
func startRunAfterUpdate(manifest SelfUpdateManifest, outcome string) (*exec.Cmd, error) {
|
|
command := exec.Command(manifest.CurrentExecutable)
|
|
command.Dir = manifest.WorkingDirectory
|
|
environment := cleanUpdateEnvironment(os.Environ())
|
|
environment = append(environment, "RUN_MODE=worker", "RUN_UPDATE_JOB_ID="+manifest.JobID, "RUN_UPDATE_OUTCOME="+outcome, "RUN_UPDATE_ATTEMPT="+strconv.Itoa(manifest.Attempt), "RUN_UPDATE_LEASE_TOKEN="+manifest.LeaseToken)
|
|
if outcome == "succeeded" {
|
|
environment = append(environment, "RUN_VERSION="+manifest.TargetRelease, "RUN_UPDATE_HEALTH_FILE="+manifest.HealthFile)
|
|
}
|
|
command.Env = environment
|
|
command.Stdout = io.Discard
|
|
command.Stderr = io.Discard
|
|
if err := command.Start(); err != nil {
|
|
return command, err
|
|
}
|
|
return command, nil
|
|
}
|
|
|
|
func MarkSelfUpdateHealthy(path string) error {
|
|
if strings.TrimSpace(path) == "" {
|
|
return nil
|
|
}
|
|
return writeRuntimeAtomicFile(path, []byte("healthy\n"), 0o600)
|
|
}
|
|
|
|
func loadSelfUpdateManifest(path string) (SelfUpdateManifest, error) {
|
|
body, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return SelfUpdateManifest{}, err
|
|
}
|
|
var manifest SelfUpdateManifest
|
|
if err := json.Unmarshal(body, &manifest); err != nil {
|
|
return SelfUpdateManifest{}, fmt.Errorf("decode self-update manifest: %w", err)
|
|
}
|
|
if manifest.Version != selfUpdateManifestVersion || manifest.JobID == "" || manifest.Attempt <= 0 || manifest.LeaseToken == "" || manifest.ArtifactID == "" || !validSHA256(manifest.ArtifactChecksum) || manifest.ArtifactSizeBytes <= 0 || manifest.ArtifactSizeBytes > maxSelfUpdateBytes || !protocol.ValidLogicalFileKey(manifest.TargetRelease) {
|
|
return SelfUpdateManifest{}, fmt.Errorf("self-update manifest is invalid")
|
|
}
|
|
return manifest, nil
|
|
}
|
|
|
|
func persistSelfUpdateManifest(path string, manifest SelfUpdateManifest) error {
|
|
body, err := json.MarshalIndent(manifest, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return writeRuntimeAtomicFile(path, body, 0o600)
|
|
}
|
|
|
|
func checksumFile(path string) (string, int64, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
defer file.Close()
|
|
hash := sha256.New()
|
|
size, err := io.Copy(hash, io.LimitReader(file, maxSelfUpdateBytes+1))
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if size > maxSelfUpdateBytes {
|
|
return "", size, fmt.Errorf("file exceeds self-update size limit")
|
|
}
|
|
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), size, nil
|
|
}
|
|
|
|
func copyExecutable(source, destination string) error {
|
|
input, err := os.Open(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer input.Close()
|
|
temporary := destination + ".update-tmp"
|
|
output, err := os.OpenFile(temporary, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(output, io.LimitReader(input, maxSelfUpdateBytes+1)); err != nil {
|
|
_ = output.Close()
|
|
_ = os.Remove(temporary)
|
|
return err
|
|
}
|
|
if err := output.Sync(); err != nil {
|
|
_ = output.Close()
|
|
_ = os.Remove(temporary)
|
|
return err
|
|
}
|
|
if err := output.Close(); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return err
|
|
}
|
|
if err := os.Rename(temporary, destination); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return err
|
|
}
|
|
return os.Chmod(destination, 0o700)
|
|
}
|
|
|
|
func cleanUpdateEnvironment(environment []string) []string {
|
|
blocked := map[string]bool{"RUN_UPDATE_MANIFEST": true, "RUN_UPDATE_HEALTH_FILE": true, "RUN_UPDATE_HEALTH_TIMEOUT_MS": true, "RUN_UPDATE_JOB_ID": true, "RUN_UPDATE_OUTCOME": true, "RUN_UPDATE_ATTEMPT": true, "RUN_UPDATE_LEASE_TOKEN": true, "RUN_MODE": true, "RUN_VERSION": true}
|
|
out := make([]string, 0, len(environment))
|
|
for _, entry := range environment {
|
|
key, _, _ := strings.Cut(entry, "=")
|
|
if !blocked[key] {
|
|
out = append(out, entry)
|
|
}
|
|
}
|
|
return out
|
|
}
|