480 lines
17 KiB
Go
480 lines
17 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
const distributionArtifactChunkSize = 1024 * 1024
|
|
|
|
type packageConfigPayload struct {
|
|
Kind string `json:"kind"`
|
|
ServerInstanceID string `json:"serverInstanceId"`
|
|
PluginID string `json:"pluginId"`
|
|
RunEndpointID string `json:"runEndpointId,omitempty"`
|
|
ProfileKey string `json:"profileKey,omitempty"`
|
|
TargetOS string `json:"targetOs"`
|
|
TargetArch string `json:"targetArch"`
|
|
SecretRef string `json:"secretRef"`
|
|
KeyGeneration int `json:"keyGeneration"`
|
|
AuthKey string `json:"authKey"`
|
|
}
|
|
|
|
func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
|
return lifecycleFailure("unsafe_distribution_build", err.Error())
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return distributionBuildFailure("build_unregistered", "Run worker is not registered")
|
|
}
|
|
input, err := worker.client.GetDistributionBuildInput(ctx, protocol.DistributionBuildInputRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
JobID: assignment.JobID,
|
|
LeaseToken: assignment.LeaseToken,
|
|
Attempt: assignment.Attempt,
|
|
})
|
|
if err != nil {
|
|
return distributionBuildFailure("build_input_failed", "could not load authenticated build input")
|
|
}
|
|
if err := validateDistributionBuildInput(assignment, input); err != nil {
|
|
return distributionBuildFailure("unsafe_build_input", err.Error())
|
|
}
|
|
|
|
report := func(percent int, message string) error {
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
response, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
JobID: assignment.JobID,
|
|
LeaseToken: assignment.LeaseToken,
|
|
Attempt: assignment.Attempt,
|
|
Progress: protocol.RunJobProgressReport{Percent: percent, Message: message},
|
|
Sequence: worker.nextProgressSequence(assignment.ProgressSequence),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !response.Accepted {
|
|
return fmt.Errorf("distribution build progress was not accepted")
|
|
}
|
|
assignment = response.Job
|
|
return worker.journal.Store(assignment)
|
|
}
|
|
|
|
workspace := distributionBuildWorkspace(worker.cfg.WorkspaceRoot, input.PluginID, assignment.JobID)
|
|
if err := os.RemoveAll(workspace); err != nil {
|
|
return distributionBuildFailure("workspace_prepare_failed", "could not reset isolated build workspace")
|
|
}
|
|
if err := os.MkdirAll(workspace, 0o700); err != nil {
|
|
return distributionBuildFailure("workspace_prepare_failed", "could not create isolated build workspace")
|
|
}
|
|
defer os.RemoveAll(workspace)
|
|
|
|
if err := report(12, "git_sync: preparing approved source"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report source preparation")
|
|
}
|
|
sourceRoot, err := worker.prepareDistributionSource(ctx, workspace, input)
|
|
if err != nil {
|
|
return distributionBuildFailure("git_sync_failed", "approved source checkout failed")
|
|
}
|
|
|
|
if err := report(28, "env_check: validating Go build environment"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report environment check")
|
|
}
|
|
if err := fixedCommand(ctx, sourceRoot, nil, "go", "version"); err != nil {
|
|
return distributionBuildFailure("env_check_failed", "Go build environment is unavailable")
|
|
}
|
|
|
|
if err := report(45, "deps_download: downloading Go modules"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report dependency download")
|
|
}
|
|
buildEnv := []string{"GOOS=" + input.TargetOS, "GOARCH=" + input.TargetArch, "CGO_ENABLED=0"}
|
|
if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "mod", "download"); err != nil {
|
|
return distributionBuildFailure("deps_download_failed", "Go module download failed")
|
|
}
|
|
|
|
if err := report(65, "build_compile: compiling target executable"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report compilation")
|
|
}
|
|
binaryPath := filepath.Join(workspace, input.OutputFilename)
|
|
entry := "./cmd/run"
|
|
ldflags := buildRunLDFlags(input, distributionBuildPlatformURL(worker, input))
|
|
if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "build", "-trimpath", "-ldflags", ldflags, "-o", binaryPath, entry); err != nil {
|
|
return distributionBuildFailure("build_compile_failed", "Go compilation failed")
|
|
}
|
|
|
|
if err := report(82, "package_finalize: creating distribution archive"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report packaging")
|
|
}
|
|
if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, binaryPath); err != nil {
|
|
return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed")
|
|
}
|
|
if err := report(96, "package_finalize: artifact upload completed"); err != nil {
|
|
return distributionBuildFailure("progress_report_failed", "could not report artifact upload")
|
|
}
|
|
return LifecycleExecutionResult{
|
|
State: lifecycleResultStateSucceeded,
|
|
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"},
|
|
ResultRef: "artifact://" + input.ArtifactID,
|
|
Message: "distribution build completed",
|
|
}
|
|
}
|
|
|
|
func (worker *Worker) prepareDistributionSource(_ context.Context, workspace string, input protocol.DistributionBuildInputResponse) (string, error) {
|
|
root, err := filepath.Abs(worker.cfg.BuildSourceRoot)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
root, err = filepath.EvalSymlinks(root)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil {
|
|
return "", err
|
|
}
|
|
isolatedSource := filepath.Join(workspace, "source")
|
|
if err := copyDistributionSource(root, isolatedSource, workspace); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeRunWorkspaceSeedConfig(isolatedSource, input.WorkspaceSeed); err != nil {
|
|
return "", err
|
|
}
|
|
return isolatedSource, nil
|
|
}
|
|
|
|
func writeRunWorkspaceSeedConfig(sourceRoot string, encodedSeed string) error {
|
|
encodedSeed = strings.TrimSpace(encodedSeed)
|
|
if encodedSeed == "" {
|
|
return nil
|
|
}
|
|
if _, err := base64.StdEncoding.DecodeString(encodedSeed); err != nil {
|
|
return fmt.Errorf("workspace seed is invalid")
|
|
}
|
|
configDir := filepath.Join(sourceRoot, "config")
|
|
if err := os.MkdirAll(configDir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
body := fmt.Sprintf("package config\n\nfunc init() { BuildWorkspaceSeed = %q }\n", encodedSeed)
|
|
return os.WriteFile(filepath.Join(configDir, "workspace_seed_generated.go"), []byte(body), 0o600)
|
|
}
|
|
|
|
func copyDistributionSource(sourceRoot string, destinationRoot string, workspace string) error {
|
|
workspace, err := filepath.Abs(workspace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return filepath.WalkDir(sourceRoot, func(path string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if path == workspace || strings.HasPrefix(path, workspace+string(filepath.Separator)) {
|
|
if entry.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
relative, err := filepath.Rel(sourceRoot, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if relative == "." {
|
|
return os.MkdirAll(destinationRoot, 0o700)
|
|
}
|
|
if entry.Name() == ".git" && entry.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
if entry.Type()&os.ModeSymlink != 0 {
|
|
return fmt.Errorf("trusted build source contains a symbolic link")
|
|
}
|
|
destination := filepath.Join(destinationRoot, relative)
|
|
if entry.IsDir() {
|
|
return os.MkdirAll(destination, 0o700)
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return nil
|
|
}
|
|
input, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
input.Close()
|
|
return err
|
|
}
|
|
_, copyErr := io.Copy(output, input)
|
|
inputCloseErr := input.Close()
|
|
closeErr := output.Close()
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
if inputCloseErr != nil {
|
|
return inputCloseErr
|
|
}
|
|
return closeErr
|
|
})
|
|
}
|
|
|
|
func distributionBuildPlatformURL(worker *Worker, input protocol.DistributionBuildInputResponse) string {
|
|
if value := strings.TrimSpace(input.PlatformURL); value != "" {
|
|
return value
|
|
}
|
|
return worker.clientPlatformURL()
|
|
}
|
|
|
|
func buildRunLDFlags(input protocol.DistributionBuildInputResponse, platformURL string) string {
|
|
values := map[string]string{
|
|
"BuildMode": "worker",
|
|
"BuildPlatformURL": platformURL,
|
|
"BuildRunEndpointID": input.RunEndpointID,
|
|
"BuildDisplayName": "Run-" + input.ServerInstanceID,
|
|
"BuildRegistrationToken": input.AuthKey,
|
|
"BuildServerInstanceID": input.ServerInstanceID,
|
|
"BuildPluginID": input.PluginID,
|
|
"BuildComponentKind": input.ComponentKind,
|
|
"BuildComponentKey": runBuildComponentKey(input),
|
|
"BuildKeyGeneration": fmt.Sprint(input.KeyGeneration),
|
|
"BuildVersion": input.TargetRelease,
|
|
}
|
|
flags := []string{"-s", "-w"}
|
|
for _, name := range []string{"BuildMode", "BuildPlatformURL", "BuildRunEndpointID", "BuildDisplayName", "BuildRegistrationToken", "BuildServerInstanceID", "BuildPluginID", "BuildComponentKind", "BuildComponentKey", "BuildKeyGeneration", "BuildVersion"} {
|
|
flags = append(flags, "-X", "browser.local/run/config."+name+"="+values[name])
|
|
}
|
|
return strings.Join(flags, " ")
|
|
}
|
|
|
|
func runBuildComponentKey(protocol.DistributionBuildInputResponse) string {
|
|
return ""
|
|
}
|
|
|
|
func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment protocol.RunJobAssignment, artifactID string, artifactPath string) error {
|
|
checksum, sizeBytes, err := checksumFile(artifactPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opened, err := worker.client.OpenArtifactTransfer(ctx, protocol.ArtifactTransferOpenRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
ArtifactID: artifactID,
|
|
Direction: "upload",
|
|
OwnerKind: "job",
|
|
OwnerID: assignment.JobID,
|
|
SizeBytes: sizeBytes,
|
|
ChunkSizeBytes: distributionArtifactChunkSize,
|
|
Checksum: checksum,
|
|
IdempotencyKey: "distribution-build:" + assignment.JobID,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
received := map[int]bool{}
|
|
for _, index := range opened.ReceivedChunkIndexes {
|
|
received[index] = true
|
|
}
|
|
file, err := os.Open(artifactPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
buffer := make([]byte, distributionArtifactChunkSize)
|
|
for index, offset := 0, int64(0); offset < sizeBytes; index, offset = index+1, offset+int64(distributionArtifactChunkSize) {
|
|
if received[index] {
|
|
continue
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
length := distributionArtifactChunkSize
|
|
if remaining := sizeBytes - offset; remaining < int64(length) {
|
|
length = int(remaining)
|
|
}
|
|
read, err := file.ReadAt(buffer[:length], offset)
|
|
if err != nil && !(err == io.EOF && read == length) {
|
|
return err
|
|
}
|
|
if read != length {
|
|
return fmt.Errorf("distribution artifact chunk is shorter than expected")
|
|
}
|
|
chunk := buffer[:length]
|
|
if _, err := worker.client.UploadArtifactChunk(ctx, protocol.ArtifactChunkUploadRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
TransferID: opened.TransferID,
|
|
ArtifactID: artifactID,
|
|
ChunkIndex: index,
|
|
Offset: offset,
|
|
SizeBytes: len(chunk),
|
|
Checksum: bytesChecksum(chunk),
|
|
Payload: chunk,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
state, err = worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
completed, err := worker.client.CompleteArtifactTransfer(ctx, protocol.ArtifactTransferCompleteRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
TransferID: opened.TransferID,
|
|
ArtifactID: artifactID,
|
|
Checksum: checksum,
|
|
SizeBytes: sizeBytes,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !completed.Completed || completed.Artifact.State != "available" {
|
|
return fmt.Errorf("artifact transfer did not complete")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateDistributionBuildInput(assignment protocol.RunJobAssignment, input protocol.DistributionBuildInputResponse) error {
|
|
if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID {
|
|
return fmt.Errorf("build input scope does not match job")
|
|
}
|
|
if input.ComponentKind != "run" {
|
|
return fmt.Errorf("build component kind is unsupported")
|
|
}
|
|
if !protocol.ValidLogicalFileKey(input.RunEndpointID) {
|
|
return fmt.Errorf("generated Run endpoint identity is unsafe")
|
|
}
|
|
if strings.TrimSpace(input.PluginID) == "" {
|
|
return fmt.Errorf("build plugin id is required")
|
|
}
|
|
if input.TargetOS != "windows" && input.TargetOS != "linux" && input.TargetOS != "darwin" {
|
|
return fmt.Errorf("target OS is unsupported")
|
|
}
|
|
if input.TargetArch != "amd64" && input.TargetArch != "arm64" {
|
|
return fmt.Errorf("target architecture is unsupported")
|
|
}
|
|
if input.ComponentKind == "run" && !protocol.ValidLogicalFileKey(input.TargetRelease) {
|
|
return fmt.Errorf("target release is unsafe")
|
|
}
|
|
if input.ComponentKind == "run" && input.PackageFormat != "raw-executable" {
|
|
return fmt.Errorf("run package format must be raw-executable")
|
|
}
|
|
if input.ComponentKind == "run" && !validDistributionPlatformURL(input.PlatformURL) {
|
|
return fmt.Errorf("run platform URL is invalid")
|
|
}
|
|
if input.ComponentKind == "run" && strings.TrimSpace(input.WorkspaceSeed) != "" {
|
|
if _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(input.WorkspaceSeed)); err != nil {
|
|
return fmt.Errorf("run workspace seed is invalid")
|
|
}
|
|
}
|
|
if strings.TrimSpace(input.ArtifactID) == "" || strings.TrimSpace(input.OutputFilename) == "" || strings.TrimSpace(input.AuthKey) == "" {
|
|
return fmt.Errorf("build input is incomplete")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validDistributionPlatformURL(value string) bool {
|
|
parsed, err := url.ParseRequestURI(strings.TrimSpace(value))
|
|
return err == nil && (parsed.Scheme == "https" || parsed.Scheme == "http") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
|
|
}
|
|
|
|
func fixedCommand(ctx context.Context, dir string, extraEnv []string, name string, args ...string) error {
|
|
startedAt := time.Now()
|
|
commandLine := distributionCommandLine(name, args)
|
|
log.Printf("RUN phase=distribution_build.command status=starting workdir=%s command=%s envKeys=%s", safeOptional(dir), commandLine, envKeysSummary(extraEnvMap(extraEnv), nil))
|
|
command := exec.CommandContext(ctx, name, args...)
|
|
command.Dir = dir
|
|
command.Env = append(os.Environ(), extraEnv...)
|
|
command.Stdout = io.Discard
|
|
command.Stderr = io.Discard
|
|
if err := command.Run(); err != nil {
|
|
log.Printf("RUN phase=distribution_build.command status=failed command=%s durationMs=%d error=%s", commandLine, time.Since(startedAt).Milliseconds(), err.Error())
|
|
return err
|
|
}
|
|
log.Printf("RUN phase=distribution_build.command status=complete command=%s durationMs=%d", commandLine, time.Since(startedAt).Milliseconds())
|
|
return nil
|
|
}
|
|
|
|
func distributionCommandLine(name string, args []string) string {
|
|
parts := append([]string{name}, args...)
|
|
return quotedCommandLine(parts)
|
|
}
|
|
|
|
func extraEnvMap(entries []string) map[string]string {
|
|
if len(entries) == 0 {
|
|
return nil
|
|
}
|
|
env := make(map[string]string, len(entries))
|
|
for _, entry := range entries {
|
|
key, value, ok := strings.Cut(entry, "=")
|
|
if !ok {
|
|
key = entry
|
|
value = ""
|
|
}
|
|
env[key] = value
|
|
}
|
|
return env
|
|
}
|
|
|
|
func distributionBuildWorkspace(workspaceRoot string, pluginID string, jobID string) string {
|
|
return filepath.Join(workspaceRoot, "distribution-builds", safeWorkspaceName(pluginID), safeWorkspaceName(jobID))
|
|
}
|
|
|
|
func bytesChecksum(payload []byte) string {
|
|
sum := sha256.Sum256(payload)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func safeWorkspaceName(value string) string {
|
|
var builder strings.Builder
|
|
for _, char := range value {
|
|
if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '-' || char == '_' {
|
|
builder.WriteRune(char)
|
|
} else {
|
|
builder.WriteByte('-')
|
|
}
|
|
}
|
|
return builder.String()
|
|
}
|
|
|
|
func distributionBuildFailure(code string, message string) LifecycleExecutionResult {
|
|
return LifecycleExecutionResult{
|
|
State: lifecycleResultStateFailed,
|
|
Progress: protocol.RunJobProgressReport{Percent: 100, Message: message},
|
|
Message: message,
|
|
ErrorCode: code,
|
|
}
|
|
}
|
|
|
|
func (worker *Worker) clientPlatformURL() string {
|
|
if client, ok := worker.client.(interface{ BaseURL() string }); ok {
|
|
return client.BaseURL()
|
|
}
|
|
return worker.cfg.PlatformURL
|
|
}
|