Keep run logs opaque and streamline transfers

runtime/distribution_build.go

@@ -1,9 +1,6 @@
 package runtime
 
 import (
-	"archive/tar"
-	"archive/zip"
-	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
@@ -19,7 +16,6 @@ import (
 	"strings"
 	"time"
 
-	"browser.local/run/config"
 	"browser.local/run/protocol"
 )
 
@@ -108,15 +104,6 @@ func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment p
 		return distributionBuildFailure("env_check_failed", "Go build environment is unavailable")
 	}
 
-	configPath := ""
-	if input.ComponentKind == "client-manager" {
-		var err error
-		configPath, err = writeDistributionConfig(sourceRoot, input, worker.clientPlatformURL())
-		if err != nil {
-			return distributionBuildFailure("config_injection_failed", "could not inject scoped component configuration")
-		}
-	}
-
 	if err := report(45, "deps_download: downloading Go modules"); err != nil {
 		return distributionBuildFailure("progress_report_failed", "could not report dependency download")
 	}
@@ -129,14 +116,8 @@ func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment p
 		return distributionBuildFailure("progress_report_failed", "could not report compilation")
 	}
 	binaryPath := filepath.Join(workspace, input.OutputFilename)
-	entry := "."
-	if input.ComponentKind == "run" {
-		entry = "./cmd/run"
-	}
-	ldflags := "-s -w"
-	if input.ComponentKind == "run" {
-		ldflags = buildRunLDFlags(input, distributionBuildPlatformURL(worker, input))
-	}
+	entry := "./cmd/run"
+	ldflags := buildRunLDFlags(input, distributionBuildPlatformURL(worker, input))
 	if err := fixedCommand(ctx, sourceRoot, buildEnv, "go", "build", "-trimpath", "-ldflags", ldflags, "-o", binaryPath, entry); err != nil {
 		return distributionBuildFailure("build_compile_failed", "Go compilation failed")
 	}
@@ -144,33 +125,7 @@ func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment p
 	if err := report(82, "package_finalize: creating distribution archive"); err != nil {
 		return distributionBuildFailure("progress_report_failed", "could not report packaging")
 	}
-	if input.ComponentKind == "run" {
-		payload, err := os.ReadFile(binaryPath)
-		if err != nil {
-			return distributionBuildFailure("package_finalize_failed", "run executable could not be read")
-		}
-		if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil {
-			return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed")
-		}
-		if err := report(96, "package_finalize: artifact upload completed"); err != nil {
-			return distributionBuildFailure("progress_report_failed", "could not report artifact upload")
-		}
-		return LifecycleExecutionResult{
-			State:     lifecycleResultStateSucceeded,
-			Progress:  protocol.RunJobProgressReport{Percent: 100, Message: "package_finalize: build artifact available"},
-			ResultRef: "artifact://" + input.ArtifactID,
-			Message:   "distribution build completed",
-		}
-	}
-	archivePath := filepath.Join(workspace, archiveFilename(input))
-	if err := createDistributionArchive(archivePath, input.PackageFormat, binaryPath, configPath); err != nil {
-		return distributionBuildFailure("package_finalize_failed", "distribution archive creation failed")
-	}
-	payload, err := os.ReadFile(archivePath)
-	if err != nil {
-		return distributionBuildFailure("package_finalize_failed", "distribution archive could not be read")
-	}
-	if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, payload); err != nil {
+	if err := worker.uploadDistributionArtifact(ctx, assignment, input.ArtifactID, binaryPath); err != nil {
 		return distributionBuildFailure("artifact_upload_failed", "distribution artifact upload failed")
 	}
 	if err := report(96, "package_finalize: artifact upload completed"); err != nil {
@@ -184,45 +139,26 @@ func (worker *Worker) executeDistributionBuild(ctx context.Context, assignment p
 	}
 }
 
-func (worker *Worker) prepareDistributionSource(ctx context.Context, workspace string, input protocol.DistributionBuildInputResponse) (string, error) {
-	if input.ComponentKind == "run" {
-		root, err := filepath.Abs(worker.cfg.BuildSourceRoot)
-		if err != nil {
-			return "", err
-		}
-		root, err = filepath.EvalSymlinks(root)
-		if err != nil {
-			return "", err
-		}
-		if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil {
-			return "", err
-		}
-		isolatedSource := filepath.Join(workspace, "source")
-		if err := copyDistributionSource(root, isolatedSource, workspace); err != nil {
-			return "", err
-		}
-		if err := writeRunWorkspaceSeedConfig(isolatedSource, input.WorkspaceSeed); err != nil {
-			return "", err
-		}
-		return isolatedSource, nil
-	}
-	checkout := filepath.Join(workspace, "source")
-	if err := os.MkdirAll(checkout, 0o700); err != nil {
+func (worker *Worker) prepareDistributionSource(_ context.Context, workspace string, input protocol.DistributionBuildInputResponse) (string, error) {
+	root, err := filepath.Abs(worker.cfg.BuildSourceRoot)
+	if err != nil {
 		return "", err
 	}
-	if err := fixedCommand(ctx, checkout, nil, "git", "init", "--quiet"); err != nil {
+	root, err = filepath.EvalSymlinks(root)
+	if err != nil {
 		return "", err
 	}
-	if err := fixedCommand(ctx, checkout, nil, "git", "remote", "add", "origin", input.RepositoryURL); err != nil {
+	if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil {
 		return "", err
 	}
-	if err := fixedCommand(ctx, checkout, nil, "git", "fetch", "--quiet", "--depth", "1", "origin", input.SourceRevision); err != nil {
+	isolatedSource := filepath.Join(workspace, "source")
+	if err := copyDistributionSource(root, isolatedSource, workspace); err != nil {
 		return "", err
 	}
-	if err := fixedCommand(ctx, checkout, nil, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD"); err != nil {
+	if err := writeRunWorkspaceSeedConfig(isolatedSource, input.WorkspaceSeed); err != nil {
 		return "", err
 	}
-	return checkout, nil
+	return isolatedSource, nil
 }
 
 func writeRunWorkspaceSeedConfig(sourceRoot string, encodedSeed string) error {
@@ -302,18 +238,6 @@ func copyDistributionSource(sourceRoot string, destinationRoot string, workspace
 	})
 }
 
-func writeDistributionConfig(sourceRoot string, input protocol.DistributionBuildInputResponse, platformURL string) (string, error) {
-	if input.ComponentKind == "client-manager" {
-		content := fmt.Sprintf("server_url: %q\nserver_instance_id: %q\nscum_client_credential: %q\nscum_client_name: %q\nscum_client_version: %q\nscum_client_machine_label: %q\nftp_provider: 3\n",
-			platformURL, input.ServerInstanceID, input.AuthKey, input.ProfileKey, "platform-build", "managed-client")
-		if err := os.WriteFile(filepath.Join(sourceRoot, "config.yaml"), []byte(content), 0o600); err != nil {
-			return "", err
-		}
-		return filepath.Join(sourceRoot, "config.yaml"), nil
-	}
-	return "", fmt.Errorf("run distributions do not use sidecar package config")
-}
-
 func distributionBuildPlatformURL(worker *Worker, input protocol.DistributionBuildInputResponse) string {
 	if value := strings.TrimSpace(input.PlatformURL); value != "" {
 		return value
@@ -342,15 +266,15 @@ func buildRunLDFlags(input protocol.DistributionBuildInputResponse, platformURL
 	return strings.Join(flags, " ")
 }
 
-func runBuildComponentKey(input protocol.DistributionBuildInputResponse) string {
-	if input.ComponentKind == config.PackageComponentRun {
-		return ""
-	}
-	return strings.TrimSpace(input.ProfileKey)
+func runBuildComponentKey(protocol.DistributionBuildInputResponse) string {
+	return ""
 }
 
-func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment protocol.RunJobAssignment, artifactID string, payload []byte) error {
-	checksum := bytesChecksum(payload)
+func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment protocol.RunJobAssignment, artifactID string, artifactPath string) error {
+	checksum, sizeBytes, err := checksumFile(artifactPath)
+	if err != nil {
+		return err
+	}
 	state, err := worker.registeredState()
 	if err != nil {
 		return err
@@ -362,7 +286,7 @@ func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment
 		Direction:      "upload",
 		OwnerKind:      "job",
 		OwnerID:        assignment.JobID,
-		SizeBytes:      int64(len(payload)),
+		SizeBytes:      sizeBytes,
 		ChunkSizeBytes: distributionArtifactChunkSize,
 		Checksum:       checksum,
 		IdempotencyKey: "distribution-build:" + assignment.JobID,
@@ -374,7 +298,13 @@ func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment
 	for _, index := range opened.ReceivedChunkIndexes {
 		received[index] = true
 	}
-	for index, offset := 0, 0; offset < len(payload); index, offset = index+1, offset+distributionArtifactChunkSize {
+	file, err := os.Open(artifactPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	buffer := make([]byte, distributionArtifactChunkSize)
+	for index, offset := 0, int64(0); offset < sizeBytes; index, offset = index+1, offset+int64(distributionArtifactChunkSize) {
 		if received[index] {
 			continue
 		}
@@ -382,18 +312,25 @@ func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment
 		if err != nil {
 			return err
 		}
-		end := offset + distributionArtifactChunkSize
-		if end > len(payload) {
-			end = len(payload)
+		length := distributionArtifactChunkSize
+		if remaining := sizeBytes - offset; remaining < int64(length) {
+			length = int(remaining)
 		}
-		chunk := payload[offset:end]
+		read, err := file.ReadAt(buffer[:length], offset)
+		if err != nil && !(err == io.EOF && read == length) {
+			return err
+		}
+		if read != length {
+			return fmt.Errorf("distribution artifact chunk is shorter than expected")
+		}
+		chunk := buffer[:length]
 		if _, err := worker.client.UploadArtifactChunk(ctx, protocol.ArtifactChunkUploadRequest{
 			RunEndpointID: state.RunEndpointID,
 			SessionToken:  state.SessionToken,
 			TransferID:    opened.TransferID,
 			ArtifactID:    artifactID,
 			ChunkIndex:    index,
-			Offset:        int64(offset),
+			Offset:        offset,
 			SizeBytes:     len(chunk),
 			Checksum:      bytesChecksum(chunk),
 			Payload:       chunk,
@@ -411,7 +348,7 @@ func (worker *Worker) uploadDistributionArtifact(ctx context.Context, assignment
 		TransferID:    opened.TransferID,
 		ArtifactID:    artifactID,
 		Checksum:      checksum,
-		SizeBytes:     int64(len(payload)),
+		SizeBytes:     sizeBytes,
 	})
 	if err != nil {
 		return err
@@ -426,24 +363,15 @@ func validateDistributionBuildInput(assignment protocol.RunJobAssignment, input
 	if input.JobID != assignment.JobID || input.ServerInstanceID != assignment.ServerInstanceID {
 		return fmt.Errorf("build input scope does not match job")
 	}
-	if input.ComponentKind != "run" && input.ComponentKind != "client-manager" {
+	if input.ComponentKind != "run" {
 		return fmt.Errorf("build component kind is unsupported")
 	}
 	if !protocol.ValidLogicalFileKey(input.RunEndpointID) {
 		return fmt.Errorf("generated Run endpoint identity is unsafe")
 	}
-	if input.ComponentKind == "client-manager" && input.RunEndpointID != assignment.RunEndpointID {
-		return fmt.Errorf("client-manager build target does not match job")
-	}
 	if strings.TrimSpace(input.PluginID) == "" {
 		return fmt.Errorf("build plugin id is required")
 	}
-	if input.ComponentKind == "client-manager" && !approvedHTTPSGitRepository(input.RepositoryURL) {
-		return fmt.Errorf("client-manager repository is not approved")
-	}
-	if input.ComponentKind == "client-manager" && strings.TrimSpace(input.SourceRevision) == "" {
-		return fmt.Errorf("client-manager source revision is required")
-	}
 	if input.TargetOS != "windows" && input.TargetOS != "linux" && input.TargetOS != "darwin" {
 		return fmt.Errorf("target OS is unsupported")
 	}
@@ -464,20 +392,12 @@ func validateDistributionBuildInput(assignment protocol.RunJobAssignment, input
 			return fmt.Errorf("run workspace seed is invalid")
 		}
 	}
-	if input.ComponentKind == "client-manager" && input.PackageFormat != "zip" && input.PackageFormat != "tar.gz" {
-		return fmt.Errorf("package format is unsupported")
-	}
 	if strings.TrimSpace(input.ArtifactID) == "" || strings.TrimSpace(input.OutputFilename) == "" || strings.TrimSpace(input.AuthKey) == "" {
 		return fmt.Errorf("build input is incomplete")
 	}
 	return nil
 }
 
-func approvedHTTPSGitRepository(value string) bool {
-	parsed, err := url.ParseRequestURI(strings.TrimSpace(value))
-	return err == nil && parsed.Scheme == "https" && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" && strings.HasSuffix(parsed.Path, ".git")
-}
-
 func validDistributionPlatformURL(value string) bool {
 	parsed, err := url.ParseRequestURI(strings.TrimSpace(value))
 	return err == nil && (parsed.Scheme == "https" || parsed.Scheme == "http") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
@@ -485,7 +405,7 @@ func validDistributionPlatformURL(value string) bool {
 
 func fixedCommand(ctx context.Context, dir string, extraEnv []string, name string, args ...string) error {
 	startedAt := time.Now()
-	commandLine := redactedDistributionCommandLine(name, args)
+	commandLine := distributionCommandLine(name, args)
 	log.Printf("RUN phase=distribution_build.command status=starting workdir=%s command=%s envKeys=%s", safeOptional(dir), commandLine, envKeysSummary(extraEnvMap(extraEnv), nil))
 	command := exec.CommandContext(ctx, name, args...)
 	command.Dir = dir
@@ -493,26 +413,16 @@ func fixedCommand(ctx context.Context, dir string, extraEnv []string, name strin
 	command.Stdout = io.Discard
 	command.Stderr = io.Discard
 	if err := command.Run(); err != nil {
-		log.Printf("RUN phase=distribution_build.command status=failed command=%s durationMs=%d error=%s", commandLine, time.Since(startedAt).Milliseconds(), RedactText(err.Error()))
+		log.Printf("RUN phase=distribution_build.command status=failed command=%s durationMs=%d error=%s", commandLine, time.Since(startedAt).Milliseconds(), err.Error())
 		return err
 	}
 	log.Printf("RUN phase=distribution_build.command status=complete command=%s durationMs=%d", commandLine, time.Since(startedAt).Milliseconds())
 	return nil
 }
 
-func redactedDistributionCommandLine(name string, args []string) string {
+func distributionCommandLine(name string, args []string) string {
 	parts := append([]string{name}, args...)
-	redacted := append([]string(nil), parts...)
-	for index, part := range redacted {
-		if part == "-ldflags" && index+1 < len(redacted) {
-			redacted[index+1] = "[redacted-ldflags]"
-			continue
-		}
-		if strings.Contains(part, "BuildRegistrationToken=") {
-			redacted[index] = "[redacted-ldflags]"
-		}
-	}
-	return redactedCommandLine(redacted)
+	return quotedCommandLine(parts)
 }
 
 func extraEnvMap(entries []string) map[string]string {
@@ -531,104 +441,6 @@ func extraEnvMap(entries []string) map[string]string {
 	return env
 }
 
-func createDistributionArchive(path string, format string, binaryPath string, configPath string) error {
-	if format == "zip" {
-		file, err := os.Create(path)
-		if err != nil {
-			return err
-		}
-		writer := zip.NewWriter(file)
-		if err := addZipFile(writer, binaryPath); err != nil {
-			writer.Close()
-			file.Close()
-			return err
-		}
-		if err := addZipFile(writer, configPath); err != nil {
-			writer.Close()
-			file.Close()
-			return err
-		}
-		if err := writer.Close(); err != nil {
-			file.Close()
-			return err
-		}
-		return file.Close()
-	}
-	file, err := os.Create(path)
-	if err != nil {
-		return err
-	}
-	gzipWriter := gzip.NewWriter(file)
-	tarWriter := tar.NewWriter(gzipWriter)
-	if err := addTarFile(tarWriter, binaryPath); err != nil {
-		tarWriter.Close()
-		gzipWriter.Close()
-		file.Close()
-		return err
-	}
-	if err := addTarFile(tarWriter, configPath); err != nil {
-		tarWriter.Close()
-		gzipWriter.Close()
-		file.Close()
-		return err
-	}
-	if err := tarWriter.Close(); err != nil {
-		gzipWriter.Close()
-		file.Close()
-		return err
-	}
-	if err := gzipWriter.Close(); err != nil {
-		file.Close()
-		return err
-	}
-	return file.Close()
-}
-
-func addZipFile(writer *zip.Writer, path string) error {
-	body, err := os.ReadFile(path)
-	if err != nil {
-		return err
-	}
-	entry, err := writer.Create(filepath.Base(path))
-	if err != nil {
-		return err
-	}
-	_, err = entry.Write(body)
-	return err
-}
-
-func addTarFile(writer *tar.Writer, path string) error {
-	info, err := os.Stat(path)
-	if err != nil {
-		return err
-	}
-	header := &tar.Header{Name: filepath.Base(path), Mode: 0o600, Size: info.Size()}
-	if strings.HasSuffix(filepath.Base(path), ".exe") || filepath.Base(path) == "run" {
-		header.Mode = 0o700
-	}
-	if err := writer.WriteHeader(header); err != nil {
-		return err
-	}
-	file, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-	_, err = io.Copy(writer, file)
-	return err
-}
-
-func archiveFilename(input protocol.DistributionBuildInputResponse) string {
-	base := "run-" + input.ServerInstanceID
-	if input.ComponentKind == "client-manager" {
-		base = input.ProfileKey + "-" + input.ServerInstanceID
-	}
-	if input.PackageFormat == "zip" {
-		return base + ".zip"
-	}
-	return base + ".tar.gz"
-}
-
 func distributionBuildWorkspace(workspaceRoot string, pluginID string, jobID string) string {
 	return filepath.Join(workspaceRoot, "distribution-builds", safeWorkspaceName(pluginID), safeWorkspaceName(jobID))
 }