fix: stream platform builder progress

2026-08-14 19:34:11 +08:00
parent a6c4cdac5d
commit 92b1159cc8
5 changed files with 284 additions and 5 deletions
@@ -3,6 +3,7 @@ package service
 import (
 	"archive/tar"
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
@@ -11,10 +12,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"browser.local/platform/domain"
@@ -33,6 +37,15 @@ type DistributionBuilder interface {
 	Build(input domain.DistributionBuildInput) ([]byte, error)
 }

+type DistributionBuildProgress struct {
+	Percent int
+	Message string
+}
+
+type distributionBuilderWithProgress interface {
+	BuildWithProgress(input domain.DistributionBuildInput, progress func(DistributionBuildProgress)) ([]byte, error)
+}
+
 // DockerDistributionBuilderConfig configures a container-per-build builder.
 type DockerDistributionBuilderConfig struct {
 	DockerBinary  string
@@ -42,6 +55,7 @@ type DockerDistributionBuilderConfig struct {
 	Timeout       time.Duration
 	PlatformURL   string
 	CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
+	CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
 }

 // DockerDistributionBuilder runs each build in a container from a pinned image,
@@ -52,6 +66,7 @@ type DockerDistributionBuilder struct {
 }

 func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *DockerDistributionBuilder {
+	customRunner := config.CommandRunner != nil
 	if strings.TrimSpace(config.DockerBinary) == "" {
 		config.DockerBinary = "docker"
 	}
@@ -61,6 +76,9 @@ func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *Docke
 	if config.CommandRunner == nil {
 		config.CommandRunner = runCommandCombined
 	}
+	if config.CommandStream == nil && !customRunner {
+		config.CommandStream = runCommandStreamCombined
+	}
 	return &DockerDistributionBuilder{config: config}
 }

@@ -68,6 +86,46 @@ func runCommandCombined(ctx context.Context, name string, args ...string) ([]byt
 	return exec.CommandContext(ctx, name, args...).CombinedOutput()
 }

+func runCommandStreamCombined(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	var mu sync.Mutex
+	var output bytes.Buffer
+	var wg sync.WaitGroup
+	capture := func(reader io.Reader) {
+		defer wg.Done()
+		scanner := bufio.NewScanner(reader)
+		scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+		for scanner.Scan() {
+			line := scanner.Text()
+			mu.Lock()
+			output.WriteString(line)
+			output.WriteByte('\n')
+			mu.Unlock()
+			if onLine != nil {
+				onLine(line)
+			}
+		}
+	}
+	wg.Add(2)
+	go capture(stdout)
+	go capture(stderr)
+	err = cmd.Wait()
+	wg.Wait()
+	return output.Bytes(), err
+}
+
 func (builder *DockerDistributionBuilder) Readiness() (bool, string) {
 	if strings.TrimSpace(builder.config.Image) == "" {
 		return false, "platform builder image is not configured"
@@ -124,9 +182,14 @@ func pinnedBuilderImage(image string) bool {
 }

 func (builder *DockerDistributionBuilder) Build(input domain.DistributionBuildInput) ([]byte, error) {
+	return builder.BuildWithProgress(input, nil)
+}
+
+func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.DistributionBuildInput, progress func(DistributionBuildProgress)) ([]byte, error) {
 	if ready, reason := builder.Readiness(); !ready {
 		return nil, validationError(reason)
 	}
+	reportBuilderProgress(progress, 8, "env_check: platform builder readiness verified")
 	sourceDir, err := filepath.Abs(strings.TrimSpace(builder.config.SourceDir))
 	if err != nil {
 		return nil, validationError("platform builder run source directory is invalid")
@@ -150,6 +213,7 @@ func (builder *DockerDistributionBuilder) Build(input domain.DistributionBuildIn
 		}
 	}
 	defer func() { _ = os.RemoveAll(jobDir) }()
+	reportBuilderProgress(progress, 14, "env_check: platform builder workspace prepared")

 	// The auth key reaches the container through a per-job input file, never
 	// through a job-channel response to a machine-side endpoint or a container
@@ -193,12 +257,15 @@ func (builder *DockerDistributionBuilder) Build(input domain.DistributionBuildIn
 	ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
 	defer cancel()
 	args := builder.containerArgs(input, sourceDir, inputDir, buildDir, outputDir, outputName)
-	if output, err := builder.config.CommandRunner(ctx, builder.config.DockerBinary, args...); err != nil {
+	reportBuilderProgress(progress, 18, "git_sync: platform builder container starting")
+	output, err := builder.runBuildCommand(ctx, args, progress)
+	if err != nil {
 		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 			return nil, validationError("platform builder timed out while building the distribution")
 		}
 		return nil, validationError("platform builder failed: " + safeBuilderFailure(output, input.AuthKey, builder.config.SourceDir, sourceDir, jobDir))
 	}
+	reportBuilderProgress(progress, 92, "package_finalize: reading platform builder output")
 	binary, err := os.ReadFile(filepath.Join(outputDir, outputName))
 	if err != nil {
 		return nil, validationError("platform builder did not produce a distribution executable")
@@ -216,14 +283,74 @@ func (builder *DockerDistributionBuilder) Build(input domain.DistributionBuildIn
 	return packageClientManagerDistribution(input.PackageFormat, outputName, binary, configPayload)
 }

+func (builder *DockerDistributionBuilder) runBuildCommand(ctx context.Context, args []string, progress func(DistributionBuildProgress)) ([]byte, error) {
+	if builder.config.CommandStream == nil {
+		return builder.config.CommandRunner(ctx, builder.config.DockerBinary, args...)
+	}
+	return builder.config.CommandStream(ctx, builder.config.DockerBinary, args, func(line string) {
+		parsed, ok := parseBuilderProgressLine(line)
+		if ok {
+			reportBuilderProgress(progress, parsed.Percent, parsed.Message)
+		}
+	})
+}
+
+const builderProgressMarker = "__platform_builder_progress__|"
+
+func parseBuilderProgressLine(line string) (DistributionBuildProgress, bool) {
+	line = strings.TrimSpace(line)
+	if !strings.HasPrefix(line, builderProgressMarker) {
+		return DistributionBuildProgress{}, false
+	}
+	rest := strings.TrimPrefix(line, builderProgressMarker)
+	parts := strings.SplitN(rest, "|", 2)
+	if len(parts) != 2 {
+		return DistributionBuildProgress{}, false
+	}
+	percent, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+	if err != nil {
+		return DistributionBuildProgress{}, false
+	}
+	message := strings.TrimSpace(parts[1])
+	if message == "" {
+		return DistributionBuildProgress{}, false
+	}
+	if percent < 1 {
+		percent = 1
+	}
+	if percent > 99 {
+		percent = 99
+	}
+	return DistributionBuildProgress{Percent: percent, Message: message}, true
+}
+
+func reportBuilderProgress(progress func(DistributionBuildProgress), percent int, message string) {
+	if progress == nil {
+		return
+	}
+	if percent < 1 {
+		percent = 1
+	}
+	if percent > 99 {
+		percent = 99
+	}
+	progress(DistributionBuildProgress{Percent: percent, Message: message})
+}
+
 const distributionBuildScript = `#!/bin/sh
 set -eu

+progress() {
+  printf '__platform_builder_progress__|%s|%s\n' "$1" "$2"
+}
+
 auth_key="$(cat /workspace/input/auth-key)"
 if [ "$COMPONENT_KIND" = "run" ]; then
+  progress 24 'git_sync: preparing run source snapshot'
  rm -rf /workspace/build/run-source
  mkdir -p /workspace/build/run-source
  cp -R /workspace/source/. /workspace/build/run-source/
+  progress 34 'env_check: injecting run build metadata'
  seed_b64="$(base64 /workspace/input/workspace-seed.json | tr -d '\n')"
  cat > /workspace/build/run-source/config/workspace_seed_generated.go <<EOF
 package config
@@ -243,8 +370,11 @@ EOF
  ldflags="$ldflags -X browser.local/run/config.BuildComponentKey=$PROFILE_KEY"
  ldflags="$ldflags -X browser.local/run/config.BuildKeyGeneration=$KEY_GENERATION"
  ldflags="$ldflags -X browser.local/run/config.BuildVersion=$TARGET_RELEASE"
+  progress 48 'build_compile: downloading Go modules'
  go mod download
+  progress 72 'build_compile: compiling run target executable'
  go build -trimpath -ldflags "$ldflags" -o "/workspace/output/$OUTPUT_FILENAME" ./cmd/run
+  progress 88 'package_finalize: run executable written'
  exit 0
 fi

@@ -257,10 +387,12 @@ case "$REPOSITORY_URL" in
  *) printf 'client-manager repository must use https\n' >&2; exit 2 ;;
 esac
 cd /workspace/build
+progress 24 'git_sync: fetching client-manager source'
 git init --quiet
 git remote add origin "$REPOSITORY_URL"
 git fetch --quiet --depth 1 origin "$SOURCE_REVISION"
 git checkout --quiet --detach FETCH_HEAD
+progress 36 'env_check: writing client-manager configuration'
 {
  printf 'server_url: "%s"\n' "$PLATFORM_URL"
  printf 'server_instance_id: "%s"\n' "$SERVER_INSTANCE_ID"
@@ -270,9 +402,12 @@ git checkout --quiet --detach FETCH_HEAD
  printf 'scum_client_machine_label: "managed-client"\n'
  printf 'ftp_provider: 3\n'
 } > config.yaml
+progress 52 'deps_download: downloading Go modules'
 go mod download
+progress 76 'build_compile: compiling client-manager executable'
 go build -trimpath -ldflags '-s -w' -o "/workspace/output/$OUTPUT_FILENAME" .
 cp config.yaml /workspace/output/config.yaml
+progress 88 'package_finalize: client-manager package inputs written'
 `

 func (builder *DockerDistributionBuilder) containerArgs(input domain.DistributionBuildInput, sourceDir string, inputDir string, buildDir string, outputDir string, outputName string) []string {
@@ -391,7 +526,7 @@ func safeBuilderFailure(output []byte, sensitiveValues ...string) string {
 	kept := make([]string, 0, len(lines))
 	for index := len(lines) - 1; index >= 0 && len(kept) < 3; index-- {
 		line := strings.TrimSpace(lines[index])
-		if line == "" || strings.Contains(line, "/workspace/input") || strings.Contains(line, "auth-key") {
+		if line == "" || strings.HasPrefix(line, builderProgressMarker) || strings.Contains(line, "/workspace/input") || strings.Contains(line, "auth-key") {
 			continue
 		}
 		line = redactBuilderHostPaths(line)