admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refresh run source during distribution builds

Browse Source
This commit is contained in:
npc0-hue
2026-08-21 23:09:28 +08:00
parent 2381b69d24
commit 0a605495af
12 changed files with 317 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/distribution_builder.go
+135 -16
View File
@@ -48,22 +48,26 @@ type distributionBuilderWithProgress interface {
// DockerDistributionBuilderConfig configures a container-per-build builder.
type DockerDistributionBuilderConfig struct {
DockerBinary string
Image string
SourceDir string
WorkspaceDir string
CacheDir string
Timeout time.Duration
PlatformURL string
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
DockerBinary string
Image string
SourceDir string
SourceRepositoryURL string
SourceRevision string
WorkspaceDir string
CacheDir string
Timeout time.Duration
PlatformURL string
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
GitCommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
}
// DockerDistributionBuilder runs each build in a container from a pinned image,
// with the run source mounted read-only and a per-job output directory mounted
// writable.
type DockerDistributionBuilder struct {
config DockerDistributionBuilderConfig
config DockerDistributionBuilderConfig
sourceMu sync.Mutex
}
func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *DockerDistributionBuilder {
@@ -77,6 +81,9 @@ func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *Docke
if config.CommandRunner == nil {
config.CommandRunner = runCommandCombined
}
if config.GitCommandRunner == nil {
config.GitCommandRunner = runCommandCombined
}
if config.CommandStream == nil && !customRunner {
config.CommandStream = runCommandStreamCombined
}
@@ -142,8 +149,13 @@ func (builder *DockerDistributionBuilder) Readiness() (bool, string) {
if err != nil {
return false, "platform builder run source directory is invalid"
}
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
return false, "platform builder run source directory does not contain a run checkout"
if _, err := os.Stat(source); err != nil && !errors.Is(err, os.ErrNotExist) {
return false, "platform builder run source directory is not accessible"
}
if strings.TrimSpace(builder.config.SourceRepositoryURL) == "" {
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
return false, "platform builder run source repository is not configured"
}
}
if strings.TrimSpace(builder.config.WorkspaceDir) == "" {
return false, "platform builder workspace directory is not configured"
@@ -199,6 +211,15 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
if err != nil {
return nil, validationError("platform builder workspace directory is invalid")
}
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
defer cancel()
if input.ComponentKind == domain.DistributionComponentRun {
// Keep a shared checkout stable while Docker reads it. The checkout is
// refreshed and exported below for this build, rather than captured when
// the platform process starts.
builder.sourceMu.Lock()
defer builder.sourceMu.Unlock()
}
cacheDir, err := builder.cacheDir(workspaceDir)
if err != nil {
return nil, validationError("platform builder cache directory is invalid")
@@ -212,15 +233,23 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
outputDir := filepath.Join(jobDir, "output")
inputDir := filepath.Join(jobDir, "input")
buildDir := filepath.Join(jobDir, "build")
sourceMountDir := filepath.Join(jobDir, "source")
goBuildCacheDir := filepath.Join(cacheDir, "go-build")
goModCacheDir := filepath.Join(cacheDir, "go-mod")
for _, directory := range []string{outputDir, inputDir, buildDir, goBuildCacheDir, goModCacheDir} {
for _, directory := range []string{outputDir, inputDir, buildDir, sourceMountDir, goBuildCacheDir, goModCacheDir} {
if err := os.MkdirAll(directory, 0o700); err != nil {
return nil, err
}
}
defer func() { _ = os.RemoveAll(jobDir) }()
reportBuilderProgress(progress, 14, "env_check: platform builder workspace prepared")
if input.ComponentKind == domain.DistributionComponentRun {
preparedSourceDir, err := builder.prepareRunSource(ctx, sourceDir, sourceMountDir, input, progress)
if err != nil {
return nil, err
}
sourceMountDir = preparedSourceDir
}
// The auth key reaches the container through a per-job input file, never
// through a job-channel response to a machine-side endpoint or a container
@@ -261,9 +290,7 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
if outputName == "" || filepath.Base(outputName) != outputName {
return nil, validationError("distribution build input has an invalid output filename")
}
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
defer cancel()
args := builder.containerArgs(input, sourceDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
args := builder.containerArgs(input, sourceMountDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
reportBuilderProgress(progress, 18, "git_sync: platform builder container starting")
output, err := builder.runBuildCommand(ctx, args, progress)
if err != nil {
@@ -290,6 +317,98 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
return packageClientManagerDistribution(input.PackageFormat, outputName, binary, configPayload)
}
// prepareRunSource implements the source phase of a Jenkins-style build. A
// configured checkout is created when absent, fetched on every build, and
// exported at FETCH_HEAD into the per-job workspace. The archive keeps local
// uncommitted files and the checkout's .git directory out of the executable.
func (builder *DockerDistributionBuilder) prepareRunSource(ctx context.Context, sourceDir, destination string, input domain.DistributionBuildInput, progress func(DistributionBuildProgress)) (string, error) {
repository := strings.TrimSpace(input.RepositoryURL)
if repository == "" {
repository = strings.TrimSpace(builder.config.SourceRepositoryURL)
}
revision := strings.TrimSpace(input.SourceRevision)
if revision == "" {
revision = strings.TrimSpace(builder.config.SourceRevision)
}
if revision == "" {
revision = "main"
}
if _, err := os.Stat(filepath.Join(sourceDir, ".git")); errors.Is(err, os.ErrNotExist) {
if repository == "" {
if _, sourceErr := os.Stat(filepath.Join(sourceDir, "go.mod")); sourceErr != nil {
return "", validationError("platform builder has no Run source checkout or repository")
}
return sourceDir, nil
}
if err := os.MkdirAll(filepath.Dir(sourceDir), 0o700); err != nil {
return "", validationError("platform builder could not create the Run source parent")
}
reportBuilderProgress(progress, 20, "git_sync: creating Run source checkout")
if _, err := builder.config.GitCommandRunner(ctx, "git", "clone", "--no-checkout", repository, sourceDir); err != nil {
return "", validationError("platform builder could not clone the Run source")
}
} else if err != nil {
return "", validationError("platform builder could not inspect the Run source checkout")
}
reportBuilderProgress(progress, 26, "git_sync: fetching Run source revision")
if _, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "fetch", "--depth", "1", "origin", revision); err != nil {
return "", validationError("platform builder could not update the Run source revision")
}
archivePayload, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "archive", "--format=tar", "FETCH_HEAD")
if err != nil {
return "", validationError("platform builder could not export the Run source revision")
}
if err := extractRunSourceArchive(archivePayload, destination); err != nil {
return "", validationError("platform builder could not prepare the Run source workspace")
}
reportBuilderProgress(progress, 34, "git_sync: Run source revision prepared")
return destination, nil
}
func extractRunSourceArchive(payload []byte, destination string) error {
reader := tar.NewReader(bytes.NewReader(payload))
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
name := filepath.Clean(filepath.FromSlash(header.Name))
if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(os.PathSeparator)) {
return errors.New("Run source archive contains an unsafe path")
}
target := filepath.Join(destination, name)
if header.FileInfo().IsDir() {
if err := os.MkdirAll(target, header.FileInfo().Mode().Perm()); err != nil {
return err
}
continue
}
if !header.FileInfo().Mode().IsRegular() {
return errors.New("Run source archive contains an unsupported file")
}
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
return err
}
file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, header.FileInfo().Mode().Perm())
if err != nil {
return err
}
_, copyErr := io.Copy(file, reader)
closeErr := file.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
}
}
func (builder *DockerDistributionBuilder) cacheDir(workspaceDir string) (string, error) {
configured := strings.TrimSpace(builder.config.CacheDir)
if configured == "" {
platform/service/distribution_builder_test.go
+81
View File
@@ -94,6 +94,66 @@ func TestDockerDistributionBuilderReadinessNamesPlatformBuilderFailures(t *testi
}
}
func TestDockerDistributionBuilderRefreshesRunSourceAtBuildTime(t *testing.T) {
sourceDir := filepath.Join(t.TempDir(), "run-checkout")
workspaceDir := t.TempDir()
var gitCalls [][]string
builder := NewDockerDistributionBuilder(DockerDistributionBuilderConfig{
DockerBinary: "docker-test",
Image: "browser-platform-distribution-builder:1.0.0",
SourceDir: sourceDir,
SourceRepositoryURL: "git@example.test:admin/run.git",
SourceRevision: "main",
WorkspaceDir: workspaceDir,
GitCommandRunner: func(_ context.Context, name string, args ...string) ([]byte, error) {
if name != "git" {
t.Fatalf("unexpected source command %q", name)
}
gitCalls = append(gitCalls, append([]string(nil), args...))
if len(args) >= 4 && args[0] == "clone" {
if err := os.MkdirAll(filepath.Join(args[len(args)-1], ".git"), 0o700); err != nil {
t.Fatalf("create fake checkout: %v", err)
}
}
if len(args) >= 4 && args[0] == "-C" && args[2] == "archive" {
return runSourceArchive(t), nil
}
return nil, nil
},
CommandRunner: func(_ context.Context, name string, args ...string) ([]byte, error) {
if name != "docker-test" {
t.Fatalf("unexpected container runtime %q", name)
}
if len(args) > 0 && (args[0] == "version" || args[0] == "image") {
return []byte("27.0.0"), nil
}
outputDir := builderMountHostPath(t, args, "/workspace/output")
if err := os.WriteFile(filepath.Join(outputDir, "run"), []byte("compiled-run"), 0o700); err != nil {
t.Fatalf("write fake build output: %v", err)
}
return nil, nil
},
})
payload, err := builder.Build(domain.DistributionBuildInput{
JobID: "job-source-refresh",
ComponentKind: domain.DistributionComponentRun,
PluginID: "game.scum",
TargetOS: "linux",
TargetArch: "amd64",
OutputFilename: "run",
AuthKey: "component-key",
})
if err != nil {
t.Fatalf("build refreshed Run source: %v", err)
}
if string(payload) != "compiled-run" {
t.Fatalf("unexpected built payload %q", payload)
}
if len(gitCalls) != 3 || gitCalls[0][0] != "clone" || gitCalls[1][0] != "-C" || gitCalls[1][2] != "fetch" || gitCalls[2][2] != "archive" {
t.Fatalf("expected clone, fetch, and archive during build, got %#v", gitCalls)
}
}
func TestDockerDistributionBuilderKeepsSecretInIsolatedInput(t *testing.T) {
sourceDir := createBuilderSource(t)
workspaceDir := t.TempDir()
@@ -143,6 +203,11 @@ func TestDockerDistributionBuilderKeepsSecretInIsolatedInput(t *testing.T) {
if bytes.Contains(script, []byte(secret)) {
t.Fatal("build script must not embed the component auth key")
}
for _, ordered := range [][2]string{{"cp -R /workspace/source/. /workspace/build/run-source/", "workspace_seed_generated.go"}, {"workspace_seed_generated.go", "go build -trimpath -ldflags"}} {
if bytes.Index(script, []byte(ordered[0])) >= bytes.Index(script, []byte(ordered[1])) {
t.Fatalf("Run build script must prepare source before injecting config and compiling: %q before %q", ordered[0], ordered[1])
}
}
planPayload, err := os.ReadFile(filepath.Join(inputDir, "autonomous-lifecycle-plan.json"))
if err != nil {
t.Fatalf("read autonomous lifecycle plan input: %v", err)
@@ -339,6 +404,22 @@ func createBuilderSource(t *testing.T) string {
return sourceDir
}
func runSourceArchive(t *testing.T) []byte {
t.Helper()
var payload bytes.Buffer
writer := tar.NewWriter(&payload)
if err := writer.WriteHeader(&tar.Header{Name: "go.mod", Mode: 0o600, Size: int64(len("module browser.local/run\n"))}); err != nil {
t.Fatalf("write source archive header: %v", err)
}
if _, err := writer.Write([]byte("module browser.local/run\n")); err != nil {
t.Fatalf("write source archive body: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close source archive: %v", err)
}
return payload.Bytes()
}
func builderMountHostPath(t *testing.T, args []string, containerSuffix string) string {
t.Helper()
for index := 0; index+1 < len(args); index++ {