admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refresh run source during distribution builds

Browse Source
This commit is contained in:
npc0-hue
2026-08-21 23:09:28 +08:00
parent 2381b69d24
commit 0a605495af
12 changed files with 317 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/distribution_builder.go
+135 -16
View File
@@ -48,22 +48,26 @@ type distributionBuilderWithProgress interface {
// DockerDistributionBuilderConfig configures a container-per-build builder.
type DockerDistributionBuilderConfig struct {
DockerBinary string
Image string
SourceDir string
WorkspaceDir string
CacheDir string
Timeout time.Duration
PlatformURL string
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
DockerBinary string
Image string
SourceDir string
SourceRepositoryURL string
SourceRevision string
WorkspaceDir string
CacheDir string
Timeout time.Duration
PlatformURL string
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
GitCommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
}
// DockerDistributionBuilder runs each build in a container from a pinned image,
// with the run source mounted read-only and a per-job output directory mounted
// writable.
type DockerDistributionBuilder struct {
config DockerDistributionBuilderConfig
config DockerDistributionBuilderConfig
sourceMu sync.Mutex
}
func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *DockerDistributionBuilder {
@@ -77,6 +81,9 @@ func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *Docke
if config.CommandRunner == nil {
config.CommandRunner = runCommandCombined
}
if config.GitCommandRunner == nil {
config.GitCommandRunner = runCommandCombined
}
if config.CommandStream == nil && !customRunner {
config.CommandStream = runCommandStreamCombined
}
@@ -142,8 +149,13 @@ func (builder *DockerDistributionBuilder) Readiness() (bool, string) {
if err != nil {
return false, "platform builder run source directory is invalid"
}
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
return false, "platform builder run source directory does not contain a run checkout"
if _, err := os.Stat(source); err != nil && !errors.Is(err, os.ErrNotExist) {
return false, "platform builder run source directory is not accessible"
}
if strings.TrimSpace(builder.config.SourceRepositoryURL) == "" {
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
return false, "platform builder run source repository is not configured"
}
}
if strings.TrimSpace(builder.config.WorkspaceDir) == "" {
return false, "platform builder workspace directory is not configured"
@@ -199,6 +211,15 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
if err != nil {
return nil, validationError("platform builder workspace directory is invalid")
}
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
defer cancel()
if input.ComponentKind == domain.DistributionComponentRun {
// Keep a shared checkout stable while Docker reads it. The checkout is
// refreshed and exported below for this build, rather than captured when
// the platform process starts.
builder.sourceMu.Lock()
defer builder.sourceMu.Unlock()
}
cacheDir, err := builder.cacheDir(workspaceDir)
if err != nil {
return nil, validationError("platform builder cache directory is invalid")
@@ -212,15 +233,23 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
outputDir := filepath.Join(jobDir, "output")
inputDir := filepath.Join(jobDir, "input")
buildDir := filepath.Join(jobDir, "build")
sourceMountDir := filepath.Join(jobDir, "source")
goBuildCacheDir := filepath.Join(cacheDir, "go-build")
goModCacheDir := filepath.Join(cacheDir, "go-mod")
for _, directory := range []string{outputDir, inputDir, buildDir, goBuildCacheDir, goModCacheDir} {
for _, directory := range []string{outputDir, inputDir, buildDir, sourceMountDir, goBuildCacheDir, goModCacheDir} {
if err := os.MkdirAll(directory, 0o700); err != nil {
return nil, err
}
}
defer func() { _ = os.RemoveAll(jobDir) }()
reportBuilderProgress(progress, 14, "env_check: platform builder workspace prepared")
if input.ComponentKind == domain.DistributionComponentRun {
preparedSourceDir, err := builder.prepareRunSource(ctx, sourceDir, sourceMountDir, input, progress)
if err != nil {
return nil, err
}
sourceMountDir = preparedSourceDir
}
// The auth key reaches the container through a per-job input file, never
// through a job-channel response to a machine-side endpoint or a container
@@ -261,9 +290,7 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
if outputName == "" || filepath.Base(outputName) != outputName {
return nil, validationError("distribution build input has an invalid output filename")
}
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
defer cancel()
args := builder.containerArgs(input, sourceDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
args := builder.containerArgs(input, sourceMountDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
reportBuilderProgress(progress, 18, "git_sync: platform builder container starting")
output, err := builder.runBuildCommand(ctx, args, progress)
if err != nil {
@@ -290,6 +317,98 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
return packageClientManagerDistribution(input.PackageFormat, outputName, binary, configPayload)
}
// prepareRunSource implements the source phase of a Jenkins-style build. A
// configured checkout is created when absent, fetched on every build, and
// exported at FETCH_HEAD into the per-job workspace. The archive keeps local
// uncommitted files and the checkout's .git directory out of the executable.
func (builder *DockerDistributionBuilder) prepareRunSource(ctx context.Context, sourceDir, destination string, input domain.DistributionBuildInput, progress func(DistributionBuildProgress)) (string, error) {
repository := strings.TrimSpace(input.RepositoryURL)
if repository == "" {
repository = strings.TrimSpace(builder.config.SourceRepositoryURL)
}
revision := strings.TrimSpace(input.SourceRevision)
if revision == "" {
revision = strings.TrimSpace(builder.config.SourceRevision)
}
if revision == "" {
revision = "main"
}
if _, err := os.Stat(filepath.Join(sourceDir, ".git")); errors.Is(err, os.ErrNotExist) {
if repository == "" {
if _, sourceErr := os.Stat(filepath.Join(sourceDir, "go.mod")); sourceErr != nil {
return "", validationError("platform builder has no Run source checkout or repository")
}
return sourceDir, nil
}
if err := os.MkdirAll(filepath.Dir(sourceDir), 0o700); err != nil {
return "", validationError("platform builder could not create the Run source parent")
}
reportBuilderProgress(progress, 20, "git_sync: creating Run source checkout")
if _, err := builder.config.GitCommandRunner(ctx, "git", "clone", "--no-checkout", repository, sourceDir); err != nil {
return "", validationError("platform builder could not clone the Run source")
}
} else if err != nil {
return "", validationError("platform builder could not inspect the Run source checkout")
}
reportBuilderProgress(progress, 26, "git_sync: fetching Run source revision")
if _, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "fetch", "--depth", "1", "origin", revision); err != nil {
return "", validationError("platform builder could not update the Run source revision")
}
archivePayload, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "archive", "--format=tar", "FETCH_HEAD")
if err != nil {
return "", validationError("platform builder could not export the Run source revision")
}
if err := extractRunSourceArchive(archivePayload, destination); err != nil {
return "", validationError("platform builder could not prepare the Run source workspace")
}
reportBuilderProgress(progress, 34, "git_sync: Run source revision prepared")
return destination, nil
}
func extractRunSourceArchive(payload []byte, destination string) error {
reader := tar.NewReader(bytes.NewReader(payload))
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
name := filepath.Clean(filepath.FromSlash(header.Name))
if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(os.PathSeparator)) {
return errors.New("Run source archive contains an unsafe path")
}
target := filepath.Join(destination, name)
if header.FileInfo().IsDir() {
if err := os.MkdirAll(target, header.FileInfo().Mode().Perm()); err != nil {
return err
}
continue
}
if !header.FileInfo().Mode().IsRegular() {
return errors.New("Run source archive contains an unsupported file")
}
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
return err
}
file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, header.FileInfo().Mode().Perm())
if err != nil {
return err
}
_, copyErr := io.Copy(file, reader)
closeErr := file.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
}
}
func (builder *DockerDistributionBuilder) cacheDir(workspaceDir string) (string, error) {
configured := strings.TrimSpace(builder.config.CacheDir)
if configured == "" {