fix: refresh run source during distribution builds
This commit is contained in:
@@ -33,6 +33,8 @@ PLATFORM_ARTIFACT_DIR=.platform-data/artifacts
|
||||
PLATFORM_BUILDER_DOCKER_BINARY=docker
|
||||
PLATFORM_BUILDER_IMAGE=browser-platform-distribution-builder:1.0.0
|
||||
PLATFORM_BUILDER_SOURCE_DIR=../run
|
||||
PLATFORM_BUILDER_SOURCE_REPOSITORY=git@git.npc0.com:admin343/run.git
|
||||
PLATFORM_BUILDER_SOURCE_REVISION=main
|
||||
PLATFORM_BUILDER_WORKSPACE_DIR=.platform-data/distribution-builds
|
||||
PLATFORM_BUILDER_TIMEOUT_SECONDS=1800
|
||||
# URL embedded into generated Run and client-manager packages.
|
||||
|
||||
+7
-3
@@ -56,7 +56,9 @@ Runtime configuration:
|
||||
- `PLATFORM_SECRET_ENVELOPE_KEY`: external secret used to derive the AES-GCM component-key envelope key; use at least 32 random characters and keep it stable across restarts.
|
||||
- `PLATFORM_BUILDER_DOCKER_BINARY`: Docker-compatible CLI used by the platform builder, default `docker`.
|
||||
- `PLATFORM_BUILDER_IMAGE`: prebuilt, explicitly versioned or digest-pinned builder image, default `browser-platform-distribution-builder:1.0.0`; floating tags such as `latest` are rejected.
|
||||
- `PLATFORM_BUILDER_SOURCE_DIR`: read-only Run source snapshot containing `go.mod`; this is required for builder readiness.
|
||||
- `PLATFORM_BUILDER_SOURCE_DIR`: persistent Run checkout location used by the platform builder. The checkout may be absent at startup; each Run build creates it when needed, fetches `PLATFORM_BUILDER_SOURCE_REVISION`, exports that revision into the isolated job workspace, injects package configuration, and then compiles.
|
||||
- `PLATFORM_BUILDER_SOURCE_REPOSITORY`: Run Git repository used when the checkout is absent or needs a build-time refresh.
|
||||
- `PLATFORM_BUILDER_SOURCE_REVISION`: branch, tag, or commit fetched for each Run build; defaults to `main`.
|
||||
- `PLATFORM_BUILDER_WORKSPACE_DIR`: private per-plugin/per-job build workspace, default `<PLATFORM_DATA_DIR>/distribution-builds`.
|
||||
- `PLATFORM_BUILDER_CACHE_DIR`: persistent Go build/module cache, default `<PLATFORM_DATA_DIR>/distribution-build-cache`; it contains no job inputs or component keys.
|
||||
- `PLATFORM_BUILDER_TIMEOUT_SECONDS`: positive build deadline, default `1800`.
|
||||
@@ -66,12 +68,14 @@ Build the dedicated toolchain image before enabling distribution generation:
|
||||
|
||||
```bash
|
||||
docker build --pull -t browser-platform-distribution-builder:1.0.0 distribution-builder
|
||||
export PLATFORM_BUILDER_SOURCE_DIR=/absolute/path/to/read-only/run-source-snapshot
|
||||
export PLATFORM_BUILDER_SOURCE_DIR=/absolute/path/to/run-checkout
|
||||
export PLATFORM_BUILDER_SOURCE_REPOSITORY=git@git.npc0.com:admin343/run.git
|
||||
export PLATFORM_BUILDER_SOURCE_REVISION=main
|
||||
export PLATFORM_BUILDER_IMAGE=browser-platform-distribution-builder:1.0.0
|
||||
go run ./cmd/platform
|
||||
```
|
||||
|
||||
Each build runs in a separate read-only container. The platform mounts source and per-job input read-only, mounts only the job build/output directories writable, and passes the component auth key through a mode-`0600` input file. The key is not sent through the machine-side job channel or Docker arguments. Production deployments may use an internal-registry `image@sha256:...` reference; the selected image must already exist in the Docker daemon because builds run with `--pull never`.
|
||||
Each build runs in a separate read-only container. For Run, the platform first exports the fetched revision into a clean per-job source directory; source and per-job input are mounted read-only, only the job build/output directories are writable, and the component auth key is passed through a mode-`0600` input file. The key is not sent through the machine-side job channel or Docker arguments. Production deployments may use an internal-registry `image@sha256:...` reference; the selected image must already exist in the Docker daemon because builds run with `--pull never`.
|
||||
|
||||
MySQL configuration example:
|
||||
|
||||
|
||||
@@ -49,12 +49,14 @@ func NewRouterFromConfig(cfg config.Config) (http.Handler, error) {
|
||||
return nil, err
|
||||
}
|
||||
core.ConfigureDistributionBuilder(service.NewDockerDistributionBuilder(service.DockerDistributionBuilderConfig{
|
||||
DockerBinary: cfg.BuilderDockerBinary,
|
||||
Image: cfg.BuilderImage,
|
||||
SourceDir: cfg.BuilderSourceDir,
|
||||
WorkspaceDir: cfg.BuilderWorkspaceDir,
|
||||
CacheDir: cfg.BuilderCacheDir,
|
||||
Timeout: time.Duration(cfg.BuilderTimeoutSeconds) * time.Second,
|
||||
DockerBinary: cfg.BuilderDockerBinary,
|
||||
Image: cfg.BuilderImage,
|
||||
SourceDir: cfg.BuilderSourceDir,
|
||||
SourceRepositoryURL: cfg.BuilderSourceRepository,
|
||||
SourceRevision: cfg.BuilderSourceRevision,
|
||||
WorkspaceDir: cfg.BuilderWorkspaceDir,
|
||||
CacheDir: cfg.BuilderCacheDir,
|
||||
Timeout: time.Duration(cfg.BuilderTimeoutSeconds) * time.Second,
|
||||
}))
|
||||
if err := core.ConfigureAIProviderMode(cfg.AIProviderMode); err != nil {
|
||||
return nil, err
|
||||
|
||||
+42
-36
@@ -12,6 +12,8 @@ const defaultAddr = ":8080"
|
||||
const defaultDataDir = ".platform-data"
|
||||
const defaultStorageBackend = "file"
|
||||
const defaultBuilderDockerBinary = "docker"
|
||||
const defaultBuilderSourceRepository = "git@git.npc0.com:admin343/run.git"
|
||||
const defaultBuilderSourceRevision = "main"
|
||||
|
||||
// defaultBuilderImage names an explicit toolchain version rather than a
|
||||
// floating tag such as latest. Operators who want digest pinning override
|
||||
@@ -20,24 +22,26 @@ const defaultBuilderImage = "browser-platform-distribution-builder:1.0.0"
|
||||
const defaultBuilderTimeoutSeconds = 1800
|
||||
|
||||
type Config struct {
|
||||
Addr string
|
||||
StorageBackend string
|
||||
MySQLDSN string
|
||||
DataDir string
|
||||
MetadataPath string
|
||||
LogDir string
|
||||
ArtifactDir string
|
||||
LogBodyBackend string
|
||||
BootstrapAdminEmail string
|
||||
BootstrapAdminPassword string
|
||||
SecretEnvelopeKey string
|
||||
AIProviderMode string
|
||||
BuilderDockerBinary string
|
||||
BuilderImage string
|
||||
BuilderSourceDir string
|
||||
BuilderWorkspaceDir string
|
||||
BuilderCacheDir string
|
||||
BuilderTimeoutSeconds int
|
||||
Addr string
|
||||
StorageBackend string
|
||||
MySQLDSN string
|
||||
DataDir string
|
||||
MetadataPath string
|
||||
LogDir string
|
||||
ArtifactDir string
|
||||
LogBodyBackend string
|
||||
BootstrapAdminEmail string
|
||||
BootstrapAdminPassword string
|
||||
SecretEnvelopeKey string
|
||||
AIProviderMode string
|
||||
BuilderDockerBinary string
|
||||
BuilderImage string
|
||||
BuilderSourceDir string
|
||||
BuilderSourceRepository string
|
||||
BuilderSourceRevision string
|
||||
BuilderWorkspaceDir string
|
||||
BuilderCacheDir string
|
||||
BuilderTimeoutSeconds int
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
@@ -78,24 +82,26 @@ func Load() Config {
|
||||
}
|
||||
|
||||
return Config{
|
||||
Addr: addr,
|
||||
StorageBackend: storageBackend,
|
||||
MySQLDSN: strings.TrimSpace(os.Getenv("PLATFORM_MYSQL_DSN")),
|
||||
DataDir: dataDir,
|
||||
MetadataPath: metadataPath,
|
||||
LogDir: logDir,
|
||||
ArtifactDir: artifactDir,
|
||||
LogBodyBackend: logBodyBackend,
|
||||
BootstrapAdminEmail: strings.TrimSpace(os.Getenv("PLATFORM_BOOTSTRAP_ADMIN_EMAIL")),
|
||||
BootstrapAdminPassword: os.Getenv("PLATFORM_BOOTSTRAP_ADMIN_PASSWORD"),
|
||||
SecretEnvelopeKey: os.Getenv("PLATFORM_SECRET_ENVELOPE_KEY"),
|
||||
AIProviderMode: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_AI_PROVIDER_MODE")), "live"),
|
||||
BuilderDockerBinary: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_DOCKER_BINARY")), defaultBuilderDockerBinary),
|
||||
BuilderImage: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_IMAGE")), defaultBuilderImage),
|
||||
BuilderSourceDir: strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_SOURCE_DIR")),
|
||||
BuilderWorkspaceDir: builderWorkspaceDir,
|
||||
BuilderCacheDir: builderCacheDir,
|
||||
BuilderTimeoutSeconds: defaultPositiveInt(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_TIMEOUT_SECONDS")), defaultBuilderTimeoutSeconds),
|
||||
Addr: addr,
|
||||
StorageBackend: storageBackend,
|
||||
MySQLDSN: strings.TrimSpace(os.Getenv("PLATFORM_MYSQL_DSN")),
|
||||
DataDir: dataDir,
|
||||
MetadataPath: metadataPath,
|
||||
LogDir: logDir,
|
||||
ArtifactDir: artifactDir,
|
||||
LogBodyBackend: logBodyBackend,
|
||||
BootstrapAdminEmail: strings.TrimSpace(os.Getenv("PLATFORM_BOOTSTRAP_ADMIN_EMAIL")),
|
||||
BootstrapAdminPassword: os.Getenv("PLATFORM_BOOTSTRAP_ADMIN_PASSWORD"),
|
||||
SecretEnvelopeKey: os.Getenv("PLATFORM_SECRET_ENVELOPE_KEY"),
|
||||
AIProviderMode: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_AI_PROVIDER_MODE")), "live"),
|
||||
BuilderDockerBinary: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_DOCKER_BINARY")), defaultBuilderDockerBinary),
|
||||
BuilderImage: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_IMAGE")), defaultBuilderImage),
|
||||
BuilderSourceDir: strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_SOURCE_DIR")),
|
||||
BuilderSourceRepository: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_SOURCE_REPOSITORY")), defaultBuilderSourceRepository),
|
||||
BuilderSourceRevision: defaultString(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_SOURCE_REVISION")), defaultBuilderSourceRevision),
|
||||
BuilderWorkspaceDir: builderWorkspaceDir,
|
||||
BuilderCacheDir: builderCacheDir,
|
||||
BuilderTimeoutSeconds: defaultPositiveInt(strings.TrimSpace(os.Getenv("PLATFORM_BUILDER_TIMEOUT_SECONDS")), defaultBuilderTimeoutSeconds),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ func TestLoadUsesDefaultAddress(t *testing.T) {
|
||||
t.Setenv("PLATFORM_BUILDER_DOCKER_BINARY", "")
|
||||
t.Setenv("PLATFORM_BUILDER_IMAGE", "")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_DIR", "")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_REPOSITORY", "")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_REVISION", "")
|
||||
t.Setenv("PLATFORM_BUILDER_WORKSPACE_DIR", "")
|
||||
t.Setenv("PLATFORM_BUILDER_CACHE_DIR", "")
|
||||
t.Setenv("PLATFORM_BUILDER_TIMEOUT_SECONDS", "")
|
||||
@@ -35,7 +37,7 @@ func TestLoadUsesDefaultAddress(t *testing.T) {
|
||||
if cfg.MetadataPath != filepath.Join(".platform-data", "metadata.json") || cfg.LogDir != filepath.Join(".platform-data", "logs") {
|
||||
t.Fatalf("unexpected default storage paths: %+v", cfg)
|
||||
}
|
||||
if cfg.BuilderDockerBinary != defaultBuilderDockerBinary || cfg.BuilderImage != defaultBuilderImage || cfg.BuilderSourceDir != "" || cfg.BuilderWorkspaceDir != filepath.Join(".platform-data", "distribution-builds") || cfg.BuilderCacheDir != filepath.Join(".platform-data", "distribution-build-cache") || cfg.BuilderTimeoutSeconds != defaultBuilderTimeoutSeconds {
|
||||
if cfg.BuilderDockerBinary != defaultBuilderDockerBinary || cfg.BuilderImage != defaultBuilderImage || cfg.BuilderSourceDir != "" || cfg.BuilderSourceRepository != defaultBuilderSourceRepository || cfg.BuilderSourceRevision != defaultBuilderSourceRevision || cfg.BuilderWorkspaceDir != filepath.Join(".platform-data", "distribution-builds") || cfg.BuilderCacheDir != filepath.Join(".platform-data", "distribution-build-cache") || cfg.BuilderTimeoutSeconds != defaultBuilderTimeoutSeconds {
|
||||
t.Fatalf("unexpected default builder config: %+v", cfg)
|
||||
}
|
||||
}
|
||||
@@ -54,6 +56,8 @@ func TestLoadUsesConfiguredAddress(t *testing.T) {
|
||||
t.Setenv("PLATFORM_BUILDER_DOCKER_BINARY", "/usr/local/bin/docker")
|
||||
t.Setenv("PLATFORM_BUILDER_IMAGE", "registry.example.test/distribution-builder:2.0.0")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_DIR", "/srv/run-source")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_REPOSITORY", "https://git.example.test/admin/run.git")
|
||||
t.Setenv("PLATFORM_BUILDER_SOURCE_REVISION", "release")
|
||||
t.Setenv("PLATFORM_BUILDER_WORKSPACE_DIR", "/srv/distribution-builds")
|
||||
t.Setenv("PLATFORM_BUILDER_CACHE_DIR", "/srv/distribution-build-cache")
|
||||
t.Setenv("PLATFORM_BUILDER_TIMEOUT_SECONDS", "900")
|
||||
@@ -65,7 +69,7 @@ func TestLoadUsesConfiguredAddress(t *testing.T) {
|
||||
if cfg.StorageBackend != "mysql" || cfg.MySQLDSN != "platform:platform@tcp(127.0.0.1:3306)/platform?parseTime=true" || cfg.DataDir != "/tmp/platform-data" || cfg.MetadataPath != "/tmp/platform-metadata.json" || cfg.LogDir != "/tmp/platform-logs" || cfg.LogBodyBackend != "file" || cfg.BootstrapAdminEmail != "admin@example.test" || cfg.BootstrapAdminPassword != "configured-secret" || cfg.SecretEnvelopeKey != "configured-envelope-key-at-least-32-bytes" {
|
||||
t.Fatalf("unexpected configured storage: %+v", cfg)
|
||||
}
|
||||
if cfg.BuilderDockerBinary != "/usr/local/bin/docker" || cfg.BuilderImage != "registry.example.test/distribution-builder:2.0.0" || cfg.BuilderSourceDir != "/srv/run-source" || cfg.BuilderWorkspaceDir != "/srv/distribution-builds" || cfg.BuilderCacheDir != "/srv/distribution-build-cache" || cfg.BuilderTimeoutSeconds != 900 {
|
||||
if cfg.BuilderDockerBinary != "/usr/local/bin/docker" || cfg.BuilderImage != "registry.example.test/distribution-builder:2.0.0" || cfg.BuilderSourceDir != "/srv/run-source" || cfg.BuilderSourceRepository != "https://git.example.test/admin/run.git" || cfg.BuilderSourceRevision != "release" || cfg.BuilderWorkspaceDir != "/srv/distribution-builds" || cfg.BuilderCacheDir != "/srv/distribution-build-cache" || cfg.BuilderTimeoutSeconds != 900 {
|
||||
t.Fatalf("unexpected configured builder: %+v", cfg)
|
||||
}
|
||||
}
|
||||
@@ -127,6 +131,8 @@ func clearPlatformEnv(t *testing.T) {
|
||||
"PLATFORM_BUILDER_DOCKER_BINARY",
|
||||
"PLATFORM_BUILDER_IMAGE",
|
||||
"PLATFORM_BUILDER_SOURCE_DIR",
|
||||
"PLATFORM_BUILDER_SOURCE_REPOSITORY",
|
||||
"PLATFORM_BUILDER_SOURCE_REVISION",
|
||||
"PLATFORM_BUILDER_WORKSPACE_DIR",
|
||||
"PLATFORM_BUILDER_CACHE_DIR",
|
||||
"PLATFORM_BUILDER_TIMEOUT_SECONDS",
|
||||
|
||||
@@ -48,22 +48,26 @@ type distributionBuilderWithProgress interface {
|
||||
|
||||
// DockerDistributionBuilderConfig configures a container-per-build builder.
|
||||
type DockerDistributionBuilderConfig struct {
|
||||
DockerBinary string
|
||||
Image string
|
||||
SourceDir string
|
||||
WorkspaceDir string
|
||||
CacheDir string
|
||||
Timeout time.Duration
|
||||
PlatformURL string
|
||||
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
|
||||
DockerBinary string
|
||||
Image string
|
||||
SourceDir string
|
||||
SourceRepositoryURL string
|
||||
SourceRevision string
|
||||
WorkspaceDir string
|
||||
CacheDir string
|
||||
Timeout time.Duration
|
||||
PlatformURL string
|
||||
CommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
CommandStream func(ctx context.Context, name string, args []string, onLine func(string)) ([]byte, error)
|
||||
GitCommandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
}
|
||||
|
||||
// DockerDistributionBuilder runs each build in a container from a pinned image,
|
||||
// with the run source mounted read-only and a per-job output directory mounted
|
||||
// writable.
|
||||
type DockerDistributionBuilder struct {
|
||||
config DockerDistributionBuilderConfig
|
||||
config DockerDistributionBuilderConfig
|
||||
sourceMu sync.Mutex
|
||||
}
|
||||
|
||||
func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *DockerDistributionBuilder {
|
||||
@@ -77,6 +81,9 @@ func NewDockerDistributionBuilder(config DockerDistributionBuilderConfig) *Docke
|
||||
if config.CommandRunner == nil {
|
||||
config.CommandRunner = runCommandCombined
|
||||
}
|
||||
if config.GitCommandRunner == nil {
|
||||
config.GitCommandRunner = runCommandCombined
|
||||
}
|
||||
if config.CommandStream == nil && !customRunner {
|
||||
config.CommandStream = runCommandStreamCombined
|
||||
}
|
||||
@@ -142,8 +149,13 @@ func (builder *DockerDistributionBuilder) Readiness() (bool, string) {
|
||||
if err != nil {
|
||||
return false, "platform builder run source directory is invalid"
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
|
||||
return false, "platform builder run source directory does not contain a run checkout"
|
||||
if _, err := os.Stat(source); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return false, "platform builder run source directory is not accessible"
|
||||
}
|
||||
if strings.TrimSpace(builder.config.SourceRepositoryURL) == "" {
|
||||
if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil {
|
||||
return false, "platform builder run source repository is not configured"
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(builder.config.WorkspaceDir) == "" {
|
||||
return false, "platform builder workspace directory is not configured"
|
||||
@@ -199,6 +211,15 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
|
||||
if err != nil {
|
||||
return nil, validationError("platform builder workspace directory is invalid")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
|
||||
defer cancel()
|
||||
if input.ComponentKind == domain.DistributionComponentRun {
|
||||
// Keep a shared checkout stable while Docker reads it. The checkout is
|
||||
// refreshed and exported below for this build, rather than captured when
|
||||
// the platform process starts.
|
||||
builder.sourceMu.Lock()
|
||||
defer builder.sourceMu.Unlock()
|
||||
}
|
||||
cacheDir, err := builder.cacheDir(workspaceDir)
|
||||
if err != nil {
|
||||
return nil, validationError("platform builder cache directory is invalid")
|
||||
@@ -212,15 +233,23 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
|
||||
outputDir := filepath.Join(jobDir, "output")
|
||||
inputDir := filepath.Join(jobDir, "input")
|
||||
buildDir := filepath.Join(jobDir, "build")
|
||||
sourceMountDir := filepath.Join(jobDir, "source")
|
||||
goBuildCacheDir := filepath.Join(cacheDir, "go-build")
|
||||
goModCacheDir := filepath.Join(cacheDir, "go-mod")
|
||||
for _, directory := range []string{outputDir, inputDir, buildDir, goBuildCacheDir, goModCacheDir} {
|
||||
for _, directory := range []string{outputDir, inputDir, buildDir, sourceMountDir, goBuildCacheDir, goModCacheDir} {
|
||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(jobDir) }()
|
||||
reportBuilderProgress(progress, 14, "env_check: platform builder workspace prepared")
|
||||
if input.ComponentKind == domain.DistributionComponentRun {
|
||||
preparedSourceDir, err := builder.prepareRunSource(ctx, sourceDir, sourceMountDir, input, progress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sourceMountDir = preparedSourceDir
|
||||
}
|
||||
|
||||
// The auth key reaches the container through a per-job input file, never
|
||||
// through a job-channel response to a machine-side endpoint or a container
|
||||
@@ -261,9 +290,7 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
|
||||
if outputName == "" || filepath.Base(outputName) != outputName {
|
||||
return nil, validationError("distribution build input has an invalid output filename")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), builder.config.Timeout)
|
||||
defer cancel()
|
||||
args := builder.containerArgs(input, sourceDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
|
||||
args := builder.containerArgs(input, sourceMountDir, inputDir, buildDir, outputDir, goBuildCacheDir, goModCacheDir, outputName)
|
||||
reportBuilderProgress(progress, 18, "git_sync: platform builder container starting")
|
||||
output, err := builder.runBuildCommand(ctx, args, progress)
|
||||
if err != nil {
|
||||
@@ -290,6 +317,98 @@ func (builder *DockerDistributionBuilder) BuildWithProgress(input domain.Distrib
|
||||
return packageClientManagerDistribution(input.PackageFormat, outputName, binary, configPayload)
|
||||
}
|
||||
|
||||
// prepareRunSource implements the source phase of a Jenkins-style build. A
|
||||
// configured checkout is created when absent, fetched on every build, and
|
||||
// exported at FETCH_HEAD into the per-job workspace. The archive keeps local
|
||||
// uncommitted files and the checkout's .git directory out of the executable.
|
||||
func (builder *DockerDistributionBuilder) prepareRunSource(ctx context.Context, sourceDir, destination string, input domain.DistributionBuildInput, progress func(DistributionBuildProgress)) (string, error) {
|
||||
repository := strings.TrimSpace(input.RepositoryURL)
|
||||
if repository == "" {
|
||||
repository = strings.TrimSpace(builder.config.SourceRepositoryURL)
|
||||
}
|
||||
revision := strings.TrimSpace(input.SourceRevision)
|
||||
if revision == "" {
|
||||
revision = strings.TrimSpace(builder.config.SourceRevision)
|
||||
}
|
||||
if revision == "" {
|
||||
revision = "main"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(sourceDir, ".git")); errors.Is(err, os.ErrNotExist) {
|
||||
if repository == "" {
|
||||
if _, sourceErr := os.Stat(filepath.Join(sourceDir, "go.mod")); sourceErr != nil {
|
||||
return "", validationError("platform builder has no Run source checkout or repository")
|
||||
}
|
||||
return sourceDir, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(sourceDir), 0o700); err != nil {
|
||||
return "", validationError("platform builder could not create the Run source parent")
|
||||
}
|
||||
reportBuilderProgress(progress, 20, "git_sync: creating Run source checkout")
|
||||
if _, err := builder.config.GitCommandRunner(ctx, "git", "clone", "--no-checkout", repository, sourceDir); err != nil {
|
||||
return "", validationError("platform builder could not clone the Run source")
|
||||
}
|
||||
} else if err != nil {
|
||||
return "", validationError("platform builder could not inspect the Run source checkout")
|
||||
}
|
||||
|
||||
reportBuilderProgress(progress, 26, "git_sync: fetching Run source revision")
|
||||
if _, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "fetch", "--depth", "1", "origin", revision); err != nil {
|
||||
return "", validationError("platform builder could not update the Run source revision")
|
||||
}
|
||||
archivePayload, err := builder.config.GitCommandRunner(ctx, "git", "-C", sourceDir, "archive", "--format=tar", "FETCH_HEAD")
|
||||
if err != nil {
|
||||
return "", validationError("platform builder could not export the Run source revision")
|
||||
}
|
||||
if err := extractRunSourceArchive(archivePayload, destination); err != nil {
|
||||
return "", validationError("platform builder could not prepare the Run source workspace")
|
||||
}
|
||||
reportBuilderProgress(progress, 34, "git_sync: Run source revision prepared")
|
||||
return destination, nil
|
||||
}
|
||||
|
||||
func extractRunSourceArchive(payload []byte, destination string) error {
|
||||
reader := tar.NewReader(bytes.NewReader(payload))
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := filepath.Clean(filepath.FromSlash(header.Name))
|
||||
if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(os.PathSeparator)) {
|
||||
return errors.New("Run source archive contains an unsafe path")
|
||||
}
|
||||
target := filepath.Join(destination, name)
|
||||
if header.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(target, header.FileInfo().Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !header.FileInfo().Mode().IsRegular() {
|
||||
return errors.New("Run source archive contains an unsupported file")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, header.FileInfo().Mode().Perm())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(file, reader)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (builder *DockerDistributionBuilder) cacheDir(workspaceDir string) (string, error) {
|
||||
configured := strings.TrimSpace(builder.config.CacheDir)
|
||||
if configured == "" {
|
||||
|
||||
@@ -94,6 +94,66 @@ func TestDockerDistributionBuilderReadinessNamesPlatformBuilderFailures(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerDistributionBuilderRefreshesRunSourceAtBuildTime(t *testing.T) {
|
||||
sourceDir := filepath.Join(t.TempDir(), "run-checkout")
|
||||
workspaceDir := t.TempDir()
|
||||
var gitCalls [][]string
|
||||
builder := NewDockerDistributionBuilder(DockerDistributionBuilderConfig{
|
||||
DockerBinary: "docker-test",
|
||||
Image: "browser-platform-distribution-builder:1.0.0",
|
||||
SourceDir: sourceDir,
|
||||
SourceRepositoryURL: "git@example.test:admin/run.git",
|
||||
SourceRevision: "main",
|
||||
WorkspaceDir: workspaceDir,
|
||||
GitCommandRunner: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
if name != "git" {
|
||||
t.Fatalf("unexpected source command %q", name)
|
||||
}
|
||||
gitCalls = append(gitCalls, append([]string(nil), args...))
|
||||
if len(args) >= 4 && args[0] == "clone" {
|
||||
if err := os.MkdirAll(filepath.Join(args[len(args)-1], ".git"), 0o700); err != nil {
|
||||
t.Fatalf("create fake checkout: %v", err)
|
||||
}
|
||||
}
|
||||
if len(args) >= 4 && args[0] == "-C" && args[2] == "archive" {
|
||||
return runSourceArchive(t), nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
CommandRunner: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
if name != "docker-test" {
|
||||
t.Fatalf("unexpected container runtime %q", name)
|
||||
}
|
||||
if len(args) > 0 && (args[0] == "version" || args[0] == "image") {
|
||||
return []byte("27.0.0"), nil
|
||||
}
|
||||
outputDir := builderMountHostPath(t, args, "/workspace/output")
|
||||
if err := os.WriteFile(filepath.Join(outputDir, "run"), []byte("compiled-run"), 0o700); err != nil {
|
||||
t.Fatalf("write fake build output: %v", err)
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
payload, err := builder.Build(domain.DistributionBuildInput{
|
||||
JobID: "job-source-refresh",
|
||||
ComponentKind: domain.DistributionComponentRun,
|
||||
PluginID: "game.scum",
|
||||
TargetOS: "linux",
|
||||
TargetArch: "amd64",
|
||||
OutputFilename: "run",
|
||||
AuthKey: "component-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("build refreshed Run source: %v", err)
|
||||
}
|
||||
if string(payload) != "compiled-run" {
|
||||
t.Fatalf("unexpected built payload %q", payload)
|
||||
}
|
||||
if len(gitCalls) != 3 || gitCalls[0][0] != "clone" || gitCalls[1][0] != "-C" || gitCalls[1][2] != "fetch" || gitCalls[2][2] != "archive" {
|
||||
t.Fatalf("expected clone, fetch, and archive during build, got %#v", gitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerDistributionBuilderKeepsSecretInIsolatedInput(t *testing.T) {
|
||||
sourceDir := createBuilderSource(t)
|
||||
workspaceDir := t.TempDir()
|
||||
@@ -143,6 +203,11 @@ func TestDockerDistributionBuilderKeepsSecretInIsolatedInput(t *testing.T) {
|
||||
if bytes.Contains(script, []byte(secret)) {
|
||||
t.Fatal("build script must not embed the component auth key")
|
||||
}
|
||||
for _, ordered := range [][2]string{{"cp -R /workspace/source/. /workspace/build/run-source/", "workspace_seed_generated.go"}, {"workspace_seed_generated.go", "go build -trimpath -ldflags"}} {
|
||||
if bytes.Index(script, []byte(ordered[0])) >= bytes.Index(script, []byte(ordered[1])) {
|
||||
t.Fatalf("Run build script must prepare source before injecting config and compiling: %q before %q", ordered[0], ordered[1])
|
||||
}
|
||||
}
|
||||
planPayload, err := os.ReadFile(filepath.Join(inputDir, "autonomous-lifecycle-plan.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read autonomous lifecycle plan input: %v", err)
|
||||
@@ -339,6 +404,22 @@ func createBuilderSource(t *testing.T) string {
|
||||
return sourceDir
|
||||
}
|
||||
|
||||
func runSourceArchive(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var payload bytes.Buffer
|
||||
writer := tar.NewWriter(&payload)
|
||||
if err := writer.WriteHeader(&tar.Header{Name: "go.mod", Mode: 0o600, Size: int64(len("module browser.local/run\n"))}); err != nil {
|
||||
t.Fatalf("write source archive header: %v", err)
|
||||
}
|
||||
if _, err := writer.Write([]byte("module browser.local/run\n")); err != nil {
|
||||
t.Fatalf("write source archive body: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close source archive: %v", err)
|
||||
}
|
||||
return payload.Bytes()
|
||||
}
|
||||
|
||||
func builderMountHostPath(t *testing.T, args []string, containerSuffix string) string {
|
||||
t.Helper()
|
||||
for index := 0; index+1 < len(args); index++ {
|
||||
|
||||
Reference in New Issue
Block a user