688 lines
30 KiB
Go
688 lines
30 KiB
Go
package service
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/platform/domain"
|
|
"browser.local/platform/repo"
|
|
)
|
|
|
|
type captureDistributionBuilder struct {
|
|
inputs chan domain.DistributionBuildInput
|
|
release <-chan struct{}
|
|
payload []byte
|
|
}
|
|
|
|
type progressDistributionBuilder struct {
|
|
started chan struct{}
|
|
release <-chan struct{}
|
|
payload []byte
|
|
}
|
|
|
|
type failingBuildProgressStore struct {
|
|
repo.Store
|
|
buildJobs repo.ClientManagerBuildJobRepository
|
|
}
|
|
|
|
func (store failingBuildProgressStore) ClientManagerBuildJobs() repo.ClientManagerBuildJobRepository {
|
|
return store.buildJobs
|
|
}
|
|
|
|
type failWhenDistributionJobRunningRepository struct {
|
|
repo.ClientManagerBuildJobRepository
|
|
jobs repo.JobRepository
|
|
err error
|
|
}
|
|
|
|
func (repository failWhenDistributionJobRunningRepository) List(filter domain.ClientManagerBuildJobFilter) ([]domain.ClientManagerBuildJob, error) {
|
|
jobs, err := repository.jobs.List(domain.JobFilter{ServerInstanceID: filter.ServerInstanceID})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, job := range jobs {
|
|
if job.Capability == domain.JobCapabilityDistributionBuild && job.State == domain.JobStateRunning {
|
|
return nil, repository.err
|
|
}
|
|
}
|
|
return repository.ClientManagerBuildJobRepository.List(filter)
|
|
}
|
|
|
|
func (builder captureDistributionBuilder) Readiness() (bool, string) {
|
|
return true, ""
|
|
}
|
|
|
|
func (builder captureDistributionBuilder) Build(input domain.DistributionBuildInput) ([]byte, error) {
|
|
builder.inputs <- input
|
|
if builder.release != nil {
|
|
<-builder.release
|
|
}
|
|
return domain.CopyBytes(builder.payload), nil
|
|
}
|
|
|
|
func (builder progressDistributionBuilder) Readiness() (bool, string) {
|
|
return true, ""
|
|
}
|
|
|
|
func (builder progressDistributionBuilder) Build(input domain.DistributionBuildInput) ([]byte, error) {
|
|
return builder.BuildWithProgress(input, nil)
|
|
}
|
|
|
|
func (builder progressDistributionBuilder) BuildWithProgress(_ domain.DistributionBuildInput, progress func(DistributionBuildProgress)) ([]byte, error) {
|
|
if progress != nil {
|
|
progress(DistributionBuildProgress{Percent: 72, Message: "build_compile: compiling run target executable"})
|
|
}
|
|
if builder.started != nil {
|
|
builder.started <- struct{}{}
|
|
}
|
|
if builder.release != nil {
|
|
<-builder.release
|
|
}
|
|
return domain.CopyBytes(builder.payload), nil
|
|
}
|
|
|
|
func TestCoreServiceKeepsPlatformBuildKeyOffMachineJobChannel(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
|
if err != nil {
|
|
t.Fatalf("get plugin fixture: %v", err)
|
|
}
|
|
plugin.LifecycleAssets = []domain.PluginAssetFile{
|
|
{Path: "actions/install.json", Content: `{"version":1,"action":"install","mode":"oneshot"}`, Mode: 0o600},
|
|
{Path: "bin/install-server", Content: "#!/usr/bin/env sh\n", Mode: 0o700},
|
|
{Path: "assets/map.bin", Content: base64.StdEncoding.EncodeToString([]byte{0xff, 0x00, 0x7f}), Encoding: "base64", Mode: 0o600},
|
|
}
|
|
plugin.RuntimeProfiles.LogSources = append(plugin.RuntimeProfiles.LogSources,
|
|
domain.RuntimeLogSource{Key: "console", Kind: "process.stdout", TargetKey: "server/process", StreamKey: "console", CursorKind: "sequence", RetentionDays: 14},
|
|
domain.RuntimeLogSource{Key: "server-events", Kind: "file.tail", TargetKey: "logs/server", StreamKey: "scum.server", CursorKind: "fingerprint", RetentionDays: 90},
|
|
)
|
|
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
|
t.Fatalf("seed plugin lifecycle assets: %v", err)
|
|
}
|
|
inputs := make(chan domain.DistributionBuildInput, 1)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
svc.ConfigureDistributionBuilder(captureDistributionBuilder{inputs: inputs, release: release, payload: []byte("captured-platform-build")})
|
|
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "platform-secret-boundary",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate platform distribution: %v", err)
|
|
}
|
|
var platformInput domain.DistributionBuildInput
|
|
select {
|
|
case platformInput = <-inputs:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("platform builder did not receive internal build input")
|
|
}
|
|
if platformInput.AuthKey == "" || platformInput.JobID != distribution.BuildJobID || platformInput.RunEndpointID != instance.RunEndpointID {
|
|
t.Fatalf("platform builder received incomplete internal input: %+v", platformInput)
|
|
}
|
|
decodedSeed, err := base64.StdEncoding.DecodeString(platformInput.WorkspaceSeed)
|
|
if err != nil {
|
|
t.Fatalf("decode workspace seed: %v", err)
|
|
}
|
|
var seedFiles []domain.PluginAssetFile
|
|
if err := json.Unmarshal(decodedSeed, &seedFiles); err != nil {
|
|
t.Fatalf("unmarshal workspace seed: %v", err)
|
|
}
|
|
if platformInput.ProfileKey != "local" || len(seedFiles) != 4 || seedFiles[1].Path != "bin/install-server" || seedFiles[1].Content == "" || seedFiles[2].Encoding != "base64" {
|
|
t.Fatalf("platform builder received incomplete plugin workspace seed: profile=%q seed=%+v", platformInput.ProfileKey, seedFiles)
|
|
}
|
|
if seedFiles[3].Path != ".platform/autonomous-lifecycle-plan.json" || seedFiles[3].Content == "" || seedFiles[3].Mode != 0o600 {
|
|
t.Fatalf("workspace seed did not include autonomous lifecycle plan file: %+v", seedFiles)
|
|
}
|
|
plan := platformInput.AutonomousLifecycle
|
|
if plan == nil || plan.SchemaVersion != "1" || plan.ServerInstanceID != instance.ID || plan.PluginID != plugin.ID || plan.ProfileKey != "local" || plan.Bootstrap == nil || plan.Bootstrap.Action != domain.ServerLifecycleActionStart || plan.Bootstrap.TargetKey != "actions/start.json" {
|
|
t.Fatalf("platform builder received incomplete autonomous lifecycle plan: %+v", plan)
|
|
}
|
|
if len(plan.DependencyProbes) != 1 || plan.DependencyProbes[0].Key != "java-runtime" || len(plan.InstallPlans) != 1 || plan.InstallPlans[0].Key != "java-install" || len(plan.LogSources) != 1 || !hasAutonomousLogSource(plan.LogSources, "process.stdout", "console") || plan.RuntimeBindings["logs/latest"] != "runtime.logs.latest" {
|
|
t.Fatalf("autonomous lifecycle plan lost plugin runtime declarations: %+v", plan)
|
|
}
|
|
if hasAutonomousLogSource(plan.LogSources, "file.tail", "latest-log") || hasAutonomousLogSource(plan.LogSources, "file.tail", "scum.server") {
|
|
t.Fatalf("autonomous lifecycle plan must not carry file-tail sources into process.start: %+v", plan.LogSources)
|
|
}
|
|
var seededPlan domain.RunAutonomousLifecyclePlan
|
|
if err := json.Unmarshal([]byte(seedFiles[3].Content), &seededPlan); err != nil {
|
|
t.Fatalf("unmarshal seeded autonomous lifecycle plan: %v", err)
|
|
}
|
|
if seededPlan.ServerInstanceID != plan.ServerInstanceID || seededPlan.Bootstrap == nil || seededPlan.Bootstrap.TargetKey != plan.Bootstrap.TargetKey {
|
|
t.Fatalf("seeded lifecycle plan differs from build input: seed=%+v input=%+v", seededPlan, plan)
|
|
}
|
|
auth, err := svc.AuthenticateComponent(domain.ComponentAuthenticationRequest{
|
|
ServerInstanceID: instance.ID,
|
|
ComponentKind: domain.DistributionComponentRun,
|
|
Generation: platformInput.KeyGeneration,
|
|
Key: platformInput.AuthKey,
|
|
})
|
|
if err != nil || !auth.Allowed {
|
|
t.Fatalf("platform builder did not receive the active plaintext component key: auth=%+v err=%v", auth, err)
|
|
}
|
|
|
|
helloRequest := validRunControlHello()
|
|
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, domain.JobCapabilityDistributionBuild)
|
|
hello, err := svc.RegisterRunHello(helloRequest)
|
|
if err != nil {
|
|
t.Fatalf("register machine endpoint: %v", err)
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{
|
|
RunEndpointID: instance.RunEndpointID,
|
|
SessionToken: hello.SessionToken,
|
|
Capabilities: []string{domain.JobCapabilityDistributionBuild},
|
|
Capacity: domain.RunCapacity{MaxJobs: 1},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("claim machine jobs: %v", err)
|
|
}
|
|
if claim.HasJob {
|
|
t.Fatalf("machine endpoint received platform-owned build job: %+v", claim)
|
|
}
|
|
machineInput, err := svc.GetDistributionBuildInput(domain.DistributionBuildInputRequest{
|
|
RunEndpointID: instance.RunEndpointID,
|
|
SessionToken: hello.SessionToken,
|
|
JobID: distribution.BuildJobID,
|
|
LeaseToken: "machine-cannot-hold-platform-build-lease",
|
|
Attempt: 1,
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("machine endpoint unexpectedly read platform build input: %+v", machineInput)
|
|
}
|
|
if machineInput.AuthKey != "" || strings.Contains(err.Error(), platformInput.AuthKey) {
|
|
t.Fatalf("machine build-input rejection leaked plaintext auth key: input=%+v err=%v", machineInput, err)
|
|
}
|
|
|
|
releaseOnce.Do(func() { close(release) })
|
|
completeDistributionBuild(t, svc, distribution, nil)
|
|
}
|
|
|
|
func hasAutonomousLogSource(sources []domain.RunAutonomousLogSource, kind string, streamKey string) bool {
|
|
for _, source := range sources {
|
|
if source.Kind == kind && source.StreamKey == streamKey {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestCoreServiceRunDistributionDefaultsEmptyDeploymentProfileToPluginLifecycleProfile(t *testing.T) {
|
|
svc := newTestCoreService()
|
|
plugin := scumDeploymentTestPlugin()
|
|
plugin.Name = "SCUM"
|
|
plugin.Version = "0.1.1"
|
|
plugin.ServerType = "scum"
|
|
plugin.Status = domain.GamePluginStatusInstalled
|
|
plugin.SupportedOS = []string{"windows"}
|
|
plugin.DeclaredPermissions = []string{"server.run.distribution"}
|
|
plugin.LifecycleActions = domain.PluginLifecycleActions{Install: "actions/install.json", Start: "actions/start.json", Stop: "actions/stop.json", Status: "actions/status.json"}
|
|
plugin.RuntimeProfiles.LifecycleProfiles = []domain.RuntimeLifecycleProfile{{Key: "run-local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop, domain.LifecycleCapabilityStatus}, ActionRefs: domain.PluginLifecycleActions{Install: "actions/install.json", Start: "actions/start.json", Stop: "actions/stop.json", Status: "actions/status.json"}, Platforms: []string{"windows"}}}
|
|
requireManualSCUMRuntimeBindings(&plugin, "run-local")
|
|
if err := svc.store.GamePlugins().Create(plugin); err != nil {
|
|
t.Fatalf("create SCUM plugin: %v", err)
|
|
}
|
|
session := createServiceUserAndLogin(t, svc, domain.User{ID: "scum-default-profile-owner", DisplayName: "SCUM Default Profile Owner", Email: "scum-default-profile@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
|
created, err := svc.CreateServerInstanceWorkflowForSession(session, domain.ServerLifecycleCreate{
|
|
ID: "scum-default-profile-package", PluginID: plugin.ID, Name: "SCUM Default Profile Package", IdempotencyKey: "scum-default-profile-package-create",
|
|
Deployment: domain.ServerDeploymentDefinition{Mode: domain.ServerDeploymentModeGuided, ServerRoot: `D:\scum-default-profile`, CreateInputs: map[string]string{"serverName": "Moon", "gamePort": "27000", "queryPort": "27015", "maxPlayers": "128"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create SCUM guided draft without explicit profile: %v", err)
|
|
}
|
|
if created.Instance.Deployment.ProfileKey != "run-local" {
|
|
t.Fatalf("expected create defaults to persist plugin lifecycle profile, got %+v", created.Instance.Deployment)
|
|
}
|
|
|
|
legacy := created.Instance
|
|
legacy.Deployment.ProfileKey = ""
|
|
if err := svc.store.ServerInstances().Update(legacy); err != nil {
|
|
t.Fatalf("simulate legacy empty deployment profile: %v", err)
|
|
}
|
|
inputs := make(chan domain.DistributionBuildInput, 1)
|
|
svc.ConfigureDistributionBuilder(captureDistributionBuilder{inputs: inputs, payload: []byte("profile-default-build")})
|
|
if _, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{ServerInstanceID: legacy.ID, TargetOS: "windows", TargetArch: "amd64", IdempotencyKey: "scum-default-profile-package"}); err != nil {
|
|
t.Fatalf("generate Run distribution with empty deployment profile: %v", err)
|
|
}
|
|
var platformInput domain.DistributionBuildInput
|
|
select {
|
|
case platformInput = <-inputs:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("platform builder did not receive profile-defaulted input")
|
|
}
|
|
plan := platformInput.AutonomousLifecycle
|
|
if platformInput.ProfileKey != "run-local" || plan == nil || plan.ProfileKey != "run-local" || plan.Deployment == nil || plan.Deployment.ProfileKey != "run-local" {
|
|
t.Fatalf("expected package context to default empty deployment profile to run-local, input=%+v plan=%+v", platformInput, plan)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceBuildsWithoutRegisteredDistributionWorker(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
bootstrapEndpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
|
if err != nil {
|
|
t.Fatalf("get bootstrap endpoint: %v", err)
|
|
}
|
|
capabilities := bootstrapEndpoint.Capabilities[:0]
|
|
for _, capability := range bootstrapEndpoint.Capabilities {
|
|
if capability != domain.JobCapabilityDistributionBuild {
|
|
capabilities = append(capabilities, capability)
|
|
}
|
|
}
|
|
bootstrapEndpoint.Capabilities = capabilities
|
|
bootstrapEndpoint.Status = domain.RunEndpointStatusOffline
|
|
if err := svc.store.RunEndpoints().Update(bootstrapEndpoint); err != nil {
|
|
t.Fatalf("remove build worker capability: %v", err)
|
|
}
|
|
instance.RunEndpointID = dedicatedRunEndpointID(instance.ID)
|
|
instance.DeploymentTargetID = ""
|
|
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
|
t.Fatalf("prepare unregistered dedicated Run binding: %v", err)
|
|
}
|
|
if _, err := svc.store.RunEndpoints().Get(instance.RunEndpointID); !errors.Is(err, repo.ErrNotFound) {
|
|
t.Fatalf("dedicated Run must be unregistered before generation, got %v", err)
|
|
}
|
|
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "windows",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "no-machine-distribution-worker",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate without registered distribution worker: %v", err)
|
|
}
|
|
distribution = completeDistributionBuild(t, svc, distribution, nil)
|
|
if distribution.RunEndpointID != instance.RunEndpointID || distribution.Status != domain.DistributionStatusAvailable {
|
|
t.Fatalf("unexpected platform-built distribution: %+v", distribution)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceGeneratedRunOnlyEndpointCanGenerateAnotherRun(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
instance.State = domain.ServerInstanceStateFailed
|
|
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
|
t.Fatalf("prepare server for dedicated Run generation: %v", err)
|
|
}
|
|
|
|
first, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "generated-run-only-first",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate first dedicated Run: %v", err)
|
|
}
|
|
first = completeDistributionBuild(t, svc, first, nil)
|
|
packageConfig := readGeneratedPackageConfig(t, svc, session, first.ArtifactID)
|
|
instance, err = svc.GetServerInstance(instance.ID)
|
|
if err != nil {
|
|
t.Fatalf("get server after Run build: %v", err)
|
|
}
|
|
|
|
helloRequest := validRunControlHello()
|
|
helloRequest.RunEndpointID = first.RunEndpointID
|
|
helloRequest.RegistrationToken = packageConfig.AuthKey
|
|
helloRequest.ServerInstanceID = instance.ID
|
|
helloRequest.PluginID = instance.PluginID
|
|
helloRequest.ComponentKind = domain.DistributionComponentRun
|
|
helloRequest.KeyGeneration = packageConfig.KeyGeneration
|
|
capabilities := helloRequest.CapabilityReport.Capabilities[:0]
|
|
for _, capability := range helloRequest.CapabilityReport.Capabilities {
|
|
if capability != domain.JobCapabilityDistributionBuild {
|
|
capabilities = append(capabilities, capability)
|
|
}
|
|
}
|
|
helloRequest.CapabilityReport.Capabilities = capabilities
|
|
registered, err := svc.RegisterRunHello(helloRequest)
|
|
if err != nil || !registered.Accepted {
|
|
t.Fatalf("register generated Run: result=%+v err=%v", registered, err)
|
|
}
|
|
online, err := svc.store.RunEndpoints().List(domain.RunEndpointFilter{Status: domain.RunEndpointStatusOnline})
|
|
if err != nil || len(online) != 1 || online[0].ID != first.RunEndpointID {
|
|
t.Fatalf("expected generated Run to be the only online endpoint: endpoints=%+v err=%v", online, err)
|
|
}
|
|
for _, capability := range online[0].Capabilities {
|
|
if capability == domain.JobCapabilityDistributionBuild {
|
|
t.Fatalf("generated Run must not advertise distribution build: %+v", online[0])
|
|
}
|
|
}
|
|
claim, err := svc.ClaimRunJob(domain.RunJobClaim{
|
|
RunEndpointID: first.RunEndpointID,
|
|
SessionToken: registered.SessionToken,
|
|
Capabilities: []string{domain.JobCapabilityDistributionBuild},
|
|
Capacity: domain.RunCapacity{MaxJobs: 1},
|
|
})
|
|
if err != nil || claim.HasJob {
|
|
t.Fatalf("generated Run must not receive platform build work: claim=%+v err=%v", claim, err)
|
|
}
|
|
|
|
second, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "generated-run-only-second",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("regenerate with generated Run as only endpoint: %v", err)
|
|
}
|
|
second = completeDistributionBuild(t, svc, second, nil)
|
|
job, err := svc.GetJob(second.BuildJobID)
|
|
if err != nil || job.RunEndpointID != platformDistributionBuilderEndpointID || job.State != domain.JobStateSucceeded || second.Status != domain.DistributionStatusAvailable {
|
|
t.Fatalf("expected platform-built regenerated Run: job=%+v distribution=%+v err=%v", job, second, err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceDoesNotDuplicateInFlightPlatformBuild(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
inputs := make(chan domain.DistributionBuildInput, 2)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
svc.ConfigureDistributionBuilder(captureDistributionBuilder{inputs: inputs, release: release, payload: []byte("idempotent-platform-build")})
|
|
request := domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "same-platform-build",
|
|
}
|
|
|
|
first, err := svc.GenerateRunDistributionForSession(session, request)
|
|
if err != nil {
|
|
t.Fatalf("generate first distribution: %v", err)
|
|
}
|
|
select {
|
|
case <-inputs:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first platform build did not start")
|
|
}
|
|
second, err := svc.GenerateRunDistributionForSession(session, request)
|
|
if err != nil {
|
|
t.Fatalf("repeat idempotent generation: %v", err)
|
|
}
|
|
if second.ID != first.ID || second.BuildJobID != first.BuildJobID {
|
|
t.Fatalf("idempotent generation returned different work: first=%+v second=%+v", first, second)
|
|
}
|
|
select {
|
|
case duplicate := <-inputs:
|
|
t.Fatalf("idempotent generation started duplicate platform build: %+v", duplicate)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
releaseOnce.Do(func() { close(release) })
|
|
completeDistributionBuild(t, svc, first, nil)
|
|
}
|
|
|
|
func TestCoreServiceProjectsPlatformBuilderProgressBeforeCompletion(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
started := make(chan struct{}, 1)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
svc.ConfigureDistributionBuilder(progressDistributionBuilder{started: started, release: release, payload: []byte("progress-platform-build")})
|
|
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "platform-builder-progress",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate platform distribution: %v", err)
|
|
}
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("platform builder did not report progress")
|
|
}
|
|
job, err := svc.GetJob(distribution.BuildJobID)
|
|
if err != nil {
|
|
t.Fatalf("get progress job: %v", err)
|
|
}
|
|
if job.State != domain.JobStateRunning || job.Progress.Percent != 72 || job.Progress.Message != "build_compile: compiling run target executable" {
|
|
t.Fatalf("expected in-flight platform builder progress, got %+v", job)
|
|
}
|
|
releaseOnce.Do(func() { close(release) })
|
|
completeDistributionBuild(t, svc, distribution, nil)
|
|
}
|
|
|
|
func TestCoreServiceTerminalizesPlatformBuildWhenRunningProjectionFails(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
const leakedDetail = "sensitive-builder-token /private/platform/build/input/auth-key"
|
|
baseStore := svc.store
|
|
svc.store = failingBuildProgressStore{
|
|
Store: baseStore,
|
|
buildJobs: failWhenDistributionJobRunningRepository{
|
|
ClientManagerBuildJobRepository: baseStore.ClientManagerBuildJobs(),
|
|
jobs: baseStore.Jobs(),
|
|
err: errors.New(leakedDetail),
|
|
},
|
|
}
|
|
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "running-projection-failure-terminalizes",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("queue platform distribution: %v", err)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
job, jobErr := svc.GetJob(distribution.BuildJobID)
|
|
updated, distributionErr := svc.store.RunDistributions().Get(distribution.ID)
|
|
if jobErr != nil || distributionErr != nil {
|
|
t.Fatalf("read failed build state: jobErr=%v distributionErr=%v", jobErr, distributionErr)
|
|
}
|
|
if isTerminalJobState(job.State) {
|
|
if job.State != domain.JobStateFailed || updated.Status != domain.DistributionStatusFailed {
|
|
t.Fatalf("projection failure did not end in failed state: job=%+v distribution=%+v", job, updated)
|
|
}
|
|
if strings.Contains(job.Progress.Message, "sensitive-builder-token") || strings.Contains(job.Progress.Message, "/private/") || strings.Contains(job.TerminalFingerprint, "sensitive-builder-token") {
|
|
t.Fatalf("terminal build state leaked internal failure details: %+v", job)
|
|
}
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatal("platform build remained non-terminal after running projection failure")
|
|
}
|
|
|
|
func TestCoreServiceDiscardsBuildCompletedAfterKeyReset(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
inputs := make(chan domain.DistributionBuildInput, 1)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
svc.ConfigureDistributionBuilder(captureDistributionBuilder{inputs: inputs, release: release, payload: []byte("stale-key-build")})
|
|
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "reset-during-platform-build",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate distribution: %v", err)
|
|
}
|
|
select {
|
|
case <-inputs:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("platform builder did not start")
|
|
}
|
|
if _, err := svc.ResetComponentKeyForSession(session, domain.ComponentKeyResetRequest{
|
|
ServerInstanceID: instance.ID,
|
|
ComponentKind: domain.DistributionComponentRun,
|
|
}); err != nil {
|
|
t.Fatalf("reset component key during build: %v", err)
|
|
}
|
|
releaseOnce.Do(func() { close(release) })
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
job, jobErr := svc.GetJob(distribution.BuildJobID)
|
|
updated, distributionErr := svc.store.RunDistributions().Get(distribution.ID)
|
|
if jobErr != nil || distributionErr != nil {
|
|
t.Fatalf("read stale build state: jobErr=%v distributionErr=%v", jobErr, distributionErr)
|
|
}
|
|
if job.State == domain.JobStateFailed {
|
|
if updated.Status != domain.DistributionStatusRevoked || !strings.Contains(job.Progress.Message, "no longer current") {
|
|
t.Fatalf("stale build did not remain revoked: job=%+v distribution=%+v", job, updated)
|
|
}
|
|
if artifact, artifactErr := svc.GetArtifact(distribution.ArtifactID); artifactErr == nil && artifact.State == domain.ArtifactStateAvailable {
|
|
t.Fatalf("stale-key build published an available artifact: %+v", artifact)
|
|
}
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatal("stale-key platform build did not terminate")
|
|
}
|
|
|
|
func TestCoreServicePreservesSucceededArtifactOnDuplicatePlatformResult(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "preserve-duplicate-platform-result",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("generate distribution: %v", err)
|
|
}
|
|
distribution = completeDistributionBuild(t, svc, distribution, nil)
|
|
job, err := svc.GetJob(distribution.BuildJobID)
|
|
if err != nil {
|
|
t.Fatalf("get completed build job: %v", err)
|
|
}
|
|
if job.State != domain.JobStateSucceeded || job.ResultRef != "artifact://"+distribution.ArtifactID {
|
|
t.Fatalf("expected succeeded build job with distribution artifact ref, got %+v", job)
|
|
}
|
|
if err := svc.succeedDistributionBuildJob(job, distribution.ArtifactID); err != nil {
|
|
t.Fatalf("replay succeeded platform result: %v", err)
|
|
}
|
|
artifact, err := svc.GetArtifact(distribution.ArtifactID)
|
|
if err != nil {
|
|
t.Fatalf("get distribution artifact: %v", err)
|
|
}
|
|
if artifact.State != domain.ArtifactStateAvailable {
|
|
t.Fatalf("duplicate platform result expired active artifact: %+v", artifact)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceResumesPendingPlatformBuildAfterEnvelopeConfiguration(t *testing.T) {
|
|
store := repo.NewMemoryStore()
|
|
artifactStore := NewMemoryArtifactBodyStore()
|
|
first, err := NewCoreServiceWithDurableStores(store, NewMemoryLogBodyStore(), artifactStore)
|
|
if err != nil {
|
|
t.Fatalf("create first durable CoreService: %v", err)
|
|
}
|
|
const envelopeKey = "restart-test-secret-envelope-key-must-remain-stable"
|
|
if err := first.ConfigureSecretEnvelopeKey(envelopeKey); err != nil {
|
|
t.Fatalf("configure first envelope key: %v", err)
|
|
}
|
|
plugin, endpoint := createPluginAndRunEndpoint(t, first)
|
|
plugin.SupportedOS = []string{"linux"}
|
|
plugin.DeclaredPermissions = append(plugin.DeclaredPermissions, "server.run.distribution")
|
|
plugin.BridgeActions = append(plugin.BridgeActions, string(domain.PluginBridgeActionRunDistribution))
|
|
if err := first.store.GamePlugins().Update(plugin); err != nil {
|
|
t.Fatalf("enable distribution fixture plugin: %v", err)
|
|
}
|
|
session := createServiceUserAndLogin(t, first, domain.User{
|
|
ID: "restart-distribution-owner",
|
|
DisplayName: "Restart Distribution Owner",
|
|
Email: "restart-distribution-owner@example.test",
|
|
Roles: []string{"server-owner"},
|
|
PasswordHash: "secret-password",
|
|
})
|
|
instance, err := first.CreateServerInstanceForSession(session, domain.ServerInstance{
|
|
ID: "server-restart-distribution",
|
|
PluginID: plugin.ID,
|
|
RunEndpointID: endpoint.ID,
|
|
Name: "Restart Distribution Server",
|
|
State: domain.ServerInstanceStateReady,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create restart distribution server: %v", err)
|
|
}
|
|
createCompleteRuntimeBinding(t, first, instance, "local")
|
|
|
|
inputs := make(chan domain.DistributionBuildInput, 1)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
first.ConfigureDistributionBuilder(captureDistributionBuilder{inputs: inputs, release: release, payload: []byte("first-process-output")})
|
|
distribution, err := first.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "resume-after-envelope-configuration",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("queue durable platform build: %v", err)
|
|
}
|
|
select {
|
|
case <-inputs:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first CoreService did not begin the pending build")
|
|
}
|
|
|
|
restarted, err := NewCoreServiceWithDurableStores(store, NewMemoryLogBodyStore(), artifactStore)
|
|
if err != nil {
|
|
t.Fatalf("create restarted durable CoreService: %v", err)
|
|
}
|
|
if err := restarted.ConfigureSecretEnvelopeKey(envelopeKey); err != nil {
|
|
t.Fatalf("configure restarted envelope key: %v", err)
|
|
}
|
|
restarted.ConfigureDistributionBuilder(staticDistributionBuilder{payload: []byte("recovered-platform-build")})
|
|
distribution = completeDistributionBuild(t, restarted, distribution, nil)
|
|
job, err := restarted.GetJob(distribution.BuildJobID)
|
|
if err != nil || job.State != domain.JobStateSucceeded || job.RunEndpointID != platformDistributionBuilderEndpointID {
|
|
t.Fatalf("expected recovered platform build to succeed: job=%+v err=%v", job, err)
|
|
}
|
|
artifact, err := restarted.GetArtifact(distribution.ArtifactID)
|
|
if err != nil || artifact.State != domain.ArtifactStateAvailable || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != job.ID {
|
|
t.Fatalf("expected recovered job-owned artifact: artifact=%+v err=%v", artifact, err)
|
|
}
|
|
}
|
|
|
|
func TestCoreServiceReportsPlatformBuilderUnavailable(t *testing.T) {
|
|
svc, session, instance := newDistributionTestFixture(t)
|
|
svc.ConfigureDistributionBuilder(staticDistributionBuilder{err: errors.New("platform builder image is unavailable")})
|
|
|
|
actions, err := svc.GetServerRuntimeActionsForSession(session, instance.ID)
|
|
if err != nil {
|
|
t.Fatalf("get runtime actions: %v", err)
|
|
}
|
|
seen := 0
|
|
for _, action := range actions.Actions {
|
|
if action.Key != "generate-run" && action.Key != "generate-client-manager" {
|
|
continue
|
|
}
|
|
seen++
|
|
if action.Available || !strings.Contains(action.Reason, "platform builder") || strings.Contains(action.Reason, "run endpoint") {
|
|
t.Fatalf("expected explicit platform builder unavailable reason, got %+v", action)
|
|
}
|
|
}
|
|
if seen != 2 {
|
|
t.Fatalf("expected both build actions, got %+v", actions.Actions)
|
|
}
|
|
|
|
_, err = svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
|
ServerInstanceID: instance.ID,
|
|
TargetOS: "linux",
|
|
TargetArch: "amd64",
|
|
IdempotencyKey: "builder-unavailable",
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "platform builder") || strings.Contains(err.Error(), "run endpoint") {
|
|
t.Fatalf("expected generation to report platform builder failure, got %v", err)
|
|
}
|
|
}
|