fix local run startup and stale dispatch
This commit is contained in:
@@ -1534,9 +1534,9 @@ func TestProductionOperationsGovernanceRoutesAreDurableAndRedacted(t *testing.T)
|
|||||||
if len(capacity.Endpoints) == 0 {
|
if len(capacity.Endpoints) == 0 {
|
||||||
t.Fatalf("expected persisted capacity endpoints, got %+v", capacity)
|
t.Fatalf("expected persisted capacity endpoints, got %+v", capacity)
|
||||||
}
|
}
|
||||||
decision := postOKJSONWithAuth[dto.CapacityAdmissionDecisionResponse](t, router, "/api/v1/production/capacity/admission", dto.CapacityAdmissionRequest{ServerInstanceID: serverID, Capability: domain.JobCapabilityConfigWrite, IdempotencyKey: "api-capacity-check"}, adminSession)
|
decision := postOKJSONWithAuth[dto.CapacityAdmissionDecisionResponse](t, router, "/api/v1/production/capacity/admission", dto.CapacityAdmissionRequest{ServerInstanceID: serverID, Capability: domain.JobCapabilityRemoteRunRCONCommand, IdempotencyKey: "api-capacity-check"}, adminSession)
|
||||||
if decision.Accepted || decision.AlertID == "" || decision.AuditEventID == "" {
|
if decision.Accepted || decision.AlertID == "" || decision.AuditEventID == "" {
|
||||||
t.Fatalf("expected stale endpoint admission to create durable evidence, got %+v", decision)
|
t.Fatalf("expected unavailable capability admission to create durable evidence, got %+v", decision)
|
||||||
}
|
}
|
||||||
alerts := getJSONWithAuth[dto.AlertListResponse](t, router, "/api/v1/alerts?state=active", adminSession)
|
alerts := getJSONWithAuth[dto.AlertListResponse](t, router, "/api/v1/alerts?state=active", adminSession)
|
||||||
if alerts.Count == 0 {
|
if alerts.Count == 0 {
|
||||||
|
|||||||
@@ -479,7 +479,7 @@ func (svc *CoreService) authorizeClientManagerLifecycle(sessionID, serverInstanc
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, capability); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, capability); err != nil {
|
||||||
_ = svc.recordAuditEvent(user.ID, deniedAction, "server-instance", instance.ID, domain.AuditResultDenied, "client-manager lifecycle denied: assigned Run endpoint is offline or unsupported")
|
_ = svc.recordAuditEvent(user.ID, deniedAction, "server-instance", instance.ID, domain.AuditResultDenied, "client-manager lifecycle denied: assigned Run endpoint is offline or unsupported")
|
||||||
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
|
||||||
}
|
}
|
||||||
@@ -1119,11 +1119,11 @@ func (svc *CoreService) clientManagerRuntimeActionProjection(instance domain.Ser
|
|||||||
}
|
}
|
||||||
baseReason := fallbackReason(!hasLifecycle || !bindingsComplete, "plugin does not declare a complete Client Manager lifecycle", bindingReason)
|
baseReason := fallbackReason(!hasLifecycle || !bindingsComplete, "plugin does not declare a complete Client Manager lifecycle", bindingReason)
|
||||||
return []domain.ServerRuntimeAction{
|
return []domain.ServerRuntimeAction{
|
||||||
runtimeAction("deploy-client-manager", "Deploy client manager", hasLifecycle && bindingsComplete && endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), "run endpoint cannot deploy client managers", baseReason)),
|
runtimeAction("deploy-client-manager", "Deploy client manager", hasLifecycle && bindingsComplete && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), "run endpoint cannot deploy client managers", baseReason)),
|
||||||
runtimeAction("control-client-manager", "Control client manager", hasLifecycle && hasInstalled && bindingsComplete && endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), fallbackReason(!hasInstalled || !endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), "client manager is not installed or endpoint cannot control it", baseReason)),
|
runtimeAction("control-client-manager", "Control client manager", hasLifecycle && hasInstalled && bindingsComplete && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), fallbackReason(!hasInstalled || !svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), "client manager is not installed or endpoint cannot control it", baseReason)),
|
||||||
runtimeAction("update-client-manager", "Update client manager", hasLifecycle && hasInstalled && hasUpdateCandidate && endpointSupports(endpoint, domain.JobCapabilityClientManagerUpdate), "no compatible update candidate is available"),
|
runtimeAction("update-client-manager", "Update client manager", hasLifecycle && hasInstalled && hasUpdateCandidate && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerUpdate), "no compatible update candidate is available"),
|
||||||
runtimeAction("rollback-client-manager", "Rollback client manager", hasLifecycle && hasPrevious && endpointSupports(endpoint, domain.JobCapabilityClientManagerRollback), "no retained previous deployment is available"),
|
runtimeAction("rollback-client-manager", "Rollback client manager", hasLifecycle && hasPrevious && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerRollback), "no retained previous deployment is available"),
|
||||||
runtimeAction("uninstall-client-manager", "Uninstall client manager", hasLifecycle && hasInstalled && endpointSupports(endpoint, domain.JobCapabilityClientManagerUninstall), "client manager is not installed or endpoint cannot uninstall it"),
|
runtimeAction("uninstall-client-manager", "Uninstall client manager", hasLifecycle && hasInstalled && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerUninstall), "client manager is not installed or endpoint cannot uninstall it"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -209,6 +209,14 @@ func TestClientManagerLifecycleRejectsCrossScopeStaleAndRevokedIdentity(t *testi
|
|||||||
|
|
||||||
func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution {
|
func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("get lifecycle Run endpoint: %v", err)
|
||||||
|
}
|
||||||
|
endpoint.LastHeartbeatAt = svc.now()
|
||||||
|
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||||
|
t.Fatalf("refresh lifecycle Run heartbeat: %v", err)
|
||||||
|
}
|
||||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("get lifecycle plugin: %v", err)
|
t.Fatalf("get lifecycle plugin: %v", err)
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, requ
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunDistribution{}, err
|
return domain.RunDistribution{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
|
||||||
return domain.RunDistribution{}, err
|
return domain.RunDistribution{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,7 +166,7 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.ClientManagerDistribution{}, err
|
return domain.ClientManagerDistribution{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
|
||||||
return domain.ClientManagerDistribution{}, err
|
return domain.ClientManagerDistribution{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -462,17 +462,17 @@ func (svc *CoreService) GetServerRuntimeActionsForSession(sessionID string, serv
|
|||||||
RunEndpointID: endpoint.ID,
|
RunEndpointID: endpoint.ID,
|
||||||
RunStatus: endpoint.Status,
|
RunStatus: endpoint.Status,
|
||||||
Actions: []domain.ServerRuntimeAction{
|
Actions: []domain.ServerRuntimeAction{
|
||||||
runtimeAction("generate-run", "Generate run", pluginDeclares(plugin, "server.run.distribution") && endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
|
runtimeAction("generate-run", "Generate run", pluginDeclares(plugin, "server.run.distribution") && svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
|
||||||
runtimeAction("download-run", "Download run", hasAvailableRunPackage, "run package has not been generated"),
|
runtimeAction("download-run", "Download run", hasAvailableRunPackage, "run package has not been generated"),
|
||||||
runtimeAction("push-run-update", "Push run update", pluginDeclares(plugin, "server.run.distribution") && endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate), "run endpoint cannot self-update", bindingReason)),
|
runtimeAction("push-run-update", "Push run update", pluginDeclares(plugin, "server.run.distribution") && svc.endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !svc.endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate), "run endpoint cannot self-update", bindingReason)),
|
||||||
runtimeAction("reset-run-key", "Reset run key", pluginDeclares(plugin, "server.run.distribution"), "plugin permission is not declared"),
|
runtimeAction("reset-run-key", "Reset run key", pluginDeclares(plugin, "server.run.distribution"), "plugin permission is not declared"),
|
||||||
runtimeAction("generate-client-manager", "Generate client manager", pluginDeclares(plugin, "server.client-manager.manage") && endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.client-manager.manage") || !endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
|
runtimeAction("generate-client-manager", "Generate client manager", pluginDeclares(plugin, "server.client-manager.manage") && svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.client-manager.manage") || !svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
|
||||||
runtimeAction("download-client-manager", "Download client manager", hasAvailableClientPackage, "client-manager package has not been generated"),
|
runtimeAction("download-client-manager", "Download client manager", hasAvailableClientPackage, "client-manager package has not been generated"),
|
||||||
runtimeAction("reset-client-manager-key", "Reset client-manager key", pluginDeclares(plugin, "server.client-manager.manage"), "client-manager permission is not declared"),
|
runtimeAction("reset-client-manager-key", "Reset client-manager key", pluginDeclares(plugin, "server.client-manager.manage"), "client-manager permission is not declared"),
|
||||||
runtimeAction("dependencies-check", "Check dependencies", dependencyPermissionDeclared && endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck), "run endpoint cannot check dependencies", bindingReason))),
|
runtimeAction("dependencies-check", "Check dependencies", dependencyPermissionDeclared && svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck), "run endpoint cannot check dependencies", bindingReason))),
|
||||||
runtimeAction("dependencies-install", "Install dependencies", dependencyPermissionDeclared && endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall), "run endpoint cannot install dependencies", bindingReason))),
|
runtimeAction("dependencies-install", "Install dependencies", dependencyPermissionDeclared && svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall), "run endpoint cannot install dependencies", bindingReason))),
|
||||||
runtimeAction("live-logs", "Live logs", pluginSupports(plugin, "logs.read"), "plugin does not declare live logs"),
|
runtimeAction("live-logs", "Live logs", pluginSupports(plugin, "logs.read"), "plugin does not declare live logs"),
|
||||||
runtimeAction("historical-logs", "Historical logs", endpointSupports(endpoint, domain.JobCapabilityLogsBackfill) && bindingsComplete, fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityLogsBackfill), "run endpoint cannot backfill logs", bindingReason)),
|
runtimeAction("historical-logs", "Historical logs", svc.endpointSupports(endpoint, domain.JobCapabilityLogsBackfill) && bindingsComplete, fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityLogsBackfill), "run endpoint cannot backfill logs", bindingReason)),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
actions.Actions = append(actions.Actions, svc.clientManagerRuntimeActionProjection(instance, plugin, endpoint, bindingsComplete, bindingReason)...)
|
actions.Actions = append(actions.Actions, svc.clientManagerRuntimeActionProjection(instance, plugin, endpoint, bindingsComplete, bindingReason)...)
|
||||||
@@ -1211,10 +1211,13 @@ func pluginSupports(plugin domain.GamePlugin, capability string) bool {
|
|||||||
return containsString(plugin.RequiredRunCapabilities, capability)
|
return containsString(plugin.RequiredRunCapabilities, capability)
|
||||||
}
|
}
|
||||||
|
|
||||||
func endpointSupports(endpoint domain.RunEndpoint, capability string) bool {
|
func (svc *CoreService) endpointSupports(endpoint domain.RunEndpoint, capability string) bool {
|
||||||
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
if !svc.runEndpointHeartbeatCurrent(endpoint) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
return containsString(endpoint.Capabilities, capability)
|
return containsString(endpoint.Capabilities, capability)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"browser.local/platform/domain"
|
"browser.local/platform/domain"
|
||||||
"browser.local/platform/repo"
|
"browser.local/platform/repo"
|
||||||
@@ -96,6 +97,31 @@ func TestCoreServiceGeneratesRunDistributionWithEncryptedSingletonKey(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCoreServiceRejectsDistributionBuildForStaleRunEndpoint(t *testing.T) {
|
||||||
|
svc, session, instance := newDistributionTestFixture(t)
|
||||||
|
svc.now = func() time.Time { return fixedTime.Add(capacityHeartbeatStaleAfter + time.Second) }
|
||||||
|
|
||||||
|
_, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
||||||
|
ServerInstanceID: instance.ID,
|
||||||
|
TargetOS: "linux",
|
||||||
|
TargetArch: "amd64",
|
||||||
|
IdempotencyKey: "idem-stale-run-endpoint",
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "heartbeat is stale") {
|
||||||
|
t.Fatalf("expected stale Run endpoint rejection, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
actions, err := svc.GetServerRuntimeActionsForSession(session, instance.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("get runtime actions: %v", err)
|
||||||
|
}
|
||||||
|
for _, action := range actions.Actions {
|
||||||
|
if action.Key == "generate-run" && action.Available {
|
||||||
|
t.Fatalf("stale Run endpoint must not expose generate-run as available: %+v", action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCoreServiceRuntimeActionsGateDependenciesOnPluginPermission(t *testing.T) {
|
func TestCoreServiceRuntimeActionsGateDependenciesOnPluginPermission(t *testing.T) {
|
||||||
svc, session, instance := newDistributionTestFixture(t)
|
svc, session, instance := newDistributionTestFixture(t)
|
||||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||||
|
|||||||
@@ -1509,6 +1509,9 @@ func (svc *CoreService) CreateRunEndpoint(endpoint domain.RunEndpoint) (domain.R
|
|||||||
if endpoint.Status == "" {
|
if endpoint.Status == "" {
|
||||||
endpoint.Status = domain.RunEndpointStatusOnline
|
endpoint.Status = domain.RunEndpointStatusOnline
|
||||||
}
|
}
|
||||||
|
if endpoint.LastHeartbeatAt.IsZero() && (endpoint.Status == domain.RunEndpointStatusOnline || endpoint.Status == domain.RunEndpointStatusDegraded) {
|
||||||
|
endpoint.LastHeartbeatAt = svc.now()
|
||||||
|
}
|
||||||
if err := validator.ValidateRunEndpoint(endpoint); err != nil {
|
if err := validator.ValidateRunEndpoint(endpoint); err != nil {
|
||||||
return domain.RunEndpoint{}, err
|
return domain.RunEndpoint{}, err
|
||||||
}
|
}
|
||||||
@@ -2158,7 +2161,7 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.Job{}, fmt.Errorf("get run endpoint dependency: %w", err)
|
return domain.Job{}, fmt.Errorf("get run endpoint dependency: %w", err)
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, job.Capability); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, job.Capability); err != nil {
|
||||||
return domain.Job{}, err
|
return domain.Job{}, err
|
||||||
}
|
}
|
||||||
if job.ServerInstanceID != "" {
|
if job.ServerInstanceID != "" {
|
||||||
@@ -2281,16 +2284,26 @@ func (svc *CoreService) ListAuditEvents(filter domain.AuditEventFilter) ([]domai
|
|||||||
return svc.store.AuditEvents().List(filter)
|
return svc.store.AuditEvents().List(filter)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRunnableEndpoint(endpoint domain.RunEndpoint, capability string) error {
|
func (svc *CoreService) validateRunnableEndpoint(endpoint domain.RunEndpoint, capability string) error {
|
||||||
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
|
||||||
return validationError("run endpoint must be online or degraded")
|
return validationError("run endpoint must be online or degraded")
|
||||||
}
|
}
|
||||||
|
if !svc.runEndpointHeartbeatCurrent(endpoint) {
|
||||||
|
return validationError("run endpoint heartbeat is stale")
|
||||||
|
}
|
||||||
if len(validator.MissingCapabilities(endpoint.Capabilities, []string{capability})) > 0 {
|
if len(validator.MissingCapabilities(endpoint.Capabilities, []string{capability})) > 0 {
|
||||||
return validationError("run endpoint missing required capability: " + capability)
|
return validationError("run endpoint missing required capability: " + capability)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (svc *CoreService) runEndpointHeartbeatCurrent(endpoint domain.RunEndpoint) bool {
|
||||||
|
if endpoint.LastHeartbeatAt.IsZero() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !svc.now().After(endpoint.LastHeartbeatAt.Add(capacityHeartbeatStaleAfter))
|
||||||
|
}
|
||||||
|
|
||||||
func validateJobServerTarget(job domain.Job, instance domain.ServerInstance, plugin domain.GamePlugin) error {
|
func validateJobServerTarget(job domain.Job, instance domain.ServerInstance, plugin domain.GamePlugin) error {
|
||||||
if instance.State == domain.ServerInstanceStateDeleted {
|
if instance.State == domain.ServerInstanceStateDeleted {
|
||||||
return validationError("server instance must not be deleted")
|
return validationError("server instance must not be deleted")
|
||||||
|
|||||||
@@ -1444,6 +1444,7 @@ func createPluginAndRunEndpoint(t *testing.T, svc *CoreService) (domain.GamePlug
|
|||||||
Version: "0.1.0",
|
Version: "0.1.0",
|
||||||
Capabilities: []string{"process.install", "process.start", "process.stop", "logs.read", "config.write", "files.read", "files.write"},
|
Capabilities: []string{"process.install", "process.start", "process.stop", "logs.read", "config.write", "files.read", "files.write"},
|
||||||
Capacity: domain.RunCapacity{MaxJobs: 4},
|
Capacity: domain.RunCapacity{MaxJobs: 4},
|
||||||
|
LastHeartbeatAt: svc.now(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("create run endpoint fixture: %v", err)
|
t.Fatalf("create run endpoint fixture: %v", err)
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func (svc *CoreService) CreateServerInstanceWorkflow(create domain.ServerLifecyc
|
|||||||
if err := validator.ValidateServerInstanceDependencies(instance, plugin, endpoint); err != nil {
|
if err := validator.ValidateServerInstanceDependencies(instance, plugin, endpoint); err != nil {
|
||||||
return domain.ServerLifecycleResult{}, err
|
return domain.ServerLifecycleResult{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
|
||||||
return domain.ServerLifecycleResult{}, err
|
return domain.ServerLifecycleResult{}, err
|
||||||
}
|
}
|
||||||
if err := svc.validateLifecycleIdempotency(instance.RunEndpointID, create.IdempotencyKey, instance.ID, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
|
if err := svc.validateLifecycleIdempotency(instance.RunEndpointID, create.IdempotencyKey, instance.ID, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
|
||||||
@@ -177,7 +177,7 @@ func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLif
|
|||||||
if err := svc.requireCompleteRuntimeBindings(instance.OwnerUserID, instance.ID, "server.lifecycle."+string(action)+".denied"); err != nil {
|
if err := svc.requireCompleteRuntimeBindings(instance.OwnerUserID, instance.ID, "server.lifecycle."+string(action)+".denied"); err != nil {
|
||||||
return domain.ServerLifecycleResult{}, err
|
return domain.ServerLifecycleResult{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
|
||||||
return domain.ServerLifecycleResult{}, err
|
return domain.ServerLifecycleResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ func (svc *CoreService) resolveSourceRCONDispatch(instance domain.ServerInstance
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return sourceRCONDispatchResolution{}, err
|
return sourceRCONDispatchResolution{}, err
|
||||||
}
|
}
|
||||||
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityRemoteRunRCONCommand); err != nil {
|
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityRemoteRunRCONCommand); err != nil {
|
||||||
return sourceRCONDispatchResolution{}, err
|
return sourceRCONDispatchResolution{}, err
|
||||||
}
|
}
|
||||||
if !strings.EqualFold(endpoint.Platform, "windows") || !strings.EqualFold(endpoint.Architecture, "amd64") {
|
if !strings.EqualFold(endpoint.Platform, "windows") || !strings.EqualFold(endpoint.Architecture, "amd64") {
|
||||||
|
|||||||
@@ -83,6 +83,19 @@ start_self_hosted_stack() {
|
|||||||
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform.pid"
|
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform.pid"
|
||||||
wait_for_url platform "$PLATFORM_URL/healthz" 45
|
wait_for_url platform "$PLATFORM_URL/healthz" 45
|
||||||
|
|
||||||
|
printf 'self-starting platform_web for local debug smoke\n'
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
exec env \
|
||||||
|
PLATFORM_API_PROXY="$PLATFORM_API_PROXY" \
|
||||||
|
VITE_PLATFORM_API_BASE_URL="$VITE_PLATFORM_API_BASE_URL" \
|
||||||
|
VITE_ENABLE_LOCAL_AUTH_FALLBACK="$VITE_ENABLE_LOCAL_AUTH_FALLBACK" \
|
||||||
|
npm --prefix platform_web run dev -- --port "$LOCAL_DEBUG_WEB_PORT"
|
||||||
|
) >"$LOCAL_DEBUG_LOG_DIR/platform_web.log" 2>&1 &
|
||||||
|
SELF_STARTED_PIDS+=("$!")
|
||||||
|
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform_web.pid"
|
||||||
|
wait_for_url platform_web "$(local_debug_web_url)" 45
|
||||||
|
|
||||||
printf 'self-starting run worker for local debug smoke\n'
|
printf 'self-starting run worker for local debug smoke\n'
|
||||||
local_debug_build_bootstrap_run
|
local_debug_build_bootstrap_run
|
||||||
(
|
(
|
||||||
@@ -106,19 +119,6 @@ start_self_hosted_stack() {
|
|||||||
) >"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 &
|
) >"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 &
|
||||||
SELF_STARTED_PIDS+=("$!")
|
SELF_STARTED_PIDS+=("$!")
|
||||||
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/run.pid"
|
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/run.pid"
|
||||||
|
|
||||||
printf 'self-starting platform_web for local debug smoke\n'
|
|
||||||
(
|
|
||||||
cd "$ROOT_DIR"
|
|
||||||
exec env \
|
|
||||||
PLATFORM_API_PROXY="$PLATFORM_API_PROXY" \
|
|
||||||
VITE_PLATFORM_API_BASE_URL="$VITE_PLATFORM_API_BASE_URL" \
|
|
||||||
VITE_ENABLE_LOCAL_AUTH_FALLBACK="$VITE_ENABLE_LOCAL_AUTH_FALLBACK" \
|
|
||||||
npm --prefix platform_web run dev -- --port "$LOCAL_DEBUG_WEB_PORT"
|
|
||||||
) >"$LOCAL_DEBUG_LOG_DIR/platform_web.log" 2>&1 &
|
|
||||||
SELF_STARTED_PIDS+=("$!")
|
|
||||||
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform_web.pid"
|
|
||||||
wait_for_url platform_web "$(local_debug_web_url)" 45
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then
|
if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then
|
||||||
|
|||||||
@@ -19,6 +19,23 @@ managed_pid_running() {
|
|||||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
|
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pid_is_descendant_of() {
|
||||||
|
local pid="$1"
|
||||||
|
local ancestor="$2"
|
||||||
|
local parent
|
||||||
|
|
||||||
|
while [[ "$pid" =~ ^[0-9]+$ && "$pid" -gt 1 ]]; do
|
||||||
|
if [[ "$pid" == "$ancestor" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
parent="$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]')"
|
||||||
|
[[ -n "$parent" && "$parent" != "$pid" ]] || return 1
|
||||||
|
pid="$parent"
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
port_listener_pid() {
|
port_listener_pid() {
|
||||||
local port="$1"
|
local port="$1"
|
||||||
lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true
|
lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true
|
||||||
@@ -51,9 +68,13 @@ ensure_port_available() {
|
|||||||
if [[ -z "$listener_pid" ]]; then
|
if [[ -z "$listener_pid" ]]; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
if managed_pid_running "$pid_file" && [[ "$listener_pid" == "$(cat "$pid_file")" ]]; then
|
if managed_pid_running "$pid_file"; then
|
||||||
|
local managed_pid
|
||||||
|
managed_pid="$(cat "$pid_file")"
|
||||||
|
if pid_is_descendant_of "$listener_pid" "$managed_pid"; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
if listener_owned_by_local_debug "$listener_pid"; then
|
if listener_owned_by_local_debug "$listener_pid"; then
|
||||||
printf 'stopping stale %s listener pid %s on port %s\n' "$name" "$listener_pid" "$port"
|
printf 'stopping stale %s listener pid %s on port %s\n' "$name" "$listener_pid" "$port"
|
||||||
kill "$listener_pid" 2>/dev/null || true
|
kill "$listener_pid" 2>/dev/null || true
|
||||||
@@ -117,20 +138,49 @@ wait_for_run_registration() {
|
|||||||
local run_pid_file="$LOCAL_DEBUG_PID_DIR/run.pid"
|
local run_pid_file="$LOCAL_DEBUG_PID_DIR/run.pid"
|
||||||
printf 'waiting for run endpoint %s registration\n' "$RUN_ENDPOINT_ID"
|
printf 'waiting for run endpoint %s registration\n' "$RUN_ENDPOINT_ID"
|
||||||
for _ in $(seq 1 "$attempts"); do
|
for _ in $(seq 1 "$attempts"); do
|
||||||
if [[ -f "$PLATFORM_METADATA_PATH" ]] && grep -Fq "\"ID\": \"$RUN_ENDPOINT_ID\"" "$PLATFORM_METADATA_PATH"; then
|
|
||||||
printf 'run endpoint %s is registered\n' "$RUN_ENDPOINT_ID"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if ! managed_pid_running "$run_pid_file"; then
|
if ! managed_pid_running "$run_pid_file"; then
|
||||||
printf 'run exited before endpoint registration; see %s\n' "$LOCAL_DEBUG_LOG_DIR/run.log" >&2
|
printf 'run exited before endpoint registration; see %s\n' "$LOCAL_DEBUG_LOG_DIR/run.log" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
if run_endpoint_has_recent_heartbeat; then
|
||||||
|
printf 'run endpoint %s is registered\n' "$RUN_ENDPOINT_ID"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
printf 'run endpoint %s did not register before timeout\n' "$RUN_ENDPOINT_ID" >&2
|
printf 'run endpoint %s did not register before timeout\n' "$RUN_ENDPOINT_ID" >&2
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_endpoint_has_recent_heartbeat() {
|
||||||
|
[[ -f "$PLATFORM_METADATA_PATH" ]] || return 1
|
||||||
|
node -e '
|
||||||
|
const fs = require("fs");
|
||||||
|
const metadata = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
|
||||||
|
const endpointID = process.argv[2];
|
||||||
|
function findEndpoint(value) {
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
if ((value.ID || value.id) === endpointID && (value.LastHeartbeatAt || value.lastHeartbeatAt)) return value;
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
for (const item of value) {
|
||||||
|
const found = findEndpoint(item);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (const item of Object.values(value)) {
|
||||||
|
const found = findEndpoint(item);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const endpoint = findEndpoint(metadata);
|
||||||
|
const heartbeat = endpoint && (endpoint.LastHeartbeatAt || endpoint.lastHeartbeatAt);
|
||||||
|
const ageMilliseconds = heartbeat ? Date.now() - Date.parse(heartbeat) : Number.POSITIVE_INFINITY;
|
||||||
|
process.exit(Number.isFinite(ageMilliseconds) && ageMilliseconds >= 0 && ageMilliseconds < 30000 ? 0 : 1);
|
||||||
|
' "$PLATFORM_METADATA_PATH" "$RUN_ENDPOINT_ID"
|
||||||
|
}
|
||||||
|
|
||||||
printf 'local debug root: %s\n' "$LOCAL_DEBUG_ROOT"
|
printf 'local debug root: %s\n' "$LOCAL_DEBUG_ROOT"
|
||||||
printf 'platform: %s\n' "$(local_debug_platform_url)"
|
printf 'platform: %s\n' "$(local_debug_platform_url)"
|
||||||
printf 'platform_web: %s\n' "$(local_debug_web_url)"
|
printf 'platform_web: %s\n' "$(local_debug_web_url)"
|
||||||
@@ -160,6 +210,14 @@ start_service platform "$ROOT_DIR/platform" env \
|
|||||||
|
|
||||||
wait_for_url platform "$(local_debug_platform_url)/healthz"
|
wait_for_url platform "$(local_debug_platform_url)/healthz"
|
||||||
|
|
||||||
|
start_service platform_web "$ROOT_DIR" env \
|
||||||
|
PLATFORM_API_PROXY="$PLATFORM_API_PROXY" \
|
||||||
|
VITE_PLATFORM_API_BASE_URL="$VITE_PLATFORM_API_BASE_URL" \
|
||||||
|
VITE_ENABLE_LOCAL_AUTH_FALLBACK="$VITE_ENABLE_LOCAL_AUTH_FALLBACK" \
|
||||||
|
npm --prefix platform_web run dev -- --host "0.0.0.0" --port "$LOCAL_DEBUG_WEB_PORT"
|
||||||
|
|
||||||
|
wait_for_url platform_web "$(local_debug_web_url)"
|
||||||
|
|
||||||
local_debug_build_bootstrap_run
|
local_debug_build_bootstrap_run
|
||||||
start_service run "$(dirname "$RUN_BOOTSTRAP_BIN")" env \
|
start_service run "$(dirname "$RUN_BOOTSTRAP_BIN")" env \
|
||||||
GOCACHE="$GOCACHE" \
|
GOCACHE="$GOCACHE" \
|
||||||
@@ -180,14 +238,6 @@ start_service run "$(dirname "$RUN_BOOTSTRAP_BIN")" env \
|
|||||||
|
|
||||||
wait_for_run_registration
|
wait_for_run_registration
|
||||||
|
|
||||||
start_service platform_web "$ROOT_DIR" env \
|
|
||||||
PLATFORM_API_PROXY="$PLATFORM_API_PROXY" \
|
|
||||||
VITE_PLATFORM_API_BASE_URL="$VITE_PLATFORM_API_BASE_URL" \
|
|
||||||
VITE_ENABLE_LOCAL_AUTH_FALLBACK="$VITE_ENABLE_LOCAL_AUTH_FALLBACK" \
|
|
||||||
npm --prefix platform_web run dev -- --host "0.0.0.0" --port "$LOCAL_DEBUG_WEB_PORT"
|
|
||||||
|
|
||||||
wait_for_url platform_web "$(local_debug_web_url)"
|
|
||||||
|
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
|
|
||||||
Local debug stack requested.
|
Local debug stack requested.
|
||||||
|
|||||||
Reference in New Issue
Block a user