admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix local run startup and stale dispatch

Browse Source
This commit is contained in:
npc0-hue
2026-07-23 23:56:10 +08:00
parent 215f6e68c5
commit 2b1f553df0
11 changed files with 155 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/api/resource_handlers_test.go
+2 -2
View File
@@ -1534,9 +1534,9 @@ func TestProductionOperationsGovernanceRoutesAreDurableAndRedacted(t *testing.T)
if len(capacity.Endpoints) == 0 {
t.Fatalf("expected persisted capacity endpoints, got %+v", capacity)
}
decision := postOKJSONWithAuth[dto.CapacityAdmissionDecisionResponse](t, router, "/api/v1/production/capacity/admission", dto.CapacityAdmissionRequest{ServerInstanceID: serverID, Capability: domain.JobCapabilityConfigWrite, IdempotencyKey: "api-capacity-check"}, adminSession)
decision := postOKJSONWithAuth[dto.CapacityAdmissionDecisionResponse](t, router, "/api/v1/production/capacity/admission", dto.CapacityAdmissionRequest{ServerInstanceID: serverID, Capability: domain.JobCapabilityRemoteRunRCONCommand, IdempotencyKey: "api-capacity-check"}, adminSession)
if decision.Accepted || decision.AlertID == "" || decision.AuditEventID == "" {
t.Fatalf("expected stale endpoint admission to create durable evidence, got %+v", decision)
t.Fatalf("expected unavailable capability admission to create durable evidence, got %+v", decision)
}
alerts := getJSONWithAuth[dto.AlertListResponse](t, router, "/api/v1/alerts?state=active", adminSession)
if alerts.Count == 0 {
platform/service/client_manager_lifecycle.go
+6 -6
View File
@@ -479,7 +479,7 @@ func (svc *CoreService) authorizeClientManagerLifecycle(sessionID, serverInstanc
if err != nil {
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
}
if err := validateRunnableEndpoint(endpoint, capability); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, capability); err != nil {
_ = svc.recordAuditEvent(user.ID, deniedAction, "server-instance", instance.ID, domain.AuditResultDenied, "client-manager lifecycle denied: assigned Run endpoint is offline or unsupported")
return domain.User{}, domain.ServerInstance{}, domain.GamePlugin{}, domain.RuntimeClientManagerProfile{}, domain.RunEndpoint{}, err
}
@@ -1119,11 +1119,11 @@ func (svc *CoreService) clientManagerRuntimeActionProjection(instance domain.Ser
}
baseReason := fallbackReason(!hasLifecycle || !bindingsComplete, "plugin does not declare a complete Client Manager lifecycle", bindingReason)
return []domain.ServerRuntimeAction{
runtimeAction("deploy-client-manager", "Deploy client manager", hasLifecycle && bindingsComplete && endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), "run endpoint cannot deploy client managers", baseReason)),
runtimeAction("control-client-manager", "Control client manager", hasLifecycle && hasInstalled && bindingsComplete && endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), fallbackReason(!hasInstalled || !endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), "client manager is not installed or endpoint cannot control it", baseReason)),
runtimeAction("update-client-manager", "Update client manager", hasLifecycle && hasInstalled && hasUpdateCandidate && endpointSupports(endpoint, domain.JobCapabilityClientManagerUpdate), "no compatible update candidate is available"),
runtimeAction("rollback-client-manager", "Rollback client manager", hasLifecycle && hasPrevious && endpointSupports(endpoint, domain.JobCapabilityClientManagerRollback), "no retained previous deployment is available"),
runtimeAction("uninstall-client-manager", "Uninstall client manager", hasLifecycle && hasInstalled && endpointSupports(endpoint, domain.JobCapabilityClientManagerUninstall), "client manager is not installed or endpoint cannot uninstall it"),
runtimeAction("deploy-client-manager", "Deploy client manager", hasLifecycle && bindingsComplete && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerDeploy), "run endpoint cannot deploy client managers", baseReason)),
runtimeAction("control-client-manager", "Control client manager", hasLifecycle && hasInstalled && bindingsComplete && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), fallbackReason(!hasInstalled || !svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerControl), "client manager is not installed or endpoint cannot control it", baseReason)),
runtimeAction("update-client-manager", "Update client manager", hasLifecycle && hasInstalled && hasUpdateCandidate && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerUpdate), "no compatible update candidate is available"),
runtimeAction("rollback-client-manager", "Rollback client manager", hasLifecycle && hasPrevious && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerRollback), "no retained previous deployment is available"),
runtimeAction("uninstall-client-manager", "Uninstall client manager", hasLifecycle && hasInstalled && svc.endpointSupports(endpoint, domain.JobCapabilityClientManagerUninstall), "client manager is not installed or endpoint cannot uninstall it"),
}
}
platform/service/client_manager_lifecycle_test.go
+8
View File
@@ -209,6 +209,14 @@ func TestClientManagerLifecycleRejectsCrossScopeStaleAndRevokedIdentity(t *testi
func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution {
t.Helper()
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
if err != nil {
t.Fatalf("get lifecycle Run endpoint: %v", err)
}
endpoint.LastHeartbeatAt = svc.now()
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
t.Fatalf("refresh lifecycle Run heartbeat: %v", err)
}
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
if err != nil {
t.Fatalf("get lifecycle plugin: %v", err)
platform/service/distributions.go
+12 -9
View File
@@ -47,7 +47,7 @@ func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, requ
if err != nil {
return domain.RunDistribution{}, err
}
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
return domain.RunDistribution{}, err
}
@@ -166,7 +166,7 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
if err != nil {
return domain.ClientManagerDistribution{}, err
}
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityDistributionBuild); err != nil {
return domain.ClientManagerDistribution{}, err
}
@@ -462,17 +462,17 @@ func (svc *CoreService) GetServerRuntimeActionsForSession(sessionID string, serv
RunEndpointID: endpoint.ID,
RunStatus: endpoint.Status,
Actions: []domain.ServerRuntimeAction{
runtimeAction("generate-run", "Generate run", pluginDeclares(plugin, "server.run.distribution") && endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
runtimeAction("generate-run", "Generate run", pluginDeclares(plugin, "server.run.distribution") && svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
runtimeAction("download-run", "Download run", hasAvailableRunPackage, "run package has not been generated"),
runtimeAction("push-run-update", "Push run update", pluginDeclares(plugin, "server.run.distribution") && endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate), "run endpoint cannot self-update", bindingReason)),
runtimeAction("push-run-update", "Push run update", pluginDeclares(plugin, "server.run.distribution") && svc.endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.run.distribution") || !svc.endpointSupports(endpoint, domain.JobCapabilityRunSelfUpdate), "run endpoint cannot self-update", bindingReason)),
runtimeAction("reset-run-key", "Reset run key", pluginDeclares(plugin, "server.run.distribution"), "plugin permission is not declared"),
runtimeAction("generate-client-manager", "Generate client manager", pluginDeclares(plugin, "server.client-manager.manage") && endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.client-manager.manage") || !endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
runtimeAction("generate-client-manager", "Generate client manager", pluginDeclares(plugin, "server.client-manager.manage") && svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild) && bindingsComplete, fallbackReason(!pluginDeclares(plugin, "server.client-manager.manage") || !svc.endpointSupports(endpoint, domain.JobCapabilityDistributionBuild), "run endpoint cannot build distributions", bindingReason)),
runtimeAction("download-client-manager", "Download client manager", hasAvailableClientPackage, "client-manager package has not been generated"),
runtimeAction("reset-client-manager-key", "Reset client-manager key", pluginDeclares(plugin, "server.client-manager.manage"), "client-manager permission is not declared"),
runtimeAction("dependencies-check", "Check dependencies", dependencyPermissionDeclared && endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck), "run endpoint cannot check dependencies", bindingReason))),
runtimeAction("dependencies-install", "Install dependencies", dependencyPermissionDeclared && endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall), "run endpoint cannot install dependencies", bindingReason))),
runtimeAction("dependencies-check", "Check dependencies", dependencyPermissionDeclared && svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesCheck), "run endpoint cannot check dependencies", bindingReason))),
runtimeAction("dependencies-install", "Install dependencies", dependencyPermissionDeclared && svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall) && bindingsComplete, fallbackReason(!dependencyPermissionDeclared, "plugin permission is not declared", fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityDependenciesInstall), "run endpoint cannot install dependencies", bindingReason))),
runtimeAction("live-logs", "Live logs", pluginSupports(plugin, "logs.read"), "plugin does not declare live logs"),
runtimeAction("historical-logs", "Historical logs", endpointSupports(endpoint, domain.JobCapabilityLogsBackfill) && bindingsComplete, fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityLogsBackfill), "run endpoint cannot backfill logs", bindingReason)),
runtimeAction("historical-logs", "Historical logs", svc.endpointSupports(endpoint, domain.JobCapabilityLogsBackfill) && bindingsComplete, fallbackReason(!svc.endpointSupports(endpoint, domain.JobCapabilityLogsBackfill), "run endpoint cannot backfill logs", bindingReason)),
},
}
actions.Actions = append(actions.Actions, svc.clientManagerRuntimeActionProjection(instance, plugin, endpoint, bindingsComplete, bindingReason)...)
@@ -1211,10 +1211,13 @@ func pluginSupports(plugin domain.GamePlugin, capability string) bool {
return containsString(plugin.RequiredRunCapabilities, capability)
}
func endpointSupports(endpoint domain.RunEndpoint, capability string) bool {
func (svc *CoreService) endpointSupports(endpoint domain.RunEndpoint, capability string) bool {
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
return false
}
if !svc.runEndpointHeartbeatCurrent(endpoint) {
return false
}
return containsString(endpoint.Capabilities, capability)
}
platform/service/distributions_test.go
+26
View File
@@ -4,6 +4,7 @@ import (
"errors"
"strings"
"testing"
"time"
"browser.local/platform/domain"
"browser.local/platform/repo"
@@ -96,6 +97,31 @@ func TestCoreServiceGeneratesRunDistributionWithEncryptedSingletonKey(t *testing
}
}
func TestCoreServiceRejectsDistributionBuildForStaleRunEndpoint(t *testing.T) {
svc, session, instance := newDistributionTestFixture(t)
svc.now = func() time.Time { return fixedTime.Add(capacityHeartbeatStaleAfter + time.Second) }
_, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
ServerInstanceID: instance.ID,
TargetOS: "linux",
TargetArch: "amd64",
IdempotencyKey: "idem-stale-run-endpoint",
})
if err == nil || !strings.Contains(err.Error(), "heartbeat is stale") {
t.Fatalf("expected stale Run endpoint rejection, got %v", err)
}
actions, err := svc.GetServerRuntimeActionsForSession(session, instance.ID)
if err != nil {
t.Fatalf("get runtime actions: %v", err)
}
for _, action := range actions.Actions {
if action.Key == "generate-run" && action.Available {
t.Fatalf("stale Run endpoint must not expose generate-run as available: %+v", action)
}
}
}
func TestCoreServiceRuntimeActionsGateDependenciesOnPluginPermission(t *testing.T) {
svc, session, instance := newDistributionTestFixture(t)
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
platform/service/resources.go
+15 -2
View File
@@ -1509,6 +1509,9 @@ func (svc *CoreService) CreateRunEndpoint(endpoint domain.RunEndpoint) (domain.R
if endpoint.Status == "" {
endpoint.Status = domain.RunEndpointStatusOnline
}
if endpoint.LastHeartbeatAt.IsZero() && (endpoint.Status == domain.RunEndpointStatusOnline || endpoint.Status == domain.RunEndpointStatusDegraded) {
endpoint.LastHeartbeatAt = svc.now()
}
if err := validator.ValidateRunEndpoint(endpoint); err != nil {
return domain.RunEndpoint{}, err
}
@@ -2158,7 +2161,7 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
if err != nil {
return domain.Job{}, fmt.Errorf("get run endpoint dependency: %w", err)
}
if err := validateRunnableEndpoint(endpoint, job.Capability); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, job.Capability); err != nil {
return domain.Job{}, err
}
if job.ServerInstanceID != "" {
@@ -2281,16 +2284,26 @@ func (svc *CoreService) ListAuditEvents(filter domain.AuditEventFilter) ([]domai
return svc.store.AuditEvents().List(filter)
}
func validateRunnableEndpoint(endpoint domain.RunEndpoint, capability string) error {
func (svc *CoreService) validateRunnableEndpoint(endpoint domain.RunEndpoint, capability string) error {
if endpoint.Status != domain.RunEndpointStatusOnline && endpoint.Status != domain.RunEndpointStatusDegraded {
return validationError("run endpoint must be online or degraded")
}
if !svc.runEndpointHeartbeatCurrent(endpoint) {
return validationError("run endpoint heartbeat is stale")
}
if len(validator.MissingCapabilities(endpoint.Capabilities, []string{capability})) > 0 {
return validationError("run endpoint missing required capability: " + capability)
}
return nil
}
func (svc *CoreService) runEndpointHeartbeatCurrent(endpoint domain.RunEndpoint) bool {
if endpoint.LastHeartbeatAt.IsZero() {
return false
}
return !svc.now().After(endpoint.LastHeartbeatAt.Add(capacityHeartbeatStaleAfter))
}
func validateJobServerTarget(job domain.Job, instance domain.ServerInstance, plugin domain.GamePlugin) error {
if instance.State == domain.ServerInstanceStateDeleted {
return validationError("server instance must not be deleted")
platform/service/resources_test.go
+6 -5
View File
@@ -1439,11 +1439,12 @@ func createPluginAndRunEndpoint(t *testing.T, svc *CoreService) (domain.GamePlug
}
endpoint, err := svc.CreateRunEndpoint(domain.RunEndpoint{
ID: "run-local",
DisplayName: "Local Run",
Version: "0.1.0",
Capabilities: []string{"process.install", "process.start", "process.stop", "logs.read", "config.write", "files.read", "files.write"},
Capacity: domain.RunCapacity{MaxJobs: 4},
ID: "run-local",
DisplayName: "Local Run",
Version: "0.1.0",
Capabilities: []string{"process.install", "process.start", "process.stop", "logs.read", "config.write", "files.read", "files.write"},
Capacity: domain.RunCapacity{MaxJobs: 4},
LastHeartbeatAt: svc.now(),
})
if err != nil {
t.Fatalf("create run endpoint fixture: %v", err)
platform/service/server_lifecycle.go
+2 -2
View File
@@ -49,7 +49,7 @@ func (svc *CoreService) CreateServerInstanceWorkflow(create domain.ServerLifecyc
if err := validator.ValidateServerInstanceDependencies(instance, plugin, endpoint); err != nil {
return domain.ServerLifecycleResult{}, err
}
if err := validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
return domain.ServerLifecycleResult{}, err
}
if err := svc.validateLifecycleIdempotency(instance.RunEndpointID, create.IdempotencyKey, instance.ID, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
@@ -177,7 +177,7 @@ func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLif
if err := svc.requireCompleteRuntimeBindings(instance.OwnerUserID, instance.ID, "server.lifecycle."+string(action)+".denied"); err != nil {
return domain.ServerLifecycleResult{}, err
}
if err := validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
return domain.ServerLifecycleResult{}, err
}
platform/service/source_rcon.go
+1 -1
View File
@@ -171,7 +171,7 @@ func (svc *CoreService) resolveSourceRCONDispatch(instance domain.ServerInstance
if err != nil {
return sourceRCONDispatchResolution{}, err
}
if err := validateRunnableEndpoint(endpoint, domain.JobCapabilityRemoteRunRCONCommand); err != nil {
if err := svc.validateRunnableEndpoint(endpoint, domain.JobCapabilityRemoteRunRCONCommand); err != nil {
return sourceRCONDispatchResolution{}, err
}
if !strings.EqualFold(endpoint.Platform, "windows") || !strings.EqualFold(endpoint.Architecture, "amd64") {