fix: recover run runtime state and logs

This commit is contained in:
npc0-hue
2026-08-07 12:01:26 +08:00
parent 3818fa0344
commit b789925ae5
23 changed files with 297 additions and 78 deletions
@@ -8,29 +8,29 @@
## 1. Platform Contracts And Projections ## 1. Platform Contracts And Projections
- [ ] 1.1 Add typed domain, DTO, API, validation, and protocol contracts for signed Run log-stream progress queries scoped to the authenticated Run endpoint and bound server instance. - [x] 1.1 Add typed domain, DTO, API, validation, and protocol contracts for signed Run log-stream progress queries scoped to the authenticated Run endpoint and bound server instance.
- [ ] 1.2 Implement Platform log-stream progress lookup that returns only the latest acknowledged sequence and cannot disclose log bodies, host paths, credentials, or another server's stream metadata. - [x] 1.2 Implement Platform log-stream progress lookup that returns only the latest acknowledged sequence and cannot disclose log bodies, host paths, credentials, or another server's stream metadata.
- [ ] 1.3 Extend Run lifecycle observations with stable managed-process identity and ordering data, then make lifecycle projection idempotent and reject stale state regressions. - [x] 1.3 Extend Run lifecycle observations with stable managed-process identity and ordering data, then make lifecycle projection idempotent and reject stale state regressions.
- [ ] 1.4 Project autonomous Run recovered and exit observations through the existing signed lifecycle channel, preserving requested-stop versus unexpected-exit classification. - [x] 1.4 Project autonomous Run recovered and exit observations through the existing signed lifecycle channel, preserving requested-stop versus unexpected-exit classification.
- [ ] 1.5 Expose a server runtime observation view that combines the persisted lifecycle projection with generic bound-endpoint heartbeat freshness without changing lifecycle state solely because Run is unavailable. - [ ] 1.5 Expose a server runtime observation view that combines the persisted lifecycle projection with generic bound-endpoint heartbeat freshness without changing lifecycle state solely because Run is unavailable.
- [ ] 1.6 Add focused Platform tests for report ordering/idempotency, authorization and scope of log progress, progress values after durable ingest, and fresh versus unverified runtime observation. - [x] 1.6 Add focused Platform tests for report ordering/idempotency, authorization and scope of log progress, progress values after durable ingest, and fresh versus unverified runtime observation.
## 2. Independent Run Recovery ## 2. Independent Run Recovery
- [ ] 2.1 Update the shared/copyable Run-Platform protocol types and API client in `git@git.npc0.com:admin343/run.git` for lifecycle observation ordering and signed log-stream progress reconciliation. - [x] 2.1 Update the shared/copyable Run-Platform protocol types and API client in `git@git.npc0.com:admin343/run.git` for lifecycle observation ordering and signed log-stream progress reconciliation.
- [ ] 2.2 Add atomic per-stream allocated and acknowledged watermark persistence to the Run log spool, including restart loading and acknowledgement-before-segment-deletion ordering. - [x] 2.2 Add atomic per-stream allocated and acknowledged watermark persistence to the Run log spool, including restart loading and acknowledgement-before-segment-deletion ordering.
- [ ] 2.3 Replace the worker-global in-memory log counter with stream-specific allocation restored from the spool watermark and pending durable segments. - [x] 2.3 Replace the worker-global in-memory log counter with stream-specific allocation restored from the spool watermark and pending durable segments.
- [ ] 2.4 Reconcile signed Platform stream progress before a stable Run-bound stream with no local watermark emits new entries; cover newly created and recreated-spool cases. - [x] 2.4 Reconcile signed Platform stream progress before a stable Run-bound stream with no local watermark emits new entries; cover newly created and recreated-spool cases.
- [ ] 2.5 Classify acknowledged-range conflicts and sequence gaps as durable recovery failures, quarantine the affected spool segment with redacted diagnostics, and resume only after safe watermark reconciliation. - [x] 2.5 Classify acknowledged-range conflicts and sequence gaps as durable recovery failures, quarantine the affected spool segment with redacted diagnostics, and resume only after safe watermark reconciliation.
- [ ] 2.6 Make autonomous process supervision report observed exit and startup-recovery transitions through the lifecycle channel, with retry-safe process identity and ordering metadata. - [x] 2.6 Make autonomous process supervision report observed exit and startup-recovery transitions through the lifecycle channel, with retry-safe process identity and ordering metadata.
- [ ] 2.7 Define and test graceful Run shutdown behavior that preserves durable state and never reports a server stop unless its generic supervisor observed that process state. - [x] 2.7 Define and test graceful Run shutdown behavior that preserves durable state and never reports a server stop unless its generic supervisor observed that process state.
- [ ] 2.8 Add Run unit tests for per-stream interleaving, restart continuity, missing-watermark progress lookup, conflict quarantine, process exit reporting, and Windows supervisor recovery. - [ ] 2.8 Add Run unit tests for per-stream interleaving, restart continuity, missing-watermark progress lookup, conflict quarantine, process exit reporting, and Windows supervisor recovery.
## 3. Management Runtime Presentation ## 3. Management Runtime Presentation
- [ ] 3.1 Extend Platform Web API types and server-management contracts to consume lifecycle projection and runtime observation freshness separately. - [x] 3.1 Extend Platform Web API types and server-management contracts to consume lifecycle projection and runtime observation freshness separately.
- [ ] 3.2 Update server list and server detail status UI so stale `running` is presented as last observed with a Run offline/unverified qualifier, not confirmed online. - [ ] 3.2 Update server list and server detail status UI so stale `running` is presented as last observed with a Run offline/unverified qualifier, not confirmed online.
- [ ] 3.3 Update the management terminal header and empty/error states to show that live output awaits Run recovery while preserving accepted bounded SSE history. - [x] 3.3 Update the management terminal header and empty/error states to show that live output awaits Run recovery while preserving accepted bounded SSE history.
- [ ] 3.4 Add focused frontend tests for fresh, stale, offline, and recovered Run observations plus terminal presentation during log recovery. - [ ] 3.4 Add focused frontend tests for fresh, stale, offline, and recovered Run observations plus terminal presentation during log recovery.
## 4. Cross-Repository Verification And Release ## 4. Cross-Repository Verification And Release
+19
View File
@@ -137,6 +137,7 @@ func (h *coreHandlers) register(mux *http.ServeMux) {
mux.HandleFunc("/api/v1/run/jobs/cancel", h.requireRunSignature(h.runJobCancelPoll)) mux.HandleFunc("/api/v1/run/jobs/cancel", h.requireRunSignature(h.runJobCancelPoll))
mux.HandleFunc("/api/v1/run/jobs/reconcile", h.requireRunSignature(h.runJobReconcile)) mux.HandleFunc("/api/v1/run/jobs/reconcile", h.requireRunSignature(h.runJobReconcile))
mux.HandleFunc("/api/v1/run/logs/batches", h.requireRunSignature(h.runLogBatchIngest)) mux.HandleFunc("/api/v1/run/logs/batches", h.requireRunSignature(h.runLogBatchIngest))
mux.HandleFunc("/api/v1/run/logs/progress", h.requireRunSignature(h.runLogStreamProgress))
mux.HandleFunc("/api/v1/run/artifacts/open", h.requireRunSignature(h.runArtifactOpen)) mux.HandleFunc("/api/v1/run/artifacts/open", h.requireRunSignature(h.runArtifactOpen))
mux.HandleFunc("/api/v1/run/artifacts/chunks", h.requireRunSignature(h.runArtifactChunkUpload)) mux.HandleFunc("/api/v1/run/artifacts/chunks", h.requireRunSignature(h.runArtifactChunkUpload))
mux.HandleFunc("/api/v1/run/artifacts/status", h.requireRunSignature(h.runArtifactStatus)) mux.HandleFunc("/api/v1/run/artifacts/status", h.requireRunSignature(h.runArtifactStatus))
@@ -2037,6 +2038,24 @@ func (h *coreHandlers) runLogBatchIngest(w http.ResponseWriter, r *http.Request)
writeJSON(w, http.StatusOK, dto.LogBatchIngestFromDomain(result)) writeJSON(w, http.StatusOK, dto.LogBatchIngestFromDomain(result))
} }
func (h *coreHandlers) runLogStreamProgress(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeMethodNotAllowed(w, http.MethodPost)
return
}
request, err := decodeJSON[dto.RunLogStreamProgressRequest](r)
if err != nil {
writeDecodeError(w, err)
return
}
result, err := h.core.GetRunLogStreamProgress(request.ToDomain())
if err != nil {
writeServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, dto.RunLogStreamProgressFromDomain(result))
}
// runArtifactOpen godoc // runArtifactOpen godoc
// @Summary Open run artifact upload transfer // @Summary Open run artifact upload transfer
// @Description Lets a registered run endpoint open a resumable upload transfer for a scoped artifact owner. // @Description Lets a registered run endpoint open a resumable upload transfer for a scoped artifact owner.
+3
View File
@@ -62,6 +62,9 @@ type RunLifecycleReport struct {
Progress RunJobProgressReport Progress RunJobProgressReport
Message string Message string
ErrorCode string ErrorCode string
ManagedProcessID string
ObservationSeq uint64
ObservedAt time.Time
ExecutionResult JobExecutionResult ExecutionResult JobExecutionResult
} }
+18
View File
@@ -48,6 +48,24 @@ type LogStreamCursorResult struct {
LatestSeq uint64 LatestSeq uint64
} }
// RunLogStreamProgress is a signed Run-only request for sequence recovery.
// It deliberately exposes no log body or host-local information.
type RunLogStreamProgress struct {
RunEndpointID string
SessionToken string
ServerInstanceID string
LogStreamID string
}
type RunLogStreamProgressResult struct {
Accepted bool
RunEndpointID string
ServerInstanceID string
LogStreamID string
LatestSeq uint64
ServerTime time.Time
}
type LogStreamEvent struct { type LogStreamEvent struct {
ServerInstanceID string ServerInstanceID string
Stream LogStream Stream LogStream
+3
View File
@@ -783,6 +783,9 @@ type ServerInstance struct {
OwnerUserID string OwnerUserID string
AdminUserIDs []string AdminUserIDs []string
State ServerInstanceState State ServerInstanceState
LifecycleProcessID string
LifecycleObservationSeq uint64
LifecycleObservedAt time.Time
ConfigVersion int ConfigVersion int
ConfigKey string ConfigKey string
ConfigContent string ConfigContent string
+6
View File
@@ -66,6 +66,9 @@ type RunLifecycleReportRequest struct {
Progress JobProgressBody `json:"progress"` Progress JobProgressBody `json:"progress"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
ErrorCode string `json:"errorCode,omitempty"` ErrorCode string `json:"errorCode,omitempty"`
ManagedProcessID string `json:"managedProcessId,omitempty"`
ObservationSeq uint64 `json:"observationSeq,omitempty"`
ObservedAt time.Time `json:"observedAt,omitempty"`
ExecutionResult RunJobExecutionResultBody `json:"executionResult,omitempty"` ExecutionResult RunJobExecutionResultBody `json:"executionResult,omitempty"`
} }
@@ -122,6 +125,9 @@ func (request RunLifecycleReportRequest) ToDomain() domain.RunLifecycleReport {
Progress: progressReportToDomain(request.Progress), Progress: progressReportToDomain(request.Progress),
Message: request.Message, Message: request.Message,
ErrorCode: request.ErrorCode, ErrorCode: request.ErrorCode,
ManagedProcessID: request.ManagedProcessID,
ObservationSeq: request.ObservationSeq,
ObservedAt: request.ObservedAt,
ExecutionResult: domain.JobExecutionResult{Kind: request.ExecutionResult.Kind, ProcessState: request.ExecutionResult.ProcessState, ExitClassification: request.ExecutionResult.ExitClassification, ExitCode: request.ExecutionResult.ExitCode, Version: request.ExecutionResult.Version, Checksum: request.ExecutionResult.Checksum, SizeBytes: request.ExecutionResult.SizeBytes, AuditSummary: request.ExecutionResult.AuditSummary, Content: request.ExecutionResult.Content, ServerDeploymentEvidence: serverDeploymentEvidenceToDomain(request.ExecutionResult.ServerDeploymentEvidence), DeploymentReceipt: deploymentReceiptToDomain(request.ExecutionResult.DeploymentReceipt)}, ExecutionResult: domain.JobExecutionResult{Kind: request.ExecutionResult.Kind, ProcessState: request.ExecutionResult.ProcessState, ExitClassification: request.ExecutionResult.ExitClassification, ExitCode: request.ExecutionResult.ExitCode, Version: request.ExecutionResult.Version, Checksum: request.ExecutionResult.Checksum, SizeBytes: request.ExecutionResult.SizeBytes, AuditSummary: request.ExecutionResult.AuditSummary, Content: request.ExecutionResult.Content, ServerDeploymentEvidence: serverDeploymentEvidenceToDomain(request.ExecutionResult.ServerDeploymentEvidence), DeploymentReceipt: deploymentReceiptToDomain(request.ExecutionResult.DeploymentReceipt)},
} }
} }
+24
View File
@@ -53,6 +53,22 @@ type LogStreamCursorResponse struct {
LatestSeq uint64 `json:"latestSeq"` LatestSeq uint64 `json:"latestSeq"`
} }
type RunLogStreamProgressRequest struct {
RunEndpointID string `json:"runEndpointId"`
SessionToken string `json:"sessionToken"`
ServerInstanceID string `json:"serverInstanceId"`
LogStreamID string `json:"logStreamId"`
}
type RunLogStreamProgressResponse struct {
Accepted bool `json:"accepted"`
RunEndpointID string `json:"runEndpointId"`
ServerInstanceID string `json:"serverInstanceId"`
LogStreamID string `json:"logStreamId"`
LatestSeq uint64 `json:"latestSeq"`
ServerTime time.Time `json:"serverTime"`
}
type LogStreamEventResponse struct { type LogStreamEventResponse struct {
ServerInstanceID string `json:"serverInstanceId"` ServerInstanceID string `json:"serverInstanceId"`
StreamID string `json:"streamId"` StreamID string `json:"streamId"`
@@ -92,6 +108,10 @@ func (request LogStreamCursorRequest) ToDomain() domain.LogStreamCursorQuery {
} }
} }
func (request RunLogStreamProgressRequest) ToDomain() domain.RunLogStreamProgress {
return domain.RunLogStreamProgress{RunEndpointID: request.RunEndpointID, SessionToken: request.SessionToken, ServerInstanceID: request.ServerInstanceID, LogStreamID: request.LogStreamID}
}
func LogBatchIngestFromDomain(result domain.LogBatchIngestResult) LogBatchIngestResponse { func LogBatchIngestFromDomain(result domain.LogBatchIngestResult) LogBatchIngestResponse {
return LogBatchIngestResponse{ return LogBatchIngestResponse{
Accepted: result.Accepted, Accepted: result.Accepted,
@@ -114,6 +134,10 @@ func LogStreamCursorFromDomain(result domain.LogStreamCursorResult) LogStreamCur
} }
} }
func RunLogStreamProgressFromDomain(result domain.RunLogStreamProgressResult) RunLogStreamProgressResponse {
return RunLogStreamProgressResponse{Accepted: result.Accepted, RunEndpointID: result.RunEndpointID, ServerInstanceID: result.ServerInstanceID, LogStreamID: result.LogStreamID, LatestSeq: result.LatestSeq, ServerTime: result.ServerTime}
}
func LogStreamEventFromDomain(event domain.LogStreamEvent) LogStreamEventResponse { func LogStreamEventFromDomain(event domain.LogStreamEvent) LogStreamEventResponse {
event = domain.CopyLogStreamEvent(event) event = domain.CopyLogStreamEvent(event)
return LogStreamEventResponse{ return LogStreamEventResponse{
+2
View File
@@ -587,6 +587,7 @@ type ServerInstanceResponse struct {
OwnerUserID string `json:"ownerUserId,omitempty"` OwnerUserID string `json:"ownerUserId,omitempty"`
AdminUserIDs []string `json:"adminUserIds"` AdminUserIDs []string `json:"adminUserIds"`
State domain.ServerInstanceState `json:"state"` State domain.ServerInstanceState `json:"state"`
LifecycleObservedAt *time.Time `json:"lifecycleObservedAt,omitempty"`
ConfigVersion int `json:"configVersion"` ConfigVersion int `json:"configVersion"`
ConfigKey string `json:"configKey,omitempty"` ConfigKey string `json:"configKey,omitempty"`
ConfigChecksum string `json:"configChecksum,omitempty"` ConfigChecksum string `json:"configChecksum,omitempty"`
@@ -1640,6 +1641,7 @@ func ServerInstanceFromDomain(instance domain.ServerInstance) ServerInstanceResp
OwnerUserID: instance.OwnerUserID, OwnerUserID: instance.OwnerUserID,
AdminUserIDs: adminUserIDs, AdminUserIDs: adminUserIDs,
State: instance.State, State: instance.State,
LifecycleObservedAt: optionalTime(instance.LifecycleObservedAt),
ConfigVersion: instance.ConfigVersion, ConfigVersion: instance.ConfigVersion,
ConfigKey: instance.ConfigKey, ConfigKey: instance.ConfigKey,
ConfigChecksum: instance.ConfigChecksum, ConfigChecksum: instance.ConfigChecksum,
+9
View File
@@ -217,6 +217,9 @@ type ServerInstance struct {
AdminUserIDs []string `json:"adminUserIds" db:"admin_user_ids"` AdminUserIDs []string `json:"adminUserIds" db:"admin_user_ids"`
// State is the server lifecycle state. // State is the server lifecycle state.
State domain.ServerInstanceState `json:"state" db:"state"` State domain.ServerInstanceState `json:"state" db:"state"`
LifecycleProcessID string `json:"lifecycleProcessId,omitempty" db:"lifecycle_process_id"`
LifecycleObservationSeq uint64 `json:"lifecycleObservationSeq,omitempty" db:"lifecycle_observation_seq"`
LifecycleObservedAt time.Time `json:"lifecycleObservedAt,omitempty" db:"lifecycle_observed_at"`
// ConfigVersion is the platform-managed optimistic concurrency version. // ConfigVersion is the platform-managed optimistic concurrency version.
ConfigVersion int `json:"configVersion" db:"config_version"` ConfigVersion int `json:"configVersion" db:"config_version"`
// ConfigKey is the logical configuration target, never a host path. // ConfigKey is the logical configuration target, never a host path.
@@ -748,6 +751,9 @@ func ServerInstanceFromDomain(instance domain.ServerInstance) ServerInstance {
OwnerUserID: instance.OwnerUserID, OwnerUserID: instance.OwnerUserID,
AdminUserIDs: domain.CopyStringSlice(instance.AdminUserIDs), AdminUserIDs: domain.CopyStringSlice(instance.AdminUserIDs),
State: instance.State, State: instance.State,
LifecycleProcessID: instance.LifecycleProcessID,
LifecycleObservationSeq: instance.LifecycleObservationSeq,
LifecycleObservedAt: instance.LifecycleObservedAt,
ConfigVersion: instance.ConfigVersion, ConfigVersion: instance.ConfigVersion,
ConfigKey: instance.ConfigKey, ConfigKey: instance.ConfigKey,
ConfigContent: instance.ConfigContent, ConfigContent: instance.ConfigContent,
@@ -769,6 +775,9 @@ func (instance ServerInstance) ToDomain() domain.ServerInstance {
OwnerUserID: instance.OwnerUserID, OwnerUserID: instance.OwnerUserID,
AdminUserIDs: domain.CopyStringSlice(instance.AdminUserIDs), AdminUserIDs: domain.CopyStringSlice(instance.AdminUserIDs),
State: instance.State, State: instance.State,
LifecycleProcessID: instance.LifecycleProcessID,
LifecycleObservationSeq: instance.LifecycleObservationSeq,
LifecycleObservedAt: instance.LifecycleObservedAt,
ConfigVersion: instance.ConfigVersion, ConfigVersion: instance.ConfigVersion,
ConfigKey: instance.ConfigKey, ConfigKey: instance.ConfigKey,
ConfigContent: instance.ConfigContent, ConfigContent: instance.ConfigContent,
+2 -1
View File
@@ -63,7 +63,7 @@ Platform-owned Run distribution builds embed an autonomous lifecycle plan for th
The plan is build input for the generated package, not a machine-side job-channel payload. Generated Run registration must not be treated as a trigger to enqueue `process.start`, `process.install`, or `process.status` work; Platform state converges from Run heartbeats, logs, lifecycle reports, supervised process facts, and terminal job/report messages. Platform and Run must not add game-specific hardcoding to interpret the plan. The plan is build input for the generated package, not a machine-side job-channel payload. Generated Run registration must not be treated as a trigger to enqueue `process.start`, `process.install`, or `process.status` work; Platform state converges from Run heartbeats, logs, lifecycle reports, supervised process facts, and terminal job/report messages. Platform and Run must not add game-specific hardcoding to interpret the plan.
Autonomous lifecycle reports use `POST /api/v1/run/lifecycle/report` with the active Run session and signed envelope when required. The route accepts only bounded terminal lifecycle facts for `process.install`, `process.start`, `process.stop`, or `process.status`; it validates the server/run binding, records audit evidence, and projects server state from Run-reported process facts without creating or completing a Platform job. Autonomous lifecycle reports use `POST /api/v1/run/lifecycle/report` with the active Run session and signed envelope when required. The route accepts only bounded terminal lifecycle facts for `process.install`, `process.start`, `process.stop`, or `process.status`; it validates the server/run binding, records audit evidence, and projects server state from Run-reported process facts without creating or completing a Platform job. A managed-process report includes an opaque `managedProcessId`, monotonic `observationSeq`, and `observedAt`; retries are idempotent and a lower sequence cannot regress a newer fact for that process.
## Log Ingest ## Log Ingest
@@ -77,6 +77,7 @@ Named log DTOs:
- `LogBatchIngestRequest` - `LogBatchIngestRequest`
- `LogBatchIngestResponse` - `LogBatchIngestResponse`
- `RunLogStreamProgressRequest` / `RunLogStreamProgressResponse`: signed Run-only sequence recovery for a server-bound `run.<endpoint>.<server>.*` stream. The response contains only the latest acknowledged sequence.
- `LogEntry` - `LogEntry`
- `LogStreamCursorRequest` - `LogStreamCursorRequest`
- `LogStreamCursorResponse` - `LogStreamCursorResponse`
+25
View File
@@ -4,6 +4,7 @@ import (
"errors" "errors"
"strings" "strings"
"testing" "testing"
"time"
"browser.local/platform/domain" "browser.local/platform/domain"
"browser.local/platform/repo" "browser.local/platform/repo"
@@ -464,6 +465,30 @@ func TestCoreServiceRunLifecycleReportProjectsGeneratedRunFacts(t *testing.T) {
} }
} }
func TestCoreServiceRejectsStaleManagedProcessObservation(t *testing.T) {
svc := newTestCoreService()
plugin := createGeneratedRunStatusPlugin(t, svc)
instance := domain.ServerInstance{ID: "managed-observation-order", PluginID: plugin.ID, PluginVersion: plugin.Version, RunEndpointID: dedicatedRunEndpointID("managed-observation-order"), Name: "Managed Observation Order", State: domain.ServerInstanceStateDraft, ConfigVersion: 1}
if err := svc.store.ServerInstances().Create(instance); err != nil {
t.Fatalf("create server: %v", err)
}
registered := registerGeneratedRunForStatusTest(t, svc, instance, plugin.ID)
observedAt := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
report := func(sequence uint64, state string, classification string) {
t.Helper()
if _, err := svc.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, ServerInstanceID: instance.ID, Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100}, ManagedProcessID: "sha256:managed-process", ObservationSeq: sequence, ObservedAt: observedAt.Add(time.Duration(sequence) * time.Second), ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: state, ExitClassification: classification}}); err != nil {
t.Fatalf("report sequence %d: %v", sequence, err)
}
}
report(1, "running", "")
report(2, "exited", "unexpected-exit")
report(1, "running", "")
stored, err := svc.GetServerInstance(instance.ID)
if err != nil || stored.State != domain.ServerInstanceStateFailed || stored.LifecycleObservationSeq != 2 {
t.Fatalf("stale running observation must not regress exit projection: server=%+v err=%v", stored, err)
}
}
func TestCoreServiceGeneratedRunRegistrationDoesNotDispatchStatusReconciliation(t *testing.T) { func TestCoreServiceGeneratedRunRegistrationDoesNotDispatchStatusReconciliation(t *testing.T) {
svc := newTestCoreService() svc := newTestCoreService()
plugin := createGeneratedRunStatusPlugin(t, svc) plugin := createGeneratedRunStatusPlugin(t, svc)
+30
View File
@@ -219,6 +219,36 @@ func (svc *CoreService) QueryLogStream(query domain.LogStreamCursorQuery) (domai
}), nil }), nil
} }
func (svc *CoreService) GetRunLogStreamProgress(request domain.RunLogStreamProgress) (domain.RunLogStreamProgressResult, error) {
if err := validator.ValidateRunLogStreamProgress(request); err != nil {
return domain.RunLogStreamProgressResult{}, err
}
if _, err := svc.validatedRunSession(request.RunEndpointID, request.SessionToken); err != nil {
return domain.RunLogStreamProgressResult{}, err
}
instance, err := svc.store.ServerInstances().Get(request.ServerInstanceID)
if err != nil {
return domain.RunLogStreamProgressResult{}, err
}
if instance.RunEndpointID != request.RunEndpointID {
return domain.RunLogStreamProgressResult{}, validationError("runEndpointId must match server instance")
}
if !strings.HasPrefix(request.LogStreamID, "run."+request.RunEndpointID+"."+request.ServerInstanceID+".") {
return domain.RunLogStreamProgressResult{}, validationError("logStreamId is not a bound run stream")
}
latest := uint64(0)
stream, err := svc.store.LogStreams().Get(request.LogStreamID)
if err == nil {
if stream.ServerInstanceID != instance.ID {
return domain.RunLogStreamProgressResult{}, validationError("logStreamId does not belong to server instance")
}
latest = stream.LatestSeq
} else if !errors.Is(err, repo.ErrNotFound) {
return domain.RunLogStreamProgressResult{}, err
}
return domain.RunLogStreamProgressResult{Accepted: true, RunEndpointID: request.RunEndpointID, ServerInstanceID: instance.ID, LogStreamID: request.LogStreamID, LatestSeq: latest, ServerTime: svc.now()}, nil
}
func validateLogBatchStream(batch domain.LogBatchIngest, stream domain.LogStream) error { func validateLogBatchStream(batch domain.LogBatchIngest, stream domain.LogStream) error {
if stream.ID != batch.LogStreamID { if stream.ID != batch.LogStreamID {
return validationError("logStreamId must match stream") return validationError("logStreamId must match stream")
+20
View File
@@ -39,6 +39,26 @@ func TestCoreServiceIngestsLogBatchAndQueriesCursor(t *testing.T) {
} }
} }
func TestCoreServiceReturnsBoundRunLogStreamProgress(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
streamID := "run.run-local.server-1.stdout"
if _, err := svc.CreateLogStream(domain.LogStream{ID: streamID, ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"}); err != nil {
t.Fatalf("create run stream: %v", err)
}
batch := validLogBatch(t, sessionToken, 1, 2)
batch.LogStreamID = streamID
if _, err := svc.IngestLogBatch(batch); err != nil {
t.Fatalf("ingest run stream: %v", err)
}
progress, err := svc.GetRunLogStreamProgress(domain.RunLogStreamProgress{RunEndpointID: "run-local", SessionToken: sessionToken, ServerInstanceID: "server-1", LogStreamID: streamID})
if err != nil || !progress.Accepted || progress.LatestSeq != 2 {
t.Fatalf("unexpected progress: %+v err=%v", progress, err)
}
if _, err := svc.GetRunLogStreamProgress(domain.RunLogStreamProgress{RunEndpointID: "run-local", SessionToken: sessionToken, ServerInstanceID: "server-1", LogStreamID: "run.run-local.other.stdout"}); err == nil {
t.Fatal("expected progress scope validation failure")
}
}
func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) { func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t) svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc) createLogStreamFixture(t, svc)
+1
View File
@@ -209,6 +209,7 @@ type Core interface {
SubscribeLogEvents(string) (LogEventSubscription, error) SubscribeLogEvents(string) (LogEventSubscription, error)
SubscribeLogEventsForSession(string, string) (LogEventSubscription, error) SubscribeLogEventsForSession(string, string) (LogEventSubscription, error)
IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error) IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error)
GetRunLogStreamProgress(domain.RunLogStreamProgress) (domain.RunLogStreamProgressResult, error)
QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error) QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
ListGamePlayersForSession(string, domain.GamePlayerFilter) ([]domain.GamePlayer, error) ListGamePlayersForSession(string, domain.GamePlayerFilter) ([]domain.GamePlayer, error)
GetGamePlayerProfileForSession(string, string) (domain.GamePlayerProfile, error) GetGamePlayerProfileForSession(string, string) (domain.GamePlayerProfile, error)
@@ -29,8 +29,20 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
stamp := svc.now() stamp := svc.now()
nextState, projected := lifecycleProjectedState(report.Capability, report.State, report.ExecutionResult) nextState, projected := lifecycleProjectedState(report.Capability, report.State, report.ExecutionResult)
if lifecycleObservationIsStale(instance, report) {
projected = false
nextState = instance.State
}
if projected { if projected {
instance.State = nextState instance.State = nextState
if report.ManagedProcessID != "" {
instance.LifecycleProcessID = report.ManagedProcessID
instance.LifecycleObservationSeq = report.ObservationSeq
instance.LifecycleObservedAt = report.ObservedAt
if instance.LifecycleObservedAt.IsZero() {
instance.LifecycleObservedAt = stamp
}
}
instance.UpdatedAt = stamp instance.UpdatedAt = stamp
if err := validator.ValidateServerInstance(instance); err != nil { if err := validator.ValidateServerInstance(instance); err != nil {
return domain.RunLifecycleReportResult{}, err return domain.RunLifecycleReportResult{}, err
@@ -49,6 +61,16 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
return domain.CopyRunLifecycleReportResult(domain.RunLifecycleReportResult{Accepted: true, RunEndpointID: report.RunEndpointID, ServerInstanceID: report.ServerInstanceID, ProjectedState: nextState, ServerTime: stamp}), nil return domain.CopyRunLifecycleReportResult(domain.RunLifecycleReportResult{Accepted: true, RunEndpointID: report.RunEndpointID, ServerInstanceID: report.ServerInstanceID, ProjectedState: nextState, ServerTime: stamp}), nil
} }
func lifecycleObservationIsStale(instance domain.ServerInstance, report domain.RunLifecycleReport) bool {
if report.ManagedProcessID == "" || instance.LifecycleProcessID == "" {
return false
}
if report.ManagedProcessID == instance.LifecycleProcessID {
return report.ObservationSeq <= instance.LifecycleObservationSeq
}
return !report.ObservedAt.IsZero() && !instance.LifecycleObservedAt.IsZero() && report.ObservedAt.Before(instance.LifecycleObservedAt)
}
func lifecycleReportSummary(report domain.RunLifecycleReport, projectedState domain.ServerInstanceState, projected bool) string { func lifecycleReportSummary(report domain.RunLifecycleReport, projectedState domain.ServerInstanceState, projected bool) string {
for _, candidate := range []string{report.ExecutionResult.AuditSummary, report.Progress.Message, report.Message, report.ErrorCode} { for _, candidate := range []string{report.ExecutionResult.AuditSummary, report.Progress.Message, report.Message, report.ErrorCode} {
if strings.TrimSpace(candidate) != "" { if strings.TrimSpace(candidate) != "" {
+6
View File
@@ -69,6 +69,12 @@ func ValidateRunLifecycleReport(report domain.RunLifecycleReport) error {
violations = appendProgressViolations(violations, report.Progress) violations = appendProgressViolations(violations, report.Progress)
violations = appendMessageLength(violations, "message", report.Message) violations = appendMessageLength(violations, "message", report.Message)
violations = appendMessageLength(violations, "errorCode", report.ErrorCode) violations = appendMessageLength(violations, "errorCode", report.ErrorCode)
if report.ManagedProcessID != "" && report.ObservationSeq == 0 {
violations = append(violations, "observationSeq is required with managedProcessId")
}
if report.ObservationSeq > 0 && report.ManagedProcessID == "" {
violations = append(violations, "managedProcessId is required with observationSeq")
}
if len([]byte(report.ExecutionResult.Content)) > maxJobChannelMessageLength*256 { if len([]byte(report.ExecutionResult.Content)) > maxJobChannelMessageLength*256 {
violations = append(violations, "executionResult.content is too large") violations = append(violations, "executionResult.content is too large")
} }
+9
View File
@@ -87,6 +87,15 @@ func ValidateLogStreamCursorQuery(query domain.LogStreamCursorQuery) error {
return finish(violations) return finish(violations)
} }
func ValidateRunLogStreamProgress(request domain.RunLogStreamProgress) error {
var violations []string
violations = appendRequired(violations, "runEndpointId", request.RunEndpointID)
violations = appendRequired(violations, "sessionToken", request.SessionToken)
violations = appendRequired(violations, "serverInstanceId", request.ServerInstanceID)
violations = appendRequired(violations, "logStreamId", request.LogStreamID)
return finish(violations)
}
func LogEntriesChecksum(entries []domain.LogEntry) (string, error) { func LogEntriesChecksum(entries []domain.LogEntry) (string, error) {
stable := make([]logEntryChecksumBody, len(entries)) stable := make([]logEntryChecksumBody, len(entries))
for i, entry := range entries { for i, entry := range entries {
+1
View File
@@ -462,6 +462,7 @@ export interface ServerInstanceResponse {
ownerUserId?: string; ownerUserId?: string;
adminUserIds: string[]; adminUserIds: string[];
state: ServerInstanceState; state: ServerInstanceState;
lifecycleObservedAt?: string;
configVersion: number; configVersion: number;
configKey?: string; configKey?: string;
configChecksum?: string; configChecksum?: string;
@@ -314,7 +314,7 @@ export function ServerManagementTerminalDrawer({ open, serverId, serverName, plu
function handleTerminalScroll() { function handleTerminalScroll() {
const output = outputRef.current; const output = outputRef.current;
if (!output) return; if (!output || initialHistoryPendingRef.current) return;
const nextFollowLatest = output.scrollHeight - output.clientHeight - output.scrollTop <= 24; const nextFollowLatest = output.scrollHeight - output.clientHeight - output.scrollTop <= 24;
followLatestRef.current = nextFollowLatest; followLatestRef.current = nextFollowLatest;
setFollowLatest(nextFollowLatest); setFollowLatest(nextFollowLatest);
@@ -369,7 +369,7 @@ export function ServerManagementTerminalDrawer({ open, serverId, serverName, plu
<div className="terminal-output-topbar"> <div className="terminal-output-topbar">
<div> <div>
<strong>{serverName}</strong> <strong>{serverName}</strong>
<span> + SSE · {streams.status === "ready" ? `${terminalStreams.length} 个日志源` : streams.status === "loading" ? "读取日志源" : "日志源异常"} · {followLatest ? "自动置底" : "已解锁滚动"}</span> <span> + SSE · Run · {streams.status === "ready" ? `${terminalStreams.length} 个日志源` : streams.status === "loading" ? "读取日志源" : "日志源异常"} · {followLatest ? "自动置底" : "已解锁滚动"}</span>
</div> </div>
<div> <div>
<button type="button" className="terminal-output-action" onClick={clearTerminalBuffer}><Trash2 size={14} /><span></span></button> <button type="button" className="terminal-output-action" onClick={clearTerminalBuffer}><Trash2 size={14} /><span></span></button>
@@ -378,7 +378,7 @@ export function ServerManagementTerminalDrawer({ open, serverId, serverName, plu
</div> </div>
<div ref={outputRef} className="terminal-output" role="log" aria-live="polite" onScroll={handleTerminalScroll}> <div ref={outputRef} className="terminal-output" role="log" aria-live="polite" onScroll={handleTerminalScroll}>
{streams.status === "error" && <div className="terminal-line terminal-line-error"><time>{new Date().toLocaleTimeString()}</time><span className="terminal-stream">LOGS</span><span className="terminal-text">{streams.reason}</span></div>} {streams.status === "error" && <div className="terminal-line terminal-line-error"><time>{new Date().toLocaleTimeString()}</time><span className="terminal-stream">LOGS</span><span className="terminal-text">{streams.reason}</span></div>}
{streams.status === "ready" && streams.data.length === 0 && <div className="terminal-line terminal-line-warn"><time>{new Date().toLocaleTimeString()}</time><span className="terminal-stream">LOGS</span><span className="terminal-text"> Run </span></div>} {streams.status === "ready" && streams.data.length === 0 && <div className="terminal-line terminal-line-warn"><time>{new Date().toLocaleTimeString()}</time><span className="terminal-stream">LOGS</span><span className="terminal-text">Run </span></div>}
{lines.map((line) => <div key={line.id} className={`terminal-line terminal-line-${line.tone}`}><time>{line.at}</time><span className="terminal-stream">{line.streamKey || line.level || "LOG"}</span><span className="terminal-text">{line.text}</span></div>)} {lines.map((line) => <div key={line.id} className={`terminal-line terminal-line-${line.tone}`}><time>{line.at}</time><span className="terminal-stream">{line.streamKey || line.level || "LOG"}</span><span className="terminal-text">{line.text}</span></div>)}
</div> </div>
</section> </section>
@@ -1,7 +1,7 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { canStartServer, canStopServer } from "./serverManagement"; import { canStartServer, canStopServer, runtimeObservationFreshness } from "./serverManagement";
import type { ServerInstanceState } from "../api/types"; import type { RunEndpointResponse, ServerInstanceResponse, ServerInstanceState } from "../api/types";
describe("server management lifecycle contracts", () => { describe("server management lifecycle contracts", () => {
it("allows explicit starts from recoverable non-running states", () => { it("allows explicit starts from recoverable non-running states", () => {
@@ -16,4 +16,12 @@ describe("server management lifecycle contracts", () => {
expect(canStopServer("running")).toBe(true); expect(canStopServer("running")).toBe(true);
expect(canStopServer("failed")).toBe(false); expect(canStopServer("failed")).toBe(false);
}); });
it("distinguishes a fresh Run observation from an unverified historical lifecycle state", () => {
const instance = { id: "server-1", state: "running", runEndpointId: "run-1" } as ServerInstanceResponse;
const endpoint = { id: "run-1", status: "online", lastHeartbeatAt: "2026-08-07T10:00:00Z" } as RunEndpointResponse;
expect(runtimeObservationFreshness(instance, endpoint, Date.parse("2026-08-07T10:00:30Z"))).toBe("fresh");
expect(runtimeObservationFreshness(instance, endpoint, Date.parse("2026-08-07T10:01:00Z"))).toBe("unverified");
expect(runtimeObservationFreshness(instance, { ...endpoint, status: "offline" }, Date.parse("2026-08-07T10:00:01Z"))).toBe("unverified");
});
}); });
@@ -61,6 +61,14 @@ export interface ServerManagementSummary {
failed: number; failed: number;
} }
export type RuntimeObservationFreshness = "fresh" | "unverified";
export function runtimeObservationFreshness(instance: ServerInstanceResponse, endpoint: RunEndpointResponse | undefined, now = Date.now()): RuntimeObservationFreshness {
if (!endpoint || endpoint.status !== "online") return "unverified";
const heartbeat = Date.parse(endpoint.lastHeartbeatAt);
return Number.isFinite(heartbeat) && now-heartbeat <= 45_000 ? "fresh" : "unverified";
}
export const emptyServerCreateForm: ServerCreateFormState = { export const emptyServerCreateForm: ServerCreateFormState = {
id: "", id: "",
name: "", name: "",
+2 -1
View File
@@ -1,4 +1,4 @@
import type { JobResponse, ServerInstanceResponse, ServerMetricsResponse, UserContactProfile, UserStatus, UserThemePreferenceResponse } from "../api/types"; import type { JobResponse, RunEndpointResponse, ServerInstanceResponse, ServerMetricsResponse, UserContactProfile, UserStatus, UserThemePreferenceResponse } from "../api/types";
export type WorkspaceRole = "platformAdmin" | "serverOwner" | "serverAdmin"; export type WorkspaceRole = "platformAdmin" | "serverOwner" | "serverAdmin";
@@ -90,6 +90,7 @@ export interface OperationRecord {
export interface ServerCardView { export interface ServerCardView {
instance: ServerInstanceResponse; instance: ServerInstanceResponse;
endpoint?: RunEndpointResponse;
metrics?: ServerMetricsResponse; metrics?: ServerMetricsResponse;
pendingJobs: number; pendingJobs: number;
activeJobs?: number; activeJobs?: number;
+6 -3
View File
@@ -25,6 +25,7 @@ import {
defaultServerCreateForm, defaultServerCreateForm,
endpointLabel, endpointLabel,
pluginCreateInputDefaults, pluginCreateInputDefaults,
runtimeObservationFreshness,
type ServerCreateFormState type ServerCreateFormState
} from "../contracts/serverManagement"; } from "../contracts/serverManagement";
import { summarizeServerOperations } from "../contracts/operationsConsole"; import { summarizeServerOperations } from "../contracts/operationsConsole";
@@ -154,13 +155,14 @@ export function ServersPage({ session, operations, onNavigate }: PageComponentPr
() => () =>
summarizeServerOperations(instances, metrics, jobs).map((summary) => ({ summarizeServerOperations(instances, metrics, jobs).map((summary) => ({
instance: summary.instance, instance: summary.instance,
endpoint: endpoints.find((endpoint) => endpoint.id === summary.instance.runEndpointId),
metrics: summary.metrics, metrics: summary.metrics,
pendingJobs: summary.activeJobs, pendingJobs: summary.activeJobs,
activeJobs: summary.activeJobs, activeJobs: summary.activeJobs,
failedJobs: summary.failedJobs, failedJobs: summary.failedJobs,
latestJob: summary.latestJob latestJob: summary.latestJob
})), })),
[instances, jobs, metrics] [endpoints, instances, jobs, metrics]
); );
const visibleCards = useMemo(() => filterServerCards(cards, keyword, statusFilter), [cards, keyword, statusFilter]); const visibleCards = useMemo(() => filterServerCards(cards, keyword, statusFilter), [cards, keyword, statusFilter]);
@@ -725,8 +727,9 @@ interface ServerCardProps {
} }
function ServerCard({ card, metricsPending, metricsUnavailable, canManage, deleteDisabledReason, onOpen, onEdit, onQuickAction, onDelete }: ServerCardProps) { function ServerCard({ card, metricsPending, metricsUnavailable, canManage, deleteDisabledReason, onOpen, onEdit, onQuickAction, onDelete }: ServerCardProps) {
const { instance, metrics, pendingJobs, failedJobs = 0 } = card; const { instance, endpoint, metrics, pendingJobs, failedJobs = 0 } = card;
const online = serverIsOnline(instance.state); const online = serverIsOnline(instance.state);
const freshness = runtimeObservationFreshness(instance, endpoint);
const canDelete = deleteDisabledReason === ""; const canDelete = deleteDisabledReason === "";
const canOpenActions = canManage || canDelete; const canOpenActions = canManage || canDelete;
const metricsWaiting = metrics?.source === "run-metrics-pending"; const metricsWaiting = metrics?.source === "run-metrics-pending";
@@ -815,7 +818,7 @@ function ServerCard({ card, metricsPending, metricsUnavailable, canManage, delet
<strong>{instance.name}</strong> <strong>{instance.name}</strong>
<span className="provider-id">{instance.id}</span> <span className="provider-id">{instance.id}</span>
</span> </span>
<span className={cx("status-pill", statusClass(instance.state))}>{stateLabel(instance.state)}</span> <span className={cx("status-pill", freshness === "fresh" ? statusClass(instance.state) : "status-disabled")}>{freshness === "fresh" ? stateLabel(instance.state) : `最后观测:${stateLabel(instance.state)}Run 未验证)`}</span>
</div> </div>
<div className="server-card-stats"> <div className="server-card-stats">
<span className="server-card-stat"> <span className="server-card-stat">