admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: recover run runtime state and logs

Browse Source
This commit is contained in:
npc0-hue
2026-08-07 12:01:26 +08:00
parent 3818fa0344
commit b789925ae5
23 changed files with 297 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/control_test.go
+25
View File
@@ -4,6 +4,7 @@ import (
"errors"
"strings"
"testing"
"time"
"browser.local/platform/domain"
"browser.local/platform/repo"
@@ -464,6 +465,30 @@ func TestCoreServiceRunLifecycleReportProjectsGeneratedRunFacts(t *testing.T) {
}
}
func TestCoreServiceRejectsStaleManagedProcessObservation(t *testing.T) {
svc := newTestCoreService()
plugin := createGeneratedRunStatusPlugin(t, svc)
instance := domain.ServerInstance{ID: "managed-observation-order", PluginID: plugin.ID, PluginVersion: plugin.Version, RunEndpointID: dedicatedRunEndpointID("managed-observation-order"), Name: "Managed Observation Order", State: domain.ServerInstanceStateDraft, ConfigVersion: 1}
if err := svc.store.ServerInstances().Create(instance); err != nil {
t.Fatalf("create server: %v", err)
}
registered := registerGeneratedRunForStatusTest(t, svc, instance, plugin.ID)
observedAt := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
report := func(sequence uint64, state string, classification string) {
t.Helper()
if _, err := svc.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: instance.RunEndpointID, SessionToken: registered.SessionToken, ServerInstanceID: instance.ID, Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100}, ManagedProcessID: "sha256:managed-process", ObservationSeq: sequence, ObservedAt: observedAt.Add(time.Duration(sequence) * time.Second), ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: state, ExitClassification: classification}}); err != nil {
t.Fatalf("report sequence %d: %v", sequence, err)
}
}
report(1, "running", "")
report(2, "exited", "unexpected-exit")
report(1, "running", "")
stored, err := svc.GetServerInstance(instance.ID)
if err != nil || stored.State != domain.ServerInstanceStateFailed || stored.LifecycleObservationSeq != 2 {
t.Fatalf("stale running observation must not regress exit projection: server=%+v err=%v", stored, err)
}
}
func TestCoreServiceGeneratedRunRegistrationDoesNotDispatchStatusReconciliation(t *testing.T) {
svc := newTestCoreService()
plugin := createGeneratedRunStatusPlugin(t, svc)
platform/service/log_ingest.go
+30
View File
@@ -219,6 +219,36 @@ func (svc *CoreService) QueryLogStream(query domain.LogStreamCursorQuery) (domai
}), nil
}
func (svc *CoreService) GetRunLogStreamProgress(request domain.RunLogStreamProgress) (domain.RunLogStreamProgressResult, error) {
if err := validator.ValidateRunLogStreamProgress(request); err != nil {
return domain.RunLogStreamProgressResult{}, err
}
if _, err := svc.validatedRunSession(request.RunEndpointID, request.SessionToken); err != nil {
return domain.RunLogStreamProgressResult{}, err
}
instance, err := svc.store.ServerInstances().Get(request.ServerInstanceID)
if err != nil {
return domain.RunLogStreamProgressResult{}, err
}
if instance.RunEndpointID != request.RunEndpointID {
return domain.RunLogStreamProgressResult{}, validationError("runEndpointId must match server instance")
}
if !strings.HasPrefix(request.LogStreamID, "run."+request.RunEndpointID+"."+request.ServerInstanceID+".") {
return domain.RunLogStreamProgressResult{}, validationError("logStreamId is not a bound run stream")
}
latest := uint64(0)
stream, err := svc.store.LogStreams().Get(request.LogStreamID)
if err == nil {
if stream.ServerInstanceID != instance.ID {
return domain.RunLogStreamProgressResult{}, validationError("logStreamId does not belong to server instance")
}
latest = stream.LatestSeq
} else if !errors.Is(err, repo.ErrNotFound) {
return domain.RunLogStreamProgressResult{}, err
}
return domain.RunLogStreamProgressResult{Accepted: true, RunEndpointID: request.RunEndpointID, ServerInstanceID: instance.ID, LogStreamID: request.LogStreamID, LatestSeq: latest, ServerTime: svc.now()}, nil
}
func validateLogBatchStream(batch domain.LogBatchIngest, stream domain.LogStream) error {
if stream.ID != batch.LogStreamID {
return validationError("logStreamId must match stream")
platform/service/log_ingest_test.go
+20
View File
@@ -39,6 +39,26 @@ func TestCoreServiceIngestsLogBatchAndQueriesCursor(t *testing.T) {
}
}
func TestCoreServiceReturnsBoundRunLogStreamProgress(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
streamID := "run.run-local.server-1.stdout"
if _, err := svc.CreateLogStream(domain.LogStream{ID: streamID, ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"}); err != nil {
t.Fatalf("create run stream: %v", err)
}
batch := validLogBatch(t, sessionToken, 1, 2)
batch.LogStreamID = streamID
if _, err := svc.IngestLogBatch(batch); err != nil {
t.Fatalf("ingest run stream: %v", err)
}
progress, err := svc.GetRunLogStreamProgress(domain.RunLogStreamProgress{RunEndpointID: "run-local", SessionToken: sessionToken, ServerInstanceID: "server-1", LogStreamID: streamID})
if err != nil || !progress.Accepted || progress.LatestSeq != 2 {
t.Fatalf("unexpected progress: %+v err=%v", progress, err)
}
if _, err := svc.GetRunLogStreamProgress(domain.RunLogStreamProgress{RunEndpointID: "run-local", SessionToken: sessionToken, ServerInstanceID: "server-1", LogStreamID: "run.run-local.other.stdout"}); err == nil {
t.Fatal("expected progress scope validation failure")
}
}
func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
platform/service/resources.go
+1
View File
@@ -209,6 +209,7 @@ type Core interface {
SubscribeLogEvents(string) (LogEventSubscription, error)
SubscribeLogEventsForSession(string, string) (LogEventSubscription, error)
IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error)
GetRunLogStreamProgress(domain.RunLogStreamProgress) (domain.RunLogStreamProgressResult, error)
QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
ListGamePlayersForSession(string, domain.GamePlayerFilter) ([]domain.GamePlayer, error)
GetGamePlayerProfileForSession(string, string) (domain.GamePlayerProfile, error)
platform/service/server_lifecycle_projection.go
+22
View File
@@ -29,8 +29,20 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
stamp := svc.now()
nextState, projected := lifecycleProjectedState(report.Capability, report.State, report.ExecutionResult)
if lifecycleObservationIsStale(instance, report) {
projected = false
nextState = instance.State
}
if projected {
instance.State = nextState
if report.ManagedProcessID != "" {
instance.LifecycleProcessID = report.ManagedProcessID
instance.LifecycleObservationSeq = report.ObservationSeq
instance.LifecycleObservedAt = report.ObservedAt
if instance.LifecycleObservedAt.IsZero() {
instance.LifecycleObservedAt = stamp
}
}
instance.UpdatedAt = stamp
if err := validator.ValidateServerInstance(instance); err != nil {
return domain.RunLifecycleReportResult{}, err
@@ -49,6 +61,16 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
return domain.CopyRunLifecycleReportResult(domain.RunLifecycleReportResult{Accepted: true, RunEndpointID: report.RunEndpointID, ServerInstanceID: report.ServerInstanceID, ProjectedState: nextState, ServerTime: stamp}), nil
}
func lifecycleObservationIsStale(instance domain.ServerInstance, report domain.RunLifecycleReport) bool {
if report.ManagedProcessID == "" || instance.LifecycleProcessID == "" {
return false
}
if report.ManagedProcessID == instance.LifecycleProcessID {
return report.ObservationSeq <= instance.LifecycleObservationSeq
}
return !report.ObservedAt.IsZero() && !instance.LifecycleObservedAt.IsZero() && report.ObservedAt.Before(instance.LifecycleObservedAt)
}
func lifecycleReportSummary(report domain.RunLifecycleReport, projectedState domain.ServerInstanceState, projected bool) string {
for _, candidate := range []string{report.ExecutionResult.AuditSummary, report.Progress.Message, report.Message, report.ErrorCode} {
if strings.TrimSpace(candidate) != "" {