Implement platform management features

This commit is contained in:
npc0-hue
2026-08-17 08:48:45 +08:00
parent 65353cf269
commit 302f1f64b7
38 changed files with 2185 additions and 110 deletions
@@ -5,6 +5,7 @@ import (
"net/http"
"strings"
"testing"
"time"
"browser.local/platform/domain"
"browser.local/platform/dto"
@@ -67,6 +68,8 @@ func TestRunChannelAPIInterleavedRequestsMutateIndependentState(t *testing.T) {
logBatch := validLogBatchRequest(t, hello.SessionToken, 1, 1)
logBatch.LogStreamID = "log-channel-isolation"
logBatch.LogSessionID = ""
logBatch.SessionStartedAt = time.Time{}
logRecorder := performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", logBatch)
assertStatus(t, logRecorder, http.StatusOK)
logAck := decodeBody[dto.LogBatchIngestResponse](t, logRecorder)
+219 -31
View File
@@ -18,6 +18,7 @@ const (
defaultLogEventHistoryLimit = 100
maxLogEventHistoryLimit = 10000
logEventHeartbeatInterval = 15 * time.Second
managedLogSessionIDPrefix = "log-session:"
)
// serverLogEvents streams platform-accepted server log history and live append events for the terminal drawer.
@@ -26,7 +27,7 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
writeMethodNotAllowed(w, http.MethodGet)
return
}
instance, streams, subscription, err := h.openLogEventSubscription(r)
instance, streams, liveEligible, subscription, err := h.openLogEventSubscription(r)
if err != nil {
writeServiceError(w, err)
return
@@ -46,25 +47,17 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
historyLimit := parseLogEventHistoryLimit(r.URL.Query().Get("historyLimit"))
for _, stream := range streams {
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(stream)); err != nil {
return
}
active := supervisedLogSession{}
if liveEligible {
active = activeSupervisedLogSession(streams)
}
if historyLimit > 0 {
history, err := h.loadLogEventHistory(streams, historyLimit)
if err != nil {
_ = writeSSEJSON(w, "error", "", map[string]string{"message": err.Error()})
flusher.Flush()
return
}
for _, event := range history {
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
return
}
}
emittedThrough, err := h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
if err != nil {
_ = writeSSEJSON(w, "error", "", map[string]string{"message": err.Error()})
flusher.Flush()
return
}
if err := writeSSEJSON(w, "ready", "", dto.LogStreamEventsReadyResponse{ServerInstanceID: instance.ID, StreamCount: len(streams), ServerTime: time.Now().UTC()}); err != nil {
if err := writeSSEJSON(w, "ready", "", dto.LogStreamEventsReadyResponse{ServerInstanceID: instance.ID, StreamCount: len(active.streams), ServerTime: time.Now().UTC()}); err != nil {
return
}
flusher.Flush()
@@ -75,13 +68,85 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
return
case event, ok := <-subscription.Events:
case subscriptionEvent, ok := <-subscription.Events:
if !ok {
return
}
if subscriptionEvent.Kind == service.LogEventSubscriptionEventProcessState {
if subscriptionEvent.ServerInstanceID != instance.ID {
continue
}
if subscriptionEvent.ProcessState != domain.ServerInstanceStateRunning {
liveEligible = false
if active.sessionID == "" {
continue
}
active = supervisedLogSession{}
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
if err != nil {
return
}
flusher.Flush()
continue
}
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
if err != nil {
return
}
next := supervisedLogSession{}
if liveEligible {
next = activeSupervisedLogSession(streams)
}
if sameSupervisedLogSession(active, next) {
continue
}
active = next
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
if err != nil {
return
}
flusher.Flush()
continue
}
if subscriptionEvent.Kind != service.LogEventSubscriptionEventLog || !liveEligible {
continue
}
event := subscriptionEvent.LogEvent
candidate := activeSupervisedLogSession([]domain.LogStream{event.Stream})
if candidate.sessionID != "" && newerLogSession(candidate, active) {
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
if err != nil {
return
}
next := supervisedLogSession{}
if liveEligible {
next = activeSupervisedLogSession(streams)
}
if !sameSupervisedLogSession(active, next) {
active = next
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
if err != nil {
return
}
flusher.Flush()
}
}
if !active.contains(event.Stream) {
continue
}
if !active.hasStream(event.Stream.ID) {
active.streams = append(active.streams, event.Stream)
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(event.Stream)); err != nil {
return
}
}
if event.Entry.Seq <= emittedThrough[event.Stream.ID] {
continue
}
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
return
}
emittedThrough[event.Stream.ID] = event.Entry.Seq
flusher.Flush()
case <-heartbeat.C:
if _, err := fmt.Fprintf(w, ": heartbeat %s\n\n", time.Now().UTC().Format(time.RFC3339)); err != nil {
@@ -92,34 +157,157 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
}
}
func (h *coreHandlers) openLogEventSubscription(r *http.Request) (domain.ServerInstance, []domain.LogStream, service.LogEventSubscription, error) {
type supervisedLogSession struct {
sessionID string
startedAt time.Time
streams []domain.LogStream
}
func activeSupervisedLogSession(streams []domain.LogStream) supervisedLogSession {
active := supervisedLogSession{}
for _, stream := range streams {
if stream.Source != domain.LogStreamSourceProcess || strings.TrimSpace(stream.LogSessionID) == "" || stream.SessionStartedAt.IsZero() {
continue
}
candidate := supervisedLogSession{sessionID: stream.LogSessionID, startedAt: stream.SessionStartedAt}
if active.sessionID == "" || newerLogSession(candidate, active) {
active = candidate
}
}
if active.sessionID == "" {
return active
}
for _, stream := range streams {
if stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == active.sessionID && stream.SessionStartedAt.Equal(active.startedAt) {
active.streams = append(active.streams, stream)
}
}
return active
}
func newerLogSession(candidate supervisedLogSession, current supervisedLogSession) bool {
if candidate.sessionID == "" || candidate.sessionID == current.sessionID {
return false
}
if current.sessionID == "" {
return true
}
if !candidate.startedAt.Equal(current.startedAt) {
return candidate.startedAt.After(current.startedAt)
}
return candidate.sessionID > current.sessionID
}
func sameSupervisedLogSession(left supervisedLogSession, right supervisedLogSession) bool {
return left.sessionID == right.sessionID && left.startedAt.Equal(right.startedAt)
}
func (session supervisedLogSession) contains(stream domain.LogStream) bool {
return session.sessionID != "" && stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == session.sessionID && stream.SessionStartedAt.Equal(session.startedAt)
}
func (session supervisedLogSession) hasStream(streamID string) bool {
for _, stream := range session.streams {
if stream.ID == streamID {
return true
}
}
return false
}
func (h *coreHandlers) writeCurrentLogSession(w http.ResponseWriter, serverInstanceID string, active supervisedLogSession, historyLimit int) (map[string]uint64, error) {
emittedThrough := make(map[string]uint64, len(active.streams))
if err := writeSSEJSON(w, "session", "", dto.LogStreamEventsSessionResponse{ServerInstanceID: serverInstanceID, LogSessionID: active.sessionID, SessionStartedAt: active.startedAt, StreamCount: len(active.streams), ServerTime: time.Now().UTC()}); err != nil {
return nil, err
}
for _, stream := range active.streams {
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(stream)); err != nil {
return nil, err
}
}
if historyLimit == 0 || len(active.streams) == 0 {
return emittedThrough, nil
}
history, err := h.loadLogEventHistory(active.streams, historyLimit)
if err != nil {
return nil, err
}
for _, event := range history {
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
return nil, err
}
if event.Entry.Seq > emittedThrough[event.Stream.ID] {
emittedThrough[event.Stream.ID] = event.Entry.Seq
}
}
return emittedThrough, nil
}
func (h *coreHandlers) openLogEventSubscription(r *http.Request) (domain.ServerInstance, []domain.LogStream, bool, service.LogEventSubscription, error) {
var instance domain.ServerInstance
var streams []domain.LogStream
var liveEligible bool
var subscription service.LogEventSubscription
var err error
if h.enforceAuthorization {
sessionID := bearerToken(r)
instance, err = h.core.GetServerInstanceForSession(sessionID, r.PathValue("id"))
if err != nil {
return domain.ServerInstance{}, nil, subscription, err
}
streams, err = h.core.ListLogStreamsForSession(sessionID, domain.LogStreamFilter{ServerInstanceID: instance.ID})
if err != nil {
return domain.ServerInstance{}, nil, subscription, err
return domain.ServerInstance{}, nil, false, subscription, err
}
subscription, err = h.core.SubscribeLogEventsForSession(sessionID, instance.ID)
if err != nil {
return domain.ServerInstance{}, nil, false, subscription, err
}
} else {
instance, err = h.core.GetServerInstance(r.PathValue("id"))
if err != nil {
return domain.ServerInstance{}, nil, subscription, err
}
streams, err = h.core.ListLogStreams(domain.LogStreamFilter{ServerInstanceID: instance.ID})
if err != nil {
return domain.ServerInstance{}, nil, subscription, err
return domain.ServerInstance{}, nil, false, subscription, err
}
subscription, err = h.core.SubscribeLogEvents(instance.ID)
if err != nil {
return domain.ServerInstance{}, nil, false, subscription, err
}
}
return instance, streams, subscription, err
if err == nil {
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
}
if err != nil && subscription.Close != nil {
subscription.Close()
}
return instance, streams, liveEligible, subscription, err
}
func (h *coreHandlers) loadLiveLogSnapshot(serverInstanceID string) ([]domain.LogStream, bool, error) {
instance, err := h.core.GetServerInstance(serverInstanceID)
if err != nil {
return nil, false, err
}
if instance.State != domain.ServerInstanceStateRunning || strings.TrimSpace(instance.RunEndpointID) == "" {
return nil, false, nil
}
endpoint, err := h.core.GetRunEndpoint(instance.RunEndpointID)
if err != nil {
return nil, false, err
}
if endpoint.Status != domain.RunEndpointStatusOnline {
return nil, false, nil
}
logSessionID := strings.TrimPrefix(instance.LifecycleProcessID, managedLogSessionIDPrefix)
if logSessionID == instance.LifecycleProcessID || strings.TrimSpace(logSessionID) == "" {
return nil, false, nil
}
streams, err := h.core.ListLogStreams(domain.LogStreamFilter{ServerInstanceID: instance.ID})
if err != nil {
return nil, false, err
}
current := make([]domain.LogStream, 0, len(streams))
for _, stream := range streams {
if stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == logSessionID {
current = append(current, stream)
}
}
return current, true, nil
}
func (h *coreHandlers) loadLogEventHistory(streams []domain.LogStream, limit int) ([]domain.LogStreamEvent, error) {
+261 -1
View File
@@ -1,18 +1,61 @@
package api
import (
"bufio"
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"browser.local/platform/domain"
"browser.local/platform/dto"
"browser.local/platform/repo"
"browser.local/platform/service"
"browser.local/platform/validator"
)
type logStreamListHookCore struct {
service.Core
once sync.Once
hook func()
}
type ssePipeResponseWriter struct {
header http.Header
pipe *io.PipeWriter
status chan int
once sync.Once
}
func newSSEPipeResponseWriter() (*ssePipeResponseWriter, *io.PipeReader) {
reader, writer := io.Pipe()
return &ssePipeResponseWriter{header: make(http.Header), pipe: writer, status: make(chan int, 1)}, reader
}
func (writer *ssePipeResponseWriter) Header() http.Header { return writer.header }
func (writer *ssePipeResponseWriter) WriteHeader(status int) {
writer.once.Do(func() { writer.status <- status })
}
func (writer *ssePipeResponseWriter) Write(payload []byte) (int, error) {
writer.WriteHeader(http.StatusOK)
return writer.pipe.Write(payload)
}
func (writer *ssePipeResponseWriter) Flush() {}
func (writer *ssePipeResponseWriter) Close() error { return writer.pipe.Close() }
func (core *logStreamListHookCore) ListLogStreams(filter domain.LogStreamFilter) ([]domain.LogStream, error) {
core.once.Do(core.hook)
return core.Core.ListLogStreams(filter)
}
func TestLogIngestAPIWorkflow(t *testing.T) {
router := newTestRouter()
hello := createLogIngestAPIFixtures(t, router)
@@ -57,6 +100,7 @@ func TestLogEventsSSEUsesServerWideNewestHistory(t *testing.T) {
hello := createLogIngestAPIFixtures(t, router)
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", dto.LogStreamCreateRequest{
ID: "log-2", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stderr",
LogSessionID: "session-current", SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default",
})
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequestForStream(t, hello.SessionToken, "log-1", "stdout", 1, 2, 0)), http.StatusOK)
@@ -70,6 +114,213 @@ func TestLogEventsSSEUsesServerWideNewestHistory(t *testing.T) {
}
}
func TestLogEventsSSEExcludesOlderSupervisedSessions(t *testing.T) {
router := newTestRouter()
hello := createLogIngestAPIFixtures(t, router)
oldStream := dto.LogStreamCreateRequest{ID: "log-old", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", LogSessionID: "session-old", SessionStartedAt: time.Date(2026, 7, 3, 11, 0, 0, 0, time.UTC), StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"}
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", oldStream)
oldBatch := validLogBatchRequestForStream(t, hello.SessionToken, "log-old", "stdout", 1, 1, 0)
oldBatch.LogSessionID = "session-old"
oldBatch.SessionStartedAt = oldStream.SessionStartedAt
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", oldBatch), http.StatusOK)
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
recorder := performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10")
body := recorder.Body.String()
if !strings.Contains(body, `"logSessionId":"session-current"`) || strings.Contains(body, `"streamId":"log-old"`) {
t.Fatalf("expected only current session in live SSE, body=%s", body)
}
}
func TestLogEventsSSEInitialSnapshotRequiresRunningOnlineProcessFact(t *testing.T) {
router := newTestRouter()
hello := createLogIngestAPIFixtures(t, router)
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
stopped := dto.RunLifecycleReportRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: dto.JobProgressBody{Percent: 100}, ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop"}}
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", stopped), http.StatusOK)
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
running := stopped
running.ManagedProcessID = "log-session:session-missing"
running.ObservationSeq = 1
running.ObservedAt = time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
running.ExecutionResult = dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"}
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", running), http.StatusOK)
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/control/heartbeat", dto.RunControlHeartbeatRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, Version: "0.1.0", Status: domain.RunEndpointStatusOffline, CapabilityFingerprint: "cap-logs", Capacity: dto.RunCapacityResponse{MaxJobs: 1}}), http.StatusOK)
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
}
func TestLogEventsSSEClearsStoppedSessionAndRestoresRunningSessionWithoutReconnect(t *testing.T) {
router := newTestRouter()
hello := createLogIngestAPIFixtures(t, router)
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
request := httptest.NewRequest(http.MethodGet, "/api/v1/server-instances/server-1/logs/events?historyLimit=10", nil).WithContext(ctx)
streamWriter, streamReader := newSSEPipeResponseWriter()
done := make(chan struct{})
go func() {
router.ServeHTTP(streamWriter, request)
_ = streamWriter.Close()
close(done)
}()
t.Cleanup(func() {
cancel()
_ = streamReader.Close()
<-done
})
if status := <-streamWriter.status; status != http.StatusOK {
t.Fatalf("unexpected SSE status: %d", status)
}
reader := bufio.NewReader(streamReader)
assertSSEEvent(t, reader, "session", `"logSessionId":"session-current"`)
assertSSEEvent(t, reader, "stream", `"id":"log-1"`)
assertSSEEvent(t, reader, "log", `"seq":1`)
assertSSEEvent(t, reader, "ready", `"streamCount":1`)
statusReport := dto.RunLifecycleReportRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: dto.JobProgressBody{Percent: 100}, ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop"}}
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", statusReport), http.StatusOK)
name, data := readSSEEvent(t, reader)
if name != "session" || strings.Contains(data, `"logSessionId"`) || !strings.Contains(data, `"streamCount":0`) {
t.Fatalf("expected stopped process to emit an empty session boundary, name=%q data=%s", name, data)
}
nextStartedAt := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
next := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stdout", "stdout", 1, 1, 20)
next.LogSessionID = "session-next"
next.SessionStartedAt = nextStartedAt
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", next), http.StatusOK)
statusReport.ManagedProcessID = "log-session:session-next"
statusReport.ObservationSeq = 1
statusReport.ObservedAt = nextStartedAt
statusReport.ExecutionResult = dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"}
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", statusReport), http.StatusOK)
assertSSEEvent(t, reader, "session", `"logSessionId":"session-next"`)
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stdout"`)
assertSSEEvent(t, reader, "log", `"streamId":"run.run-local.server-1.session-next.stdout"`)
}
func TestLogEventsSSEOrdersReplaySessionSwitchAndAdditionalStreamWithoutDuplicates(t *testing.T) {
core := service.NewCoreService(repo.NewMemoryStore())
if err := core.SeedLocalPlatformAdmin(); err != nil {
t.Fatalf("seed platform admin: %v", err)
}
setupRouter := NewTestRouterWithCore(core)
hello := createLogIngestAPIFixtures(t, setupRouter)
initial := validLogBatchRequest(t, hello.SessionToken, 1, 1)
hookResult := make(chan error, 1)
hooked := &logStreamListHookCore{Core: core, hook: func() {
_, err := core.IngestLogBatch(initial.ToDomain())
hookResult <- err
}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
request := httptest.NewRequest(http.MethodGet, "/api/v1/server-instances/server-1/logs/events?historyLimit=10", nil).WithContext(ctx)
streamWriter, streamReader := newSSEPipeResponseWriter()
done := make(chan struct{})
go func() {
NewTestRouterWithCore(hooked).ServeHTTP(streamWriter, request)
_ = streamWriter.Close()
close(done)
}()
t.Cleanup(func() {
cancel()
_ = streamReader.Close()
<-done
})
if status := <-streamWriter.status; status != http.StatusOK {
t.Fatalf("unexpected SSE status: %d", status)
}
reader := bufio.NewReader(streamReader)
assertSSEEvent(t, reader, "session", `"logSessionId":"session-current"`)
assertSSEEvent(t, reader, "stream", `"id":"log-1"`)
assertSSEEvent(t, reader, "log", `"seq":1`)
assertSSEEvent(t, reader, "ready", `"streamCount":1`)
if err := <-hookResult; err != nil {
t.Fatalf("ingest during stream snapshot: %v", err)
}
nextStartedAt := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
next := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stdout", "stdout", 1, 1, 20)
next.LogSessionID = "session-next"
next.SessionStartedAt = nextStartedAt
if _, err := core.IngestLogBatch(next.ToDomain()); err != nil {
t.Fatalf("ingest next session stdout: %v", err)
}
if _, err := core.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, ManagedProcessID: "log-session:session-next", ObservationSeq: 1, ObservedAt: nextStartedAt, ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running"}}); err != nil {
t.Fatalf("report next managed process: %v", err)
}
assertSSEEvent(t, reader, "session", `"logSessionId":"session-next"`)
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stdout"`)
assertSSEEvent(t, reader, "log", `"seq":1`)
stderr := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stderr", "stderr", 1, 1, 21)
stderr.LogSessionID = "session-next"
stderr.SessionStartedAt = nextStartedAt
if _, err := core.IngestLogBatch(stderr.ToDomain()); err != nil {
t.Fatalf("ingest next session stderr: %v", err)
}
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stderr"`)
assertSSEEvent(t, reader, "log", `"streamKey":"stderr"`)
delayedOld := validLogBatchRequest(t, hello.SessionToken, 2, 2)
if _, err := core.IngestLogBatch(delayedOld.ToDomain()); err != nil {
t.Fatalf("ingest delayed old session batch: %v", err)
}
nextLive := validLogBatchRequestForStream(t, hello.SessionToken, next.LogStreamID, "stdout", 2, 2, 22)
nextLive.LogSessionID = "session-next"
nextLive.SessionStartedAt = nextStartedAt
if _, err := core.IngestLogBatch(nextLive.ToDomain()); err != nil {
t.Fatalf("ingest next session live append: %v", err)
}
assertSSEEvent(t, reader, "log", `"streamId":"run.run-local.server-1.session-next.stdout"`)
}
func assertSSEEvent(t *testing.T, reader *bufio.Reader, wantName string, wantData string) {
t.Helper()
name, data := readSSEEvent(t, reader)
if name != wantName || !strings.Contains(data, wantData) {
t.Fatalf("unexpected SSE event: name=%q data=%s; want name=%q containing %s", name, data, wantName, wantData)
}
}
func assertEmptyInitialLogSession(t *testing.T, recorder *httptest.ResponseRecorder) {
t.Helper()
assertStatus(t, recorder, http.StatusOK)
body := recorder.Body.String()
if !strings.Contains(body, "event: session") || strings.Contains(body, `"logSessionId"`) || strings.Contains(body, "event: log") || !strings.Contains(body, `"streamCount":0`) {
t.Fatalf("expected an empty initial live session, body=%s", body)
}
}
func readSSEEvent(t *testing.T, reader *bufio.Reader) (string, string) {
t.Helper()
for {
name, data := "", ""
for {
line, err := reader.ReadString('\n')
if err != nil {
t.Fatalf("read SSE event: %v", err)
}
line = strings.TrimRight(line, "\r\n")
if line == "" {
if name != "" {
return name, data
}
break
}
if strings.HasPrefix(line, "event: ") {
name = strings.TrimPrefix(line, "event: ")
}
if strings.HasPrefix(line, "data: ") {
data = strings.TrimPrefix(line, "data: ")
}
}
}
}
func performCancelledSSE(t *testing.T, router http.Handler, path string) *httptest.ResponseRecorder {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
@@ -110,15 +361,22 @@ func createLogIngestAPIFixtures(t *testing.T, router http.Handler) dto.RunContro
hello := decodeBody[dto.RunControlHelloResponse](t, performRunControlHello(t, router, helloRequest))
adminSession := createAdminSession(t, router)
postJSON[dto.GamePluginResponse](t, router, "/api/v1/game-plugins", validGamePluginRequest())
postJSONWithAuth[dto.ServerInstanceResponse](t, router, "/api/v1/server-instances", dto.ServerInstanceCreateRequest{ID: "server-1", PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM #1"}, adminSession)
postJSONWithAuth[dto.ServerInstanceResponse](t, router, "/api/v1/server-instances", dto.ServerInstanceCreateRequest{ID: "server-1", PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM #1", State: domain.ServerInstanceStateRunning}, adminSession)
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", dto.LogStreamCreateRequest{
ID: "log-1",
ServerInstanceID: "server-1",
Source: domain.LogStreamSourceProcess,
StreamKey: "stdout",
LogSessionID: "session-current",
SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
StorageBackend: domain.LogStorageBackendLocalSegments,
RetentionPolicy: "default",
})
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", dto.RunLifecycleReportRequest{
RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded,
ManagedProcessID: "log-session:session-current", ObservationSeq: 1, ObservedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"},
}), http.StatusOK)
return hello
}
@@ -146,6 +404,8 @@ func validLogBatchRequestForStream(t *testing.T, sessionToken string, streamID s
ServerInstanceID: "server-1",
StreamKey: streamKey,
Source: domain.LogStreamSourceProcess,
LogSessionID: "session-current",
SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
FirstSeq: firstSeq,
LastSeq: lastSeq,
Compression: "none",
+2
View File
@@ -18,6 +18,8 @@ type LogBatchIngest struct {
ServerInstanceID string
StreamKey string
Source LogStreamSource
LogSessionID string
SessionStartedAt time.Time
FirstSeq uint64
LastSeq uint64
Compression string
+2
View File
@@ -1507,6 +1507,8 @@ type LogStream struct {
ServerInstanceID string
Source LogStreamSource
StreamKey string
LogSessionID string
SessionStartedAt time.Time
LatestSeq uint64
StorageBackend LogStorageBackend
RetentionPolicy string
+16
View File
@@ -22,6 +22,8 @@ type LogBatchIngestRequest struct {
ServerInstanceID string `json:"serverInstanceId"`
StreamKey string `json:"streamKey"`
Source domain.LogStreamSource `json:"source"`
LogSessionID string `json:"logSessionId,omitempty"`
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
FirstSeq uint64 `json:"firstSeq"`
LastSeq uint64 `json:"lastSeq"`
Compression string `json:"compression"`
@@ -74,6 +76,8 @@ type LogStreamEventResponse struct {
StreamID string `json:"streamId"`
Source domain.LogStreamSource `json:"source"`
StreamKey string `json:"streamKey"`
LogSessionID string `json:"logSessionId,omitempty"`
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
LatestSeq uint64 `json:"latestSeq"`
Entry LogEntryBody `json:"entry"`
}
@@ -84,6 +88,14 @@ type LogStreamEventsReadyResponse struct {
ServerTime time.Time `json:"serverTime"`
}
type LogStreamEventsSessionResponse struct {
ServerInstanceID string `json:"serverInstanceId"`
LogSessionID string `json:"logSessionId,omitempty"`
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
StreamCount int `json:"streamCount"`
ServerTime time.Time `json:"serverTime"`
}
func (request LogBatchIngestRequest) ToDomain() domain.LogBatchIngest {
return domain.LogBatchIngest{
RunEndpointID: request.RunEndpointID,
@@ -92,6 +104,8 @@ func (request LogBatchIngestRequest) ToDomain() domain.LogBatchIngest {
ServerInstanceID: request.ServerInstanceID,
StreamKey: request.StreamKey,
Source: request.Source,
LogSessionID: request.LogSessionID,
SessionStartedAt: request.SessionStartedAt,
FirstSeq: request.FirstSeq,
LastSeq: request.LastSeq,
Compression: request.Compression,
@@ -145,6 +159,8 @@ func LogStreamEventFromDomain(event domain.LogStreamEvent) LogStreamEventRespons
StreamID: event.Stream.ID,
Source: event.Stream.Source,
StreamKey: event.Stream.StreamKey,
LogSessionID: event.Stream.LogSessionID,
SessionStartedAt: event.Stream.SessionStartedAt,
LatestSeq: event.LatestSeq,
Entry: logEntryFromDomain(event.Entry),
}
+8
View File
@@ -893,6 +893,8 @@ type LogStreamCreateRequest struct {
ServerInstanceID string `json:"serverInstanceId"`
Source domain.LogStreamSource `json:"source"`
StreamKey string `json:"streamKey"`
LogSessionID string `json:"logSessionId,omitempty"`
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
StorageBackend domain.LogStorageBackend `json:"storageBackend"`
RetentionPolicy string `json:"retentionPolicy"`
}
@@ -902,6 +904,8 @@ type LogStreamResponse struct {
ServerInstanceID string `json:"serverInstanceId"`
Source domain.LogStreamSource `json:"source"`
StreamKey string `json:"streamKey"`
LogSessionID string `json:"logSessionId,omitempty"`
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
LatestSeq uint64 `json:"latestSeq"`
StorageBackend domain.LogStorageBackend `json:"storageBackend"`
RetentionPolicy string `json:"retentionPolicy"`
@@ -1363,6 +1367,8 @@ func (request LogStreamCreateRequest) ToDomain() domain.LogStream {
ServerInstanceID: request.ServerInstanceID,
Source: request.Source,
StreamKey: request.StreamKey,
LogSessionID: request.LogSessionID,
SessionStartedAt: request.SessionStartedAt,
StorageBackend: request.StorageBackend,
RetentionPolicy: request.RetentionPolicy,
}
@@ -1938,6 +1944,8 @@ func LogStreamFromDomain(stream domain.LogStream) LogStreamResponse {
ServerInstanceID: stream.ServerInstanceID,
Source: stream.Source,
StreamKey: stream.StreamKey,
LogSessionID: stream.LogSessionID,
SessionStartedAt: stream.SessionStartedAt,
LatestSeq: stream.LatestSeq,
StorageBackend: stream.StorageBackend,
RetentionPolicy: stream.RetentionPolicy,
+4
View File
@@ -415,6 +415,10 @@ type LogStream struct {
Source domain.LogStreamSource `json:"source" db:"source"`
// StreamKey is stable within the server instance.
StreamKey string `json:"streamKey" db:"stream_key"`
// LogSessionID groups plugin-declared supervised-process streams for one process generation.
LogSessionID string `json:"logSessionId,omitempty" db:"log_session_id"`
// SessionStartedAt is Run's persisted start timestamp for the supervised process generation.
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty" db:"session_started_at"`
// LatestSeq is the latest accepted sequence number.
LatestSeq uint64 `json:"latestSeq" db:"latest_seq"`
// StorageBackend identifies the log body backend.
@@ -66,7 +66,17 @@ func (svc *CoreService) enqueueDistributionBuild(job domain.Job) {
delete(svc.distributionBuilds, job.ID)
svc.distributionBuildMu.Unlock()
}()
_ = svc.executeDistributionBuild(job)
defer func() {
if recover() != nil {
_ = svc.failDistributionBuildJob(job, "platform builder failed unexpectedly")
}
}()
if err := svc.executeDistributionBuild(job); err != nil {
// executeDistributionBuild may fail after persisting the running state.
// A second terminalization attempt is idempotent when the specific
// failure path already marked the job failed.
_ = svc.failDistributionBuildJob(job, "platform builder failed before completion")
}
}()
}
@@ -99,7 +109,7 @@ func (svc *CoreService) executeDistributionBuild(job domain.Job) error {
payload, buildErr = builder.Build(input)
}
if buildErr != nil {
return svc.failDistributionBuildJob(job, builderJobFailureMessage(buildErr))
return svc.failDistributionBuildJob(job, builderJobFailureMessage(buildErr, input.AuthKey))
}
if _, err := svc.platformDistributionBuildInput(job); err != nil {
return svc.failDistributionBuildJob(job, "platform builder discarded output because the component key is no longer current")
@@ -25,6 +25,34 @@ type progressDistributionBuilder struct {
payload []byte
}
type failingBuildProgressStore struct {
repo.Store
buildJobs repo.ClientManagerBuildJobRepository
}
func (store failingBuildProgressStore) ClientManagerBuildJobs() repo.ClientManagerBuildJobRepository {
return store.buildJobs
}
type failWhenDistributionJobRunningRepository struct {
repo.ClientManagerBuildJobRepository
jobs repo.JobRepository
err error
}
func (repository failWhenDistributionJobRunningRepository) List(filter domain.ClientManagerBuildJobFilter) ([]domain.ClientManagerBuildJob, error) {
jobs, err := repository.jobs.List(domain.JobFilter{ServerInstanceID: filter.ServerInstanceID})
if err != nil {
return nil, err
}
for _, job := range jobs {
if job.Capability == domain.JobCapabilityDistributionBuild && job.State == domain.JobStateRunning {
return nil, repository.err
}
}
return repository.ClientManagerBuildJobRepository.List(filter)
}
func (builder captureDistributionBuilder) Readiness() (bool, string) {
return true, ""
}
@@ -379,6 +407,49 @@ func TestCoreServiceProjectsPlatformBuilderProgressBeforeCompletion(t *testing.T
completeDistributionBuild(t, svc, distribution, nil)
}
func TestCoreServiceTerminalizesPlatformBuildWhenRunningProjectionFails(t *testing.T) {
svc, session, instance := newDistributionTestFixture(t)
const leakedDetail = "sensitive-builder-token /private/platform/build/input/auth-key"
baseStore := svc.store
svc.store = failingBuildProgressStore{
Store: baseStore,
buildJobs: failWhenDistributionJobRunningRepository{
ClientManagerBuildJobRepository: baseStore.ClientManagerBuildJobs(),
jobs: baseStore.Jobs(),
err: errors.New(leakedDetail),
},
}
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
ServerInstanceID: instance.ID,
TargetOS: "linux",
TargetArch: "amd64",
IdempotencyKey: "running-projection-failure-terminalizes",
})
if err != nil {
t.Fatalf("queue platform distribution: %v", err)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
job, jobErr := svc.GetJob(distribution.BuildJobID)
updated, distributionErr := svc.store.RunDistributions().Get(distribution.ID)
if jobErr != nil || distributionErr != nil {
t.Fatalf("read failed build state: jobErr=%v distributionErr=%v", jobErr, distributionErr)
}
if isTerminalJobState(job.State) {
if job.State != domain.JobStateFailed || updated.Status != domain.DistributionStatusFailed {
t.Fatalf("projection failure did not end in failed state: job=%+v distribution=%+v", job, updated)
}
if strings.Contains(job.Progress.Message, "sensitive-builder-token") || strings.Contains(job.Progress.Message, "/private/") || strings.Contains(job.TerminalFingerprint, "sensitive-builder-token") {
t.Fatalf("terminal build state leaked internal failure details: %+v", job)
}
return
}
time.Sleep(time.Millisecond)
}
t.Fatal("platform build remained non-terminal after running projection failure")
}
func TestCoreServiceDiscardsBuildCompletedAfterKeyReset(t *testing.T) {
svc, session, instance := newDistributionTestFixture(t)
inputs := make(chan domain.DistributionBuildInput, 1)
+2 -2
View File
@@ -553,11 +553,11 @@ func redactBuilderHostPaths(line string) string {
return strings.Join(fields, " ")
}
func builderJobFailureMessage(err error) string {
func builderJobFailureMessage(err error, sensitiveValues ...string) string {
if err == nil {
return "platform builder failed"
}
message := strings.TrimSpace(err.Error())
message := safeBuilderFailure([]byte(err.Error()), sensitiveValues...)
if message == "" {
return "platform builder failed"
}
@@ -289,6 +289,14 @@ func TestDockerDistributionBuilderRedactsFailureAndTimeout(t *testing.T) {
}
}
func TestBuilderJobFailureMessageRedactsSensitiveValuesAndHostPaths(t *testing.T) {
const secret = "sensitive-component-key"
message := builderJobFailureMessage(errors.New(secret+" /private/platform/build/input/auth-key"), secret)
if strings.Contains(message, secret) || strings.Contains(message, "/private/") || strings.Contains(message, "auth-key") {
t.Fatalf("builder job failure leaked sensitive details: %s", message)
}
}
func TestPackageClientManagerDistributionProducesProtectedArchives(t *testing.T) {
for _, packageFormat := range []string{"zip", "tar.gz"} {
t.Run(packageFormat, func(t *testing.T) {
+43 -11
View File
@@ -8,14 +8,28 @@ import (
const logEventSubscriberBuffer = 512
type LogEventSubscriptionEventKind string
const (
LogEventSubscriptionEventLog LogEventSubscriptionEventKind = "log"
LogEventSubscriptionEventProcessState LogEventSubscriptionEventKind = "process-state"
)
type LogEventSubscriptionEvent struct {
Kind LogEventSubscriptionEventKind
LogEvent domain.LogStreamEvent
ServerInstanceID string
ProcessState domain.ServerInstanceState
}
type LogEventSubscription struct {
Events <-chan domain.LogStreamEvent
Events <-chan LogEventSubscriptionEvent
Close func()
}
type logEventSubscriber struct {
serverInstanceID string
events chan domain.LogStreamEvent
events chan LogEventSubscriptionEvent
}
func (svc *CoreService) SubscribeLogEvents(serverInstanceID string) (LogEventSubscription, error) {
@@ -26,7 +40,7 @@ func (svc *CoreService) SubscribeLogEvents(serverInstanceID string) (LogEventSub
if _, err := svc.store.ServerInstances().Get(serverInstanceID); err != nil {
return LogEventSubscription{}, err
}
events := make(chan domain.LogStreamEvent, logEventSubscriberBuffer)
events := make(chan LogEventSubscriptionEvent, logEventSubscriberBuffer)
svc.logEventMu.Lock()
svc.logEventSubscriberSeq++
id := svc.logEventSubscriberSeq
@@ -55,18 +69,36 @@ func (svc *CoreService) publishLogEvents(stream domain.LogStream, entries []doma
if len(entries) == 0 {
return
}
events := make([]domain.LogStreamEvent, len(entries))
events := make([]LogEventSubscriptionEvent, len(entries))
for index, entry := range entries {
events[index] = domain.CopyLogStreamEvent(domain.LogStreamEvent{
ServerInstanceID: stream.ServerInstanceID,
Stream: stream,
Entry: entry,
LatestSeq: stream.LatestSeq,
})
events[index] = LogEventSubscriptionEvent{
Kind: LogEventSubscriptionEventLog,
LogEvent: domain.CopyLogStreamEvent(domain.LogStreamEvent{
ServerInstanceID: stream.ServerInstanceID,
Stream: stream,
Entry: entry,
LatestSeq: stream.LatestSeq,
}),
}
}
svc.publishLogSubscriptionEvents(stream.ServerInstanceID, events)
}
func (svc *CoreService) publishLogProcessState(instance domain.ServerInstance) {
svc.publishLogSubscriptionEvents(instance.ID, []LogEventSubscriptionEvent{{
Kind: LogEventSubscriptionEventProcessState,
ServerInstanceID: instance.ID,
ProcessState: instance.State,
}})
}
func (svc *CoreService) publishLogSubscriptionEvents(serverInstanceID string, events []LogEventSubscriptionEvent) {
if len(events) == 0 {
return
}
svc.logEventMu.Lock()
for id, subscriber := range svc.logEventSubscribers {
if subscriber.serverInstanceID != stream.ServerInstanceID {
if subscriber.serverInstanceID != serverInstanceID {
continue
}
dropped := false
+23 -13
View File
@@ -20,6 +20,9 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
if err := svc.validateRunSession(batch.RunEndpointID, batch.SessionToken); err != nil {
return domain.LogBatchIngestResult{}, err
}
lock := svc.logIngestLock(batch.ServerInstanceID)
lock.Lock()
defer lock.Unlock()
stamp := svc.now()
stream, err := svc.store.LogStreams().Get(batch.LogStreamID)
@@ -58,7 +61,7 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
}
storedBatch := domain.CopyLogBatchIngest(batch)
sanitizeGamePlayerNetworkFields(&storedBatch)
sanitizeLogNetworkFields(&storedBatch)
record := domain.CopyLogBatchRecord(domain.LogBatchRecord{
Checksum: batch.Checksum,
FirstSeq: batch.FirstSeq,
@@ -96,7 +99,7 @@ func (svc *CoreService) ensureJobLogStreamForBatch(batch domain.LogBatchIngest,
if job.ServerInstanceID != batch.ServerInstanceID || job.RunEndpointID != batch.RunEndpointID {
return validationError("log batch job scope does not match stream")
}
return svc.ensureJobLogStreams(job, stamp)
return svc.ensureJobLogStreamsUnlocked(job, stamp)
}
func (svc *CoreService) ensureLogStreamForBatch(batch domain.LogBatchIngest, stamp time.Time) error {
@@ -107,7 +110,7 @@ func (svc *CoreService) ensureLogStreamForBatch(batch domain.LogBatchIngest, sta
if job.ServerInstanceID != batch.ServerInstanceID || job.RunEndpointID != batch.RunEndpointID {
return validationError("log batch job scope does not match stream")
}
return svc.ensureJobLogStreams(job, stamp)
return svc.ensureJobLogStreamsUnlocked(job, stamp)
}
if !errors.Is(err, repo.ErrNotFound) || !strings.HasPrefix(jobID, "autonomous-") {
return err
@@ -120,8 +123,14 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
if batch.Source != domain.LogStreamSourceProcess && batch.Source != domain.LogStreamSourceFile && batch.Source != domain.LogStreamSourceManagementProgram {
return repo.ErrNotFound
}
if batch.LogStreamID != runLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.StreamKey) && !legacyAutonomousLogStream(batch) {
return repo.ErrNotFound
expectedStreamID := runLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.StreamKey)
if batch.LogSessionID != "" {
expectedStreamID = runSessionLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.LogSessionID, batch.StreamKey)
}
if batch.LogStreamID != expectedStreamID {
if batch.LogSessionID != "" || !legacyAutonomousLogStream(batch) {
return repo.ErrNotFound
}
}
instance, err := svc.store.ServerInstances().Get(batch.ServerInstanceID)
if err != nil {
@@ -130,7 +139,7 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
if instance.RunEndpointID != batch.RunEndpointID {
return validationError("server instance run endpoint must match log batch endpoint")
}
_, err = svc.CreateLogStream(domain.LogStream{ID: batch.LogStreamID, ServerInstanceID: batch.ServerInstanceID, Source: batch.Source, StreamKey: batch.StreamKey, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default", CreatedAt: stamp, UpdatedAt: stamp})
_, err = svc.createLogStream(domain.LogStream{ID: batch.LogStreamID, ServerInstanceID: batch.ServerInstanceID, Source: batch.Source, StreamKey: batch.StreamKey, LogSessionID: batch.LogSessionID, SessionStartedAt: batch.SessionStartedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default", CreatedAt: stamp, UpdatedAt: stamp})
if errors.Is(err, repo.ErrDuplicate) {
return nil
}
@@ -166,18 +175,16 @@ func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIn
return false
}
// sanitizeGamePlayerNetworkFields removes raw network material before the durable log body is written.
func sanitizeGamePlayerNetworkFields(batch *domain.LogBatchIngest) {
// sanitizeLogNetworkFields removes raw network material before the durable log body is written.
func sanitizeLogNetworkFields(batch *domain.LogBatchIngest) {
for index := range batch.Entries {
fields := batch.Entries[index].Fields
if fields == nil {
continue
}
if fields["eventType"] == "scum.login" {
delete(fields, "networkFingerprint")
delete(fields, "ip")
delete(fields, "ipAddress")
}
delete(fields, "networkFingerprint")
delete(fields, "ip")
delete(fields, "ipAddress")
}
}
@@ -249,5 +256,8 @@ func validateLogBatchStream(batch domain.LogBatchIngest, stream domain.LogStream
if stream.Source != batch.Source {
return validationError("source must match stream")
}
if stream.LogSessionID != batch.LogSessionID || !stream.SessionStartedAt.Equal(batch.SessionStartedAt) {
return validationError("log session metadata must match stream")
}
return nil
}
@@ -0,0 +1,31 @@
package service
import (
"testing"
"browser.local/platform/domain"
)
func TestSanitizeLogNetworkFieldsIsGameAgnostic(t *testing.T) {
batch := domain.LogBatchIngest{Entries: []domain.LogEntry{
{Fields: map[string]string{
"eventType": "game.session.opened",
"networkFingerprint": "fingerprint",
"ip": "192.0.2.1",
"ipAddress": "2001:db8::1",
"playerId": "player-1",
}},
}}
sanitizeLogNetworkFields(&batch)
fields := batch.Entries[0].Fields
for _, key := range []string{"networkFingerprint", "ip", "ipAddress"} {
if _, exists := fields[key]; exists {
t.Fatalf("expected %s to be removed", key)
}
}
if fields["playerId"] != "player-1" {
t.Fatal("expected unrelated fields to be preserved")
}
}
+83 -1
View File
@@ -3,6 +3,7 @@ package service
import (
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -74,7 +75,7 @@ func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
}
select {
case event := <-subscription.Events:
if event.Stream.ID != "log-1" || event.Entry.Seq != 1 || event.LatestSeq != 1 {
if event.Kind != LogEventSubscriptionEventLog || event.LogEvent.Stream.ID != "log-1" || event.LogEvent.Entry.Seq != 1 || event.LogEvent.LatestSeq != 1 {
t.Fatalf("unexpected log event: %+v", event)
}
case <-time.After(time.Second):
@@ -91,6 +92,76 @@ func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
}
}
func TestCoreServicePersistsAndEnforcesImmutableProcessLogSessionMetadata(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
startedAt := time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
streamID := runSessionLogStreamID("run-local", "server-1", "session-a", "stdout")
batch := validLogBatch(t, sessionToken, 1, 1)
batch.LogStreamID = streamID
batch.LogSessionID = "session-a"
batch.SessionStartedAt = startedAt
if _, err := svc.IngestLogBatch(batch); err != nil {
t.Fatalf("ingest session-scoped process batch: %v", err)
}
stream, err := svc.GetLogStream(streamID)
if err != nil || stream.LogSessionID != "session-a" || !stream.SessionStartedAt.Equal(startedAt) {
t.Fatalf("session metadata was not persisted: stream=%+v err=%v", stream, err)
}
conflict := validLogBatch(t, sessionToken, 2, 2)
conflict.LogStreamID = streamID
conflict.LogSessionID = "session-a"
conflict.SessionStartedAt = startedAt.Add(time.Second)
if _, err := svc.IngestLogBatch(conflict); err == nil || !strings.Contains(err.Error(), "metadata must match") {
t.Fatalf("expected immutable stream metadata rejection, got %v", err)
}
legacy := createLogStreamFixture(t, svc)
legacyBatch := validLogBatch(t, sessionToken, 1, 1)
legacyBatch.LogStreamID = legacy.ID
legacyBatch.LogSessionID = "session-a"
legacyBatch.SessionStartedAt = startedAt
if _, err := svc.IngestLogBatch(legacyBatch); err == nil || !strings.Contains(err.Error(), "metadata must match") {
t.Fatalf("expected legacy stream to reject attached session metadata, got %v", err)
}
}
func TestCoreServiceSerializesConsistentSessionStreamCreation(t *testing.T) {
svc, _ := newRegisteredLogIngestService(t)
startedAt := time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
streams := []domain.LogStream{
{ID: "session-stream-stdout", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", LogSessionID: "session-a", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"},
{ID: "session-stream-stderr", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stderr", LogSessionID: "session-a", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"},
}
errorsByStream := make(chan error, len(streams))
var wait sync.WaitGroup
for _, stream := range streams {
stream := stream
wait.Add(1)
go func() {
defer wait.Done()
_, err := svc.CreateLogStream(stream)
errorsByStream <- err
}()
}
wait.Wait()
close(errorsByStream)
for err := range errorsByStream {
if err != nil {
t.Fatalf("create consistent session stream: %v", err)
}
}
_, err := svc.CreateLogStream(domain.LogStream{ID: "session-stream-conflict", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "console", LogSessionID: "session-a", SessionStartedAt: startedAt.Add(time.Second), StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"})
if err == nil || !strings.Contains(err.Error(), "conflicts") {
t.Fatalf("expected conflicting session timestamp rejection, got %v", err)
}
_, err = svc.CreateLogStream(domain.LogStream{ID: "session-file-tail", ServerInstanceID: "server-1", Source: domain.LogStreamSourceFile, StreamKey: "file", LogSessionID: "session-file", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"})
if err == nil || !strings.Contains(err.Error(), "only valid for process") {
t.Fatalf("expected file-tail session metadata rejection, got %v", err)
}
}
func TestCoreServiceLogBatchDuplicateAck(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
@@ -346,6 +417,17 @@ func TestCoreServiceAcceptsLegacyAutonomousJobLogStreamWithoutPlatformJob(t *tes
}
}
func TestCoreServiceRejectsSessionMetadataOnLegacyAutonomousStreamID(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
batch := validLogBatch(t, sessionToken, 1, 1)
batch.LogStreamID = jobLogStreamID("autonomous-bootstrap-start", "stdout")
batch.LogSessionID = "session-a"
batch.SessionStartedAt = time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
if _, err := svc.IngestLogBatch(batch); err == nil {
t.Fatal("expected session-scoped batch with legacy autonomous stream ID to be rejected")
}
}
func TestCoreServiceRejectsOutOfOrderAndConflictingLogBatches(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
+48 -1
View File
@@ -233,6 +233,7 @@ type CoreService struct {
bridgeMu sync.Mutex
bridgeSeq uint64
logStore LogBodyStore
logIngestMu [64]sync.Mutex
logEventMu sync.Mutex
logEventSubscribers map[uint64]logEventSubscriber
logEventSubscriberSeq uint64
@@ -2476,6 +2477,13 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
}
func (svc *CoreService) ensureJobLogStreams(job domain.Job, stamp time.Time) error {
lock := svc.logIngestLock(job.ServerInstanceID)
lock.Lock()
defer lock.Unlock()
return svc.ensureJobLogStreamsUnlocked(job, stamp)
}
func (svc *CoreService) ensureJobLogStreamsUnlocked(job domain.Job, stamp time.Time) error {
if strings.TrimSpace(job.ServerInstanceID) == "" || strings.TrimSpace(job.ID) == "" {
return nil
}
@@ -2524,7 +2532,7 @@ func (svc *CoreService) ensureJobLogStreams(job domain.Job, stamp time.Time) err
CreatedAt: stamp,
UpdatedAt: stamp,
}
if _, err := svc.CreateLogStream(stream); err != nil && !errors.Is(err, repo.ErrDuplicate) {
if _, err := svc.createLogStream(stream); err != nil && !errors.Is(err, repo.ErrDuplicate) {
return err
}
}
@@ -2539,6 +2547,10 @@ func runLogStreamID(runEndpointID string, serverInstanceID string, streamKey str
return fmt.Sprintf("run.%s.%s.%s", runEndpointID, serverInstanceID, streamKey)
}
func runSessionLogStreamID(runEndpointID string, serverInstanceID string, logSessionID string, streamKey string) string {
return fmt.Sprintf("run.%s.%s.%s.%s", runEndpointID, serverInstanceID, logSessionID, streamKey)
}
func (svc *CoreService) GetJob(id string) (domain.Job, error) {
job, err := svc.store.Jobs().Get(id)
if err != nil {
@@ -2587,6 +2599,22 @@ func (svc *CoreService) ListArtifacts(filter domain.ArtifactFilter) ([]domain.Ar
}
func (svc *CoreService) CreateLogStream(stream domain.LogStream) (domain.LogStream, error) {
lock := svc.logIngestLock(stream.ServerInstanceID)
lock.Lock()
defer lock.Unlock()
return svc.createLogStream(stream)
}
func (svc *CoreService) logIngestLock(serverInstanceID string) *sync.Mutex {
hash := uint32(2166136261)
for index := 0; index < len(serverInstanceID); index++ {
hash ^= uint32(serverInstanceID[index])
hash *= 16777619
}
return &svc.logIngestMu[hash%uint32(len(svc.logIngestMu))]
}
func (svc *CoreService) createLogStream(stream domain.LogStream) (domain.LogStream, error) {
instance, err := svc.store.ServerInstances().Get(stream.ServerInstanceID)
if err != nil {
return domain.LogStream{}, fmt.Errorf("get server instance dependency: %w", err)
@@ -2604,12 +2632,31 @@ func (svc *CoreService) CreateLogStream(stream domain.LogStream) (domain.LogStre
if err := validator.ValidateLogStream(stream); err != nil {
return domain.LogStream{}, err
}
if err := svc.validateLogStreamSession(stream); err != nil {
return domain.LogStream{}, err
}
if err := svc.store.LogStreams().Create(stream); err != nil {
return domain.LogStream{}, err
}
return domain.CopyLogStream(stream), nil
}
func (svc *CoreService) validateLogStreamSession(stream domain.LogStream) error {
if stream.LogSessionID == "" {
return nil
}
streams, err := svc.store.LogStreams().List(domain.LogStreamFilter{ServerInstanceID: stream.ServerInstanceID})
if err != nil {
return err
}
for _, existing := range streams {
if existing.LogSessionID == stream.LogSessionID && !existing.SessionStartedAt.Equal(stream.SessionStartedAt) {
return validationError("log session metadata conflicts with an existing stream")
}
}
return nil
}
func (svc *CoreService) GetLogStream(id string) (domain.LogStream, error) {
return svc.store.LogStreams().Get(id)
}
+9
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"browser.local/platform/domain"
"browser.local/platform/repo"
@@ -255,6 +256,14 @@ func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLif
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
return domain.ServerLifecycleResult{}, err
}
if action == domain.ServerLifecycleActionStart {
instance.LifecycleProcessID = ""
instance.LifecycleObservationSeq = 0
instance.LifecycleObservedAt = time.Time{}
if err := svc.store.ServerInstances().Update(instance); err != nil {
return domain.ServerLifecycleResult{}, err
}
}
job, err := svc.dispatchLifecycleJob(instance, action, command.IdempotencyKey)
if err != nil {
@@ -50,6 +50,7 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
if err := svc.store.ServerInstances().Update(instance); err != nil {
return domain.RunLifecycleReportResult{}, err
}
svc.publishLogProcessState(instance)
}
auditResult := domain.AuditResultSuccess
if report.State == domain.JobStateFailed || report.State == domain.JobStateCancelled {
@@ -140,6 +141,7 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
if err := svc.store.ServerInstances().Update(instance); err != nil {
return err
}
svc.publishLogProcessState(instance)
auditResult := domain.AuditResultSuccess
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
auditResult = domain.AuditResultFailed
+56
View File
@@ -3,6 +3,7 @@ package service
import (
"strings"
"testing"
"time"
"browser.local/platform/domain"
)
@@ -103,6 +104,61 @@ func TestLifecycleProjectedStateUsesRunProcessFacts(t *testing.T) {
}
}
func TestLifecycleJobResultsPublishProcessStateEvents(t *testing.T) {
svc, sessionToken := newLifecycleRunService(t)
createLifecyclePlugin(t, svc)
created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-state-events", PluginID: "server.scum", RunEndpointID: "run-local", Name: "State Events", IdempotencyKey: "state-events-create", ProfileKey: "local"})
if err != nil {
t.Fatalf("create lifecycle server: %v", err)
}
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, created.Instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
ready, err := svc.GetServerInstance(created.Instance.ID)
if err != nil {
t.Fatalf("get ready server: %v", err)
}
if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "state-events-start"}); err != nil {
t.Fatalf("dispatch start: %v", err)
}
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, ready.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
subscription, err := svc.SubscribeLogEvents(ready.ID)
if err != nil {
t.Fatalf("subscribe state events: %v", err)
}
defer subscription.Close()
running, err := svc.GetServerInstance(ready.ID)
if err != nil {
t.Fatalf("get running server: %v", err)
}
if _, err := svc.StopServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: running.ID, ExpectedConfigVersion: running.ConfigVersion, IdempotencyKey: "state-events-stop"}); err != nil {
t.Fatalf("dispatch stop: %v", err)
}
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, running.ID, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)
assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateStopped)
stopped, err := svc.GetServerInstance(running.ID)
if err != nil {
t.Fatalf("get stopped server: %v", err)
}
if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: stopped.ID, ExpectedConfigVersion: stopped.ConfigVersion, IdempotencyKey: "state-events-restart"}); err != nil {
t.Fatalf("dispatch restart: %v", err)
}
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, stopped.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateRunning)
}
func assertLogProcessStateEvent(t *testing.T, subscription LogEventSubscription, want domain.ServerInstanceState) {
t.Helper()
select {
case event := <-subscription.Events:
if event.Kind != LogEventSubscriptionEventProcessState || event.ProcessState != want {
t.Fatalf("unexpected process state event: %+v", event)
}
case <-time.After(time.Second):
t.Fatalf("expected process state event %q", want)
}
}
func TestCoreServiceCreatesTargetBoundDraftAndRequiresDedicatedRunRegistration(t *testing.T) {
svc, _ := newLifecycleRunService(t)
plugin := createLifecyclePlugin(t, svc)
+8
View File
@@ -27,6 +27,14 @@ func ValidateLogBatchIngest(batch domain.LogBatchIngest) error {
if !validLogStreamSource(batch.Source) {
violations = append(violations, "source is invalid")
}
hasSessionID := strings.TrimSpace(batch.LogSessionID) != ""
hasSessionStart := !batch.SessionStartedAt.IsZero()
if hasSessionID != hasSessionStart {
violations = append(violations, "logSessionId and sessionStartedAt must be provided together")
}
if (hasSessionID || hasSessionStart) && batch.Source != domain.LogStreamSourceProcess {
violations = append(violations, "log session metadata is only valid for process streams")
}
if batch.FirstSeq == 0 || batch.LastSeq == 0 {
violations = append(violations, "sequence range must be positive")
}
+8
View File
@@ -1667,6 +1667,14 @@ func ValidateLogStream(stream domain.LogStream) error {
if !validLogStorageBackend(stream.StorageBackend) {
violations = append(violations, "storageBackend is invalid")
}
hasSessionID := strings.TrimSpace(stream.LogSessionID) != ""
hasSessionStart := !stream.SessionStartedAt.IsZero()
if hasSessionID != hasSessionStart {
violations = append(violations, "logSessionId and sessionStartedAt must be provided together")
}
if (hasSessionID || hasSessionStart) && stream.Source != domain.LogStreamSourceProcess {
violations = append(violations, "log session metadata is only valid for process streams")
}
return finish(violations)
}