Implement platform management features
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/dto"
|
||||
@@ -67,6 +68,8 @@ func TestRunChannelAPIInterleavedRequestsMutateIndependentState(t *testing.T) {
|
||||
|
||||
logBatch := validLogBatchRequest(t, hello.SessionToken, 1, 1)
|
||||
logBatch.LogStreamID = "log-channel-isolation"
|
||||
logBatch.LogSessionID = ""
|
||||
logBatch.SessionStartedAt = time.Time{}
|
||||
logRecorder := performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", logBatch)
|
||||
assertStatus(t, logRecorder, http.StatusOK)
|
||||
logAck := decodeBody[dto.LogBatchIngestResponse](t, logRecorder)
|
||||
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
defaultLogEventHistoryLimit = 100
|
||||
maxLogEventHistoryLimit = 10000
|
||||
logEventHeartbeatInterval = 15 * time.Second
|
||||
managedLogSessionIDPrefix = "log-session:"
|
||||
)
|
||||
|
||||
// serverLogEvents streams platform-accepted server log history and live append events for the terminal drawer.
|
||||
@@ -26,7 +27,7 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
|
||||
writeMethodNotAllowed(w, http.MethodGet)
|
||||
return
|
||||
}
|
||||
instance, streams, subscription, err := h.openLogEventSubscription(r)
|
||||
instance, streams, liveEligible, subscription, err := h.openLogEventSubscription(r)
|
||||
if err != nil {
|
||||
writeServiceError(w, err)
|
||||
return
|
||||
@@ -46,25 +47,17 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
historyLimit := parseLogEventHistoryLimit(r.URL.Query().Get("historyLimit"))
|
||||
for _, stream := range streams {
|
||||
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(stream)); err != nil {
|
||||
return
|
||||
}
|
||||
active := supervisedLogSession{}
|
||||
if liveEligible {
|
||||
active = activeSupervisedLogSession(streams)
|
||||
}
|
||||
if historyLimit > 0 {
|
||||
history, err := h.loadLogEventHistory(streams, historyLimit)
|
||||
if err != nil {
|
||||
_ = writeSSEJSON(w, "error", "", map[string]string{"message": err.Error()})
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
for _, event := range history {
|
||||
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
emittedThrough, err := h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
|
||||
if err != nil {
|
||||
_ = writeSSEJSON(w, "error", "", map[string]string{"message": err.Error()})
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
if err := writeSSEJSON(w, "ready", "", dto.LogStreamEventsReadyResponse{ServerInstanceID: instance.ID, StreamCount: len(streams), ServerTime: time.Now().UTC()}); err != nil {
|
||||
if err := writeSSEJSON(w, "ready", "", dto.LogStreamEventsReadyResponse{ServerInstanceID: instance.ID, StreamCount: len(active.streams), ServerTime: time.Now().UTC()}); err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
@@ -75,13 +68,85 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case event, ok := <-subscription.Events:
|
||||
case subscriptionEvent, ok := <-subscription.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if subscriptionEvent.Kind == service.LogEventSubscriptionEventProcessState {
|
||||
if subscriptionEvent.ServerInstanceID != instance.ID {
|
||||
continue
|
||||
}
|
||||
if subscriptionEvent.ProcessState != domain.ServerInstanceStateRunning {
|
||||
liveEligible = false
|
||||
if active.sessionID == "" {
|
||||
continue
|
||||
}
|
||||
active = supervisedLogSession{}
|
||||
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
continue
|
||||
}
|
||||
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
next := supervisedLogSession{}
|
||||
if liveEligible {
|
||||
next = activeSupervisedLogSession(streams)
|
||||
}
|
||||
if sameSupervisedLogSession(active, next) {
|
||||
continue
|
||||
}
|
||||
active = next
|
||||
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
continue
|
||||
}
|
||||
if subscriptionEvent.Kind != service.LogEventSubscriptionEventLog || !liveEligible {
|
||||
continue
|
||||
}
|
||||
event := subscriptionEvent.LogEvent
|
||||
candidate := activeSupervisedLogSession([]domain.LogStream{event.Stream})
|
||||
if candidate.sessionID != "" && newerLogSession(candidate, active) {
|
||||
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
next := supervisedLogSession{}
|
||||
if liveEligible {
|
||||
next = activeSupervisedLogSession(streams)
|
||||
}
|
||||
if !sameSupervisedLogSession(active, next) {
|
||||
active = next
|
||||
emittedThrough, err = h.writeCurrentLogSession(w, instance.ID, active, historyLimit)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
if !active.contains(event.Stream) {
|
||||
continue
|
||||
}
|
||||
if !active.hasStream(event.Stream.ID) {
|
||||
active.streams = append(active.streams, event.Stream)
|
||||
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(event.Stream)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if event.Entry.Seq <= emittedThrough[event.Stream.ID] {
|
||||
continue
|
||||
}
|
||||
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
|
||||
return
|
||||
}
|
||||
emittedThrough[event.Stream.ID] = event.Entry.Seq
|
||||
flusher.Flush()
|
||||
case <-heartbeat.C:
|
||||
if _, err := fmt.Fprintf(w, ": heartbeat %s\n\n", time.Now().UTC().Format(time.RFC3339)); err != nil {
|
||||
@@ -92,34 +157,157 @@ func (h *coreHandlers) serverLogEvents(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *coreHandlers) openLogEventSubscription(r *http.Request) (domain.ServerInstance, []domain.LogStream, service.LogEventSubscription, error) {
|
||||
type supervisedLogSession struct {
|
||||
sessionID string
|
||||
startedAt time.Time
|
||||
streams []domain.LogStream
|
||||
}
|
||||
|
||||
func activeSupervisedLogSession(streams []domain.LogStream) supervisedLogSession {
|
||||
active := supervisedLogSession{}
|
||||
for _, stream := range streams {
|
||||
if stream.Source != domain.LogStreamSourceProcess || strings.TrimSpace(stream.LogSessionID) == "" || stream.SessionStartedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
candidate := supervisedLogSession{sessionID: stream.LogSessionID, startedAt: stream.SessionStartedAt}
|
||||
if active.sessionID == "" || newerLogSession(candidate, active) {
|
||||
active = candidate
|
||||
}
|
||||
}
|
||||
if active.sessionID == "" {
|
||||
return active
|
||||
}
|
||||
for _, stream := range streams {
|
||||
if stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == active.sessionID && stream.SessionStartedAt.Equal(active.startedAt) {
|
||||
active.streams = append(active.streams, stream)
|
||||
}
|
||||
}
|
||||
return active
|
||||
}
|
||||
|
||||
func newerLogSession(candidate supervisedLogSession, current supervisedLogSession) bool {
|
||||
if candidate.sessionID == "" || candidate.sessionID == current.sessionID {
|
||||
return false
|
||||
}
|
||||
if current.sessionID == "" {
|
||||
return true
|
||||
}
|
||||
if !candidate.startedAt.Equal(current.startedAt) {
|
||||
return candidate.startedAt.After(current.startedAt)
|
||||
}
|
||||
return candidate.sessionID > current.sessionID
|
||||
}
|
||||
|
||||
func sameSupervisedLogSession(left supervisedLogSession, right supervisedLogSession) bool {
|
||||
return left.sessionID == right.sessionID && left.startedAt.Equal(right.startedAt)
|
||||
}
|
||||
|
||||
func (session supervisedLogSession) contains(stream domain.LogStream) bool {
|
||||
return session.sessionID != "" && stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == session.sessionID && stream.SessionStartedAt.Equal(session.startedAt)
|
||||
}
|
||||
|
||||
func (session supervisedLogSession) hasStream(streamID string) bool {
|
||||
for _, stream := range session.streams {
|
||||
if stream.ID == streamID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *coreHandlers) writeCurrentLogSession(w http.ResponseWriter, serverInstanceID string, active supervisedLogSession, historyLimit int) (map[string]uint64, error) {
|
||||
emittedThrough := make(map[string]uint64, len(active.streams))
|
||||
if err := writeSSEJSON(w, "session", "", dto.LogStreamEventsSessionResponse{ServerInstanceID: serverInstanceID, LogSessionID: active.sessionID, SessionStartedAt: active.startedAt, StreamCount: len(active.streams), ServerTime: time.Now().UTC()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, stream := range active.streams {
|
||||
if err := writeSSEJSON(w, "stream", "", dto.LogStreamFromDomain(stream)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if historyLimit == 0 || len(active.streams) == 0 {
|
||||
return emittedThrough, nil
|
||||
}
|
||||
history, err := h.loadLogEventHistory(active.streams, historyLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, event := range history {
|
||||
if err := writeSSEJSON(w, "log", logEventID(event), dto.LogStreamEventFromDomain(event)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if event.Entry.Seq > emittedThrough[event.Stream.ID] {
|
||||
emittedThrough[event.Stream.ID] = event.Entry.Seq
|
||||
}
|
||||
}
|
||||
return emittedThrough, nil
|
||||
}
|
||||
|
||||
func (h *coreHandlers) openLogEventSubscription(r *http.Request) (domain.ServerInstance, []domain.LogStream, bool, service.LogEventSubscription, error) {
|
||||
var instance domain.ServerInstance
|
||||
var streams []domain.LogStream
|
||||
var liveEligible bool
|
||||
var subscription service.LogEventSubscription
|
||||
var err error
|
||||
if h.enforceAuthorization {
|
||||
sessionID := bearerToken(r)
|
||||
instance, err = h.core.GetServerInstanceForSession(sessionID, r.PathValue("id"))
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, subscription, err
|
||||
}
|
||||
streams, err = h.core.ListLogStreamsForSession(sessionID, domain.LogStreamFilter{ServerInstanceID: instance.ID})
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, subscription, err
|
||||
return domain.ServerInstance{}, nil, false, subscription, err
|
||||
}
|
||||
subscription, err = h.core.SubscribeLogEventsForSession(sessionID, instance.ID)
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, false, subscription, err
|
||||
}
|
||||
} else {
|
||||
instance, err = h.core.GetServerInstance(r.PathValue("id"))
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, subscription, err
|
||||
}
|
||||
streams, err = h.core.ListLogStreams(domain.LogStreamFilter{ServerInstanceID: instance.ID})
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, subscription, err
|
||||
return domain.ServerInstance{}, nil, false, subscription, err
|
||||
}
|
||||
subscription, err = h.core.SubscribeLogEvents(instance.ID)
|
||||
if err != nil {
|
||||
return domain.ServerInstance{}, nil, false, subscription, err
|
||||
}
|
||||
}
|
||||
return instance, streams, subscription, err
|
||||
if err == nil {
|
||||
streams, liveEligible, err = h.loadLiveLogSnapshot(instance.ID)
|
||||
}
|
||||
if err != nil && subscription.Close != nil {
|
||||
subscription.Close()
|
||||
}
|
||||
return instance, streams, liveEligible, subscription, err
|
||||
}
|
||||
|
||||
func (h *coreHandlers) loadLiveLogSnapshot(serverInstanceID string) ([]domain.LogStream, bool, error) {
|
||||
instance, err := h.core.GetServerInstance(serverInstanceID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if instance.State != domain.ServerInstanceStateRunning || strings.TrimSpace(instance.RunEndpointID) == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
endpoint, err := h.core.GetRunEndpoint(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if endpoint.Status != domain.RunEndpointStatusOnline {
|
||||
return nil, false, nil
|
||||
}
|
||||
logSessionID := strings.TrimPrefix(instance.LifecycleProcessID, managedLogSessionIDPrefix)
|
||||
if logSessionID == instance.LifecycleProcessID || strings.TrimSpace(logSessionID) == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
streams, err := h.core.ListLogStreams(domain.LogStreamFilter{ServerInstanceID: instance.ID})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
current := make([]domain.LogStream, 0, len(streams))
|
||||
for _, stream := range streams {
|
||||
if stream.Source == domain.LogStreamSourceProcess && stream.LogSessionID == logSessionID {
|
||||
current = append(current, stream)
|
||||
}
|
||||
}
|
||||
return current, true, nil
|
||||
}
|
||||
|
||||
func (h *coreHandlers) loadLogEventHistory(streams []domain.LogStream, limit int) ([]domain.LogStreamEvent, error) {
|
||||
|
||||
@@ -1,18 +1,61 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/dto"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/service"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
type logStreamListHookCore struct {
|
||||
service.Core
|
||||
once sync.Once
|
||||
hook func()
|
||||
}
|
||||
|
||||
type ssePipeResponseWriter struct {
|
||||
header http.Header
|
||||
pipe *io.PipeWriter
|
||||
status chan int
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func newSSEPipeResponseWriter() (*ssePipeResponseWriter, *io.PipeReader) {
|
||||
reader, writer := io.Pipe()
|
||||
return &ssePipeResponseWriter{header: make(http.Header), pipe: writer, status: make(chan int, 1)}, reader
|
||||
}
|
||||
|
||||
func (writer *ssePipeResponseWriter) Header() http.Header { return writer.header }
|
||||
|
||||
func (writer *ssePipeResponseWriter) WriteHeader(status int) {
|
||||
writer.once.Do(func() { writer.status <- status })
|
||||
}
|
||||
|
||||
func (writer *ssePipeResponseWriter) Write(payload []byte) (int, error) {
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
return writer.pipe.Write(payload)
|
||||
}
|
||||
|
||||
func (writer *ssePipeResponseWriter) Flush() {}
|
||||
|
||||
func (writer *ssePipeResponseWriter) Close() error { return writer.pipe.Close() }
|
||||
|
||||
func (core *logStreamListHookCore) ListLogStreams(filter domain.LogStreamFilter) ([]domain.LogStream, error) {
|
||||
core.once.Do(core.hook)
|
||||
return core.Core.ListLogStreams(filter)
|
||||
}
|
||||
|
||||
func TestLogIngestAPIWorkflow(t *testing.T) {
|
||||
router := newTestRouter()
|
||||
hello := createLogIngestAPIFixtures(t, router)
|
||||
@@ -57,6 +100,7 @@ func TestLogEventsSSEUsesServerWideNewestHistory(t *testing.T) {
|
||||
hello := createLogIngestAPIFixtures(t, router)
|
||||
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", dto.LogStreamCreateRequest{
|
||||
ID: "log-2", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stderr",
|
||||
LogSessionID: "session-current", SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default",
|
||||
})
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequestForStream(t, hello.SessionToken, "log-1", "stdout", 1, 2, 0)), http.StatusOK)
|
||||
@@ -70,6 +114,213 @@ func TestLogEventsSSEUsesServerWideNewestHistory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogEventsSSEExcludesOlderSupervisedSessions(t *testing.T) {
|
||||
router := newTestRouter()
|
||||
hello := createLogIngestAPIFixtures(t, router)
|
||||
oldStream := dto.LogStreamCreateRequest{ID: "log-old", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", LogSessionID: "session-old", SessionStartedAt: time.Date(2026, 7, 3, 11, 0, 0, 0, time.UTC), StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"}
|
||||
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", oldStream)
|
||||
oldBatch := validLogBatchRequestForStream(t, hello.SessionToken, "log-old", "stdout", 1, 1, 0)
|
||||
oldBatch.LogSessionID = "session-old"
|
||||
oldBatch.SessionStartedAt = oldStream.SessionStartedAt
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", oldBatch), http.StatusOK)
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
|
||||
|
||||
recorder := performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10")
|
||||
body := recorder.Body.String()
|
||||
if !strings.Contains(body, `"logSessionId":"session-current"`) || strings.Contains(body, `"streamId":"log-old"`) {
|
||||
t.Fatalf("expected only current session in live SSE, body=%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogEventsSSEInitialSnapshotRequiresRunningOnlineProcessFact(t *testing.T) {
|
||||
router := newTestRouter()
|
||||
hello := createLogIngestAPIFixtures(t, router)
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
|
||||
|
||||
stopped := dto.RunLifecycleReportRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: dto.JobProgressBody{Percent: 100}, ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop"}}
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", stopped), http.StatusOK)
|
||||
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
|
||||
|
||||
running := stopped
|
||||
running.ManagedProcessID = "log-session:session-missing"
|
||||
running.ObservationSeq = 1
|
||||
running.ObservedAt = time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
|
||||
running.ExecutionResult = dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"}
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", running), http.StatusOK)
|
||||
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/control/heartbeat", dto.RunControlHeartbeatRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, Version: "0.1.0", Status: domain.RunEndpointStatusOffline, CapabilityFingerprint: "cap-logs", Capacity: dto.RunCapacityResponse{MaxJobs: 1}}), http.StatusOK)
|
||||
assertEmptyInitialLogSession(t, performCancelledSSE(t, router, "/api/v1/server-instances/server-1/logs/events?historyLimit=10"))
|
||||
}
|
||||
|
||||
func TestLogEventsSSEClearsStoppedSessionAndRestoresRunningSessionWithoutReconnect(t *testing.T) {
|
||||
router := newTestRouter()
|
||||
hello := createLogIngestAPIFixtures(t, router)
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", validLogBatchRequest(t, hello.SessionToken, 1, 1)), http.StatusOK)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/server-instances/server-1/logs/events?historyLimit=10", nil).WithContext(ctx)
|
||||
streamWriter, streamReader := newSSEPipeResponseWriter()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
router.ServeHTTP(streamWriter, request)
|
||||
_ = streamWriter.Close()
|
||||
close(done)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
_ = streamReader.Close()
|
||||
<-done
|
||||
})
|
||||
if status := <-streamWriter.status; status != http.StatusOK {
|
||||
t.Fatalf("unexpected SSE status: %d", status)
|
||||
}
|
||||
reader := bufio.NewReader(streamReader)
|
||||
assertSSEEvent(t, reader, "session", `"logSessionId":"session-current"`)
|
||||
assertSSEEvent(t, reader, "stream", `"id":"log-1"`)
|
||||
assertSSEEvent(t, reader, "log", `"seq":1`)
|
||||
assertSSEEvent(t, reader, "ready", `"streamCount":1`)
|
||||
|
||||
statusReport := dto.RunLifecycleReportRequest{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, Progress: dto.JobProgressBody{Percent: 100}, ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "stopped", ExitClassification: "requested-stop"}}
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", statusReport), http.StatusOK)
|
||||
name, data := readSSEEvent(t, reader)
|
||||
if name != "session" || strings.Contains(data, `"logSessionId"`) || !strings.Contains(data, `"streamCount":0`) {
|
||||
t.Fatalf("expected stopped process to emit an empty session boundary, name=%q data=%s", name, data)
|
||||
}
|
||||
|
||||
nextStartedAt := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
|
||||
next := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stdout", "stdout", 1, 1, 20)
|
||||
next.LogSessionID = "session-next"
|
||||
next.SessionStartedAt = nextStartedAt
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/logs/batches", next), http.StatusOK)
|
||||
|
||||
statusReport.ManagedProcessID = "log-session:session-next"
|
||||
statusReport.ObservationSeq = 1
|
||||
statusReport.ObservedAt = nextStartedAt
|
||||
statusReport.ExecutionResult = dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"}
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", statusReport), http.StatusOK)
|
||||
assertSSEEvent(t, reader, "session", `"logSessionId":"session-next"`)
|
||||
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stdout"`)
|
||||
assertSSEEvent(t, reader, "log", `"streamId":"run.run-local.server-1.session-next.stdout"`)
|
||||
}
|
||||
|
||||
func TestLogEventsSSEOrdersReplaySessionSwitchAndAdditionalStreamWithoutDuplicates(t *testing.T) {
|
||||
core := service.NewCoreService(repo.NewMemoryStore())
|
||||
if err := core.SeedLocalPlatformAdmin(); err != nil {
|
||||
t.Fatalf("seed platform admin: %v", err)
|
||||
}
|
||||
setupRouter := NewTestRouterWithCore(core)
|
||||
hello := createLogIngestAPIFixtures(t, setupRouter)
|
||||
initial := validLogBatchRequest(t, hello.SessionToken, 1, 1)
|
||||
hookResult := make(chan error, 1)
|
||||
hooked := &logStreamListHookCore{Core: core, hook: func() {
|
||||
_, err := core.IngestLogBatch(initial.ToDomain())
|
||||
hookResult <- err
|
||||
}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/server-instances/server-1/logs/events?historyLimit=10", nil).WithContext(ctx)
|
||||
streamWriter, streamReader := newSSEPipeResponseWriter()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
NewTestRouterWithCore(hooked).ServeHTTP(streamWriter, request)
|
||||
_ = streamWriter.Close()
|
||||
close(done)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
_ = streamReader.Close()
|
||||
<-done
|
||||
})
|
||||
if status := <-streamWriter.status; status != http.StatusOK {
|
||||
t.Fatalf("unexpected SSE status: %d", status)
|
||||
}
|
||||
reader := bufio.NewReader(streamReader)
|
||||
assertSSEEvent(t, reader, "session", `"logSessionId":"session-current"`)
|
||||
assertSSEEvent(t, reader, "stream", `"id":"log-1"`)
|
||||
assertSSEEvent(t, reader, "log", `"seq":1`)
|
||||
assertSSEEvent(t, reader, "ready", `"streamCount":1`)
|
||||
if err := <-hookResult; err != nil {
|
||||
t.Fatalf("ingest during stream snapshot: %v", err)
|
||||
}
|
||||
|
||||
nextStartedAt := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC)
|
||||
next := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stdout", "stdout", 1, 1, 20)
|
||||
next.LogSessionID = "session-next"
|
||||
next.SessionStartedAt = nextStartedAt
|
||||
if _, err := core.IngestLogBatch(next.ToDomain()); err != nil {
|
||||
t.Fatalf("ingest next session stdout: %v", err)
|
||||
}
|
||||
if _, err := core.ReportRunLifecycle(domain.RunLifecycleReport{RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded, ManagedProcessID: "log-session:session-next", ObservationSeq: 1, ObservedAt: nextStartedAt, ExecutionResult: domain.JobExecutionResult{Kind: "process", ProcessState: "running"}}); err != nil {
|
||||
t.Fatalf("report next managed process: %v", err)
|
||||
}
|
||||
assertSSEEvent(t, reader, "session", `"logSessionId":"session-next"`)
|
||||
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stdout"`)
|
||||
assertSSEEvent(t, reader, "log", `"seq":1`)
|
||||
|
||||
stderr := validLogBatchRequestForStream(t, hello.SessionToken, "run.run-local.server-1.session-next.stderr", "stderr", 1, 1, 21)
|
||||
stderr.LogSessionID = "session-next"
|
||||
stderr.SessionStartedAt = nextStartedAt
|
||||
if _, err := core.IngestLogBatch(stderr.ToDomain()); err != nil {
|
||||
t.Fatalf("ingest next session stderr: %v", err)
|
||||
}
|
||||
assertSSEEvent(t, reader, "stream", `"id":"run.run-local.server-1.session-next.stderr"`)
|
||||
assertSSEEvent(t, reader, "log", `"streamKey":"stderr"`)
|
||||
|
||||
delayedOld := validLogBatchRequest(t, hello.SessionToken, 2, 2)
|
||||
if _, err := core.IngestLogBatch(delayedOld.ToDomain()); err != nil {
|
||||
t.Fatalf("ingest delayed old session batch: %v", err)
|
||||
}
|
||||
nextLive := validLogBatchRequestForStream(t, hello.SessionToken, next.LogStreamID, "stdout", 2, 2, 22)
|
||||
nextLive.LogSessionID = "session-next"
|
||||
nextLive.SessionStartedAt = nextStartedAt
|
||||
if _, err := core.IngestLogBatch(nextLive.ToDomain()); err != nil {
|
||||
t.Fatalf("ingest next session live append: %v", err)
|
||||
}
|
||||
assertSSEEvent(t, reader, "log", `"streamId":"run.run-local.server-1.session-next.stdout"`)
|
||||
}
|
||||
|
||||
func assertSSEEvent(t *testing.T, reader *bufio.Reader, wantName string, wantData string) {
|
||||
t.Helper()
|
||||
name, data := readSSEEvent(t, reader)
|
||||
if name != wantName || !strings.Contains(data, wantData) {
|
||||
t.Fatalf("unexpected SSE event: name=%q data=%s; want name=%q containing %s", name, data, wantName, wantData)
|
||||
}
|
||||
}
|
||||
|
||||
func assertEmptyInitialLogSession(t *testing.T, recorder *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
assertStatus(t, recorder, http.StatusOK)
|
||||
body := recorder.Body.String()
|
||||
if !strings.Contains(body, "event: session") || strings.Contains(body, `"logSessionId"`) || strings.Contains(body, "event: log") || !strings.Contains(body, `"streamCount":0`) {
|
||||
t.Fatalf("expected an empty initial live session, body=%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func readSSEEvent(t *testing.T, reader *bufio.Reader) (string, string) {
|
||||
t.Helper()
|
||||
for {
|
||||
name, data := "", ""
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatalf("read SSE event: %v", err)
|
||||
}
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
if line == "" {
|
||||
if name != "" {
|
||||
return name, data
|
||||
}
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(line, "event: ") {
|
||||
name = strings.TrimPrefix(line, "event: ")
|
||||
}
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
data = strings.TrimPrefix(line, "data: ")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func performCancelledSSE(t *testing.T, router http.Handler, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -110,15 +361,22 @@ func createLogIngestAPIFixtures(t *testing.T, router http.Handler) dto.RunContro
|
||||
hello := decodeBody[dto.RunControlHelloResponse](t, performRunControlHello(t, router, helloRequest))
|
||||
adminSession := createAdminSession(t, router)
|
||||
postJSON[dto.GamePluginResponse](t, router, "/api/v1/game-plugins", validGamePluginRequest())
|
||||
postJSONWithAuth[dto.ServerInstanceResponse](t, router, "/api/v1/server-instances", dto.ServerInstanceCreateRequest{ID: "server-1", PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM #1"}, adminSession)
|
||||
postJSONWithAuth[dto.ServerInstanceResponse](t, router, "/api/v1/server-instances", dto.ServerInstanceCreateRequest{ID: "server-1", PluginID: "server.scum", RunEndpointID: "run-local", Name: "SCUM #1", State: domain.ServerInstanceStateRunning}, adminSession)
|
||||
postJSON[dto.LogStreamResponse](t, router, "/api/v1/log-streams", dto.LogStreamCreateRequest{
|
||||
ID: "log-1",
|
||||
ServerInstanceID: "server-1",
|
||||
Source: domain.LogStreamSourceProcess,
|
||||
StreamKey: "stdout",
|
||||
LogSessionID: "session-current",
|
||||
SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
StorageBackend: domain.LogStorageBackendLocalSegments,
|
||||
RetentionPolicy: "default",
|
||||
})
|
||||
assertStatus(t, performJSON(t, router, http.MethodPost, "/api/v1/run/lifecycle/report", dto.RunLifecycleReportRequest{
|
||||
RunEndpointID: "run-local", SessionToken: hello.SessionToken, ServerInstanceID: "server-1", Capability: domain.LifecycleCapabilityStatus, State: domain.JobStateSucceeded,
|
||||
ManagedProcessID: "log-session:session-current", ObservationSeq: 1, ObservedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
ExecutionResult: dto.RunJobExecutionResultBody{Kind: "process", ProcessState: "running"},
|
||||
}), http.StatusOK)
|
||||
return hello
|
||||
}
|
||||
|
||||
@@ -146,6 +404,8 @@ func validLogBatchRequestForStream(t *testing.T, sessionToken string, streamID s
|
||||
ServerInstanceID: "server-1",
|
||||
StreamKey: streamKey,
|
||||
Source: domain.LogStreamSourceProcess,
|
||||
LogSessionID: "session-current",
|
||||
SessionStartedAt: time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC),
|
||||
FirstSeq: firstSeq,
|
||||
LastSeq: lastSeq,
|
||||
Compression: "none",
|
||||
|
||||
@@ -18,6 +18,8 @@ type LogBatchIngest struct {
|
||||
ServerInstanceID string
|
||||
StreamKey string
|
||||
Source LogStreamSource
|
||||
LogSessionID string
|
||||
SessionStartedAt time.Time
|
||||
FirstSeq uint64
|
||||
LastSeq uint64
|
||||
Compression string
|
||||
|
||||
@@ -1507,6 +1507,8 @@ type LogStream struct {
|
||||
ServerInstanceID string
|
||||
Source LogStreamSource
|
||||
StreamKey string
|
||||
LogSessionID string
|
||||
SessionStartedAt time.Time
|
||||
LatestSeq uint64
|
||||
StorageBackend LogStorageBackend
|
||||
RetentionPolicy string
|
||||
|
||||
@@ -22,6 +22,8 @@ type LogBatchIngestRequest struct {
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
Source domain.LogStreamSource `json:"source"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
|
||||
FirstSeq uint64 `json:"firstSeq"`
|
||||
LastSeq uint64 `json:"lastSeq"`
|
||||
Compression string `json:"compression"`
|
||||
@@ -74,6 +76,8 @@ type LogStreamEventResponse struct {
|
||||
StreamID string `json:"streamId"`
|
||||
Source domain.LogStreamSource `json:"source"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
|
||||
LatestSeq uint64 `json:"latestSeq"`
|
||||
Entry LogEntryBody `json:"entry"`
|
||||
}
|
||||
@@ -84,6 +88,14 @@ type LogStreamEventsReadyResponse struct {
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
type LogStreamEventsSessionResponse struct {
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
|
||||
StreamCount int `json:"streamCount"`
|
||||
ServerTime time.Time `json:"serverTime"`
|
||||
}
|
||||
|
||||
func (request LogBatchIngestRequest) ToDomain() domain.LogBatchIngest {
|
||||
return domain.LogBatchIngest{
|
||||
RunEndpointID: request.RunEndpointID,
|
||||
@@ -92,6 +104,8 @@ func (request LogBatchIngestRequest) ToDomain() domain.LogBatchIngest {
|
||||
ServerInstanceID: request.ServerInstanceID,
|
||||
StreamKey: request.StreamKey,
|
||||
Source: request.Source,
|
||||
LogSessionID: request.LogSessionID,
|
||||
SessionStartedAt: request.SessionStartedAt,
|
||||
FirstSeq: request.FirstSeq,
|
||||
LastSeq: request.LastSeq,
|
||||
Compression: request.Compression,
|
||||
@@ -145,6 +159,8 @@ func LogStreamEventFromDomain(event domain.LogStreamEvent) LogStreamEventRespons
|
||||
StreamID: event.Stream.ID,
|
||||
Source: event.Stream.Source,
|
||||
StreamKey: event.Stream.StreamKey,
|
||||
LogSessionID: event.Stream.LogSessionID,
|
||||
SessionStartedAt: event.Stream.SessionStartedAt,
|
||||
LatestSeq: event.LatestSeq,
|
||||
Entry: logEntryFromDomain(event.Entry),
|
||||
}
|
||||
|
||||
@@ -893,6 +893,8 @@ type LogStreamCreateRequest struct {
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
Source domain.LogStreamSource `json:"source"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
|
||||
StorageBackend domain.LogStorageBackend `json:"storageBackend"`
|
||||
RetentionPolicy string `json:"retentionPolicy"`
|
||||
}
|
||||
@@ -902,6 +904,8 @@ type LogStreamResponse struct {
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
Source domain.LogStreamSource `json:"source"`
|
||||
StreamKey string `json:"streamKey"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty"`
|
||||
LatestSeq uint64 `json:"latestSeq"`
|
||||
StorageBackend domain.LogStorageBackend `json:"storageBackend"`
|
||||
RetentionPolicy string `json:"retentionPolicy"`
|
||||
@@ -1363,6 +1367,8 @@ func (request LogStreamCreateRequest) ToDomain() domain.LogStream {
|
||||
ServerInstanceID: request.ServerInstanceID,
|
||||
Source: request.Source,
|
||||
StreamKey: request.StreamKey,
|
||||
LogSessionID: request.LogSessionID,
|
||||
SessionStartedAt: request.SessionStartedAt,
|
||||
StorageBackend: request.StorageBackend,
|
||||
RetentionPolicy: request.RetentionPolicy,
|
||||
}
|
||||
@@ -1938,6 +1944,8 @@ func LogStreamFromDomain(stream domain.LogStream) LogStreamResponse {
|
||||
ServerInstanceID: stream.ServerInstanceID,
|
||||
Source: stream.Source,
|
||||
StreamKey: stream.StreamKey,
|
||||
LogSessionID: stream.LogSessionID,
|
||||
SessionStartedAt: stream.SessionStartedAt,
|
||||
LatestSeq: stream.LatestSeq,
|
||||
StorageBackend: stream.StorageBackend,
|
||||
RetentionPolicy: stream.RetentionPolicy,
|
||||
|
||||
@@ -415,6 +415,10 @@ type LogStream struct {
|
||||
Source domain.LogStreamSource `json:"source" db:"source"`
|
||||
// StreamKey is stable within the server instance.
|
||||
StreamKey string `json:"streamKey" db:"stream_key"`
|
||||
// LogSessionID groups plugin-declared supervised-process streams for one process generation.
|
||||
LogSessionID string `json:"logSessionId,omitempty" db:"log_session_id"`
|
||||
// SessionStartedAt is Run's persisted start timestamp for the supervised process generation.
|
||||
SessionStartedAt time.Time `json:"sessionStartedAt,omitempty" db:"session_started_at"`
|
||||
// LatestSeq is the latest accepted sequence number.
|
||||
LatestSeq uint64 `json:"latestSeq" db:"latest_seq"`
|
||||
// StorageBackend identifies the log body backend.
|
||||
|
||||
@@ -66,7 +66,17 @@ func (svc *CoreService) enqueueDistributionBuild(job domain.Job) {
|
||||
delete(svc.distributionBuilds, job.ID)
|
||||
svc.distributionBuildMu.Unlock()
|
||||
}()
|
||||
_ = svc.executeDistributionBuild(job)
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
_ = svc.failDistributionBuildJob(job, "platform builder failed unexpectedly")
|
||||
}
|
||||
}()
|
||||
if err := svc.executeDistributionBuild(job); err != nil {
|
||||
// executeDistributionBuild may fail after persisting the running state.
|
||||
// A second terminalization attempt is idempotent when the specific
|
||||
// failure path already marked the job failed.
|
||||
_ = svc.failDistributionBuildJob(job, "platform builder failed before completion")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -99,7 +109,7 @@ func (svc *CoreService) executeDistributionBuild(job domain.Job) error {
|
||||
payload, buildErr = builder.Build(input)
|
||||
}
|
||||
if buildErr != nil {
|
||||
return svc.failDistributionBuildJob(job, builderJobFailureMessage(buildErr))
|
||||
return svc.failDistributionBuildJob(job, builderJobFailureMessage(buildErr, input.AuthKey))
|
||||
}
|
||||
if _, err := svc.platformDistributionBuildInput(job); err != nil {
|
||||
return svc.failDistributionBuildJob(job, "platform builder discarded output because the component key is no longer current")
|
||||
|
||||
@@ -25,6 +25,34 @@ type progressDistributionBuilder struct {
|
||||
payload []byte
|
||||
}
|
||||
|
||||
type failingBuildProgressStore struct {
|
||||
repo.Store
|
||||
buildJobs repo.ClientManagerBuildJobRepository
|
||||
}
|
||||
|
||||
func (store failingBuildProgressStore) ClientManagerBuildJobs() repo.ClientManagerBuildJobRepository {
|
||||
return store.buildJobs
|
||||
}
|
||||
|
||||
type failWhenDistributionJobRunningRepository struct {
|
||||
repo.ClientManagerBuildJobRepository
|
||||
jobs repo.JobRepository
|
||||
err error
|
||||
}
|
||||
|
||||
func (repository failWhenDistributionJobRunningRepository) List(filter domain.ClientManagerBuildJobFilter) ([]domain.ClientManagerBuildJob, error) {
|
||||
jobs, err := repository.jobs.List(domain.JobFilter{ServerInstanceID: filter.ServerInstanceID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if job.Capability == domain.JobCapabilityDistributionBuild && job.State == domain.JobStateRunning {
|
||||
return nil, repository.err
|
||||
}
|
||||
}
|
||||
return repository.ClientManagerBuildJobRepository.List(filter)
|
||||
}
|
||||
|
||||
func (builder captureDistributionBuilder) Readiness() (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
@@ -379,6 +407,49 @@ func TestCoreServiceProjectsPlatformBuilderProgressBeforeCompletion(t *testing.T
|
||||
completeDistributionBuild(t, svc, distribution, nil)
|
||||
}
|
||||
|
||||
func TestCoreServiceTerminalizesPlatformBuildWhenRunningProjectionFails(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
const leakedDetail = "sensitive-builder-token /private/platform/build/input/auth-key"
|
||||
baseStore := svc.store
|
||||
svc.store = failingBuildProgressStore{
|
||||
Store: baseStore,
|
||||
buildJobs: failWhenDistributionJobRunningRepository{
|
||||
ClientManagerBuildJobRepository: baseStore.ClientManagerBuildJobs(),
|
||||
jobs: baseStore.Jobs(),
|
||||
err: errors.New(leakedDetail),
|
||||
},
|
||||
}
|
||||
|
||||
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
TargetOS: "linux",
|
||||
TargetArch: "amd64",
|
||||
IdempotencyKey: "running-projection-failure-terminalizes",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("queue platform distribution: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
job, jobErr := svc.GetJob(distribution.BuildJobID)
|
||||
updated, distributionErr := svc.store.RunDistributions().Get(distribution.ID)
|
||||
if jobErr != nil || distributionErr != nil {
|
||||
t.Fatalf("read failed build state: jobErr=%v distributionErr=%v", jobErr, distributionErr)
|
||||
}
|
||||
if isTerminalJobState(job.State) {
|
||||
if job.State != domain.JobStateFailed || updated.Status != domain.DistributionStatusFailed {
|
||||
t.Fatalf("projection failure did not end in failed state: job=%+v distribution=%+v", job, updated)
|
||||
}
|
||||
if strings.Contains(job.Progress.Message, "sensitive-builder-token") || strings.Contains(job.Progress.Message, "/private/") || strings.Contains(job.TerminalFingerprint, "sensitive-builder-token") {
|
||||
t.Fatalf("terminal build state leaked internal failure details: %+v", job)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatal("platform build remained non-terminal after running projection failure")
|
||||
}
|
||||
|
||||
func TestCoreServiceDiscardsBuildCompletedAfterKeyReset(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
inputs := make(chan domain.DistributionBuildInput, 1)
|
||||
|
||||
@@ -553,11 +553,11 @@ func redactBuilderHostPaths(line string) string {
|
||||
return strings.Join(fields, " ")
|
||||
}
|
||||
|
||||
func builderJobFailureMessage(err error) string {
|
||||
func builderJobFailureMessage(err error, sensitiveValues ...string) string {
|
||||
if err == nil {
|
||||
return "platform builder failed"
|
||||
}
|
||||
message := strings.TrimSpace(err.Error())
|
||||
message := safeBuilderFailure([]byte(err.Error()), sensitiveValues...)
|
||||
if message == "" {
|
||||
return "platform builder failed"
|
||||
}
|
||||
|
||||
@@ -289,6 +289,14 @@ func TestDockerDistributionBuilderRedactsFailureAndTimeout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuilderJobFailureMessageRedactsSensitiveValuesAndHostPaths(t *testing.T) {
|
||||
const secret = "sensitive-component-key"
|
||||
message := builderJobFailureMessage(errors.New(secret+" /private/platform/build/input/auth-key"), secret)
|
||||
if strings.Contains(message, secret) || strings.Contains(message, "/private/") || strings.Contains(message, "auth-key") {
|
||||
t.Fatalf("builder job failure leaked sensitive details: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageClientManagerDistributionProducesProtectedArchives(t *testing.T) {
|
||||
for _, packageFormat := range []string{"zip", "tar.gz"} {
|
||||
t.Run(packageFormat, func(t *testing.T) {
|
||||
|
||||
@@ -8,14 +8,28 @@ import (
|
||||
|
||||
const logEventSubscriberBuffer = 512
|
||||
|
||||
type LogEventSubscriptionEventKind string
|
||||
|
||||
const (
|
||||
LogEventSubscriptionEventLog LogEventSubscriptionEventKind = "log"
|
||||
LogEventSubscriptionEventProcessState LogEventSubscriptionEventKind = "process-state"
|
||||
)
|
||||
|
||||
type LogEventSubscriptionEvent struct {
|
||||
Kind LogEventSubscriptionEventKind
|
||||
LogEvent domain.LogStreamEvent
|
||||
ServerInstanceID string
|
||||
ProcessState domain.ServerInstanceState
|
||||
}
|
||||
|
||||
type LogEventSubscription struct {
|
||||
Events <-chan domain.LogStreamEvent
|
||||
Events <-chan LogEventSubscriptionEvent
|
||||
Close func()
|
||||
}
|
||||
|
||||
type logEventSubscriber struct {
|
||||
serverInstanceID string
|
||||
events chan domain.LogStreamEvent
|
||||
events chan LogEventSubscriptionEvent
|
||||
}
|
||||
|
||||
func (svc *CoreService) SubscribeLogEvents(serverInstanceID string) (LogEventSubscription, error) {
|
||||
@@ -26,7 +40,7 @@ func (svc *CoreService) SubscribeLogEvents(serverInstanceID string) (LogEventSub
|
||||
if _, err := svc.store.ServerInstances().Get(serverInstanceID); err != nil {
|
||||
return LogEventSubscription{}, err
|
||||
}
|
||||
events := make(chan domain.LogStreamEvent, logEventSubscriberBuffer)
|
||||
events := make(chan LogEventSubscriptionEvent, logEventSubscriberBuffer)
|
||||
svc.logEventMu.Lock()
|
||||
svc.logEventSubscriberSeq++
|
||||
id := svc.logEventSubscriberSeq
|
||||
@@ -55,18 +69,36 @@ func (svc *CoreService) publishLogEvents(stream domain.LogStream, entries []doma
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
events := make([]domain.LogStreamEvent, len(entries))
|
||||
events := make([]LogEventSubscriptionEvent, len(entries))
|
||||
for index, entry := range entries {
|
||||
events[index] = domain.CopyLogStreamEvent(domain.LogStreamEvent{
|
||||
ServerInstanceID: stream.ServerInstanceID,
|
||||
Stream: stream,
|
||||
Entry: entry,
|
||||
LatestSeq: stream.LatestSeq,
|
||||
})
|
||||
events[index] = LogEventSubscriptionEvent{
|
||||
Kind: LogEventSubscriptionEventLog,
|
||||
LogEvent: domain.CopyLogStreamEvent(domain.LogStreamEvent{
|
||||
ServerInstanceID: stream.ServerInstanceID,
|
||||
Stream: stream,
|
||||
Entry: entry,
|
||||
LatestSeq: stream.LatestSeq,
|
||||
}),
|
||||
}
|
||||
}
|
||||
svc.publishLogSubscriptionEvents(stream.ServerInstanceID, events)
|
||||
}
|
||||
|
||||
func (svc *CoreService) publishLogProcessState(instance domain.ServerInstance) {
|
||||
svc.publishLogSubscriptionEvents(instance.ID, []LogEventSubscriptionEvent{{
|
||||
Kind: LogEventSubscriptionEventProcessState,
|
||||
ServerInstanceID: instance.ID,
|
||||
ProcessState: instance.State,
|
||||
}})
|
||||
}
|
||||
|
||||
func (svc *CoreService) publishLogSubscriptionEvents(serverInstanceID string, events []LogEventSubscriptionEvent) {
|
||||
if len(events) == 0 {
|
||||
return
|
||||
}
|
||||
svc.logEventMu.Lock()
|
||||
for id, subscriber := range svc.logEventSubscribers {
|
||||
if subscriber.serverInstanceID != stream.ServerInstanceID {
|
||||
if subscriber.serverInstanceID != serverInstanceID {
|
||||
continue
|
||||
}
|
||||
dropped := false
|
||||
|
||||
@@ -20,6 +20,9 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
|
||||
if err := svc.validateRunSession(batch.RunEndpointID, batch.SessionToken); err != nil {
|
||||
return domain.LogBatchIngestResult{}, err
|
||||
}
|
||||
lock := svc.logIngestLock(batch.ServerInstanceID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
stamp := svc.now()
|
||||
stream, err := svc.store.LogStreams().Get(batch.LogStreamID)
|
||||
@@ -58,7 +61,7 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
|
||||
}
|
||||
|
||||
storedBatch := domain.CopyLogBatchIngest(batch)
|
||||
sanitizeGamePlayerNetworkFields(&storedBatch)
|
||||
sanitizeLogNetworkFields(&storedBatch)
|
||||
record := domain.CopyLogBatchRecord(domain.LogBatchRecord{
|
||||
Checksum: batch.Checksum,
|
||||
FirstSeq: batch.FirstSeq,
|
||||
@@ -96,7 +99,7 @@ func (svc *CoreService) ensureJobLogStreamForBatch(batch domain.LogBatchIngest,
|
||||
if job.ServerInstanceID != batch.ServerInstanceID || job.RunEndpointID != batch.RunEndpointID {
|
||||
return validationError("log batch job scope does not match stream")
|
||||
}
|
||||
return svc.ensureJobLogStreams(job, stamp)
|
||||
return svc.ensureJobLogStreamsUnlocked(job, stamp)
|
||||
}
|
||||
|
||||
func (svc *CoreService) ensureLogStreamForBatch(batch domain.LogBatchIngest, stamp time.Time) error {
|
||||
@@ -107,7 +110,7 @@ func (svc *CoreService) ensureLogStreamForBatch(batch domain.LogBatchIngest, sta
|
||||
if job.ServerInstanceID != batch.ServerInstanceID || job.RunEndpointID != batch.RunEndpointID {
|
||||
return validationError("log batch job scope does not match stream")
|
||||
}
|
||||
return svc.ensureJobLogStreams(job, stamp)
|
||||
return svc.ensureJobLogStreamsUnlocked(job, stamp)
|
||||
}
|
||||
if !errors.Is(err, repo.ErrNotFound) || !strings.HasPrefix(jobID, "autonomous-") {
|
||||
return err
|
||||
@@ -120,8 +123,14 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
|
||||
if batch.Source != domain.LogStreamSourceProcess && batch.Source != domain.LogStreamSourceFile && batch.Source != domain.LogStreamSourceManagementProgram {
|
||||
return repo.ErrNotFound
|
||||
}
|
||||
if batch.LogStreamID != runLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.StreamKey) && !legacyAutonomousLogStream(batch) {
|
||||
return repo.ErrNotFound
|
||||
expectedStreamID := runLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.StreamKey)
|
||||
if batch.LogSessionID != "" {
|
||||
expectedStreamID = runSessionLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.LogSessionID, batch.StreamKey)
|
||||
}
|
||||
if batch.LogStreamID != expectedStreamID {
|
||||
if batch.LogSessionID != "" || !legacyAutonomousLogStream(batch) {
|
||||
return repo.ErrNotFound
|
||||
}
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(batch.ServerInstanceID)
|
||||
if err != nil {
|
||||
@@ -130,7 +139,7 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
|
||||
if instance.RunEndpointID != batch.RunEndpointID {
|
||||
return validationError("server instance run endpoint must match log batch endpoint")
|
||||
}
|
||||
_, err = svc.CreateLogStream(domain.LogStream{ID: batch.LogStreamID, ServerInstanceID: batch.ServerInstanceID, Source: batch.Source, StreamKey: batch.StreamKey, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default", CreatedAt: stamp, UpdatedAt: stamp})
|
||||
_, err = svc.createLogStream(domain.LogStream{ID: batch.LogStreamID, ServerInstanceID: batch.ServerInstanceID, Source: batch.Source, StreamKey: batch.StreamKey, LogSessionID: batch.LogSessionID, SessionStartedAt: batch.SessionStartedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default", CreatedAt: stamp, UpdatedAt: stamp})
|
||||
if errors.Is(err, repo.ErrDuplicate) {
|
||||
return nil
|
||||
}
|
||||
@@ -166,18 +175,16 @@ func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIn
|
||||
return false
|
||||
}
|
||||
|
||||
// sanitizeGamePlayerNetworkFields removes raw network material before the durable log body is written.
|
||||
func sanitizeGamePlayerNetworkFields(batch *domain.LogBatchIngest) {
|
||||
// sanitizeLogNetworkFields removes raw network material before the durable log body is written.
|
||||
func sanitizeLogNetworkFields(batch *domain.LogBatchIngest) {
|
||||
for index := range batch.Entries {
|
||||
fields := batch.Entries[index].Fields
|
||||
if fields == nil {
|
||||
continue
|
||||
}
|
||||
if fields["eventType"] == "scum.login" {
|
||||
delete(fields, "networkFingerprint")
|
||||
delete(fields, "ip")
|
||||
delete(fields, "ipAddress")
|
||||
}
|
||||
delete(fields, "networkFingerprint")
|
||||
delete(fields, "ip")
|
||||
delete(fields, "ipAddress")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,5 +256,8 @@ func validateLogBatchStream(batch domain.LogBatchIngest, stream domain.LogStream
|
||||
if stream.Source != batch.Source {
|
||||
return validationError("source must match stream")
|
||||
}
|
||||
if stream.LogSessionID != batch.LogSessionID || !stream.SessionStartedAt.Equal(batch.SessionStartedAt) {
|
||||
return validationError("log session metadata must match stream")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func TestSanitizeLogNetworkFieldsIsGameAgnostic(t *testing.T) {
|
||||
batch := domain.LogBatchIngest{Entries: []domain.LogEntry{
|
||||
{Fields: map[string]string{
|
||||
"eventType": "game.session.opened",
|
||||
"networkFingerprint": "fingerprint",
|
||||
"ip": "192.0.2.1",
|
||||
"ipAddress": "2001:db8::1",
|
||||
"playerId": "player-1",
|
||||
}},
|
||||
}}
|
||||
|
||||
sanitizeLogNetworkFields(&batch)
|
||||
|
||||
fields := batch.Entries[0].Fields
|
||||
for _, key := range []string{"networkFingerprint", "ip", "ipAddress"} {
|
||||
if _, exists := fields[key]; exists {
|
||||
t.Fatalf("expected %s to be removed", key)
|
||||
}
|
||||
}
|
||||
if fields["playerId"] != "player-1" {
|
||||
t.Fatal("expected unrelated fields to be preserved")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -74,7 +75,7 @@ func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case event := <-subscription.Events:
|
||||
if event.Stream.ID != "log-1" || event.Entry.Seq != 1 || event.LatestSeq != 1 {
|
||||
if event.Kind != LogEventSubscriptionEventLog || event.LogEvent.Stream.ID != "log-1" || event.LogEvent.Entry.Seq != 1 || event.LogEvent.LatestSeq != 1 {
|
||||
t.Fatalf("unexpected log event: %+v", event)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
@@ -91,6 +92,76 @@ func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServicePersistsAndEnforcesImmutableProcessLogSessionMetadata(t *testing.T) {
|
||||
svc, sessionToken := newRegisteredLogIngestService(t)
|
||||
startedAt := time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
|
||||
streamID := runSessionLogStreamID("run-local", "server-1", "session-a", "stdout")
|
||||
batch := validLogBatch(t, sessionToken, 1, 1)
|
||||
batch.LogStreamID = streamID
|
||||
batch.LogSessionID = "session-a"
|
||||
batch.SessionStartedAt = startedAt
|
||||
if _, err := svc.IngestLogBatch(batch); err != nil {
|
||||
t.Fatalf("ingest session-scoped process batch: %v", err)
|
||||
}
|
||||
stream, err := svc.GetLogStream(streamID)
|
||||
if err != nil || stream.LogSessionID != "session-a" || !stream.SessionStartedAt.Equal(startedAt) {
|
||||
t.Fatalf("session metadata was not persisted: stream=%+v err=%v", stream, err)
|
||||
}
|
||||
|
||||
conflict := validLogBatch(t, sessionToken, 2, 2)
|
||||
conflict.LogStreamID = streamID
|
||||
conflict.LogSessionID = "session-a"
|
||||
conflict.SessionStartedAt = startedAt.Add(time.Second)
|
||||
if _, err := svc.IngestLogBatch(conflict); err == nil || !strings.Contains(err.Error(), "metadata must match") {
|
||||
t.Fatalf("expected immutable stream metadata rejection, got %v", err)
|
||||
}
|
||||
|
||||
legacy := createLogStreamFixture(t, svc)
|
||||
legacyBatch := validLogBatch(t, sessionToken, 1, 1)
|
||||
legacyBatch.LogStreamID = legacy.ID
|
||||
legacyBatch.LogSessionID = "session-a"
|
||||
legacyBatch.SessionStartedAt = startedAt
|
||||
if _, err := svc.IngestLogBatch(legacyBatch); err == nil || !strings.Contains(err.Error(), "metadata must match") {
|
||||
t.Fatalf("expected legacy stream to reject attached session metadata, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceSerializesConsistentSessionStreamCreation(t *testing.T) {
|
||||
svc, _ := newRegisteredLogIngestService(t)
|
||||
startedAt := time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
|
||||
streams := []domain.LogStream{
|
||||
{ID: "session-stream-stdout", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stdout", LogSessionID: "session-a", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"},
|
||||
{ID: "session-stream-stderr", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "stderr", LogSessionID: "session-a", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"},
|
||||
}
|
||||
errorsByStream := make(chan error, len(streams))
|
||||
var wait sync.WaitGroup
|
||||
for _, stream := range streams {
|
||||
stream := stream
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
_, err := svc.CreateLogStream(stream)
|
||||
errorsByStream <- err
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
close(errorsByStream)
|
||||
for err := range errorsByStream {
|
||||
if err != nil {
|
||||
t.Fatalf("create consistent session stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := svc.CreateLogStream(domain.LogStream{ID: "session-stream-conflict", ServerInstanceID: "server-1", Source: domain.LogStreamSourceProcess, StreamKey: "console", LogSessionID: "session-a", SessionStartedAt: startedAt.Add(time.Second), StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"})
|
||||
if err == nil || !strings.Contains(err.Error(), "conflicts") {
|
||||
t.Fatalf("expected conflicting session timestamp rejection, got %v", err)
|
||||
}
|
||||
_, err = svc.CreateLogStream(domain.LogStream{ID: "session-file-tail", ServerInstanceID: "server-1", Source: domain.LogStreamSourceFile, StreamKey: "file", LogSessionID: "session-file", SessionStartedAt: startedAt, StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"})
|
||||
if err == nil || !strings.Contains(err.Error(), "only valid for process") {
|
||||
t.Fatalf("expected file-tail session metadata rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceLogBatchDuplicateAck(t *testing.T) {
|
||||
svc, sessionToken := newRegisteredLogIngestService(t)
|
||||
createLogStreamFixture(t, svc)
|
||||
@@ -346,6 +417,17 @@ func TestCoreServiceAcceptsLegacyAutonomousJobLogStreamWithoutPlatformJob(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceRejectsSessionMetadataOnLegacyAutonomousStreamID(t *testing.T) {
|
||||
svc, sessionToken := newRegisteredLogIngestService(t)
|
||||
batch := validLogBatch(t, sessionToken, 1, 1)
|
||||
batch.LogStreamID = jobLogStreamID("autonomous-bootstrap-start", "stdout")
|
||||
batch.LogSessionID = "session-a"
|
||||
batch.SessionStartedAt = time.Date(2026, 7, 3, 12, 30, 0, 0, time.UTC)
|
||||
if _, err := svc.IngestLogBatch(batch); err == nil {
|
||||
t.Fatal("expected session-scoped batch with legacy autonomous stream ID to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceRejectsOutOfOrderAndConflictingLogBatches(t *testing.T) {
|
||||
svc, sessionToken := newRegisteredLogIngestService(t)
|
||||
createLogStreamFixture(t, svc)
|
||||
|
||||
@@ -233,6 +233,7 @@ type CoreService struct {
|
||||
bridgeMu sync.Mutex
|
||||
bridgeSeq uint64
|
||||
logStore LogBodyStore
|
||||
logIngestMu [64]sync.Mutex
|
||||
logEventMu sync.Mutex
|
||||
logEventSubscribers map[uint64]logEventSubscriber
|
||||
logEventSubscriberSeq uint64
|
||||
@@ -2476,6 +2477,13 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
|
||||
}
|
||||
|
||||
func (svc *CoreService) ensureJobLogStreams(job domain.Job, stamp time.Time) error {
|
||||
lock := svc.logIngestLock(job.ServerInstanceID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
return svc.ensureJobLogStreamsUnlocked(job, stamp)
|
||||
}
|
||||
|
||||
func (svc *CoreService) ensureJobLogStreamsUnlocked(job domain.Job, stamp time.Time) error {
|
||||
if strings.TrimSpace(job.ServerInstanceID) == "" || strings.TrimSpace(job.ID) == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -2524,7 +2532,7 @@ func (svc *CoreService) ensureJobLogStreams(job domain.Job, stamp time.Time) err
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
if _, err := svc.CreateLogStream(stream); err != nil && !errors.Is(err, repo.ErrDuplicate) {
|
||||
if _, err := svc.createLogStream(stream); err != nil && !errors.Is(err, repo.ErrDuplicate) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -2539,6 +2547,10 @@ func runLogStreamID(runEndpointID string, serverInstanceID string, streamKey str
|
||||
return fmt.Sprintf("run.%s.%s.%s", runEndpointID, serverInstanceID, streamKey)
|
||||
}
|
||||
|
||||
func runSessionLogStreamID(runEndpointID string, serverInstanceID string, logSessionID string, streamKey string) string {
|
||||
return fmt.Sprintf("run.%s.%s.%s.%s", runEndpointID, serverInstanceID, logSessionID, streamKey)
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetJob(id string) (domain.Job, error) {
|
||||
job, err := svc.store.Jobs().Get(id)
|
||||
if err != nil {
|
||||
@@ -2587,6 +2599,22 @@ func (svc *CoreService) ListArtifacts(filter domain.ArtifactFilter) ([]domain.Ar
|
||||
}
|
||||
|
||||
func (svc *CoreService) CreateLogStream(stream domain.LogStream) (domain.LogStream, error) {
|
||||
lock := svc.logIngestLock(stream.ServerInstanceID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
return svc.createLogStream(stream)
|
||||
}
|
||||
|
||||
func (svc *CoreService) logIngestLock(serverInstanceID string) *sync.Mutex {
|
||||
hash := uint32(2166136261)
|
||||
for index := 0; index < len(serverInstanceID); index++ {
|
||||
hash ^= uint32(serverInstanceID[index])
|
||||
hash *= 16777619
|
||||
}
|
||||
return &svc.logIngestMu[hash%uint32(len(svc.logIngestMu))]
|
||||
}
|
||||
|
||||
func (svc *CoreService) createLogStream(stream domain.LogStream) (domain.LogStream, error) {
|
||||
instance, err := svc.store.ServerInstances().Get(stream.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.LogStream{}, fmt.Errorf("get server instance dependency: %w", err)
|
||||
@@ -2604,12 +2632,31 @@ func (svc *CoreService) CreateLogStream(stream domain.LogStream) (domain.LogStre
|
||||
if err := validator.ValidateLogStream(stream); err != nil {
|
||||
return domain.LogStream{}, err
|
||||
}
|
||||
if err := svc.validateLogStreamSession(stream); err != nil {
|
||||
return domain.LogStream{}, err
|
||||
}
|
||||
if err := svc.store.LogStreams().Create(stream); err != nil {
|
||||
return domain.LogStream{}, err
|
||||
}
|
||||
return domain.CopyLogStream(stream), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateLogStreamSession(stream domain.LogStream) error {
|
||||
if stream.LogSessionID == "" {
|
||||
return nil
|
||||
}
|
||||
streams, err := svc.store.LogStreams().List(domain.LogStreamFilter{ServerInstanceID: stream.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, existing := range streams {
|
||||
if existing.LogSessionID == stream.LogSessionID && !existing.SessionStartedAt.Equal(stream.SessionStartedAt) {
|
||||
return validationError("log session metadata conflicts with an existing stream")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetLogStream(id string) (domain.LogStream, error) {
|
||||
return svc.store.LogStreams().Get(id)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
@@ -255,6 +256,14 @@ func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLif
|
||||
if err := svc.validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
if action == domain.ServerLifecycleActionStart {
|
||||
instance.LifecycleProcessID = ""
|
||||
instance.LifecycleObservationSeq = 0
|
||||
instance.LifecycleObservedAt = time.Time{}
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
job, err := svc.dispatchLifecycleJob(instance, action, command.IdempotencyKey)
|
||||
if err != nil {
|
||||
|
||||
@@ -50,6 +50,7 @@ func (svc *CoreService) ReportRunLifecycle(report domain.RunLifecycleReport) (do
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return domain.RunLifecycleReportResult{}, err
|
||||
}
|
||||
svc.publishLogProcessState(instance)
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if report.State == domain.JobStateFailed || report.State == domain.JobStateCancelled {
|
||||
@@ -140,6 +141,7 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
svc.publishLogProcessState(instance)
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
|
||||
auditResult = domain.AuditResultFailed
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
@@ -103,6 +104,61 @@ func TestLifecycleProjectedStateUsesRunProcessFacts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleJobResultsPublishProcessStateEvents(t *testing.T) {
|
||||
svc, sessionToken := newLifecycleRunService(t)
|
||||
createLifecyclePlugin(t, svc)
|
||||
created, err := svc.CreateServerInstanceWorkflow(domain.ServerLifecycleCreate{ID: "server-state-events", PluginID: "server.scum", RunEndpointID: "run-local", Name: "State Events", IdempotencyKey: "state-events-create", ProfileKey: "local"})
|
||||
if err != nil {
|
||||
t.Fatalf("create lifecycle server: %v", err)
|
||||
}
|
||||
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, created.Instance.ID, domain.LifecycleCapabilityInstall, domain.JobStateSucceeded)
|
||||
ready, err := svc.GetServerInstance(created.Instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get ready server: %v", err)
|
||||
}
|
||||
if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: ready.ID, ExpectedConfigVersion: ready.ConfigVersion, IdempotencyKey: "state-events-start"}); err != nil {
|
||||
t.Fatalf("dispatch start: %v", err)
|
||||
}
|
||||
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, ready.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
|
||||
|
||||
subscription, err := svc.SubscribeLogEvents(ready.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe state events: %v", err)
|
||||
}
|
||||
defer subscription.Close()
|
||||
running, err := svc.GetServerInstance(ready.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get running server: %v", err)
|
||||
}
|
||||
if _, err := svc.StopServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: running.ID, ExpectedConfigVersion: running.ConfigVersion, IdempotencyKey: "state-events-stop"}); err != nil {
|
||||
t.Fatalf("dispatch stop: %v", err)
|
||||
}
|
||||
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, running.ID, domain.LifecycleCapabilityStop, domain.JobStateSucceeded)
|
||||
assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateStopped)
|
||||
|
||||
stopped, err := svc.GetServerInstance(running.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get stopped server: %v", err)
|
||||
}
|
||||
if _, err := svc.StartServerInstance(domain.ServerLifecycleCommand{ServerInstanceID: stopped.ID, ExpectedConfigVersion: stopped.ConfigVersion, IdempotencyKey: "state-events-restart"}); err != nil {
|
||||
t.Fatalf("dispatch restart: %v", err)
|
||||
}
|
||||
claimAndCompleteLifecycleJobForServer(t, svc, sessionToken, stopped.ID, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
|
||||
assertLogProcessStateEvent(t, subscription, domain.ServerInstanceStateRunning)
|
||||
}
|
||||
|
||||
func assertLogProcessStateEvent(t *testing.T, subscription LogEventSubscription, want domain.ServerInstanceState) {
|
||||
t.Helper()
|
||||
select {
|
||||
case event := <-subscription.Events:
|
||||
if event.Kind != LogEventSubscriptionEventProcessState || event.ProcessState != want {
|
||||
t.Fatalf("unexpected process state event: %+v", event)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("expected process state event %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceCreatesTargetBoundDraftAndRequiresDedicatedRunRegistration(t *testing.T) {
|
||||
svc, _ := newLifecycleRunService(t)
|
||||
plugin := createLifecyclePlugin(t, svc)
|
||||
|
||||
@@ -27,6 +27,14 @@ func ValidateLogBatchIngest(batch domain.LogBatchIngest) error {
|
||||
if !validLogStreamSource(batch.Source) {
|
||||
violations = append(violations, "source is invalid")
|
||||
}
|
||||
hasSessionID := strings.TrimSpace(batch.LogSessionID) != ""
|
||||
hasSessionStart := !batch.SessionStartedAt.IsZero()
|
||||
if hasSessionID != hasSessionStart {
|
||||
violations = append(violations, "logSessionId and sessionStartedAt must be provided together")
|
||||
}
|
||||
if (hasSessionID || hasSessionStart) && batch.Source != domain.LogStreamSourceProcess {
|
||||
violations = append(violations, "log session metadata is only valid for process streams")
|
||||
}
|
||||
if batch.FirstSeq == 0 || batch.LastSeq == 0 {
|
||||
violations = append(violations, "sequence range must be positive")
|
||||
}
|
||||
|
||||
@@ -1667,6 +1667,14 @@ func ValidateLogStream(stream domain.LogStream) error {
|
||||
if !validLogStorageBackend(stream.StorageBackend) {
|
||||
violations = append(violations, "storageBackend is invalid")
|
||||
}
|
||||
hasSessionID := strings.TrimSpace(stream.LogSessionID) != ""
|
||||
hasSessionStart := !stream.SessionStartedAt.IsZero()
|
||||
if hasSessionID != hasSessionStart {
|
||||
violations = append(violations, "logSessionId and sessionStartedAt must be provided together")
|
||||
}
|
||||
if (hasSessionID || hasSessionStart) && stream.Source != domain.LogStreamSourceProcess {
|
||||
violations = append(violations, "log session metadata is only valid for process streams")
|
||||
}
|
||||
return finish(violations)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user