Relay current process output without spool
This commit is contained in:
@@ -309,6 +309,10 @@ func (c PlatformClient) IngestLogBatch(ctx context.Context, request protocol.Log
|
||||
return postPlatformJSON[protocol.LogBatchIngestRequest, protocol.LogBatchIngestResponse](ctx, c, "/api/v1/run/logs/batches", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) RelayLiveLogBatch(ctx context.Context, request protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) {
|
||||
return postPlatformJSON[protocol.LogBatchIngestRequest, protocol.LogBatchIngestResponse](ctx, c, "/api/v1/run/logs/relay", request)
|
||||
}
|
||||
|
||||
func (c PlatformClient) GetRunLogStreamProgress(ctx context.Context, request protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error) {
|
||||
return postPlatformJSON[protocol.RunLogStreamProgressRequest, protocol.RunLogStreamProgressResponse](ctx, c, "/api/v1/run/logs/progress", request)
|
||||
}
|
||||
|
||||
+3
-8
@@ -66,13 +66,6 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Printf("RUN phase=workspace_seed status=ready workspace=%s", diagnosticValue(cfg.WorkspaceRoot))
|
||||
log.Printf("RUN phase=log_spool status=opening path=%s", diagnosticValue(cfg.SpoolRoot))
|
||||
logSpool, err := spool.NewLogSpool(cfg.SpoolRoot)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "initialize log spool: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Printf("RUN phase=log_spool status=ready path=%s", diagnosticValue(cfg.SpoolRoot))
|
||||
log.Printf("RUN phase=artifact_queue status=opening path=%s", diagnosticValue(cfg.SpoolRoot))
|
||||
artifactQueue, err := spool.NewArtifactQueue(cfg.SpoolRoot)
|
||||
if err != nil {
|
||||
@@ -80,11 +73,13 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Printf("RUN phase=artifact_queue status=ready path=%s", diagnosticValue(cfg.SpoolRoot))
|
||||
liveLogSink := runruntime.NewLiveLogSink(client)
|
||||
defer liveLogSink.Close()
|
||||
log.Printf("RUN phase=worker_init status=starting endpoint=%s", diagnosticValue(cfg.RunEndpointID))
|
||||
worker, err := runruntime.NewWorker(
|
||||
cfg,
|
||||
client,
|
||||
runruntime.WithProcessLogSink(&runruntime.SpoolLogSink{Spool: logSpool}),
|
||||
runruntime.WithProcessLogSink(liveLogSink),
|
||||
runruntime.WithLifecycleArtifactHook(&runruntime.QueueArtifactHook{Queue: artifactQueue}),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
+3
-1
@@ -29,4 +29,6 @@ Control is the lightweight high-priority channel between run and platform.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Durable log ingest, artifact chunk transfer, and the optional game client bridge remain separate channels. The job channel is separate from control and uses `/api/v1/run/jobs/*` routes.
|
||||
Live log relay, artifact chunk transfer, and the optional game client bridge
|
||||
remain separate channels. The job channel is separate from control and uses
|
||||
`/api/v1/run/jobs/*` routes.
|
||||
|
||||
+10
-3
@@ -76,14 +76,21 @@ The executor resolves lifecycle action templates under the scoped server workspa
|
||||
- Remote database and RCON jobs must use scoped input/artifact refs rather than embedding query or command bodies in job results.
|
||||
- Run self-update, dependency, and log backfill jobs must use declared capabilities, logical target keys, scoped refs, and bounded result refs.
|
||||
- Job payloads must not include logs, artifact chunks, raw host paths, raw credentials, direct sockets, or large inline result bodies.
|
||||
- Process stdout/stderr must be redacted and written to the log spool rather than embedded in progress/result bodies.
|
||||
- Process stdout/stderr must be redacted and sent through the asynchronous live
|
||||
relay rather than embedded in progress/result bodies. Relay failure or drop
|
||||
must not block job execution or create a local log backlog.
|
||||
- Job ack/progress/result/cancel/reconcile calls are lightweight lifecycle metadata and must be able to complete while artifact/file transfer work is active or retrying.
|
||||
- Persistent control events are hints only. Run must still claim, ack, execute, and complete durable jobs through this job channel so retries, leases, and idempotency stay platform-authoritative.
|
||||
- Dependency adapters and update downloads run in the job worker while control heartbeat, cancellation polling, log spool upload, and artifact upload retain independent bounded loops.
|
||||
- Dependency adapters and update downloads run in the job worker while control
|
||||
heartbeat, cancellation polling, live log relay, and artifact upload retain
|
||||
independent bounded loops.
|
||||
- Terminal results must remain idempotent under log and artifact retry pressure and must reference artifacts by safe `artifact://...` refs rather than embedding transfer payloads.
|
||||
|
||||
## Deferred Channels
|
||||
|
||||
Durable log ingest, artifact chunk transfer, and optional game client bridge traffic remain separate channels and must not be multiplexed through job result payloads. Artifact transfer carries chunk payloads only through `/api/v1/run/artifacts/*` routes.
|
||||
Live log relay, artifact chunk transfer, and optional game client bridge traffic
|
||||
remain separate channels and must not be multiplexed through job result
|
||||
payloads. Artifact transfer carries chunk payloads only through
|
||||
`/api/v1/run/artifacts/*` routes.
|
||||
|
||||
Production code signing/KMS, rollout rings/fleet orchestration, client-manager lifecycle, plugin lifecycle, production scaling/alerts, and real AI-provider integration are explicitly outside this contract.
|
||||
|
||||
+25
-14
@@ -1,12 +1,18 @@
|
||||
# Run Log Ingest Contract
|
||||
# Run Live Log Relay Contract
|
||||
|
||||
Logs are durable historical data. They are not transported as best-effort UI messages.
|
||||
Run log traffic is a best-effort push of output observed now. It is not a
|
||||
durable log database, delivery queue, or acknowledgement-dependent lifecycle
|
||||
channel. Game plugins own durable log storage and analysis.
|
||||
|
||||
## Implemented Routes
|
||||
|
||||
- `POST /api/v1/run/logs/batches`: uploads one bounded log batch and receives an acknowledgement range.
|
||||
- `POST /api/v1/run/logs/relay`: pushes one bounded current-output batch to the
|
||||
platform relay. The platform may forward it to live subscribers, but does
|
||||
not persist the body or require a delivery acknowledgement.
|
||||
- `POST /api/v1/log-streams/query`: queries stored log entries after a stream sequence cursor.
|
||||
- `GET /api/v1/server-instances/{id}/logs/events`: browser-facing Server-Sent Events stream for replaying recent stored entries and pushing newly ingested platform log entries.
|
||||
- `GET /api/v1/server-instances/{id}/logs/events`: browser-facing Server-Sent
|
||||
Events stream for the current supervised process session. A new subscriber
|
||||
starts at the live boundary and receives no platform log history.
|
||||
|
||||
## Payloads
|
||||
|
||||
@@ -17,20 +23,25 @@ Logs are durable historical data. They are not transported as best-effort UI mes
|
||||
- `LogStreamCursorResponse`: ordered entries, next cursor, and latest acknowledged sequence.
|
||||
- `LogStreamEventResponse`: safe browser event containing server ID, stream metadata, latest sequence, and one log entry.
|
||||
|
||||
Run-assigned Platform jobs use `job.<jobId>.<streamKey>` log stream IDs. Autonomous lifecycle bootstrap is not a Platform job, so it uses `run.<runEndpointId>.<serverInstanceId>.<streamKey>` and Platform creates the server-bound stream from the signed Run batch instead of looking for a job record.
|
||||
|
||||
## Local Spool
|
||||
|
||||
Run must write unacknowledged logs to a local spool/WAL before upload. Segments may be removed only after platform acknowledgement.
|
||||
Autonomous lifecycle output uses
|
||||
`run.<runEndpointId>.<serverInstanceId>.<logSessionId>.<streamKey>` stream IDs.
|
||||
The platform creates or updates stream metadata from the signed relay batch;
|
||||
the body remains live-only.
|
||||
|
||||
## Priority
|
||||
|
||||
Log flush has higher priority than artifact transfer. Artifact work must slow down when log spool pressure rises.
|
||||
Live relay is asynchronous and lower priority than control and lifecycle
|
||||
progress. A full in-memory relay queue or a failed relay request drops the
|
||||
current event and never blocks, retries, or changes lifecycle state. Run does
|
||||
not create a log spool, resend backlog, or platform delivery watermark.
|
||||
|
||||
Log spool retry state is independent from artifact/file retry state. Acknowledged log batches may be removed even when artifact chunks are still pending, and artifact chunk acknowledgement must not alter log sequence state. Log ingest payloads carry bounded entries only and must not include artifact chunks, file bodies, host paths, raw credentials, or direct socket details.
|
||||
|
||||
The Run uploader flushes committed spool segments independently with bounded request contexts. A failed or partial acknowledgement leaves the segment pending for restart/retry; control heartbeat and job lifecycle polling do not wait for log or artifact flushes.
|
||||
Log relay payloads carry bounded redacted entries only and must not include
|
||||
artifact chunks, file bodies, host paths, raw credentials, or direct socket
|
||||
details.
|
||||
|
||||
## Browser Channel
|
||||
|
||||
Browser live tail is a platform-owned SSE fan-out from durable ingest and cursor state. External log storage backends and optional game client bridge traffic remain separate channels. Artifact transfer uses its own lower-priority channel and must not be multiplexed through log ingest.
|
||||
Browser live tail is a platform SSE fan-out from current relay events and
|
||||
process-session metadata. Historical log queries and plugin-owned storage are
|
||||
separate channels. Artifact transfer uses its own lower-priority channel and
|
||||
must not be multiplexed through live log relay.
|
||||
|
||||
+26
-46
@@ -132,7 +132,7 @@ func TestTypedProcessOutputCaptureResumesAfterRunRestart(t *testing.T) {
|
||||
waitForManagedTailers(t, restarted.managed.(*OSManagedProcessSupervisor))
|
||||
}
|
||||
|
||||
func TestManagedProcessOutputAfterCanceledRunContextIsSpooledOnRestart(t *testing.T) {
|
||||
func TestManagedProcessOutputAfterCanceledRunContextIsNotSpooledOnRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{
|
||||
@@ -199,24 +199,14 @@ func TestManagedProcessOutputAfterCanceledRunContextIsSpooledOnRestart(t *testin
|
||||
WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "new-session", Spool: restartedSpool}),
|
||||
)
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
waitForSpooledText(t, restartedSpool, "managed stdout triggered after-cancel", "managed stderr triggered after-cancel")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if pending := mustPendingLogs(t, restartedSpool); len(pending) != 0 {
|
||||
t.Fatalf("restart replayed output observed before Run restart: %+v", pending)
|
||||
}
|
||||
restartedManaged := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
resumed := waitForManagedOffsets(t, restartedManaged, scope, identity.StdoutOffset, identity.StderrOffset)
|
||||
if resumed.LogSessionID != identity.LogSessionID || resumed.StdoutOffset <= identity.StdoutOffset || resumed.StderrOffset <= identity.StderrOffset {
|
||||
t.Fatalf("restart did not retain session and commit offsets: before=%+v after=%+v", identity, resumed)
|
||||
}
|
||||
for _, batch := range mustPendingLogs(t, restartedSpool) {
|
||||
if batch.LogSessionID != identity.LogSessionID || !batch.SessionStartedAt.Equal(identity.StartedAt) {
|
||||
t.Fatalf("spooled batch lost process session metadata: %+v", batch)
|
||||
}
|
||||
if batch.FirstSeq != 1 || len(batch.Entries) == 0 || batch.Entries[0].Seq != 1 {
|
||||
t.Fatalf("fresh generation stream did not start durably at sequence 1: %+v", batch)
|
||||
}
|
||||
for index, entry := range batch.Entries {
|
||||
if entry.Seq != uint64(index+1) {
|
||||
t.Fatalf("fresh generation stream sequence is not contiguous: %+v", batch)
|
||||
}
|
||||
}
|
||||
resumed := restartedManaged.Status(ProcessIdentity{Scope: scope})
|
||||
if resumed.LogSessionID != identity.LogSessionID || resumed.StdoutOffset != identity.StdoutOffset || resumed.StderrOffset != identity.StderrOffset {
|
||||
t.Fatalf("restart changed output offsets without observing new output: before=%+v after=%+v", identity, resumed)
|
||||
}
|
||||
stop := executionAssignment(protocol.RunCapabilityProcessStop)
|
||||
stop.TargetKey = "actions/stop.json"
|
||||
@@ -293,7 +283,7 @@ func TestImmediateManagedProcessRestartTailsNewGeneration(t *testing.T) {
|
||||
waitForManagedTailers(t, managed)
|
||||
}
|
||||
|
||||
func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) {
|
||||
func TestManagedProcessRestartDoesNotRetainUndrainedRetiredGeneration(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := executionAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.ExecutionInput.LogSources = []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "game.console.stdout"}, {Kind: "process.stderr", StreamKey: "game.console.stderr"}}
|
||||
@@ -328,9 +318,9 @@ func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) {
|
||||
t.Fatalf("process restart reused log session: first=%+v second=%+v", first, second)
|
||||
}
|
||||
oldManaged.mu.Lock()
|
||||
if len(oldManaged.retired) != 1 {
|
||||
if len(oldManaged.retired) != 0 {
|
||||
oldManaged.mu.Unlock()
|
||||
t.Fatalf("expected one durable retired generation, got %+v", oldManaged.retired)
|
||||
t.Fatalf("expected no durable retired generation, got %+v", oldManaged.retired)
|
||||
}
|
||||
oldManaged.stopTailersLocked(first)
|
||||
oldManaged.stopTailersLocked(second)
|
||||
@@ -342,7 +332,10 @@ func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) {
|
||||
}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: assignment.RunEndpointID, SessionToken: "restart-session", Spool: logSpool}))
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
waitForSpooledText(t, logSpool, "retired-a", "current-b")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if pending := mustPendingLogs(t, logSpool); len(pending) != 0 {
|
||||
t.Fatalf("restart replayed retired output: %+v", pending)
|
||||
}
|
||||
restartedManaged := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
waitForManagedTailers(t, restartedManaged)
|
||||
restartedManaged.mu.Lock()
|
||||
@@ -353,7 +346,7 @@ func TestManagedProcessRestartRetainsUndrainedRetiredGeneration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedProcessSourceCursorDeduplicatesCommittedLineWithStaleJournalOffset(t *testing.T) {
|
||||
func TestManagedProcessRestartDoesNotReplayHistoricalOutput(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
stateDir := filepath.Join(root, "state")
|
||||
outputDir := filepath.Join(stateDir, "process-output")
|
||||
@@ -361,7 +354,7 @@ func TestManagedProcessSourceCursorDeduplicatesCommittedLineWithStaleJournalOffs
|
||||
t.Fatalf("create process output dir: %v", err)
|
||||
}
|
||||
line := "committed before offset\n"
|
||||
identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "server-execution", "local"), ServerInstanceID: "server-execution", RunEndpointID: "run-execution", JobID: "execution-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", LogSessionID: "session-stale-offset", PID: 12345, StartedAt: time.Now().UTC(), State: "exited", StdoutLogRef: "stale.stdout.log", StderrLogRef: "stale.stderr.log", StdoutStreamKey: "game.console.stdout", StderrStreamKey: "game.console.stderr"}
|
||||
identity := ProcessIdentity{Scope: filepath.Join(root, "instances", "server-execution", "local"), ServerInstanceID: "server-execution", RunEndpointID: "run-execution", JobID: "execution-job", Capability: protocol.RunCapabilityProcessStart, ProfileKey: "local", LogSessionID: "session-stale-offset", PID: os.Getpid(), StartedAt: time.Now().UTC(), State: "running", StdoutLogRef: "stale.stdout.log", StderrLogRef: "stale.stderr.log", StdoutStreamKey: "game.console.stdout", StderrStreamKey: "game.console.stderr"}
|
||||
if err := os.WriteFile(filepath.Join(outputDir, identity.StdoutLogRef), []byte(line), 0o600); err != nil {
|
||||
t.Fatalf("write stale stdout: %v", err)
|
||||
}
|
||||
@@ -375,34 +368,21 @@ func TestManagedProcessSourceCursorDeduplicatesCommittedLineWithStaleJournalOffs
|
||||
if err := os.WriteFile(filepath.Join(stateDir, "processes.json"), body, 0o600); err != nil {
|
||||
t.Fatalf("write process journal: %v", err)
|
||||
}
|
||||
logSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
assignment := assignmentFromProcessIdentity(identity)
|
||||
sink := &SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "old-run-session", Spool: logSpool}
|
||||
if err := sink.AppendWithCursor(context.Background(), assignment, "stdout", strings.TrimSpace(line), ProcessLogCursor{StartOffset: 0, EndOffset: int64(len(line))}); err != nil {
|
||||
t.Fatalf("commit line before offset: %v", err)
|
||||
}
|
||||
streamID := logStreamIDForAssignment(assignment, identity.StdoutStreamKey)
|
||||
if err := logSpool.Ack(protocol.LogBatchIngestResponse{LogStreamID: streamID, AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
|
||||
t.Fatalf("ack committed line: %v", err)
|
||||
}
|
||||
restartedSpool, err := spool.NewLogSpool(filepath.Join(root, "spool"))
|
||||
if err != nil {
|
||||
t.Fatalf("restart log spool: %v", err)
|
||||
}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(&SpoolLogSink{RunEndpointID: identity.RunEndpointID, SessionToken: "new-run-session", Spool: restartedSpool}))
|
||||
sink := &recordingLogSink{}
|
||||
restarted := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(sink))
|
||||
restarted.ResumeManagedProcessLogs(context.Background())
|
||||
managed := restarted.managed.(*OSManagedProcessSupervisor)
|
||||
resumed := waitForManagedOffsets(t, managed, identity.Scope, 0, -1)
|
||||
waitForManagedTailers(t, managed)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if resumed.StdoutOffset != int64(len(line)) {
|
||||
t.Fatalf("stale journal offset was not advanced: %+v", resumed)
|
||||
t.Fatalf("restart did not move live tail to the current output boundary: %+v", resumed)
|
||||
}
|
||||
if pending := mustPendingLogs(t, restartedSpool); len(pending) != 0 {
|
||||
t.Fatalf("committed source cursor was enqueued twice: %+v", pending)
|
||||
if lines := sink.snapshot(); len(lines) != 0 {
|
||||
t.Fatalf("restart replayed historical output: %+v", lines)
|
||||
}
|
||||
managed.mu.Lock()
|
||||
managed.stopTailersLocked(identity)
|
||||
managed.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestManagedProcessSupervisorMigratesLiveLegacySession(t *testing.T) {
|
||||
|
||||
@@ -205,11 +205,6 @@ func (supervisor *OSManagedProcessSupervisor) Start(ctx context.Context, command
|
||||
identity.UpdatedAt = identity.StartedAt
|
||||
identity.CommandFingerprint = fingerprintArgs(command.Args)
|
||||
previous, hadPrevious := supervisor.items[key]
|
||||
retiredKey := ""
|
||||
if hadPrevious && supervisor.hasPendingOutput(previous) {
|
||||
retiredKey = managedProcessGenerationKey(previous)
|
||||
supervisor.retired[retiredKey] = previous
|
||||
}
|
||||
supervisor.items[key] = identity
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
_ = forceManagedProcessStop(identity)
|
||||
@@ -219,9 +214,6 @@ func (supervisor *OSManagedProcessSupervisor) Start(ctx context.Context, command
|
||||
} else {
|
||||
delete(supervisor.items, key)
|
||||
}
|
||||
if retiredKey != "" {
|
||||
delete(supervisor.retired, retiredKey)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=persist_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
@@ -324,6 +316,8 @@ func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcess
|
||||
changed := false
|
||||
for _, item := range supervisor.items {
|
||||
if item.State == "running" && supervisor.isAlive(item) {
|
||||
item.StdoutOffset = supervisor.outputFileSize(item.StdoutLogRef, item.StdoutOffset)
|
||||
item.StderrOffset = supervisor.outputFileSize(item.StderrLogRef, item.StderrOffset)
|
||||
item.State = "running"
|
||||
item.ObservationSeq++
|
||||
item.UpdatedAt = now
|
||||
@@ -331,17 +325,9 @@ func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcess
|
||||
changed = true
|
||||
supervisor.startTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
} else if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
}
|
||||
}
|
||||
for key, item := range supervisor.retired {
|
||||
if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_retired_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
continue
|
||||
}
|
||||
for key := range supervisor.retired {
|
||||
delete(supervisor.retired, key)
|
||||
changed = true
|
||||
}
|
||||
@@ -350,6 +336,17 @@ func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcess
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) outputFileSize(ref string, fallback int64) int64 {
|
||||
if ref == "" {
|
||||
return fallback
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(ref)))
|
||||
if err != nil || info.Size() < fallback {
|
||||
return fallback
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Reconcile() {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
@@ -361,7 +358,9 @@ func (supervisor *OSManagedProcessSupervisor) Reconcile() {
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
item.ObservationSeq++
|
||||
supervisor.items[key] = item
|
||||
supervisor.drainTailersLocked(item)
|
||||
// A Run restart must never replay output produced before this
|
||||
// process became observable again.
|
||||
supervisor.stopTailersLocked(item)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -359,6 +359,9 @@ func (worker *Worker) registerUnlocked(ctx context.Context) error {
|
||||
}
|
||||
sink.mu.Unlock()
|
||||
}
|
||||
if sink, ok := worker.executor.logSink.(*LiveLogSink); ok {
|
||||
sink.SetSession(state.RunEndpointID, state.SessionToken)
|
||||
}
|
||||
if hook, ok := worker.executor.artifactHook.(*QueueArtifactHook); ok {
|
||||
hook.RunEndpointID = state.RunEndpointID
|
||||
hook.SessionToken = state.SessionToken
|
||||
@@ -1438,6 +1441,118 @@ type SpoolLogSink struct {
|
||||
Progress func(context.Context, string, string) (uint64, error)
|
||||
}
|
||||
|
||||
type LiveLogClient interface {
|
||||
RelayLiveLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error)
|
||||
}
|
||||
|
||||
// LiveLogSink is process-local and best-effort. It has no disk spool, no
|
||||
// resend backlog, and does not make lifecycle execution wait for the platform.
|
||||
type LiveLogSink struct {
|
||||
Client LiveLogClient
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
|
||||
mu sync.Mutex
|
||||
sequences map[string]uint64
|
||||
queue chan protocol.LogBatchIngestRequest
|
||||
closed bool
|
||||
closeOnce sync.Once
|
||||
}
|
||||
|
||||
func NewLiveLogSink(client LiveLogClient) *LiveLogSink {
|
||||
sink := &LiveLogSink{Client: client, sequences: map[string]uint64{}, queue: make(chan protocol.LogBatchIngestRequest, 512)}
|
||||
go sink.dispatch()
|
||||
return sink
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) SetSession(endpointID string, sessionToken string) {
|
||||
sink.mu.Lock()
|
||||
sink.RunEndpointID = endpointID
|
||||
sink.SessionToken = sessionToken
|
||||
sink.mu.Unlock()
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
return sink.append(assignment, stream, line)
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) AppendWithCursor(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, _ ProcessLogCursor) error {
|
||||
return sink.append(assignment, stream, line)
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) append(assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
if sink == nil {
|
||||
return nil
|
||||
}
|
||||
redactedLine := RedactText(line)
|
||||
redacted := line != redactedLine || strings.HasPrefix(redactedLine, "[redacted protected ")
|
||||
streamKey := declaredProcessStreamKey(assignment, stream)
|
||||
logStreamID := logStreamIDForAssignment(assignment, streamKey)
|
||||
sink.mu.Lock()
|
||||
defer sink.mu.Unlock()
|
||||
if sink.closed || sink.queue == nil {
|
||||
return nil
|
||||
}
|
||||
endpointID := sink.RunEndpointID
|
||||
if endpointID == "" {
|
||||
endpointID = assignment.RunEndpointID
|
||||
}
|
||||
if endpointID == "" || sink.SessionToken == "" || assignment.ServerInstanceID == "" {
|
||||
return nil
|
||||
}
|
||||
sink.sequences[logStreamID]++
|
||||
sequence := sink.sequences[logStreamID]
|
||||
batch := protocol.LogBatchIngestRequest{
|
||||
RunEndpointID: endpointID,
|
||||
SessionToken: sink.SessionToken,
|
||||
LogStreamID: logStreamID,
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
StreamKey: streamKey,
|
||||
Source: "process",
|
||||
LogSessionID: assignment.LogSessionID,
|
||||
SessionStartedAt: assignment.SessionStartedAt,
|
||||
FirstSeq: sequence,
|
||||
LastSeq: sequence,
|
||||
Compression: "none",
|
||||
Entries: []protocol.LogEntry{{Seq: sequence, Timestamp: time.Now().UTC(), Level: "info", Line: redactedLine, Redacted: redacted}},
|
||||
}
|
||||
batch.Checksum, _ = checksumForLogEntries(batch.Entries)
|
||||
select {
|
||||
case sink.queue <- batch:
|
||||
default:
|
||||
log.Printf("RUN phase=live_log_relay status=dropped stream=%s sequence=%d reason=transient_queue_full", safeOptional(logStreamID), sequence)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) dispatch() {
|
||||
for batch := range sink.queue {
|
||||
if sink.Client == nil {
|
||||
continue
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
_, err := sink.Client.RelayLiveLogBatch(ctx, batch)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=live_log_relay status=dropped stream=%s sequence=%d error=%s", safeOptional(batch.LogStreamID), batch.FirstSeq, RedactText(err.Error()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sink *LiveLogSink) Close() {
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
sink.closeOnce.Do(func() {
|
||||
sink.mu.Lock()
|
||||
sink.closed = true
|
||||
if sink.queue != nil {
|
||||
close(sink.queue)
|
||||
}
|
||||
sink.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func (sink *SpoolLogSink) Append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
return sink.append(ctx, assignment, stream, line, nil)
|
||||
}
|
||||
|
||||
@@ -404,6 +404,33 @@ func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveLogSinkQueuesCurrentBatchWithoutDurableSpool(t *testing.T) {
|
||||
client := &recordingLiveLogClient{received: make(chan protocol.LogBatchIngestRequest, 1)}
|
||||
sink := NewLiveLogSink(client)
|
||||
sink.SetSession("run-test", "session-token")
|
||||
t.Cleanup(sink.Close)
|
||||
assignment := protocol.RunJobAssignment{
|
||||
JobID: "execution-job",
|
||||
RunEndpointID: "run-test",
|
||||
ServerInstanceID: "server-worker",
|
||||
Capability: protocol.RunCapabilityProcessStart,
|
||||
LogSessionID: "generation-current",
|
||||
SessionStartedAt: workerTestTime(),
|
||||
ExecutionInput: protocol.RunJobExecutionInput{LogSources: []protocol.RuntimeLogSourcePlan{{Kind: "process.stdout", StreamKey: "game.console.stdout"}}},
|
||||
}
|
||||
if err := sink.Append(context.Background(), assignment, "stdout", "current output"); err != nil {
|
||||
t.Fatalf("append live log: %v", err)
|
||||
}
|
||||
select {
|
||||
case batch := <-client.received:
|
||||
if batch.LogStreamID != "run.run-test.server-worker.generation-current.game.console.stdout" || batch.FirstSeq != 1 || batch.LastSeq != 1 || batch.Checksum == "" || len(batch.Entries) != 1 || batch.Entries[0].Line != "current output" {
|
||||
t.Fatalf("unexpected live batch: %+v", batch)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("live log was not dispatched")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpoolLogSinkUsesRunScopedStreamForAutonomousLifecycle(t *testing.T) {
|
||||
logSpool, err := spool.NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
@@ -898,6 +925,15 @@ type recordingDurableLogClient struct {
|
||||
err error
|
||||
}
|
||||
|
||||
type recordingLiveLogClient struct {
|
||||
received chan protocol.LogBatchIngestRequest
|
||||
}
|
||||
|
||||
func (client *recordingLiveLogClient) RelayLiveLogBatch(_ context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) {
|
||||
client.received <- batch
|
||||
return protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: batch.LogStreamID, AcceptedFrom: batch.FirstSeq, AcceptedTo: batch.LastSeq}, nil
|
||||
}
|
||||
|
||||
func (client *recordingDurableLogClient) IngestLogBatch(_ context.Context, batch protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error) {
|
||||
client.batch = batch
|
||||
if client.err != nil {
|
||||
|
||||
Reference in New Issue
Block a user