Recover managed process log sessions

runtime/execution_test.go

@@ -497,6 +497,13 @@ func TestTypedProcessStartStopIsIdempotentAndReconciles(t *testing.T) {
 	if resumed.LogSessionID != first.LogSessionID {
 		t.Fatalf("expected Run restart to retain session, before=%q after=%q", first.LogSessionID, resumed.LogSessionID)
 	}
+	beforeResumeSeq := resumed.ObservationSeq
+	beforeResumeUpdatedAt := resumed.UpdatedAt
+	restarted.ResumeManagedProcessLogs(context.Background())
+	resumed = restarted.managed.(*OSManagedProcessSupervisor).Status(ProcessIdentity{Scope: processScope(root, assignment)})
+	if resumed.ObservationSeq <= beforeResumeSeq || !resumed.UpdatedAt.After(beforeResumeUpdatedAt) {
+		t.Fatalf("expected resume to refresh process observation, beforeSeq=%d after=%+v", beforeResumeSeq, resumed)
+	}
 	stop := executionAssignment(protocol.RunCapabilityProcessStop)
 	stop.TargetKey = "actions/stop.json"
 	stopped := restarted.Execute(stop)

runtime/process_supervisor.go

@@ -320,8 +320,15 @@ func (supervisor *OSManagedProcessSupervisor) Status(identity ProcessIdentity) P
 func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcessOutput) {
 	supervisor.mu.Lock()
 	defer supervisor.mu.Unlock()
+	now := time.Now().UTC()
+	changed := false
 	for _, item := range supervisor.items {
 		if item.State == "running" && supervisor.isAlive(item) {
+			item.State = "running"
+			item.ObservationSeq++
+			item.UpdatedAt = now
+			supervisor.items[item.Scope] = item
+			changed = true
 			supervisor.startTailersLocked(item, output)
 			log.Printf("RUN phase=process.managed status=resume_output pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
 		} else if supervisor.hasPendingOutput(item) {
@@ -329,7 +336,6 @@ func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcess
 			log.Printf("RUN phase=process.managed status=resume_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
 		}
 	}
-	changed := false
 	for key, item := range supervisor.retired {
 		if supervisor.hasPendingOutput(item) {
 			supervisor.startDrainTailersLocked(item, output)

runtime/worker.go

@@ -1185,14 +1185,19 @@ type durableLogClient interface {
 	IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error)
 }
 
+type durableLogProgressClient interface {
+	GetRunLogStreamProgress(context.Context, protocol.RunLogStreamProgressRequest) (protocol.RunLogStreamProgressResponse, error)
+}
+
 type durableArtifactClient interface {
 	UploadArtifactChunk(context.Context, protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error)
 }
 
 type sessionLogBatchClient struct {
-	client        durableLogClient
-	runEndpointID string
-	sessionToken  string
+	client         durableLogClient
+	progressClient durableLogProgressClient
+	runEndpointID  string
+	sessionToken   string
 }
 
 type sessionLogStreamProgressClient struct {
@@ -1250,6 +1255,26 @@ func (client sessionLogBatchClient) IngestLogBatch(ctx context.Context, batch pr
 	return response, err
 }
 
+func (client sessionLogBatchClient) LogStreamLatestSeq(ctx context.Context, batch protocol.LogBatchIngestRequest) (uint64, error) {
+	if client.progressClient == nil {
+		return 0, fmt.Errorf("platform log progress client is unavailable")
+	}
+	response, err := client.progressClient.GetRunLogStreamProgress(ctx, protocol.RunLogStreamProgressRequest{
+		RunEndpointID:    client.runEndpointID,
+		SessionToken:     client.sessionToken,
+		ServerInstanceID: batch.ServerInstanceID,
+		LogStreamID:      batch.LogStreamID,
+	})
+	if err != nil {
+		return 0, err
+	}
+	if !response.Accepted || response.LogStreamID != batch.LogStreamID {
+		return 0, fmt.Errorf("platform log stream progress response is invalid")
+	}
+	log.Printf("RUN phase=durable_uploaders.logs status=sequence_recover stream=%s localFirstSeq=%d platformLatestSeq=%d", safeOptional(batch.LogStreamID), batch.FirstSeq, response.LatestSeq)
+	return response.LatestSeq, nil
+}
+
 func logBatchNotFoundError(err error) bool {
 	var httpErr interface{ HTTPStatus() int }
 	return errors.As(err, &httpErr) && httpErr.HTTPStatus() == http.StatusNotFound
@@ -1305,6 +1330,7 @@ func (worker *Worker) runDurableUploaders(ctx context.Context, done chan<- struc
 	logSink, hasLogSink := worker.executor.logSink.(*SpoolLogSink)
 	artifactHook, hasArtifactHook := worker.executor.artifactHook.(*QueueArtifactHook)
 	logClient, hasLogClient := worker.client.(durableLogClient)
+	progressClient, _ := worker.client.(durableLogProgressClient)
 	artifactClient, hasArtifactClient := worker.client.(durableArtifactClient)
 	if (!hasLogSink || !hasLogClient) && (!hasArtifactHook || !hasArtifactClient) {
 		log.Printf("RUN phase=durable_uploaders status=disabled logs=%t artifacts=%t", hasLogSink && hasLogClient, hasArtifactHook && hasArtifactClient)
@@ -1328,7 +1354,7 @@ func (worker *Worker) runDurableUploaders(ctx context.Context, done chan<- struc
 				startedAt := time.Now()
 				log.Printf("RUN phase=durable_uploaders.logs status=flush_start timeoutMs=%d", durableUploaderFlushTimeout.Milliseconds())
 				flushCtx, cancel := context.WithTimeout(ctx, durableUploaderFlushTimeout)
-				client := sessionLogBatchClient{client: logClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken}
+				client := sessionLogBatchClient{client: logClient, progressClient: progressClient, runEndpointID: state.RunEndpointID, sessionToken: state.SessionToken}
 				flushed, err := logSink.Spool.Flush(flushCtx, client)
 				if err != nil {
 					log.Printf("RUN phase=durable_uploaders.logs status=flush_failed flushed=%d durationMs=%d error=%s", flushed, time.Since(startedAt).Milliseconds(), RedactText(err.Error()))

runtime/worker_test.go

@@ -636,6 +636,19 @@ func TestSessionLogBatchClientQuarantinesSequenceGap(t *testing.T) {
 	}
 }
 
+func TestSessionLogBatchClientRecoversLatestSeqThroughProgressEndpoint(t *testing.T) {
+	workerClient := newFakeWorkerClient()
+	workerClient.logStreamProgress = 736
+	client := sessionLogBatchClient{client: &recordingDurableLogClient{}, progressClient: workerClient, runEndpointID: "run-current", sessionToken: "token-current"}
+	latestSeq, err := client.LogStreamLatestSeq(context.Background(), protocol.LogBatchIngestRequest{LogStreamID: "run.run-current.server-worker.session.stdout", ServerInstanceID: "server-worker", FirstSeq: 10413})
+	if err != nil || latestSeq != 736 {
+		t.Fatalf("recover latest sequence: latestSeq=%d err=%v", latestSeq, err)
+	}
+	if len(workerClient.logProgressRequests) != 1 || workerClient.logProgressRequests[0].RunEndpointID != "run-current" || workerClient.logProgressRequests[0].SessionToken != "token-current" || workerClient.logProgressRequests[0].ServerInstanceID != "server-worker" || workerClient.logProgressRequests[0].LogStreamID != "run.run-current.server-worker.session.stdout" {
+		t.Fatalf("unexpected progress request: %+v", workerClient.logProgressRequests)
+	}
+}
+
 func TestSessionLogBatchClientQuarantinesSequenceConflict(t *testing.T) {
 	recorder := &recordingDurableLogClient{err: api.PlatformRequestError{Status: http.StatusBadRequest, Code: "validation_failed", Details: []string{"log batch conflicts with acknowledged range"}}}
 	client := sessionLogBatchClient{client: recorder, runEndpointID: "run-current", sessionToken: "token-current"}