Recover managed process log sessions

spool/log_spool.go

@@ -428,6 +428,10 @@ type LogBatchClient interface {
 	IngestLogBatch(context.Context, protocol.LogBatchIngestRequest) (protocol.LogBatchIngestResponse, error)
 }
 
+type LogStreamProgressClient interface {
+	LogStreamLatestSeq(context.Context, protocol.LogBatchIngestRequest) (uint64, error)
+}
+
 type PermanentLogBatchError struct {
 	Reason string
 	Err    error
@@ -479,6 +483,21 @@ func (spool LogSpool) Flush(ctx context.Context, client LogBatchClient) (int, er
 		if err != nil {
 			var permanent PermanentLogBatchError
 			if errors.As(err, &permanent) {
+				if permanent.Reason == "platform_sequence_gap" {
+					latestSeq, progressErr := recoverLogStreamLatestSeq(ctx, client, batch)
+					if progressErr != nil {
+						return acknowledged, err
+					}
+					rejected, rejectErr := spool.RejectStreamAfter(batch.LogStreamID, latestSeq, permanent.Reason)
+					if rejectErr != nil {
+						return acknowledged, rejectErr
+					}
+					if resetErr := spool.ResetStreamWatermark(batch.LogStreamID, latestSeq); resetErr != nil {
+						return acknowledged, resetErr
+					}
+					acknowledged += rejected
+					return acknowledged, nil
+				}
 				if rejectErr := spool.Reject(batch, permanent.Reason); rejectErr != nil {
 					return acknowledged, rejectErr
 				}
@@ -498,6 +517,50 @@ func (spool LogSpool) Flush(ctx context.Context, client LogBatchClient) (int, er
 	return acknowledged, nil
 }
 
+func recoverLogStreamLatestSeq(ctx context.Context, client LogBatchClient, batch protocol.LogBatchIngestRequest) (uint64, error) {
+	progressClient, ok := client.(LogStreamProgressClient)
+	if !ok {
+		return 0, fmt.Errorf("log stream progress client is required after sequence gap")
+	}
+	return progressClient.LogStreamLatestSeq(ctx, batch)
+}
+
+func (spool LogSpool) ResetStreamWatermark(logStreamID string, latestSeq uint64) error {
+	spool.mu.Lock()
+	defer spool.mu.Unlock()
+	watermark := spool.watermarks[logStreamID]
+	watermark.Allocated = latestSeq
+	watermark.Acknowledged = latestSeq
+	spool.watermarks[logStreamID] = watermark
+	return spool.persistWatermarksLocked()
+}
+
+func (spool LogSpool) RejectStreamAfter(logStreamID string, latestSeq uint64, reason string) (int, error) {
+	spool.mu.Lock()
+	defer spool.mu.Unlock()
+	segments, err := spool.pendingSegmentsLocked()
+	if err != nil {
+		return 0, err
+	}
+	rejectedDir := filepath.Join(filepath.Dir(spool.dir), "logs-rejected")
+	rejected := 0
+	for _, segment := range segments {
+		batch := segment.batch.LogBatchIngestRequest
+		if batch.LogStreamID != logStreamID || batch.FirstSeq <= latestSeq {
+			continue
+		}
+		if err := os.MkdirAll(rejectedDir, 0o755); err != nil {
+			return rejected, fmt.Errorf("create rejected log spool directory: %w", err)
+		}
+		rejectedPath := filepath.Join(rejectedDir, fmt.Sprintf("%s.%s", filepath.Base(segment.path), sanitizeSegmentName(reason)))
+		if err := os.Rename(segment.path, rejectedPath); err != nil {
+			return rejected, fmt.Errorf("move rejected log spool segment: %w", err)
+		}
+		rejected++
+	}
+	return rejected, nil
+}
+
 func (spool LogSpool) Reject(batch protocol.LogBatchIngestRequest, reason string) error {
 	spool.mu.Lock()
 	defer spool.mu.Unlock()