admin343/run
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Recover log streams stuck on straddling spool segments

Browse Source 
A segment that the platform already acknowledged could still be extended by
the next tailed line, so its checksum covered entries stored under a different
batch. The platform then rejected the same body every second while
RejectStreamAfter skipped it, because it only quarantined segments that start
after the platform latest. Every newly allocated line was quarantined by the
following recovery pass, so live log ingest never resumed.

Quarantine pending segments that reach beyond the acknowledged range, never
extend a segment the platform already stored, and log the gap recovery so a
stalled stream is diagnosable from the spool alone.
This commit is contained in:
npc0-hue
2026-09-16 13:34:45 +08:00
parent 4e88199d18
commit e73fe765c3
2 changed files with 80 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spool/log_spool.go
+17 -2
View File
@@ -230,9 +230,14 @@ func (spool LogSpool) mergeLatest(batch durableLogBatch, checksum func([]protoco
if err != nil {
return durableLogBatch{}, pendingLogSegment{}, false, err
}
acknowledged := spool.watermarks[batch.LogStreamID].Acknowledged
for index := len(pending) - 1; index >= 0; index-- {
previous := pending[index]
if spool.inflight[previous.path] || !compatibleLogBatch(previous.batch, batch) || previous.batch.LastSeq+1 != batch.FirstSeq || len(previous.batch.Entries)+len(batch.Entries) > maxAggregatedLogEntries || !compatibleSourceCursor(previous.batch.SourceCursor, batch.SourceCursor) {
// Extending a segment the platform already stored would rewrite its
// checksum for a range that can no longer be uploaded, and the segment
// then becomes permanently unresendable. Leave it for the deduplicating
// retry and start a new segment instead.
if spool.inflight[previous.path] || previous.batch.LastSeq <= acknowledged || !compatibleLogBatch(previous.batch, batch) || previous.batch.LastSeq+1 != batch.FirstSeq || len(previous.batch.Entries)+len(batch.Entries) > maxAggregatedLogEntries || !compatibleSourceCursor(previous.batch.SourceCursor, batch.SourceCursor) {
continue
}
merged := previous.batch
@@ -652,10 +657,12 @@ func (spool LogSpool) Flush(ctx context.Context, client LogBatchClient) (int, er
if progressErr != nil {
return acknowledged, err
}
log.Printf("RUN phase=log_spool.flush status=sequence_gap stream=%s platformLatestSeq=%d batchFirstSeq=%d batchLastSeq=%d", safeSpoolStreamID(batch.LogStreamID), latestSeq, batch.FirstSeq, batch.LastSeq)
rejected, rejectErr := spool.RejectStreamAfter(batch.LogStreamID, latestSeq, permanent.Reason)
if rejectErr != nil {
return acknowledged, rejectErr
}
log.Printf("RUN phase=log_spool.flush status=sequence_gap_recovered stream=%s platformLatestSeq=%d rejectedSegments=%d", safeSpoolStreamID(batch.LogStreamID), latestSeq, rejected)
if resetErr := spool.ResetStreamWatermark(batch.LogStreamID, latestSeq); resetErr != nil {
return acknowledged, resetErr
}
@@ -709,7 +716,15 @@ func (spool LogSpool) RejectStreamAfter(logStreamID string, latestSeq uint64, re
rejected := 0
for _, segment := range segments {
batch := segment.batch.LogBatchIngestRequest
if batch.LogStreamID != logStreamID || batch.FirstSeq <= latestSeq {
// A pending segment is only resendable when it lies entirely inside the
// platform's acknowledged range (it is then deduplicated by sequence
// checksum). Every segment that reaches beyond the platform latest, or
// that straddles it, can never be accepted again: its checksum covers
// entries the platform already stored under a different batch. Keeping
// such a straddling segment pending made the spool resend the same
// rejected body forever, and every newly allocated line was quarantined
// by the next recovery pass, so the stream never resumed.
if batch.LogStreamID != logStreamID || batch.LastSeq <= latestSeq {
continue
}
if err := spool.moveLogSegmentToRejectedLocked(segment.path, reason); err != nil {