Recover log streams stuck on straddling spool segments
A segment that the platform already acknowledged could still be extended by the next tailed line, so its checksum covered entries stored under a different batch. The platform then rejected the same body every second while RejectStreamAfter skipped it, because it only quarantined segments that start after the platform latest. Every newly allocated line was quarantined by the following recovery pass, so live log ingest never resumed. Quarantine pending segments that reach beyond the acknowledged range, never extend a segment the platform already stored, and log the gap recovery so a stalled stream is diagnosable from the spool alone.
This commit is contained in:
@@ -188,6 +188,69 @@ func TestLogSpoolSequenceGapResetsWatermarkToPlatformLatest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogSpoolSequenceGapRejectsStraddlingAcknowledgedSegment(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logSpool, err := NewLogSpool(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
// The acknowledged range ends inside this segment: sequences 10..13 were
|
||||
// extended after the platform had already stored sequence 10.
|
||||
if err := logSpool.Enqueue(validSpoolLogBatch(10, 13)); err != nil {
|
||||
t.Fatalf("enqueue straddling batch: %v", err)
|
||||
}
|
||||
if err := logSpool.Enqueue(validSpoolLogBatch(14, 14)); err != nil {
|
||||
t.Fatalf("enqueue later batch: %v", err)
|
||||
}
|
||||
client := &recoveringSequenceGapLogBatchClient{latestSeq: 10}
|
||||
flushed, err := logSpool.Flush(context.Background(), client)
|
||||
if err != nil || flushed != 2 {
|
||||
t.Fatalf("recover sequence gap: flushed=%d err=%v", flushed, err)
|
||||
}
|
||||
pending, err := logSpool.Pending()
|
||||
if err != nil || len(pending) != 0 {
|
||||
t.Fatalf("expected unresendable segments rejected, pending=%+v err=%v", pending, err)
|
||||
}
|
||||
rejected, err := os.ReadDir(filepath.Join(root, "logs-rejected"))
|
||||
if err != nil || len(rejected) != 2 {
|
||||
t.Fatalf("expected two rejected segments, rejected=%+v err=%v", rejected, err)
|
||||
}
|
||||
checksum := func(entries []protocol.LogEntry) (string, error) { return fmt.Sprintf("sha256:%d", len(entries)), nil }
|
||||
next := validSpoolLogBatch(0, 0)
|
||||
sequence, appended, err := logSpool.EnqueueNextAggregated(context.Background(), next, nil, nil, checksum)
|
||||
if err != nil || !appended || sequence != 11 {
|
||||
t.Fatalf("expected allocation to resume after platform latest, sequence=%d appended=%t err=%v", sequence, appended, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogSpoolDoesNotExtendAcknowledgedSegment(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logSpool, err := NewLogSpool(root)
|
||||
if err != nil {
|
||||
t.Fatalf("new log spool: %v", err)
|
||||
}
|
||||
checksum := func(entries []protocol.LogEntry) (string, error) { return fmt.Sprintf("sha256:%d", len(entries)), nil }
|
||||
if err := logSpool.EnqueueAggregated(validSpoolLogBatch(1, 1), checksum); err != nil {
|
||||
t.Fatalf("enqueue acknowledged segment: %v", err)
|
||||
}
|
||||
logSpool.mu.Lock()
|
||||
watermark := logSpool.watermarks["log-1"]
|
||||
watermark.Acknowledged = 1
|
||||
logSpool.watermarks["log-1"] = watermark
|
||||
logSpool.mu.Unlock()
|
||||
if err := logSpool.EnqueueAggregated(validSpoolLogBatch(2, 2), checksum); err != nil {
|
||||
t.Fatalf("enqueue after acknowledged segment: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(root, "logs"))
|
||||
if err != nil || len(entries) != 2 {
|
||||
t.Fatalf("expected the acknowledged segment untouched and a new segment, entries=%+v err=%v", entries, err)
|
||||
}
|
||||
pending, err := logSpool.Pending()
|
||||
if err != nil || len(pending) != 2 || pending[0].FirstSeq != 1 || pending[0].LastSeq != 1 || pending[1].FirstSeq != 2 || pending[1].LastSeq != 2 {
|
||||
t.Fatalf("acknowledged segment was extended: pending=%+v err=%v", pending, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogSpoolRestoresPendingAllocationWithoutWatermark(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
first, err := NewLogSpool(root)
|
||||
|
||||
Reference in New Issue
Block a user