first commit

run/spool/README.md

@@ -0,0 +1,19 @@
+# run/spool
+
+Local durable queues live here.
+
+Required spool areas:
+
+- `logs`: unacknowledged log segments.
+- `jobs`: accepted job journal for duplicate detection and reconciliation.
+- `artifacts`: incomplete artifact transfer state.
+
+Spool pressure must be visible in run capacity reports.
+
+## Channel Isolation
+
+- Log and artifact retry state are stored in separate spool areas and are acknowledged independently.
+- Acknowledging a log batch must not scan, remove, or block on artifact chunks.
+- Acknowledging an artifact chunk must not scan, remove, or block on log batches.
+- Control heartbeat and job ack/result payloads remain metadata-only; they must never carry spool file paths, artifact chunks, log entries, raw credentials, direct sockets, or large inline bodies.
+- Artifact transfer backlog is lower priority than log flush, job lifecycle calls, and control heartbeat.

run/spool/artifact_queue.go

@@ -0,0 +1,125 @@
+package spool
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"browser.local/run/protocol"
+)
+
+type ArtifactQueue struct {
+	dir string
+}
+
+func NewArtifactQueue(dir string) (ArtifactQueue, error) {
+	if strings.TrimSpace(dir) == "" {
+		return ArtifactQueue{}, fmt.Errorf("spool directory is required")
+	}
+	artifactDir := filepath.Join(dir, "artifacts")
+	if err := os.MkdirAll(artifactDir, 0o755); err != nil {
+		return ArtifactQueue{}, fmt.Errorf("create artifact queue: %w", err)
+	}
+	return ArtifactQueue{dir: artifactDir}, nil
+}
+
+func (queue ArtifactQueue) Enqueue(chunk protocol.ArtifactChunkUploadRequest) error {
+	path := queue.chunkPath(chunk)
+	tmp := path + ".tmp"
+	file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("open artifact queue chunk: %w", err)
+	}
+	encodeErr := json.NewEncoder(file).Encode(chunk)
+	closeErr := file.Close()
+	if encodeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("encode artifact queue chunk: %w", encodeErr)
+	}
+	if closeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("close artifact queue chunk: %w", closeErr)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("commit artifact queue chunk: %w", err)
+	}
+	return nil
+}
+
+func (queue ArtifactQueue) Pending() ([]protocol.ArtifactChunkUploadRequest, error) {
+	entries, err := os.ReadDir(queue.dir)
+	if err != nil {
+		return nil, fmt.Errorf("read artifact queue: %w", err)
+	}
+	paths := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+			continue
+		}
+		paths = append(paths, filepath.Join(queue.dir, entry.Name()))
+	}
+	sort.Strings(paths)
+	chunks := make([]protocol.ArtifactChunkUploadRequest, 0, len(paths))
+	for _, path := range paths {
+		file, err := os.Open(path)
+		if err != nil {
+			return nil, fmt.Errorf("open artifact queue chunk: %w", err)
+		}
+		var chunk protocol.ArtifactChunkUploadRequest
+		decodeErr := json.NewDecoder(file).Decode(&chunk)
+		closeErr := file.Close()
+		if decodeErr != nil {
+			return nil, fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
+		}
+		if closeErr != nil {
+			return nil, fmt.Errorf("close artifact queue chunk: %w", closeErr)
+		}
+		chunks = append(chunks, chunk)
+	}
+	return chunks, nil
+}
+
+func (queue ArtifactQueue) Ack(response protocol.ArtifactChunkUploadResponse) error {
+	if !response.Accepted {
+		return nil
+	}
+	entries, err := os.ReadDir(queue.dir)
+	if err != nil {
+		return fmt.Errorf("read artifact queue: %w", err)
+	}
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+			continue
+		}
+		path := filepath.Join(queue.dir, entry.Name())
+		file, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("open artifact queue chunk: %w", err)
+		}
+		var chunk protocol.ArtifactChunkUploadRequest
+		decodeErr := json.NewDecoder(file).Decode(&chunk)
+		closeErr := file.Close()
+		if decodeErr != nil {
+			return fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
+		}
+		if closeErr != nil {
+			return fmt.Errorf("close artifact queue chunk: %w", closeErr)
+		}
+		if chunk.TransferID == response.TransferID && chunk.ArtifactID == response.ArtifactID && chunk.ChunkIndex == response.ChunkIndex {
+			if err := os.Remove(path); err != nil {
+				return fmt.Errorf("remove acknowledged artifact queue chunk: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func (queue ArtifactQueue) chunkPath(chunk protocol.ArtifactChunkUploadRequest) string {
+	transferID := sanitizeSegmentName(chunk.TransferID)
+	artifactID := sanitizeSegmentName(chunk.ArtifactID)
+	return filepath.Join(queue.dir, fmt.Sprintf("%s-%s-%020d.json", transferID, artifactID, chunk.ChunkIndex))
+}

run/spool/artifact_queue_test.go

@@ -0,0 +1,75 @@
+package spool
+
+import (
+	"testing"
+
+	"browser.local/run/protocol"
+)
+
+func TestArtifactQueueRetainsPendingAndRemovesAcknowledgedChunk(t *testing.T) {
+	queue, err := NewArtifactQueue(t.TempDir())
+	if err != nil {
+		t.Fatalf("new artifact queue: %v", err)
+	}
+	first := validQueuedArtifactChunk(0)
+	second := validQueuedArtifactChunk(1)
+	if err := queue.Enqueue(first); err != nil {
+		t.Fatalf("enqueue first: %v", err)
+	}
+	if err := queue.Enqueue(second); err != nil {
+		t.Fatalf("enqueue second: %v", err)
+	}
+	pending, err := queue.Pending()
+	if err != nil {
+		t.Fatalf("pending before ack: %v", err)
+	}
+	if len(pending) != 2 {
+		t.Fatalf("expected two pending chunks, got %+v", pending)
+	}
+
+	if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil {
+		t.Fatalf("ack first: %v", err)
+	}
+	pending, err = queue.Pending()
+	if err != nil {
+		t.Fatalf("pending after ack: %v", err)
+	}
+	if len(pending) != 1 || pending[0].ChunkIndex != 1 {
+		t.Fatalf("expected second chunk pending, got %+v", pending)
+	}
+}
+
+func TestArtifactQueueRetainsChunkWhenAckDoesNotMatch(t *testing.T) {
+	queue, err := NewArtifactQueue(t.TempDir())
+	if err != nil {
+		t.Fatalf("new artifact queue: %v", err)
+	}
+	if err := queue.Enqueue(validQueuedArtifactChunk(0)); err != nil {
+		t.Fatalf("enqueue: %v", err)
+	}
+	if err := queue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 1}); err != nil {
+		t.Fatalf("ack mismatch: %v", err)
+	}
+	pending, err := queue.Pending()
+	if err != nil {
+		t.Fatalf("pending: %v", err)
+	}
+	if len(pending) != 1 || pending[0].ChunkIndex != 0 {
+		t.Fatalf("expected original chunk pending, got %+v", pending)
+	}
+}
+
+func validQueuedArtifactChunk(index int) protocol.ArtifactChunkUploadRequest {
+	payload := []byte{byte(index), byte(index + 1)}
+	return protocol.ArtifactChunkUploadRequest{
+		RunEndpointID: "run-local",
+		SessionToken:  "session-token",
+		TransferID:    "transfer-1",
+		ArtifactID:    "artifact-1",
+		ChunkIndex:    index,
+		Offset:        int64(index * len(payload)),
+		SizeBytes:     len(payload),
+		Checksum:      "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+		Payload:       payload,
+	}
+}

run/spool/channel_isolation_test.go

@@ -0,0 +1,85 @@
+package spool
+
+import (
+	"testing"
+
+	"browser.local/run/protocol"
+)
+
+func TestLogSpoolAckIsIndependentFromArtifactBacklog(t *testing.T) {
+	root := t.TempDir()
+	logSpool, err := NewLogSpool(root)
+	if err != nil {
+		t.Fatalf("new log spool: %v", err)
+	}
+	artifactQueue, err := NewArtifactQueue(root)
+	if err != nil {
+		t.Fatalf("new artifact queue: %v", err)
+	}
+
+	if err := logSpool.Enqueue(validSpoolLogBatch(1, 1)); err != nil {
+		t.Fatalf("enqueue log: %v", err)
+	}
+	for i := 0; i < 3; i++ {
+		if err := artifactQueue.Enqueue(validQueuedArtifactChunk(i)); err != nil {
+			t.Fatalf("enqueue artifact chunk %d: %v", i, err)
+		}
+	}
+
+	if err := logSpool.Ack(protocol.LogBatchIngestResponse{Accepted: true, LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
+		t.Fatalf("ack log batch: %v", err)
+	}
+	logs, err := logSpool.Pending()
+	if err != nil {
+		t.Fatalf("pending logs: %v", err)
+	}
+	if len(logs) != 0 {
+		t.Fatalf("expected log batch removed despite artifact backlog, got %+v", logs)
+	}
+	chunks, err := artifactQueue.Pending()
+	if err != nil {
+		t.Fatalf("pending artifacts: %v", err)
+	}
+	if len(chunks) != 3 {
+		t.Fatalf("artifact backlog should remain independent, got %+v", chunks)
+	}
+}
+
+func TestArtifactAckIsIndependentFromLogBacklog(t *testing.T) {
+	root := t.TempDir()
+	logSpool, err := NewLogSpool(root)
+	if err != nil {
+		t.Fatalf("new log spool: %v", err)
+	}
+	artifactQueue, err := NewArtifactQueue(root)
+	if err != nil {
+		t.Fatalf("new artifact queue: %v", err)
+	}
+
+	for _, batch := range []protocol.LogBatchIngestRequest{validSpoolLogBatch(1, 1), validSpoolLogBatch(2, 2)} {
+		if err := logSpool.Enqueue(batch); err != nil {
+			t.Fatalf("enqueue log batch: %v", err)
+		}
+	}
+	if err := artifactQueue.Enqueue(validQueuedArtifactChunk(0)); err != nil {
+		t.Fatalf("enqueue artifact chunk: %v", err)
+	}
+
+	if err := artifactQueue.Ack(protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: "transfer-1", ArtifactID: "artifact-1", ChunkIndex: 0}); err != nil {
+		t.Fatalf("ack artifact chunk: %v", err)
+	}
+	chunks, err := artifactQueue.Pending()
+	if err != nil {
+		t.Fatalf("pending artifacts: %v", err)
+	}
+	if len(chunks) != 0 {
+		t.Fatalf("expected artifact chunk removed despite log backlog, got %+v", chunks)
+	}
+	logs, err := logSpool.Pending()
+	if err != nil {
+		t.Fatalf("pending logs: %v", err)
+	}
+	if len(logs) != 2 {
+		t.Fatalf("log backlog should remain independent, got %+v", logs)
+	}
+}

run/spool/log_spool.go

@@ -0,0 +1,136 @@
+package spool
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"browser.local/run/protocol"
+)
+
+type LogSpool struct {
+	dir string
+}
+
+func NewLogSpool(dir string) (LogSpool, error) {
+	if strings.TrimSpace(dir) == "" {
+		return LogSpool{}, fmt.Errorf("spool directory is required")
+	}
+	logDir := filepath.Join(dir, "logs")
+	if err := os.MkdirAll(logDir, 0o755); err != nil {
+		return LogSpool{}, fmt.Errorf("create log spool: %w", err)
+	}
+	return LogSpool{dir: logDir}, nil
+}
+
+func (spool LogSpool) Enqueue(batch protocol.LogBatchIngestRequest) error {
+	path := spool.batchPath(batch)
+	tmp := path + ".tmp"
+	file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("open log spool segment: %w", err)
+	}
+	encodeErr := json.NewEncoder(file).Encode(batch)
+	closeErr := file.Close()
+	if encodeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("encode log spool segment: %w", encodeErr)
+	}
+	if closeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("close log spool segment: %w", closeErr)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("commit log spool segment: %w", err)
+	}
+	return nil
+}
+
+func (spool LogSpool) Pending() ([]protocol.LogBatchIngestRequest, error) {
+	entries, err := os.ReadDir(spool.dir)
+	if err != nil {
+		return nil, fmt.Errorf("read log spool: %w", err)
+	}
+	paths := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+			continue
+		}
+		paths = append(paths, filepath.Join(spool.dir, entry.Name()))
+	}
+	sort.Strings(paths)
+	batches := make([]protocol.LogBatchIngestRequest, 0, len(paths))
+	for _, path := range paths {
+		file, err := os.Open(path)
+		if err != nil {
+			return nil, fmt.Errorf("open log spool segment: %w", err)
+		}
+		var batch protocol.LogBatchIngestRequest
+		decodeErr := json.NewDecoder(file).Decode(&batch)
+		closeErr := file.Close()
+		if decodeErr != nil {
+			return nil, fmt.Errorf("decode log spool segment: %w", decodeErr)
+		}
+		if closeErr != nil {
+			return nil, fmt.Errorf("close log spool segment: %w", closeErr)
+		}
+		batches = append(batches, batch)
+	}
+	return batches, nil
+}
+
+func (spool LogSpool) Ack(response protocol.LogBatchIngestResponse) error {
+	entries, err := os.ReadDir(spool.dir)
+	if err != nil {
+		return fmt.Errorf("read log spool: %w", err)
+	}
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+			continue
+		}
+		path := filepath.Join(spool.dir, entry.Name())
+		file, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("open log spool segment: %w", err)
+		}
+		var batch protocol.LogBatchIngestRequest
+		decodeErr := json.NewDecoder(file).Decode(&batch)
+		closeErr := file.Close()
+		if decodeErr != nil {
+			return fmt.Errorf("decode log spool segment: %w", decodeErr)
+		}
+		if closeErr != nil {
+			return fmt.Errorf("close log spool segment: %w", closeErr)
+		}
+		if batch.LogStreamID == response.LogStreamID && batch.FirstSeq >= response.AcceptedFrom && batch.LastSeq <= response.AcceptedTo {
+			if err := os.Remove(path); err != nil {
+				return fmt.Errorf("remove acknowledged log spool segment: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func (spool LogSpool) batchPath(batch protocol.LogBatchIngestRequest) string {
+	streamID := sanitizeSegmentName(batch.LogStreamID)
+	return filepath.Join(spool.dir, fmt.Sprintf("%s-%020d-%020d.json", streamID, batch.FirstSeq, batch.LastSeq))
+}
+
+func sanitizeSegmentName(value string) string {
+	var builder strings.Builder
+	for _, r := range value {
+		if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
+			builder.WriteRune(r)
+			continue
+		}
+		builder.WriteByte('_')
+	}
+	if builder.Len() == 0 {
+		return "stream"
+	}
+	return builder.String()
+}

run/spool/log_spool_test.go

@@ -0,0 +1,81 @@
+package spool
+
+import (
+	"testing"
+	"time"
+
+	"browser.local/run/protocol"
+)
+
+func TestLogSpoolRetainsPendingAndRemovesAcknowledgedBatch(t *testing.T) {
+	spool, err := NewLogSpool(t.TempDir())
+	if err != nil {
+		t.Fatalf("new log spool: %v", err)
+	}
+	first := validSpoolLogBatch(1, 2)
+	second := validSpoolLogBatch(3, 3)
+	if err := spool.Enqueue(first); err != nil {
+		t.Fatalf("enqueue first: %v", err)
+	}
+	if err := spool.Enqueue(second); err != nil {
+		t.Fatalf("enqueue second: %v", err)
+	}
+	pending, err := spool.Pending()
+	if err != nil {
+		t.Fatalf("pending before ack: %v", err)
+	}
+	if len(pending) != 2 {
+		t.Fatalf("expected two pending batches, got %+v", pending)
+	}
+
+	if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 2}); err != nil {
+		t.Fatalf("ack first: %v", err)
+	}
+	pending, err = spool.Pending()
+	if err != nil {
+		t.Fatalf("pending after ack: %v", err)
+	}
+	if len(pending) != 1 || pending[0].FirstSeq != 3 {
+		t.Fatalf("expected second batch pending, got %+v", pending)
+	}
+}
+
+func TestLogSpoolRetainsBatchWhenAckDoesNotCoverRange(t *testing.T) {
+	spool, err := NewLogSpool(t.TempDir())
+	if err != nil {
+		t.Fatalf("new log spool: %v", err)
+	}
+	if err := spool.Enqueue(validSpoolLogBatch(1, 2)); err != nil {
+		t.Fatalf("enqueue: %v", err)
+	}
+	if err := spool.Ack(protocol.LogBatchIngestResponse{LogStreamID: "log-1", AcceptedFrom: 1, AcceptedTo: 1}); err != nil {
+		t.Fatalf("partial ack: %v", err)
+	}
+	pending, err := spool.Pending()
+	if err != nil {
+		t.Fatalf("pending: %v", err)
+	}
+	if len(pending) != 1 {
+		t.Fatalf("expected batch to remain pending, got %+v", pending)
+	}
+}
+
+func validSpoolLogBatch(firstSeq uint64, lastSeq uint64) protocol.LogBatchIngestRequest {
+	entries := make([]protocol.LogEntry, 0, lastSeq-firstSeq+1)
+	for seq := firstSeq; seq <= lastSeq; seq++ {
+		entries = append(entries, protocol.LogEntry{Seq: seq, Timestamp: time.Date(2026, 7, 3, 12, 0, int(seq), 0, time.UTC), Level: "info", Line: "line"})
+	}
+	return protocol.LogBatchIngestRequest{
+		RunEndpointID:    "run-local",
+		SessionToken:     "session-token",
+		LogStreamID:      "log-1",
+		ServerInstanceID: "server-1",
+		StreamKey:        "stdout",
+		Source:           "process",
+		FirstSeq:         firstSeq,
+		LastSeq:          lastSeq,
+		Compression:      "none",
+		Checksum:         "sha256:test",
+		Entries:          entries,
+	}
+}