Keep run logs opaque and streamline transfers

spool/artifact_queue.go

@@ -1,6 +1,7 @@
 package spool
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -37,28 +38,20 @@ func (queue ArtifactQueue) Enqueue(chunk protocol.ArtifactChunkUploadRequest) er
 	} else if !os.IsNotExist(err) {
 		return err
 	}
-	tmp := path + ".tmp"
-	file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
-	if err != nil {
-		return fmt.Errorf("open artifact queue chunk: %w", err)
-	}
-	encodeErr := json.NewEncoder(file).Encode(chunk)
-	closeErr := file.Close()
-	if encodeErr != nil {
-		_ = os.Remove(tmp)
-		return fmt.Errorf("encode artifact queue chunk: %w", encodeErr)
-	}
-	if closeErr != nil {
-		_ = os.Remove(tmp)
-		return fmt.Errorf("close artifact queue chunk: %w", closeErr)
-	}
-	if err := syncFile(tmp); err != nil {
-		_ = os.Remove(tmp)
+	payloadPath := artifactChunkPayloadPath(path)
+	if err := writeArtifactQueueFile(payloadPath, chunk.Payload); err != nil {
 		return err
 	}
-	if err := os.Rename(tmp, path); err != nil {
-		_ = os.Remove(tmp)
-		return fmt.Errorf("commit artifact queue chunk: %w", err)
+	metadata := chunk
+	metadata.Payload = nil
+	var body bytes.Buffer
+	if err := json.NewEncoder(&body).Encode(metadata); err != nil {
+		_ = os.Remove(payloadPath)
+		return fmt.Errorf("encode artifact queue chunk metadata: %w", err)
+	}
+	if err := writeArtifactQueueFile(path, body.Bytes()); err != nil {
+		_ = os.Remove(payloadPath)
+		return err
 	}
 	return nil
 }
@@ -105,6 +98,11 @@ func readArtifactChunk(path string) (protocol.ArtifactChunkUploadRequest, error)
 	if err := json.NewDecoder(file).Decode(&chunk); err != nil {
 		return protocol.ArtifactChunkUploadRequest{}, fmt.Errorf("decode artifact queue chunk: %w", err)
 	}
+	payload, err := os.ReadFile(artifactChunkPayloadPath(path))
+	if err != nil {
+		return protocol.ArtifactChunkUploadRequest{}, fmt.Errorf("read artifact queue chunk payload: %w", err)
+	}
+	chunk.Payload = payload
 	return chunk, nil
 }
 
@@ -123,18 +121,9 @@ func (queue ArtifactQueue) Pending() ([]protocol.ArtifactChunkUploadRequest, err
 	sort.Strings(paths)
 	chunks := make([]protocol.ArtifactChunkUploadRequest, 0, len(paths))
 	for _, path := range paths {
-		file, err := os.Open(path)
+		chunk, err := readArtifactChunk(path)
 		if err != nil {
-			return nil, fmt.Errorf("open artifact queue chunk: %w", err)
-		}
-		var chunk protocol.ArtifactChunkUploadRequest
-		decodeErr := json.NewDecoder(file).Decode(&chunk)
-		closeErr := file.Close()
-		if decodeErr != nil {
-			return nil, fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
-		}
-		if closeErr != nil {
-			return nil, fmt.Errorf("close artifact queue chunk: %w", closeErr)
+			return nil, err
 		}
 		chunks = append(chunks, chunk)
 	}
@@ -154,23 +143,17 @@ func (queue ArtifactQueue) Ack(response protocol.ArtifactChunkUploadResponse) er
 			continue
 		}
 		path := filepath.Join(queue.dir, entry.Name())
-		file, err := os.Open(path)
+		chunk, err := readArtifactChunk(path)
 		if err != nil {
-			return fmt.Errorf("open artifact queue chunk: %w", err)
-		}
-		var chunk protocol.ArtifactChunkUploadRequest
-		decodeErr := json.NewDecoder(file).Decode(&chunk)
-		closeErr := file.Close()
-		if decodeErr != nil {
-			return fmt.Errorf("decode artifact queue chunk: %w", decodeErr)
-		}
-		if closeErr != nil {
-			return fmt.Errorf("close artifact queue chunk: %w", closeErr)
+			return err
 		}
 		if chunk.TransferID == response.TransferID && chunk.ArtifactID == response.ArtifactID && chunk.ChunkIndex == response.ChunkIndex {
 			if err := os.Remove(path); err != nil {
 				return fmt.Errorf("remove acknowledged artifact queue chunk: %w", err)
 			}
+			if err := os.Remove(artifactChunkPayloadPath(path)); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("remove acknowledged artifact queue payload: %w", err)
+			}
 		}
 	}
 	return nil
@@ -181,3 +164,34 @@ func (queue ArtifactQueue) chunkPath(chunk protocol.ArtifactChunkUploadRequest)
 	artifactID := sanitizeSegmentName(chunk.ArtifactID)
 	return filepath.Join(queue.dir, fmt.Sprintf("%s-%s-%020d.json", transferID, artifactID, chunk.ChunkIndex))
 }
+
+func artifactChunkPayloadPath(metadataPath string) string {
+	return strings.TrimSuffix(metadataPath, ".json") + ".bin"
+}
+
+func writeArtifactQueueFile(path string, payload []byte) error {
+	tmp := path + ".tmp"
+	file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return fmt.Errorf("open artifact queue file: %w", err)
+	}
+	_, writeErr := file.Write(payload)
+	closeErr := file.Close()
+	if writeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("write artifact queue file: %w", writeErr)
+	}
+	if closeErr != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("close artifact queue file: %w", closeErr)
+	}
+	if err := syncFile(tmp); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("commit artifact queue file: %w", err)
+	}
+	return nil
+}

spool/artifact_queue_test.go

@@ -1,6 +1,9 @@
 package spool
 
 import (
+	"bytes"
+	"os"
+	"strings"
 	"testing"
 
 	"browser.local/run/protocol"
@@ -16,6 +19,17 @@ func TestArtifactQueueRetainsPendingAndRemovesAcknowledgedChunk(t *testing.T) {
 	if err := queue.Enqueue(first); err != nil {
 		t.Fatalf("enqueue first: %v", err)
 	}
+	metadata, err := os.ReadFile(queue.chunkPath(first))
+	if err != nil {
+		t.Fatalf("read queued metadata: %v", err)
+	}
+	if strings.Contains(string(metadata), "payload") {
+		t.Fatalf("queued artifact metadata must not JSON encode payload: %s", string(metadata))
+	}
+	storedPayload, err := os.ReadFile(artifactChunkPayloadPath(queue.chunkPath(first)))
+	if err != nil || !bytes.Equal(storedPayload, first.Payload) {
+		t.Fatalf("queued raw payload mismatch payload=%q err=%v", storedPayload, err)
+	}
 	if err := queue.Enqueue(second); err != nil {
 		t.Fatalf("enqueue second: %v", err)
 	}