init

runtime/log_sources.go

@@ -0,0 +1,118 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+
+	"browser.local/run/protocol"
+)
+
+type LogSourceCheckpoint struct {
+	SourceKey string
+	Offset    int64
+	Sequence  uint64
+	CursorRef string
+}
+
+type LogCheckpointStore interface {
+	GetLogCheckpoint(sourceKey string) LogSourceCheckpoint
+	PutLogCheckpoint(checkpoint LogSourceCheckpoint)
+}
+
+type MemoryLogCheckpointStore struct {
+	checkpoints map[string]LogSourceCheckpoint
+}
+
+func NewMemoryLogCheckpointStore() *MemoryLogCheckpointStore {
+	return &MemoryLogCheckpointStore{checkpoints: map[string]LogSourceCheckpoint{}}
+}
+
+func (store *MemoryLogCheckpointStore) GetLogCheckpoint(sourceKey string) LogSourceCheckpoint {
+	if store == nil || store.checkpoints == nil {
+		return LogSourceCheckpoint{SourceKey: sourceKey}
+	}
+	return store.checkpoints[sourceKey]
+}
+
+func (store *MemoryLogCheckpointStore) PutLogCheckpoint(checkpoint LogSourceCheckpoint) {
+	if store == nil {
+		return
+	}
+	if store.checkpoints == nil {
+		store.checkpoints = map[string]LogSourceCheckpoint{}
+	}
+	store.checkpoints[checkpoint.SourceKey] = checkpoint
+}
+
+func TailDeclaredFileLogSource(ctx context.Context, workspaceRoot string, assignment protocol.RunJobAssignment, source RuntimeLogSource, sink ProcessLogSink, store LogCheckpointStore) LifecycleExecutionResult {
+	if source.Kind != "file.tail" {
+		return lifecycleFailure("unsupported_log_source", "only file.tail sources are supported by the local tailer")
+	}
+	if !protocol.ValidLogicalFileKey(source.Key) || !protocol.ValidLogicalFileKey(source.TargetKey) || !protocol.ValidLogicalFileKey(source.StreamKey) {
+		return lifecycleFailure("unsafe_log_source", "log source is unsafe")
+	}
+	if sink == nil {
+		sink = NoopProcessLogSink{}
+	}
+	if store == nil {
+		store = NewMemoryLogCheckpointStore()
+	}
+	serverRoot, err := scopedServerWorkspace(workspaceRoot, assignment.ServerInstanceID)
+	if err != nil {
+		return lifecycleFailure("unsafe_log_workspace", err.Error())
+	}
+	path, err := scopedPath(serverRoot, source.TargetKey)
+	if err != nil {
+		return lifecycleFailure("unsafe_log_source", err.Error())
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return lifecycleFailure("log_source_open_failed", err.Error())
+	}
+	defer file.Close()
+
+	checkpoint := store.GetLogCheckpoint(source.Key)
+	if checkpoint.Offset > 0 {
+		if _, err := file.Seek(checkpoint.Offset, 0); err != nil {
+			return lifecycleFailure("log_source_seek_failed", err.Error())
+		}
+	}
+	body := make([]byte, maxLifecycleOutputBytes)
+	n, err := file.Read(body)
+	if err != nil && n == 0 {
+		return LifecycleExecutionResult{
+			State:     lifecycleResultStateSucceeded,
+			Progress:  protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint unchanged"},
+			ResultRef: fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID)),
+			Message:   "live log source had no new lines",
+		}
+	}
+	for _, line := range splitBoundedLines(string(body[:n])) {
+		checkpoint.Sequence++
+		if err := sink.Append(ctx, assignment, source.StreamKey, line); err != nil {
+			return lifecycleFailure("log_source_sink_failed", err.Error())
+		}
+	}
+	checkpoint.SourceKey = source.Key
+	checkpoint.Offset += int64(n)
+	checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID))
+	store.PutLogCheckpoint(checkpoint)
+	return LifecycleExecutionResult{
+		State:     lifecycleResultStateSucceeded,
+		Progress:  protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint updated"},
+		ResultRef: checkpoint.CursorRef,
+		Message:   "live log source tailed with durable offset checkpoint",
+	}
+}
+
+func RedactedLogCheckpointSummary(checkpoint LogSourceCheckpoint) string {
+	return strings.Join([]string{
+		"source=" + checkpoint.SourceKey,
+		fmt.Sprintf("offset=%d", checkpoint.Offset),
+		fmt.Sprintf("sequence=%d", checkpoint.Sequence),
+		"cursorRef=" + checkpoint.CursorRef,
+	}, " ")
+}