Add runtime SQLite query targets

This commit is contained in:
npc0-hue
2026-09-01 17:15:09 +08:00
parent 65da73eff0
commit 163fbda394
17 changed files with 708 additions and 114 deletions
+45
View File
@@ -88,6 +88,51 @@ func (worker *Worker) materializeSQLiteProbeDataTarget(ctx context.Context, assi
return matched.WorkspaceKey, nil
}
func (worker *Worker) materializeSQLiteQueryDataTarget(ctx context.Context, assignment protocol.RunJobAssignment) (string, error) {
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
if err != nil {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_plan_invalid", false)
}
if !ok || plan == nil || len(plan.DataTargets) == 0 {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_not_declared", false)
}
if plan.ServerInstanceID != assignment.ServerInstanceID || plan.RunEndpointID != assignment.RunEndpointID || plan.PluginID != assignment.ExecutionInput.PluginID {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
}
if plan.ProfileKey != "" && assignment.ExecutionInput.WorkspaceScope != "" && plan.ProfileKey != assignment.ExecutionInput.WorkspaceScope {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
}
var matched *protocol.RunAutonomousDataTarget
for i := range plan.DataTargets {
target := &plan.DataTargets[i]
if dataTargetMatchesSQLiteProbeAssignment(*target, assignment.TargetKey) {
matched = target
break
}
}
if matched == nil {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_not_declared", false)
}
if !dataTargetSupportsPlatform(matched.Platforms, runtime.GOOS) {
return assignment.TargetKey, newDataTargetMaterializeError("data_target_platform_unsupported", false)
}
manifest, err := materializeSQLiteSnapshotDataTarget(ctx, worker.cfg.WorkspaceRoot, assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope, *matched, plan.RuntimeBindings)
if err != nil {
log.Printf("RUN phase=data_target.snapshot status=failed target=%s code=%s", safeOptional(matched.Key), safeOptional(dataTargetErrorCode(err)))
return assignment.TargetKey, err
}
log.Printf("RUN phase=data_target.snapshot status=complete target=%s workspaceKey=%s bytes=%d checksum=%s", safeOptional(matched.Key), safeOptional(matched.WorkspaceKey), manifest.SnapshotSizeBytes, safeOptional(manifest.SnapshotChecksum))
return matched.WorkspaceKey, nil
}
func sqliteQueryFailureForDataTarget(err error) LifecycleExecutionResult {
var materializeErr dataTargetMaterializeError
if errors.As(err, &materializeErr) {
return lifecycleExecutionFailure(materializeErr.code, "SQLite query data target is unavailable", materializeErr.retryable)
}
return lifecycleExecutionFailure("data_target_unavailable", "SQLite query data target is unavailable", true)
}
func dataTargetMatchesSQLiteProbeAssignment(target protocol.RunAutonomousDataTarget, assignmentTargetKey string) bool {
if target.Kind != "sqlite.snapshot" || assignmentTargetKey == "" {
return false
+82
View File
@@ -93,6 +93,88 @@ func TestWorkerMaterializesMatchingDataTargetBeforeSQLiteProbe(t *testing.T) {
}
}
func TestWorkerMaterializesMatchingDataTargetBeforeSQLiteQuery(t *testing.T) {
client := newFakeWorkerClient()
cfg := workerTestConfig(t)
cfg.ServerInstanceID = "server-data-query"
cfg.PluginID = "game.example"
cfg.ComponentKind = config.PackageComponentRun
cfg.ComponentKey = "run-local"
sourceRoot := t.TempDir()
createSQLiteProbeFixture(t, filepath.Join(sourceRoot, "Saved", "SaveFiles", "current.db"))
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
SchemaVersion: "1",
ServerInstanceID: cfg.ServerInstanceID,
PluginID: cfg.PluginID,
PluginVersion: "1.0.0",
RunEndpointID: cfg.RunEndpointID,
ProfileKey: cfg.ComponentKey,
TargetOS: runtime.GOOS,
TargetArch: runtime.GOARCH,
TargetRelease: "run-dist-test",
Bootstrap: &protocol.RunAutonomousLifecycleAction{Action: "start", Operation: "start", Capability: protocol.RunCapabilityProcessStart, TargetKey: "actions/start.json"},
DataTargets: []protocol.RunAutonomousDataTarget{{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024, Platforms: []string{runtime.GOOS}}},
RuntimeBindings: map[string]string{"server-root": sourceRoot},
})
scope, err := NewWorkspaceResolver(cfg.WorkspaceRoot).Scope(cfg.ServerInstanceID, cfg.ComponentKey)
if err != nil {
t.Fatalf("create package scope: %v", err)
}
writeSQLiteQueryAsset(t, scope, "sql/members.sql", "SELECT CAST(id AS TEXT) AS memberId, name AS displayName FROM members ORDER BY id LIMIT :limit")
worker, err := NewWorker(cfg, client)
if err != nil {
t.Fatalf("new worker: %v", err)
}
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
assignment.ServerInstanceID = cfg.ServerInstanceID
assignment.RunEndpointID = cfg.RunEndpointID
assignment.TargetKey = "current-db"
assignment.InputRef = "input://plugin-query-poll/server-data-query/members"
assignment.ExecutionInput.WorkspaceScope = cfg.ComponentKey
assignment.ExecutionInput.PluginID = cfg.PluginID
assignment.ExecutionInput.RemoteAdapterKey = "current-db"
assignment.ExecutionInput.RemoteAdapterKind = "database"
assignment.ExecutionInput.TimeoutSeconds = 10
assignment.ExecutionInput.Inputs = map[string]string{"sqlRef": "sql/members.sql", "maxRows": "2"}
result := worker.executeAssignment(context.Background(), assignment)
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "sqlite.query" || !strings.Contains(result.ExecutionResult.Content, `"displayName":"alpha"`) {
serialized, _ := json.Marshal(result)
t.Fatalf("expected worker query to materialize and return rows, got %s", serialized)
}
}
func TestWorkerSQLiteQueryRequiresDeclaredDataTarget(t *testing.T) {
client := newFakeWorkerClient()
cfg := workerTestConfig(t)
cfg.ServerInstanceID = "server-data-query-missing"
cfg.PluginID = "game.example"
cfg.ComponentKind = config.PackageComponentRun
cfg.ComponentKey = "run-local"
writeAutonomousPlanFixture(t, cfg.WorkspaceRoot, cfg.ServerInstanceID, cfg.ComponentKey, protocol.RunAutonomousLifecyclePlan{
SchemaVersion: "1", ServerInstanceID: cfg.ServerInstanceID, PluginID: cfg.PluginID, PluginVersion: "1.0.0", RunEndpointID: cfg.RunEndpointID, ProfileKey: cfg.ComponentKey,
TargetOS: runtime.GOOS, TargetArch: runtime.GOARCH, TargetRelease: "run-dist-test",
})
worker, err := NewWorker(cfg, client)
if err != nil {
t.Fatalf("new worker: %v", err)
}
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
assignment.ServerInstanceID = cfg.ServerInstanceID
assignment.RunEndpointID = cfg.RunEndpointID
assignment.TargetKey = "current-db"
assignment.InputRef = "input://plugin-query-poll/server-data-query-missing/members"
assignment.ExecutionInput.WorkspaceScope = cfg.ComponentKey
assignment.ExecutionInput.PluginID = cfg.PluginID
assignment.ExecutionInput.RemoteAdapterKey = "current-db"
assignment.ExecutionInput.RemoteAdapterKind = "database"
assignment.ExecutionInput.Inputs = map[string]string{"sqlRef": "sql/members.sql", "maxRows": "2"}
if result := worker.executeAssignment(context.Background(), assignment); result.ErrorCode != "data_target_not_declared" {
t.Fatalf("expected missing target failure, got %+v", result)
}
}
func TestMaterializeSQLiteSnapshotDataTargetRejectsUnboundSourceRoot(t *testing.T) {
target := protocol.RunAutonomousDataTarget{Key: "current-db", Kind: "sqlite.snapshot", TransportKey: "current-db", SourceRootKey: "server-root", SourcePath: "Saved/SaveFiles/current.db", WorkspaceKey: "databases/current-db", RefreshPolicy: "on-demand-snapshot", MaxBytes: 128 * 1024 * 1024}
_, err := materializeSQLiteSnapshotDataTarget(context.Background(), t.TempDir(), "server-data", "run-local", target, map[string]string{})
+40 -13
View File
@@ -67,6 +67,7 @@ type LifecycleExecutor struct {
localStartupDiagnostics bool
protectedRequests *ProtectedRequestRegistry
sqliteSchemaProbe *SQLiteSchemaProbeExecutor
sqliteQuery *SQLiteQueryExecutor
metricCollector MetricCollector
}
@@ -118,6 +119,7 @@ func NewLifecycleExecutor(options ...LifecycleExecutorOption) LifecycleExecutor
executor.fileExecutor = files
}
executor.sqliteSchemaProbe = NewSQLiteSchemaProbeExecutor(executor.workspaceRoot)
executor.sqliteQuery = NewSQLiteQueryExecutor(executor.workspaceRoot)
return executor
}
@@ -389,6 +391,9 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
}
log.Printf("RUN phase=lifecycle.command status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
command.OutputLine = func(stream string, line string) {
_ = executor.logSink.Append(ctx, assignment, stream, line)
}
result, err := executor.supervisor.Run(ctx, command)
if err != nil && ctx.Err() != nil {
log.Printf("RUN phase=lifecycle.command status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error()))
@@ -399,7 +404,9 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
ErrorCode: "lifecycle_cancelled",
}
}
executor.writeProcessLogs(ctx, assignment, result)
if !result.OutputRelayed {
executor.writeProcessLogs(ctx, assignment, result)
}
if err != nil {
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error()))
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
@@ -519,7 +526,7 @@ func (executor LifecycleExecutor) writeProcessLogs(ctx context.Context, assignme
{stream: "stdout", body: result.Stdout},
{stream: "stderr", body: result.Stderr},
} {
for _, line := range splitBoundedLines(item.body) {
for _, line := range splitRawLogLines(item.body) {
_ = executor.logSink.Append(ctx, assignment, item.stream, line)
}
}
@@ -919,12 +926,14 @@ type ProcessCommand struct {
JobID string
Capability string
Action string
OutputLine func(string, string)
}
type ProcessResult struct {
ExitCode int
Stdout string
Stderr string
ExitCode int
Stdout string
Stderr string
OutputRelayed bool
}
type ProcessSupervisor interface {
@@ -969,7 +978,7 @@ func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCo
err := cmd.Wait()
stdoutWriter.Flush()
stderrWriter.Flush()
result := ProcessResult{Stdout: RedactText(stdout.String()), Stderr: RedactText(stderr.String())}
result := ProcessResult{Stdout: stdout.String(), Stderr: stderr.String(), OutputRelayed: command.OutputLine != nil}
if cmd.ProcessState != nil {
result.ExitCode = cmd.ProcessState.ExitCode()
}
@@ -1011,7 +1020,7 @@ func (writer *lifecycleOutputWriter) Write(p []byte) (int, error) {
if index < 0 {
break
}
line := strings.TrimRight(writer.pending[:index], "\r")
line := writer.pending[:index]
writer.pending = writer.pending[index+1:]
writer.logLine(line)
}
@@ -1021,19 +1030,17 @@ func (writer *lifecycleOutputWriter) Write(p []byte) (int, error) {
func (writer *lifecycleOutputWriter) Flush() {
writer.mu.Lock()
defer writer.mu.Unlock()
if strings.TrimSpace(writer.pending) == "" {
writer.pending = ""
if writer.pending == "" {
return
}
writer.logLine(strings.TrimRight(writer.pending, "\r"))
writer.logLine(writer.pending)
writer.pending = ""
}
func (writer *lifecycleOutputWriter) logLine(line string) {
if strings.TrimSpace(line) == "" {
return
if writer.command.OutputLine != nil {
writer.command.OutputLine(writer.stream, line)
}
log.Printf("RUN phase=process.command.output status=line job=%s capability=%s action=%s stream=%s line=%q", safeOptional(writer.command.JobID), safeOptional(writer.command.Capability), safeOptional(writer.command.Action), writer.stream, RedactText(line))
}
type ioLimitWriter struct {
@@ -1242,6 +1249,26 @@ func splitBoundedLines(value string) []string {
return out
}
// splitRawLogLines only removes the newline framing used by LogEntry. It
// deliberately preserves every other byte, including blank lines and spaces.
func splitRawLogLines(value string) []string {
if value == "" {
return nil
}
lines := make([]string, 0, strings.Count(value, "\n")+1)
for start := 0; start < len(value); {
end := strings.IndexByte(value[start:], '\n')
if end < 0 {
lines = append(lines, value[start:])
break
}
end += start
lines = append(lines, value[start:end])
start = end + 1
}
return lines
}
func checksumForText(value string) string {
sum := sha256.Sum256([]byte(value))
return "sha256:" + hex.EncodeToString(sum[:])
+49 -3
View File
@@ -452,7 +452,7 @@ func TestResolveRuntimeProfilesSupportsDeclaredModesAndSafeMissingKeys(t *testin
}
}
func TestTailDeclaredFileLogSourceUsesCheckpointAndRedaction(t *testing.T) {
func TestTailDeclaredFileLogSourceUsesCheckpointAndVerbatimOutput(t *testing.T) {
root := t.TempDir()
assignment := lifecycleAssignment(protocol.RunCapabilityLogsRead)
serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs")
@@ -472,8 +472,8 @@ func TestTailDeclaredFileLogSourceUsesCheckpointAndRedaction(t *testing.T) {
if result.State != "succeeded" || !strings.Contains(result.ResultRef, "live-log-checkpoint") {
t.Fatalf("expected file tail success, got %+v", result)
}
if len(sink.lines) != 2 || strings.Contains(strings.Join(sink.lines, "\n"), "password=hidden") {
t.Fatalf("expected redacted tailed lines, got %+v", sink.lines)
if len(sink.lines) != 2 || strings.Join(sink.lines, "\n") != "latest-log:first line\nlatest-log:password=hidden" {
t.Fatalf("expected verbatim tailed lines, got %+v", sink.lines)
}
checkpoint := store.GetLogCheckpoint("latest-log")
if checkpoint.Offset == 0 || checkpoint.Sequence != 2 || strings.Contains(RedactedLogCheckpointSummary(checkpoint), "/Users/") {
@@ -490,6 +490,36 @@ func TestTailDeclaredFileLogSourceUsesCheckpointAndRedaction(t *testing.T) {
}
}
func TestTailDeclaredFileLogSourceDoesNotLimitOrRewriteOutput(t *testing.T) {
root := t.TempDir()
assignment := lifecycleAssignment(protocol.RunCapabilityLogsRead)
serverRoot := filepath.Join(root, assignment.ServerInstanceID, "logs")
if err := os.MkdirAll(serverRoot, 0o755); err != nil {
t.Fatalf("create logs dir: %v", err)
}
longLine := "password=visible\r" + strings.Repeat("x", maxLifecycleOutputBytes+1)
logPath := filepath.Join(serverRoot, "current.log")
if err := os.WriteFile(logPath, []byte(longLine+"\n\nfinal"), 0o644); err != nil {
t.Fatalf("write log file: %v", err)
}
store := NewMemoryLogCheckpointStore()
sink := &recordingLogSink{}
source := RuntimeLogSource{Key: "current-log", Kind: "file.tail", TargetKey: "logs/current.log", StreamKey: "current-log", CursorKind: "offset"}
result := TailDeclaredFileLogSource(context.Background(), root, assignment, source, sink, store)
if result.State != lifecycleResultStateSucceeded || len(sink.lines) != 3 {
t.Fatalf("expected all unbounded log entries, result=%+v lines=%d", result, len(sink.lines))
}
if sink.lines[0] != "current-log:"+longLine || sink.lines[1] != "current-log:" || sink.lines[2] != "current-log:final" {
t.Fatalf("expected byte-for-byte log payloads, got lengths=%d,%d,%d", len(sink.lines[0]), len(sink.lines[1]), len(sink.lines[2]))
}
checkpoint := store.GetLogCheckpoint(source.Key)
if checkpoint.Offset != int64(len(longLine+"\n\nfinal")) || checkpoint.Sequence != 3 {
t.Fatalf("expected complete checkpoint after unbounded tail, got %+v", checkpoint)
}
}
func TestLifecycleExecutorExecutesDeclaredLogBackfillTail(t *testing.T) {
root := t.TempDir()
assignment := lifecycleAssignment(protocol.RunCapabilityLogsBackfill)
@@ -523,6 +553,22 @@ func (supervisor *recordingSupervisor) Run(_ context.Context, command ProcessCom
return ProcessResult{ExitCode: 0, Stdout: "recorded\n"}, nil
}
func TestOSProcessSupervisorRelaysRawCarriageReturn(t *testing.T) {
var output []string
result, err := (OSProcessSupervisor{}).Run(context.Background(), ProcessCommand{
Args: []string{"sh", "-c", "printf 'raw\\r\\n'"},
OutputLine: func(stream string, line string) {
output = append(output, stream+":"+line)
},
})
if err != nil || result.ExitCode != 0 {
t.Fatalf("run process: result=%+v err=%v", result, err)
}
if len(output) != 1 || output[0] != "stdout:raw\r" {
t.Fatalf("expected raw CR to be relayed, got %+v", output)
}
}
type recordingLogSink struct {
mu sync.Mutex
lines []string
+25 -17
View File
@@ -1,8 +1,10 @@
package runtime
import (
"bufio"
"context"
"fmt"
"io"
"net/url"
"os"
"strings"
@@ -80,26 +82,32 @@ func TailDeclaredFileLogSource(ctx context.Context, workspaceRoot string, assign
return lifecycleFailure("log_source_seek_failed", err.Error())
}
}
body := make([]byte, maxLifecycleOutputBytes)
n, err := file.Read(body)
if err != nil && n == 0 {
return LifecycleExecutionResult{
State: lifecycleResultStateSucceeded,
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint unchanged"},
ResultRef: fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID)),
Message: "live log source had no new lines",
reader := bufio.NewReader(file)
for {
// A newline only frames an entry. Every other byte, including CR, blank
// lines, and arbitrarily long output, remains untouched.
line, readErr := reader.ReadString('\n')
if len(line) > 0 {
checkpoint.Sequence++
if err := sink.Append(ctx, assignment, source.StreamKey, strings.TrimSuffix(line, "\n")); err != nil {
return lifecycleFailure("log_source_sink_failed", err.Error())
}
checkpoint.SourceKey = source.Key
checkpoint.Offset += int64(len(line))
checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID))
store.PutLogCheckpoint(checkpoint)
}
}
for _, line := range splitBoundedLines(string(body[:n])) {
checkpoint.Sequence++
if err := sink.Append(ctx, assignment, source.StreamKey, line); err != nil {
return lifecycleFailure("log_source_sink_failed", err.Error())
if readErr == nil {
continue
}
if readErr == io.EOF {
break
}
return lifecycleFailure("log_source_read_failed", readErr.Error())
}
if checkpoint.CursorRef == "" {
checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID))
}
checkpoint.SourceKey = source.Key
checkpoint.Offset += int64(n)
checkpoint.CursorRef = fmt.Sprintf("artifact://jobs/%s/live-log-checkpoint", url.PathEscape(assignment.JobID))
store.PutLogCheckpoint(checkpoint)
return LifecycleExecutionResult{
State: lifecycleResultStateSucceeded,
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "live log checkpoint updated"},
+11 -14
View File
@@ -535,20 +535,17 @@ func (supervisor *OSManagedProcessSupervisor) tailOutput(ctx context.Context, ta
if len(line) > 0 {
startOffset := offset
endOffset := offset + int64(len(line))
text := strings.TrimSpace(line)
if text != "" {
for {
if sinkErr := sink(identity, ManagedProcessLine{Text: text, StartOffset: startOffset, EndOffset: endOffset}); sinkErr == nil {
log.Printf("RUN phase=process.managed.output status=line job=%s pid=%d stream=%s line=%q", safeOptional(identity.JobID), identity.PID, stream, RedactText(text))
break
} else {
log.Printf("RUN phase=process.managed.output status=sink_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(sinkErr.Error()))
}
select {
case <-ctx.Done():
return
case <-time.After(managedProcessOutputRetryDelay):
}
text := strings.TrimSuffix(line, "\n")
for {
if sinkErr := sink(identity, ManagedProcessLine{Text: text, StartOffset: startOffset, EndOffset: endOffset}); sinkErr == nil {
break
} else {
log.Printf("RUN phase=process.managed.output status=sink_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(sinkErr.Error()))
}
select {
case <-ctx.Done():
return
case <-time.After(managedProcessOutputRetryDelay):
}
}
for {
+2 -38
View File
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"
"time"
@@ -154,48 +153,13 @@ func validProtectedRequestKind(kind string) bool {
func (executor LifecycleExecutor) writeProtectedProgramLogs(ctx context.Context, assignment protocol.RunJobAssignment, outcome ProtectedRequestOutcome) {
for _, item := range []struct{ stream, body string }{{"management-program.stdout", outcome.Stdout}, {"management-program.stderr", outcome.Stderr}} {
for _, line := range splitProtectedProgramLines(item.body) {
_ = executor.logSink.Append(ctx, assignment, item.stream, sanitizeProtectedProgramLogLine(line))
_ = executor.logSink.Append(ctx, assignment, item.stream, line)
}
}
}
func splitProtectedProgramLines(value string) []string {
value = strings.ToValidUTF8(strings.TrimSpace(value), "")
if len(value) > maxLifecycleOutputBytes {
value = strings.ToValidUTF8(value[:maxLifecycleOutputBytes], "")
}
lines := strings.Split(value, "\n")
bounded := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimRight(line, "\r")
if strings.TrimSpace(line) != "" {
bounded = append(bounded, line)
}
}
return bounded
}
func sanitizeProtectedProgramLogLine(line string) string {
if containsProtectedProgramPrivateText(line) {
return "[redacted protected management-program output]"
}
return RedactText(line)
}
func containsProtectedProgramPrivateText(value string) bool {
lower := strings.ToLower(value)
for _, marker := range []string{"password=", "password:", "secret=", "secret:", "token=", "token:", "credential", "bearer ", "api_key", "apikey", "dsn=", "path=", "file=", "database=", "://", "mysql:", "postgres:", "sqlite:", "file:", "socket", "named pipe"} {
if strings.Contains(lower, marker) {
return true
}
}
for _, field := range strings.Fields(value) {
field = strings.Trim(field, "\"'()[]{}<>,;")
if strings.HasPrefix(field, "/") || strings.HasPrefix(field, "./") || strings.HasPrefix(field, "../") || strings.HasPrefix(field, `\\`) || len(field) >= 3 && ((field[0] >= 'a' && field[0] <= 'z') || (field[0] >= 'A' && field[0] <= 'Z')) && field[1] == ':' && (field[2] == '/' || field[2] == '\\') {
return true
}
}
return false
return splitRawLogLines(value)
}
func protectedRequestFailure(capability string, status string, code string) LifecycleExecutionResult {
+10 -9
View File
@@ -57,22 +57,23 @@ func TestWorkerExecutesFencedProtectedProgramAndSpoolsDedicatedLogs(t *testing.T
if err != nil || entryCount != 5 {
t.Fatalf("expected five program log lines: batches=%+v err=%v", batches, err)
}
redactedEntries := 0
logLines := []string{}
for _, batch := range batches {
if batch.Source != "management-program" || batch.Source == "file" || batch.Source == "process" {
t.Fatalf("program output used wrong log source: %+v", batch)
}
for _, entry := range batch.Entries {
if entry.Redacted {
redactedEntries++
}
if strings.Contains(entry.Line, "password=hidden") || strings.Contains(entry.Line, "/Users/") || strings.Contains(entry.Line, "://") {
t.Fatalf("program log leaked protected output: %+v", entry)
t.Fatalf("program log must remain verbatim, got %+v", entry)
}
logLines = append(logLines, entry.Line)
}
}
if redactedEntries != 3 {
t.Fatalf("expected three explicitly redacted private lines, got %d", redactedEntries)
joinedLogs := strings.Join(logLines, "\n")
for _, expected := range []string{"password=hidden", "/Users/private/scum.ini", "mysql://private"} {
if !strings.Contains(joinedLogs, expected) {
t.Fatalf("program log did not preserve %q: %q", expected, joinedLogs)
}
}
serialized, err := json.Marshal([]any{client.resultRequests, client.protectedRequests, worker.journal.ActiveJobs()})
if err != nil {
@@ -166,8 +167,8 @@ func TestUnknownProtectedProgramKeepsSafeDiagnosticInProgramLogOnly(t *testing.T
if result.ErrorCode != "protected_request_unknown" || result.State != lifecycleResultStateFailed {
t.Fatalf("unexpected unknown program result: %+v", result)
}
if len(sink.lines) != 1 || !strings.HasPrefix(sink.lines[0], "management-program.stderr:") || strings.Contains(sink.lines[0], "/private/") || !strings.Contains(sink.lines[0], "[redacted protected") {
t.Fatalf("unknown program diagnostic was not safely channelized: %+v", sink.lines)
if len(sink.lines) != 1 || sink.lines[0] != "management-program.stderr:unknown field database=/private/scum.db" {
t.Fatalf("unknown program diagnostic was not faithfully channelized: %+v", sink.lines)
}
}
+312
View File
@@ -0,0 +1,312 @@
package runtime
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"strconv"
"strings"
"time"
"browser.local/run/protocol"
)
const (
defaultSQLiteQueryMaxRows = 500
maxSQLiteQueryRows = 500
maxSQLiteQueryColumns = 128
maxSQLiteQuerySQLBytes = 64 * 1024
maxSQLiteQueryResultBytes = 64 * 1024
maxSQLiteQueryTimeout = 60 * time.Second
)
var sqliteNamedParameterPattern = regexp.MustCompile(`[:@$]([A-Za-z_][A-Za-z0-9_]*)`)
// SQLiteQueryExecutor executes a package-provided, read-only SQL asset against
// a declared SQLite snapshot inside the server's scoped Run workspace.
type SQLiteQueryExecutor struct{ resolver WorkspaceResolver }
func NewSQLiteQueryExecutor(workspaceRoot string) *SQLiteQueryExecutor {
return &SQLiteQueryExecutor{resolver: NewWorkspaceResolver(workspaceRoot)}
}
func (executor *SQLiteQueryExecutor) Execute(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
if err := validateSQLiteQueryAssignment(assignment); err != nil {
return sqliteQueryFailure("invalid_request", false)
}
queryAsset, maxRows, err := sqliteQueryRequest(assignment.ExecutionInput.Inputs)
if err != nil {
return sqliteQueryFailure("invalid_request", false)
}
scope, err := executor.resolver.Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
if err != nil {
return sqliteQueryFailure("target_unavailable", false)
}
databasePath, err := executor.resolver.ExistingTarget(scope, assignment.TargetKey)
if err != nil {
return sqliteQueryFailure("target_unavailable", false)
}
sqlPath, err := executor.resolver.ExistingTarget(scope, queryAsset)
if err != nil {
return sqliteQueryFailure("sql_asset_unavailable", false)
}
statement, err := readSQLiteQueryAsset(sqlPath)
if err != nil {
return sqliteQueryFailure(sqliteQueryAssetErrorCode(err), false)
}
if !isReadOnlySQLiteStatement(statement) {
return sqliteQueryFailure("sql_asset_not_read_only", false)
}
queryCtx, cancel := sqliteQueryContext(ctx, assignment.ExecutionInput.TimeoutSeconds)
defer cancel()
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+databasePath+"?mode=ro")
if err != nil {
return sqliteQueryFailure("sqlite_open_failed", true)
}
defer database.Close()
database.SetMaxOpenConns(1)
if _, err := database.ExecContext(queryCtx, "PRAGMA query_only = ON"); err != nil {
return sqliteQueryFailure(sqliteQueryErrorCode(queryCtx, err), true)
}
rows, err := database.QueryContext(queryCtx, statement, sqliteQueryArguments(statement, maxRows)...)
if err != nil {
return sqliteQueryFailure(sqliteQueryErrorCode(queryCtx, err), true)
}
defer rows.Close()
content, err := collectSQLiteQueryRows(rows, maxRows)
if err != nil {
return sqliteQueryFailure(sqliteQueryErrorCode(queryCtx, err), false)
}
if err := rows.Err(); err != nil {
return sqliteQueryFailure(sqliteQueryErrorCode(queryCtx, err), true)
}
return LifecycleExecutionResult{
State: lifecycleResultStateSucceeded,
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "SQLite query completed"},
Message: "SQLite query completed",
ExecutionResult: protocol.RunJobExecutionResult{
Kind: "sqlite.query",
Checksum: checksumForText(content),
SizeBytes: int64(len([]byte(content))),
Summary: "bounded query-only SQLite rows",
Content: content,
},
}
}
func validateSQLiteQueryAssignment(assignment protocol.RunJobAssignment) error {
if assignment.Capability != protocol.RunCapabilityRemoteRunDBSQLiteQuery || assignment.ServerInstanceID == "" || assignment.ExecutionInput.WorkspaceScope == "" {
return fmt.Errorf("SQLite query requires a scoped database job")
}
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
return err
}
if assignment.ExecutionInput.RemoteAdapterKind != "" && assignment.ExecutionInput.RemoteAdapterKind != "database" {
return fmt.Errorf("SQLite query requires the database adapter")
}
if assignment.ExecutionInput.RemoteAdapterKey != "" && !protocol.ValidLogicalFileKey(assignment.ExecutionInput.RemoteAdapterKey) {
return fmt.Errorf("SQLite query adapter key is invalid")
}
return nil
}
func sqliteQueryRequest(inputs map[string]string) (string, int, error) {
if len(inputs) == 0 {
return "", 0, fmt.Errorf("SQLite query inputs are missing")
}
queryAsset := strings.TrimSpace(inputs["sqlRef"])
if queryAsset == "" || queryAsset != inputs["sqlRef"] || !protocol.ValidLogicalFileKey(queryAsset) || !strings.HasPrefix(queryAsset, "sql/") || !strings.HasSuffix(strings.ToLower(queryAsset), ".sql") {
return "", 0, fmt.Errorf("SQLite query SQL asset is invalid")
}
maxRows := defaultSQLiteQueryMaxRows
if rawMaxRows, exists := inputs["maxRows"]; exists {
parsed, err := strconv.Atoi(rawMaxRows)
if err != nil || parsed < 1 || parsed > maxSQLiteQueryRows {
return "", 0, fmt.Errorf("SQLite query row limit is invalid")
}
maxRows = parsed
}
return queryAsset, maxRows, nil
}
func readSQLiteQueryAsset(path string) (string, error) {
info, err := os.Stat(path)
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 || info.Size() > maxSQLiteQuerySQLBytes {
return "", errSQLiteQueryAssetInvalid
}
body, err := os.ReadFile(path)
if err != nil || len(body) == 0 || len(body) > maxSQLiteQuerySQLBytes {
return "", errSQLiteQueryAssetInvalid
}
statement := strings.TrimSpace(string(body))
if statement == "" || strings.ContainsRune(statement, '\x00') {
return "", errSQLiteQueryAssetInvalid
}
return statement, nil
}
var errSQLiteQueryAssetInvalid = errors.New("SQLite query asset is invalid")
func sqliteQueryAssetErrorCode(err error) string {
if errors.Is(err, errSQLiteQueryAssetInvalid) {
return "sql_asset_invalid"
}
return "sql_asset_unavailable"
}
func isReadOnlySQLiteStatement(statement string) bool {
trimmed := strings.TrimSpace(statement)
trimmed = strings.TrimSuffix(trimmed, ";")
if trimmed == "" || strings.Contains(trimmed, ";") {
return false
}
parts := strings.Fields(strings.ToUpper(strings.TrimSpace(stripSQLiteLeadingComments(trimmed))))
if len(parts) == 0 {
return false
}
switch parts[0] {
case "SELECT", "WITH":
return true
case "EXPLAIN":
return len(parts) > 1 && (parts[1] == "SELECT" || len(parts) > 2 && parts[1] == "QUERY" && parts[2] == "PLAN")
default:
return false
}
}
func stripSQLiteLeadingComments(statement string) string {
for {
statement = strings.TrimSpace(statement)
switch {
case strings.HasPrefix(statement, "--"):
if lineEnd := strings.IndexByte(statement, '\n'); lineEnd >= 0 {
statement = statement[lineEnd+1:]
continue
}
return ""
case strings.HasPrefix(statement, "/*"):
commentEnd := strings.Index(statement[2:], "*/")
if commentEnd < 0 {
return ""
}
statement = statement[commentEnd+4:]
continue
}
return statement
}
}
func sqliteQueryContext(ctx context.Context, timeoutSeconds int) (context.Context, context.CancelFunc) {
timeout := maxSQLiteQueryTimeout
if timeoutSeconds > 0 && time.Duration(timeoutSeconds)*time.Second < timeout {
timeout = time.Duration(timeoutSeconds) * time.Second
}
return context.WithTimeout(ctx, timeout)
}
func sqliteQueryArguments(statement string, maxRows int) []any {
seen := map[string]struct{}{}
arguments := make([]any, 0)
for _, match := range sqliteNamedParameterPattern.FindAllStringSubmatch(statement, -1) {
name := match[1]
if _, exists := seen[name]; exists {
continue
}
seen[name] = struct{}{}
value := any(nil)
if name == "limit" {
value = maxRows
}
arguments = append(arguments, sql.Named(name, value))
}
return arguments
}
func collectSQLiteQueryRows(rows *sql.Rows, maxRows int) (string, error) {
columns, err := rows.Columns()
if err != nil {
return "", err
}
if len(columns) == 0 || len(columns) > maxSQLiteQueryColumns || !uniqueSQLiteQueryColumns(columns) {
return "", errSQLiteQueryResultLimit
}
result := struct {
Rows []map[string]any `json:"rows"`
}{Rows: make([]map[string]any, 0, maxRows)}
for rows.Next() {
if len(result.Rows) >= maxRows {
return "", errSQLiteQueryResultLimit
}
values := make([]any, len(columns))
pointers := make([]any, len(values))
for index := range values {
pointers[index] = &values[index]
}
if err := rows.Scan(pointers...); err != nil {
return "", err
}
row := make(map[string]any, len(columns))
for index, column := range columns {
row[column] = sqliteQueryValue(values[index])
}
result.Rows = append(result.Rows, row)
payload, err := json.Marshal(result)
if err != nil || len(payload) > maxSQLiteQueryResultBytes {
return "", errSQLiteQueryResultLimit
}
}
payload, err := json.Marshal(result)
if err != nil || len(payload) > maxSQLiteQueryResultBytes {
return "", errSQLiteQueryResultLimit
}
return string(payload), nil
}
var errSQLiteQueryResultLimit = errors.New("SQLite query result exceeds limit")
func uniqueSQLiteQueryColumns(columns []string) bool {
seen := map[string]struct{}{}
for _, column := range columns {
if strings.TrimSpace(column) == "" {
return false
}
if _, exists := seen[column]; exists {
return false
}
seen[column] = struct{}{}
}
return true
}
func sqliteQueryValue(value any) any {
if bytes, ok := value.([]byte); ok {
return append([]byte(nil), bytes...)
}
return value
}
func sqliteQueryErrorCode(ctx context.Context, err error) string {
if errors.Is(err, errSQLiteQueryResultLimit) {
return "result_limit_exceeded"
}
if errors.Is(ctx.Err(), context.Canceled) {
return "cancelled"
}
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return "timeout"
}
lower := strings.ToLower(fmt.Sprint(err))
if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") {
return "database_busy"
}
return "sqlite_read_failed"
}
func sqliteQueryFailure(code string, retryable bool) LifecycleExecutionResult {
return lifecycleExecutionFailure(code, "SQLite query failed", retryable)
}
+103
View File
@@ -0,0 +1,103 @@
package runtime
import (
"context"
"database/sql"
"encoding/json"
"os"
"path/filepath"
"testing"
"browser.local/run/protocol"
)
func TestSQLiteQueryReturnsDeclaredRowsVerbatim(t *testing.T) {
root := t.TempDir()
assignment := sqliteQueryAssignment()
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
if err != nil {
t.Fatalf("create scope: %v", err)
}
createSQLiteQueryFixture(t, filepath.Join(scope, assignment.TargetKey))
writeSQLiteQueryAsset(t, scope, assignment.ExecutionInput.Inputs["sqlRef"], `
SELECT id AS steamId, name AS displayName, note AS originalText
FROM users
WHERE (:steamId IS NULL OR id = :steamId)
ORDER BY id
LIMIT :limit;
`)
result := NewSQLiteQueryExecutor(root).Execute(context.Background(), assignment)
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "sqlite.query" || result.ExecutionResult.Checksum == "" || result.ExecutionResult.SizeBytes <= 0 {
t.Fatalf("expected successful SQLite rows, got %+v", result)
}
var payload struct {
Rows []map[string]any `json:"rows"`
}
if err := json.Unmarshal([]byte(result.ExecutionResult.Content), &payload); err != nil {
t.Fatalf("decode rows: %v", err)
}
if len(payload.Rows) != 2 || payload.Rows[0]["steamId"] != "steam-1" || payload.Rows[0]["originalText"] != "password=opaque C:/game/users.db" {
t.Fatalf("SQLite rows were not faithfully returned: %+v", payload.Rows)
}
}
func TestSQLiteQueryRejectsUnsafeAssetsAndLimitOverrun(t *testing.T) {
root := t.TempDir()
assignment := sqliteQueryAssignment()
scope, err := NewWorkspaceResolver(root).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
if err != nil {
t.Fatalf("create scope: %v", err)
}
createSQLiteQueryFixture(t, filepath.Join(scope, assignment.TargetKey))
writeSQLiteQueryAsset(t, scope, assignment.ExecutionInput.Inputs["sqlRef"], "SELECT id AS steamId FROM users ORDER BY id")
unsafe := assignment
unsafe.ExecutionInput.Inputs = map[string]string{"sqlRef": "../outside.sql", "maxRows": "2"}
if result := NewSQLiteQueryExecutor(root).Execute(context.Background(), unsafe); result.ErrorCode != "invalid_request" {
t.Fatalf("expected unsafe asset rejection, got %+v", result)
}
limited := assignment
limited.ExecutionInput.Inputs = map[string]string{"sqlRef": assignment.ExecutionInput.Inputs["sqlRef"], "maxRows": "1"}
if result := NewSQLiteQueryExecutor(root).Execute(context.Background(), limited); result.ErrorCode != "result_limit_exceeded" {
t.Fatalf("expected result limit failure instead of truncation, got %+v", result)
}
}
func sqliteQueryAssignment() protocol.RunJobAssignment {
assignment := lifecycleAssignment(protocol.RunCapabilityRemoteRunDBSQLiteQuery)
assignment.TargetKey = "databases/current.db"
assignment.InputRef = "input://plugin-query-poll/server-1/users"
assignment.ExecutionInput.WorkspaceScope = "profile-default"
assignment.ExecutionInput.RemoteAdapterKey = "current-db"
assignment.ExecutionInput.RemoteAdapterKind = "database"
assignment.ExecutionInput.TimeoutSeconds = 10
assignment.ExecutionInput.Inputs = map[string]string{"sqlRef": "sql/users.sql", "maxRows": "2"}
return assignment
}
func createSQLiteQueryFixture(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
database, err := sql.Open(sqliteSchemaProbeDriver, path)
if err != nil {
t.Fatal(err)
}
defer database.Close()
if _, err := database.Exec(`CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT NOT NULL, note TEXT); INSERT INTO users(id, name, note) VALUES ('steam-1', 'Ada', 'password=opaque C:/game/users.db'), ('steam-2', 'Lin', 'unchanged');`); err != nil {
t.Fatal(err)
}
}
func writeSQLiteQueryAsset(t *testing.T, scope string, asset string, body string) {
t.Helper()
path := filepath.Join(scope, filepath.FromSlash(asset))
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
+15 -7
View File
@@ -735,6 +735,18 @@ func (worker *Worker) executeAssignment(ctx context.Context, assignment protocol
}
return worker.executor.sqliteSchemaProbe.Execute(ctx, assignment)
}
if assignment.Capability == protocol.RunCapabilityRemoteRunDBSQLiteQuery {
log.Printf("RUN phase=job.dispatch status=selected job=%s executor=sqlite_query", assignment.JobID)
if worker.executor.sqliteQuery == nil {
return lifecycleFailure("sqlite_query_unavailable", "SQLite query executor is unavailable")
}
targetKey, err := worker.materializeSQLiteQueryDataTarget(ctx, assignment)
if err != nil {
return sqliteQueryFailureForDataTarget(err)
}
assignment.TargetKey = targetKey
return worker.executor.sqliteQuery.Execute(ctx, assignment)
}
if isSupportedLifecycleCapability(assignment.Capability) {
log.Printf("RUN phase=job.dispatch status=selected job=%s executor=lifecycle", assignment.JobID)
return worker.executor.ExecuteContext(ctx, assignment)
@@ -1484,8 +1496,6 @@ func (sink *LiveLogSink) append(assignment protocol.RunJobAssignment, stream str
if sink == nil {
return nil
}
redactedLine := RedactText(line)
redacted := line != redactedLine || strings.HasPrefix(redactedLine, "[redacted protected ")
streamKey := declaredProcessStreamKey(assignment, stream)
logStreamID := logStreamIDForAssignment(assignment, streamKey)
sink.mu.Lock()
@@ -1514,7 +1524,7 @@ func (sink *LiveLogSink) append(assignment protocol.RunJobAssignment, stream str
FirstSeq: sequence,
LastSeq: sequence,
Compression: "none",
Entries: []protocol.LogEntry{{Seq: sequence, Timestamp: time.Now().UTC(), Level: "info", Line: redactedLine, Redacted: redacted}},
Entries: []protocol.LogEntry{{Seq: sequence, Timestamp: time.Now().UTC(), Level: "info", Line: line, Redacted: false}},
}
batch.Checksum, _ = checksumForLogEntries(batch.Entries)
select {
@@ -1564,11 +1574,9 @@ func (sink *SpoolLogSink) AppendWithCursor(ctx context.Context, assignment proto
func (sink *SpoolLogSink) append(ctx context.Context, assignment protocol.RunJobAssignment, stream string, line string, cursor *spool.LogSourceCursor) error {
sink.mu.Lock()
defer sink.mu.Unlock()
redactedLine := RedactText(line)
redacted := line != redactedLine || strings.HasPrefix(redactedLine, "[redacted protected ")
streamKey := declaredProcessStreamKey(assignment, stream)
logStreamID := logStreamIDForAssignment(assignment, streamKey)
entry := protocol.LogEntry{Timestamp: time.Now().UTC(), Level: "info", Line: redactedLine, Redacted: redacted}
entry := protocol.LogEntry{Timestamp: time.Now().UTC(), Level: "info", Line: line, Redacted: false}
source := "process"
if strings.HasPrefix(stream, "management-program.") {
source = "management-program"
@@ -1640,7 +1648,7 @@ type QueueArtifactHook struct {
func (hook QueueArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, result ProcessResult) (string, error) {
ref := fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID)
payload := []byte(RedactText(result.Stdout + result.Stderr))
payload := []byte(result.Stdout + result.Stderr)
if len(payload) == 0 {
payload = []byte("lifecycle result metadata")
}
+1 -1
View File
@@ -392,7 +392,7 @@ func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) {
if err != nil {
t.Fatalf("pending logs: %v", err)
}
if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || logs[0].StreamKey != "game.console.stdout" || containsText(logs[0].Entries[0].Line, "password=hidden") {
if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || logs[0].StreamKey != "game.console.stdout" || logs[0].Entries[0].Line != "started password=hidden" || logs[0].Entries[0].Redacted {
t.Fatalf("unexpected spooled logs: %+v", logs)
}
chunks, err := artifactQueue.Pending()