admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make log payloads opaque pass-through

Browse Source
This commit is contained in:
npc0-hue
2026-09-03 12:40:17 +08:00
parent 5b82a42cc4
commit bf3c382d15
16 changed files with 63 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/log_body_store.go
+1 -4
View File
@@ -293,10 +293,7 @@ func sameLogBatchRecord(left domain.LogBatchRecord, right domain.LogBatchRecord)
if left.FirstSeq != right.FirstSeq || left.LastSeq != right.LastSeq {
return false
}
if left.Checksum == right.Checksum {
return true
}
return len(left.Entries) == 1 && right.Checksum == validator.LogLineChecksum(left.Entries[0].Line)
return left.Checksum == right.Checksum
}
func readLogSegment(path string) (domain.LogBatchRecord, error) {
platform/service/log_ingest.go
+1 -7
View File
@@ -177,13 +177,7 @@ func jobIDFromLogBatch(batch domain.LogBatchIngest) (string, bool) {
}
func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIngest) bool {
if record.Checksum == batch.Checksum {
return true
}
if len(record.Entries) == 1 && len(batch.Entries) == 1 {
return batch.Checksum == validator.LogLineChecksum(record.Entries[0].Line)
}
return false
return record.Checksum == batch.Checksum
}
func (svc *CoreService) QueryLogStream(query domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error) {
platform/service/log_ingest_privacy_test.go
+6 -2
View File
@@ -8,7 +8,7 @@ import (
func TestLogIngestPreservesOpaqueFields(t *testing.T) {
batch := domain.LogBatchIngest{Entries: []domain.LogEntry{
{Fields: map[string]string{
{Line: "password=opaque /Users/operator/game.log tcp://127.0.0.1:7777", Fields: map[string]string{
"eventType": "game.session.opened",
"networkFingerprint": "fingerprint",
"ip": "192.0.2.1",
@@ -17,7 +17,11 @@ func TestLogIngestPreservesOpaqueFields(t *testing.T) {
}},
}}
fields := storedLogEntries(batch.Entries)[0].Fields
stored := storedLogEntries(batch.Entries)
fields := stored[0].Fields
if stored[0].Line != batch.Entries[0].Line {
t.Fatalf("expected opaque log line to be preserved, got %q", stored[0].Line)
}
for key, expected := range map[string]string{"networkFingerprint": "fingerprint", "ip": "192.0.2.1", "ipAddress": "2001:db8::1", "playerId": "player-1"} {
if fields[key] != expected {
t.Fatalf("expected opaque field %s to be preserved, got %q", key, fields[key])
platform/service/log_ingest_test.go
+6 -6
View File
@@ -236,7 +236,7 @@ func TestCoreServiceAcceptsAutoCreatedRunJobLogStreams(t *testing.T) {
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
}
ack, err := svc.IngestLogBatch(batch)
@@ -297,7 +297,7 @@ func TestCoreServiceAcceptsPluginDeclaredProcessLogStreams(t *testing.T) {
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -338,7 +338,7 @@ func TestCoreServiceRepairsMissingDeclaredProcessLogStreamOnIngest(t *testing.T)
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -363,7 +363,7 @@ func TestCoreServiceAcceptsAutonomousRunLogStreamWithoutPlatformJob(t *testing.T
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -395,7 +395,7 @@ func TestCoreServiceAcceptsAutonomousRunFileTailLogStreamWithoutPlatformJob(t *t
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -431,7 +431,7 @@ func TestCoreServiceAcceptsLegacyAutonomousJobLogStreamWithoutPlatformJob(t *tes
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
platform/service/resources.go
+1 -24
View File
@@ -2020,7 +2020,7 @@ func (svc *CoreService) GetDeclaredFileReadSnapshotForSession(sessionID string,
PluginID: base.PluginID,
Key: base.Key,
State: "ready",
Content: redactDeclaredFileReadContent(completed.ExecutionResult.Content),
Content: completed.ExecutionResult.Content,
Version: completed.ExecutionResult.Version,
Checksum: completed.ExecutionResult.Checksum,
SizeBytes: completed.ExecutionResult.SizeBytes,
@@ -2065,29 +2065,6 @@ func jobCompletedAt(job domain.Job) time.Time {
}
return job.CreatedAt
}
func redactDeclaredFileReadContent(content string) string {
lines := strings.Split(content, "\n")
for index, line := range lines {
key, _, found := strings.Cut(line, "=")
if !found || !secretLikeFileAssignmentKey(key) {
continue
}
lines[index] = key + "=<redacted>"
}
return strings.Join(lines, "\n")
}
func secretLikeFileAssignmentKey(key string) bool {
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(key), "_", ""), "-", ""))
for _, marker := range []string{"password", "passwd", "secret", "token", "apikey", "accesskey", "privatekey", "rcon"} {
if strings.Contains(normalized, marker) {
return true
}
}
return false
}
func (svc *CoreService) PreviewServerConfigWriteForSession(sessionID string, request domain.ServerConfigDiffRequest) (domain.ServerConfigDiffPreview, error) {
if request.Key == "" {
request.Key = "server.properties"
platform/service/resources_test.go
+5 -5
View File
@@ -1126,7 +1126,7 @@ func TestServerFileListFallsBackToPluginWorkspaceWithoutRunListCapability(t *tes
}
}
func TestDeclaredFileReadSnapshotProjectionStatesAndRedaction(t *testing.T) {
func TestDeclaredFileReadSnapshotProjectionStatesAndPassThroughContent(t *testing.T) {
svc := newTestCoreService()
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
plugin.FileWorkspace = scumTestFileWorkspace()
@@ -1163,13 +1163,13 @@ func TestDeclaredFileReadSnapshotProjectionStatesAndRedaction(t *testing.T) {
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-success-old", domain.JobStateSucceeded, 4, "ServerName=Old\nRconPassword=secret\n")
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-failed-newer", domain.JobStateFailed, 5, "")
snapshot, err = svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "scum-server-settings")
if err != nil || snapshot.State != "ready" || snapshot.JobID != "job-file-snapshot-success-old" || !strings.Contains(snapshot.Content, "RconPassword=<redacted>") {
t.Fatalf("expected older successful redacted result, snapshot=%+v err=%v", snapshot, err)
if err != nil || snapshot.State != "ready" || snapshot.JobID != "job-file-snapshot-success-old" || !strings.Contains(snapshot.Content, "RconPassword=secret") {
t.Fatalf("expected older successful pass-through result, snapshot=%+v err=%v", snapshot, err)
}
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-success-new", domain.JobStateSucceeded, 6, "ServerName=New\nApiToken=secret\n")
snapshot, err = svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "scum-server-settings")
if err != nil || snapshot.JobID != "job-file-snapshot-success-new" || !strings.Contains(snapshot.Content, "ServerName=New") || strings.Contains(snapshot.Content, "secret") {
t.Fatalf("expected newest successful redacted result, snapshot=%+v err=%v", snapshot, err)
if err != nil || snapshot.JobID != "job-file-snapshot-success-new" || snapshot.Content != "ServerName=New\nApiToken=secret\n" {
t.Fatalf("expected newest successful pass-through result, snapshot=%+v err=%v", snapshot, err)
}
unknownSnapshot, err := svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "logs/latest.log")
if err != nil || unknownSnapshot.State != "not-read" {
platform/service/server_files.go
+1 -1
View File
@@ -344,7 +344,7 @@ func (svc *CoreService) PrepareServerFileDownloadForSession(sessionID string, re
return domain.CopyServerFileDownloadResult(domain.ServerFileDownloadResult{Status: "ready", ServerInstanceID: ctx.Instance.ID, Key: request.Key, Filename: filename, ContentType: reference.ContentType, Checksum: reference.Checksum, SizeBytes: reference.SizeBytes, Artifact: &reference, Job: job, ReadAt: job.TerminalAt}), nil
}
if job.ExecutionResult.Content != "" {
content := redactDeclaredFileReadContent(job.ExecutionResult.Content)
content := job.ExecutionResult.Content
return domain.CopyServerFileDownloadResult(domain.ServerFileDownloadResult{Status: "ready", ServerInstanceID: ctx.Instance.ID, Key: request.Key, Filename: filename, ContentType: "text/plain; charset=utf-8", Content: content, Checksum: job.ExecutionResult.Checksum, SizeBytes: int64(len([]byte(content))), Job: job, ReadAt: job.TerminalAt}), nil
}
}
platform/validator/log_ingest.go
+1 -13
View File
@@ -72,7 +72,7 @@ func ValidateLogBatchIngest(batch domain.LogBatchIngest) error {
computed, err := LogEntriesChecksum(batch.Entries)
if err != nil {
violations = append(violations, "checksum cannot be computed")
} else if batch.Checksum != computed && !logLineChecksumMatches(batch) {
} else if batch.Checksum != computed {
violations = append(violations, "checksum does not match entries")
}
}
@@ -119,18 +119,6 @@ func LogEntriesChecksum(entries []domain.LogEntry) (string, error) {
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func logLineChecksumMatches(batch domain.LogBatchIngest) bool {
if len(batch.Entries) != 1 {
return false
}
return batch.Checksum == LogLineChecksum(batch.Entries[0].Line)
}
func LogLineChecksum(value string) string {
sum := sha256.Sum256([]byte(value))
return "sha256:" + hex.EncodeToString(sum[:])
}
type logEntryChecksumBody struct {
Seq uint64 `json:"seq"`
Timestamp string `json:"timestamp"`
platform/validator/log_ingest_test.go
+15
View File
@@ -1,6 +1,8 @@
package validator
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
@@ -22,3 +24,16 @@ func TestValidateLogBatchIngestAcceptsVerbatimBlankAndLongLines(t *testing.T) {
t.Fatalf("verbatim log batch was rejected: %v", err)
}
}
func TestValidateLogBatchIngestRejectsLineOnlyChecksumCompatibility(t *testing.T) {
entry := domain.LogEntry{Seq: 1, Timestamp: time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC), Line: "password=opaque /Users/operator/game.log"}
batch := domain.LogBatchIngest{RunEndpointID: "run-1", SessionToken: "session-1", LogStreamID: "run.run-1.server-1.stdout", ServerInstanceID: "server-1", StreamKey: "stdout", Source: domain.LogStreamSourceProcess, FirstSeq: 1, LastSeq: 1, Compression: "none", Checksum: lineOnlyChecksum(entry.Line), Entries: []domain.LogEntry{entry}}
if err := ValidateLogBatchIngest(batch); err == nil || !strings.Contains(err.Error(), "checksum") {
t.Fatalf("expected full-entry checksum rejection, got %v", err)
}
}
func lineOnlyChecksum(value string) string {
sum := sha256.Sum256([]byte(value))
return "sha256:" + hex.EncodeToString(sum[:])
}
platform_web/api/types.ts
+1
View File
@@ -691,6 +691,7 @@ export interface JobExecutionResultResponse {
checksum?: string;
sizeBytes?: number;
summary?: string;
content?: string;
}
export interface JobListResponse {
platform_web/schemas/gameClientBridge.test.ts
+9 -8
View File
@@ -63,17 +63,18 @@ describe("Game Client Bridge safe projection schema", () => {
{ sessionToken: "component-session-material" },
{ componentSession: "component-session-material" },
{ componentKey: "raw-component-key" },
{ sourceSessionId: "component-session-1" },
{ result: { ...safeCommand.result, payload: { dsn: "sqlite:///srv/scum/SCUM.db" } } },
{ result: { ...safeCommand.result, payload: { endpoint: "tcp://127.0.0.1:9999" } } },
{ result: { ...safeCommand.result, payload: { output: "/Users/operator/scum/config.yaml" } } }
{ sourceSessionId: "component-session-1" }
])("rejects forbidden command projection %#", (unsafe) => {
expect(() => parseSafeGameClientBridgeCommand({ ...safeCommand, ...unsafe })).toThrow(/forbidden|sensitive/i);
});
it("rejects credentials nested inside snapshot payloads", () => {
const unsafe = structuredClone(safeSnapshotList);
unsafe.items[0].payload = { players: [{ playerId: "player-1", credential: "raw-password" }] } as unknown as typeof unsafe.items[0]["payload"];
expect(() => parseSafeGameClientBridgeSnapshotList(unsafe)).toThrow(/forbidden/i);
it("preserves sensitive-looking command and snapshot payloads", () => {
expect(parseSafeGameClientBridgeCommand({
...safeCommand,
result: { ...safeCommand.result, payload: { dsn: "sqlite:///srv/scum/SCUM.db", endpoint: "tcp://127.0.0.1:9999", output: "/Users/operator/scum/config.yaml" } }
})).toMatchObject({ result: { payload: { dsn: "sqlite:///srv/scum/SCUM.db", endpoint: "tcp://127.0.0.1:9999", output: "/Users/operator/scum/config.yaml" } } });
const payload = { players: [{ playerId: "player-1", credential: "raw-password", note: "password=opaque" }] };
expect(parseSafeGameClientBridgeSnapshotList({ ...safeSnapshotList, items: [{ ...safeSnapshotList.items[0], payload }] })).toMatchObject({ items: [{ payload }] });
});
});
platform_web/schemas/gameClientBridge.ts
+4 -37
View File
@@ -16,7 +16,7 @@ import type {
const commandStates = new Set<GameClientBridgeCommandState>(["pending", "claimed", "succeeded", "failed", "cancelled", "expired", "unknown"]);
const resultStatuses = new Set<GameClientBridgeResultStatus>(["succeeded", "failed", "cancelled", "unknown"]);
const forbiddenKeys = new Set([
const forbiddenEnvelopeKeys = new Set([
"apikey",
"accesskey",
"accesskeyid",
@@ -47,19 +47,6 @@ const forbiddenKeys = new Set([
"storagecredential",
"token"
]);
const forbiddenFragments = [
"bearer ",
"password=",
"secret://",
"unix://",
"tcp://",
"mysql://",
"postgres://",
"sqlite://",
"rcon://"
];
const forbiddenHostPath = /(?:^|[\s"'])(?:\/[Uu]sers\/|\/home\/|\/root\/|\/var\/|\/etc\/|\/opt\/|[a-z]:[\\/]|\\\\[^\\]+\\)/;
export function parseSafeGameClientBridgeStatus(value: unknown): GameClientBridgeStatusResponse {
const record = safeObject(value, "Game Client Bridge status");
return {
@@ -191,32 +178,12 @@ function parseSnapshot(value: unknown): GameClientBridgeSnapshotResponse {
function safeObject(value: unknown, label: string): Record<string, unknown> {
const record = object(value, label);
rejectSensitiveProjection(record);
for (const key of Object.keys(record)) {
if (forbiddenEnvelopeKeys.has(key.toLowerCase().replace(/[^a-z0-9]/g, ""))) throw new Error("Game Client Bridge response contains a forbidden field");
}
return record;
}
function rejectSensitiveProjection(value: unknown): void {
if (typeof value === "string") {
const normalized = value.toLowerCase();
if (forbiddenFragments.some((fragment) => normalized.includes(fragment)) || forbiddenHostPath.test(value)) {
throw new Error("Game Client Bridge response contains sensitive connection or host data");
}
return;
}
if (Array.isArray(value)) {
value.forEach(rejectSensitiveProjection);
return;
}
if (value && typeof value === "object") {
for (const [key, child] of Object.entries(value)) {
if (forbiddenKeys.has(key.toLowerCase().replace(/[^a-z0-9]/g, ""))) {
throw new Error("Game Client Bridge response contains a forbidden field");
}
rejectSensitiveProjection(child);
}
}
}
function jsonObject(value: unknown, label: string): GameClientBridgeJsonObject {
const record = object(value, label);
return Object.fromEntries(Object.entries(record).map(([key, child]) => [key, jsonValue(child, `${label}.${key}`)]));
platform_web/schemas/jobs.test.ts
+3 -5
View File
@@ -27,14 +27,12 @@ describe("safe job projection schema", () => {
expect(() => parseSafeJobResponse({ ...retryingJob, [field]: "forbidden" })).toThrow(/forbidden field/);
});
it("accepts safe typed execution metadata without private content", () => {
it("accepts typed execution metadata and preserves content", () => {
const parsed = parseSafeJobResponse({
...retryingJob,
state: "succeeded",
executionResult: { kind: "file.write", version: 2, checksum: "sha256:" + "a".repeat(64), sizeBytes: 18, summary: "atomic compare-and-swap file write" }
executionResult: { kind: "file.read", version: 2, checksum: "sha256:" + "a".repeat(64), sizeBytes: 18, summary: "bounded regular-file read", content: "password=opaque\n/Users/operator/config.ini" }
});
expect(parsed.executionResult).toMatchObject({ kind: "file.write", version: 2, sizeBytes: 18 });
expect(parsed.executionResult).not.toHaveProperty("content");
expect(() => parseSafeJobResponse({ ...retryingJob, executionResult: { content: "private" } })).toThrow(/forbidden field/);
expect(parsed.executionResult).toMatchObject({ kind: "file.read", version: 2, sizeBytes: 18, content: "password=opaque\n/Users/operator/config.ini" });
});
});
platform_web/schemas/jobs.ts
+3 -2
View File
@@ -1,7 +1,7 @@
import type { JobResponse, JobState } from "../api/types";
const jobStates = new Set<JobState>(["queued", "accepted", "running", "retrying", "succeeded", "failed", "cancelled"]);
const forbiddenProjectionKeys = new Set(["leaseToken", "leaseTokenHash", "leaseSessionGeneration", "sessionToken", "secretRef", "hostPath", "socket", "credential", "content"]);
const forbiddenProjectionKeys = new Set(["leaseToken", "leaseTokenHash", "leaseSessionGeneration", "sessionToken", "secretRef", "hostPath", "socket", "credential"]);
export function parseSafeJobResponse(value: unknown): JobResponse {
if (!isRecord(value)) throw new Error("job projection must be an object");
@@ -57,7 +57,8 @@ function optionalExecutionResult(value: Record<string, unknown>, key: string): J
version: optionalNumber(field, "version"),
checksum: optionalString(field, "checksum"),
sizeBytes: optionalNumber(field, "sizeBytes"),
summary: optionalString(field, "summary")
summary: optionalString(field, "summary"),
content: optionalString(field, "content")
};
return result;
}
platform_web/schemas/runtimeUpdates.test.ts
+3 -3
View File
@@ -57,8 +57,8 @@ describe("safe dependency and Run update projections", () => {
})).toThrow(/forbidden field/);
});
it("rejects raw host paths or credentials hidden in safe-looking evidence", () => {
expect(() => parseSafeDependencyCatalog({
it("preserves raw host paths or credentials in evidence text", () => {
expect(parseSafeDependencyCatalog({
serverInstanceId: "server-1",
pluginId: "game.runtime",
pluginVersion: "1.0.0",
@@ -68,6 +68,6 @@ describe("safe dependency and Run update projections", () => {
probes: [{ key: "java", kind: "java.version", required: true, state: "present", evidence: "/Users/operator/private" }],
plans: [],
updatedAt: "2026-07-18T12:00:00Z"
})).toThrow(/unsafe runtime details/);
}).probes[0].evidence).toBe("/Users/operator/private");
});
});
platform_web/schemas/runtimeUpdates.ts
+3 -11
View File
@@ -31,8 +31,6 @@ const forbiddenProjectionKeys = new Set([
"bindings",
"downloadref"
]);
const unsafeProjectionText = /(?:\/Users\/|\/home\/|\/var\/run\/|[A-Za-z]:\\|unix:\/\/|tcp:\/\/|Bearer\s+|password=|token=|sk-[A-Za-z0-9_-]+)/i;
export function parseSafeDependencyCatalog(value: unknown): DependencyCatalogResponse {
const record = requiredRecordValue(value, "dependency catalog");
rejectForbiddenProjection(record);
@@ -72,9 +70,9 @@ function parseDependencyProbe(value: unknown): DependencyProbeViewResponse {
key: requiredString(record, "key"),
kind: requiredString(record, "kind"),
required: requiredBoolean(record, "required"),
minimumVersion: optionalSafeString(record, "minimumVersion"),
minimumVersion: optionalString(record, "minimumVersion"),
state,
evidence: optionalSafeString(record, "evidence"),
evidence: optionalString(record, "evidence"),
installPlanKey: optionalString(record, "installPlanKey")
};
}
@@ -125,7 +123,7 @@ function parseRunUpdate(record: Record<string, unknown>): RunUpdateJobResponse {
idempotencyKey: optionalString(record, "idempotencyKey"),
status,
phase,
message: optionalSafeString(record, "message"),
message: optionalString(record, "message"),
rollback: requiredBoolean(record, "rollback"),
createdAt: requiredString(record, "createdAt"),
updatedAt: requiredString(record, "updatedAt")
@@ -168,12 +166,6 @@ function optionalString(value: Record<string, unknown>, key: string): string | u
return field;
}
function optionalSafeString(value: Record<string, unknown>, key: string): string | undefined {
const field = optionalString(value, key);
if (field && unsafeProjectionText.test(field)) throw new Error(`${key} contains unsafe runtime details`);
return field;
}
function requiredChecksum(value: Record<string, unknown>, key: string): string {
const field = requiredString(value, key);
if (!/^sha256:[a-f0-9]{64}$/.test(field)) throw new Error(`${key} must be a SHA-256 checksum`);