admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make log payloads opaque pass-through

Browse Source
This commit is contained in:
npc0-hue
2026-09-03 12:40:17 +08:00
parent 5b82a42cc4
commit bf3c382d15
16 changed files with 63 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/log_body_store.go
+1 -4
View File
@@ -293,10 +293,7 @@ func sameLogBatchRecord(left domain.LogBatchRecord, right domain.LogBatchRecord)
if left.FirstSeq != right.FirstSeq || left.LastSeq != right.LastSeq {
return false
}
if left.Checksum == right.Checksum {
return true
}
return len(left.Entries) == 1 && right.Checksum == validator.LogLineChecksum(left.Entries[0].Line)
return left.Checksum == right.Checksum
}
func readLogSegment(path string) (domain.LogBatchRecord, error) {
platform/service/log_ingest.go
+1 -7
View File
@@ -177,13 +177,7 @@ func jobIDFromLogBatch(batch domain.LogBatchIngest) (string, bool) {
}
func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIngest) bool {
if record.Checksum == batch.Checksum {
return true
}
if len(record.Entries) == 1 && len(batch.Entries) == 1 {
return batch.Checksum == validator.LogLineChecksum(record.Entries[0].Line)
}
return false
return record.Checksum == batch.Checksum
}
func (svc *CoreService) QueryLogStream(query domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error) {
platform/service/log_ingest_privacy_test.go
+6 -2
View File
@@ -8,7 +8,7 @@ import (
func TestLogIngestPreservesOpaqueFields(t *testing.T) {
batch := domain.LogBatchIngest{Entries: []domain.LogEntry{
{Fields: map[string]string{
{Line: "password=opaque /Users/operator/game.log tcp://127.0.0.1:7777", Fields: map[string]string{
"eventType": "game.session.opened",
"networkFingerprint": "fingerprint",
"ip": "192.0.2.1",
@@ -17,7 +17,11 @@ func TestLogIngestPreservesOpaqueFields(t *testing.T) {
}},
}}
fields := storedLogEntries(batch.Entries)[0].Fields
stored := storedLogEntries(batch.Entries)
fields := stored[0].Fields
if stored[0].Line != batch.Entries[0].Line {
t.Fatalf("expected opaque log line to be preserved, got %q", stored[0].Line)
}
for key, expected := range map[string]string{"networkFingerprint": "fingerprint", "ip": "192.0.2.1", "ipAddress": "2001:db8::1", "playerId": "player-1"} {
if fields[key] != expected {
t.Fatalf("expected opaque field %s to be preserved, got %q", key, fields[key])
platform/service/log_ingest_test.go
+6 -6
View File
@@ -236,7 +236,7 @@ func TestCoreServiceAcceptsAutoCreatedRunJobLogStreams(t *testing.T) {
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
}
ack, err := svc.IngestLogBatch(batch)
@@ -297,7 +297,7 @@ func TestCoreServiceAcceptsPluginDeclaredProcessLogStreams(t *testing.T) {
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -338,7 +338,7 @@ func TestCoreServiceRepairsMissingDeclaredProcessLogStreamOnIngest(t *testing.T)
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -363,7 +363,7 @@ func TestCoreServiceAcceptsAutonomousRunLogStreamWithoutPlatformJob(t *testing.T
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -395,7 +395,7 @@ func TestCoreServiceAcceptsAutonomousRunFileTailLogStreamWithoutPlatformJob(t *t
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
@@ -431,7 +431,7 @@ func TestCoreServiceAcceptsLegacyAutonomousJobLogStreamWithoutPlatformJob(t *tes
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Checksum: checksumForEntries(t, []domain.LogEntry{entry}),
Entries: []domain.LogEntry{entry},
})
if err != nil {
platform/service/resources.go
+1 -24
View File
@@ -2020,7 +2020,7 @@ func (svc *CoreService) GetDeclaredFileReadSnapshotForSession(sessionID string,
PluginID: base.PluginID,
Key: base.Key,
State: "ready",
Content: redactDeclaredFileReadContent(completed.ExecutionResult.Content),
Content: completed.ExecutionResult.Content,
Version: completed.ExecutionResult.Version,
Checksum: completed.ExecutionResult.Checksum,
SizeBytes: completed.ExecutionResult.SizeBytes,
@@ -2065,29 +2065,6 @@ func jobCompletedAt(job domain.Job) time.Time {
}
return job.CreatedAt
}
func redactDeclaredFileReadContent(content string) string {
lines := strings.Split(content, "\n")
for index, line := range lines {
key, _, found := strings.Cut(line, "=")
if !found || !secretLikeFileAssignmentKey(key) {
continue
}
lines[index] = key + "=<redacted>"
}
return strings.Join(lines, "\n")
}
func secretLikeFileAssignmentKey(key string) bool {
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(key), "_", ""), "-", ""))
for _, marker := range []string{"password", "passwd", "secret", "token", "apikey", "accesskey", "privatekey", "rcon"} {
if strings.Contains(normalized, marker) {
return true
}
}
return false
}
func (svc *CoreService) PreviewServerConfigWriteForSession(sessionID string, request domain.ServerConfigDiffRequest) (domain.ServerConfigDiffPreview, error) {
if request.Key == "" {
request.Key = "server.properties"
platform/service/resources_test.go
+5 -5
View File
@@ -1126,7 +1126,7 @@ func TestServerFileListFallsBackToPluginWorkspaceWithoutRunListCapability(t *tes
}
}
func TestDeclaredFileReadSnapshotProjectionStatesAndRedaction(t *testing.T) {
func TestDeclaredFileReadSnapshotProjectionStatesAndPassThroughContent(t *testing.T) {
svc := newTestCoreService()
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
plugin.FileWorkspace = scumTestFileWorkspace()
@@ -1163,13 +1163,13 @@ func TestDeclaredFileReadSnapshotProjectionStatesAndRedaction(t *testing.T) {
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-success-old", domain.JobStateSucceeded, 4, "ServerName=Old\nRconPassword=secret\n")
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-failed-newer", domain.JobStateFailed, 5, "")
snapshot, err = svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "scum-server-settings")
if err != nil || snapshot.State != "ready" || snapshot.JobID != "job-file-snapshot-success-old" || !strings.Contains(snapshot.Content, "RconPassword=<redacted>") {
t.Fatalf("expected older successful redacted result, snapshot=%+v err=%v", snapshot, err)
if err != nil || snapshot.State != "ready" || snapshot.JobID != "job-file-snapshot-success-old" || !strings.Contains(snapshot.Content, "RconPassword=secret") {
t.Fatalf("expected older successful pass-through result, snapshot=%+v err=%v", snapshot, err)
}
createDeclaredFileReadJob(t, svc, instance, endpoint, "job-file-snapshot-success-new", domain.JobStateSucceeded, 6, "ServerName=New\nApiToken=secret\n")
snapshot, err = svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "scum-server-settings")
if err != nil || snapshot.JobID != "job-file-snapshot-success-new" || !strings.Contains(snapshot.Content, "ServerName=New") || strings.Contains(snapshot.Content, "secret") {
t.Fatalf("expected newest successful redacted result, snapshot=%+v err=%v", snapshot, err)
if err != nil || snapshot.JobID != "job-file-snapshot-success-new" || snapshot.Content != "ServerName=New\nApiToken=secret\n" {
t.Fatalf("expected newest successful pass-through result, snapshot=%+v err=%v", snapshot, err)
}
unknownSnapshot, err := svc.GetDeclaredFileReadSnapshotForSession(ownerSession, instance.ID, "logs/latest.log")
if err != nil || unknownSnapshot.State != "not-read" {
platform/service/server_files.go
+1 -1
View File
@@ -344,7 +344,7 @@ func (svc *CoreService) PrepareServerFileDownloadForSession(sessionID string, re
return domain.CopyServerFileDownloadResult(domain.ServerFileDownloadResult{Status: "ready", ServerInstanceID: ctx.Instance.ID, Key: request.Key, Filename: filename, ContentType: reference.ContentType, Checksum: reference.Checksum, SizeBytes: reference.SizeBytes, Artifact: &reference, Job: job, ReadAt: job.TerminalAt}), nil
}
if job.ExecutionResult.Content != "" {
content := redactDeclaredFileReadContent(job.ExecutionResult.Content)
content := job.ExecutionResult.Content
return domain.CopyServerFileDownloadResult(domain.ServerFileDownloadResult{Status: "ready", ServerInstanceID: ctx.Instance.ID, Key: request.Key, Filename: filename, ContentType: "text/plain; charset=utf-8", Content: content, Checksum: job.ExecutionResult.Checksum, SizeBytes: int64(len([]byte(content))), Job: job, ReadAt: job.TerminalAt}), nil
}
}
platform/validator/log_ingest.go
+1 -13
View File
@@ -72,7 +72,7 @@ func ValidateLogBatchIngest(batch domain.LogBatchIngest) error {
computed, err := LogEntriesChecksum(batch.Entries)
if err != nil {
violations = append(violations, "checksum cannot be computed")
} else if batch.Checksum != computed && !logLineChecksumMatches(batch) {
} else if batch.Checksum != computed {
violations = append(violations, "checksum does not match entries")
}
}
@@ -119,18 +119,6 @@ func LogEntriesChecksum(entries []domain.LogEntry) (string, error) {
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func logLineChecksumMatches(batch domain.LogBatchIngest) bool {
if len(batch.Entries) != 1 {
return false
}
return batch.Checksum == LogLineChecksum(batch.Entries[0].Line)
}
func LogLineChecksum(value string) string {
sum := sha256.Sum256([]byte(value))
return "sha256:" + hex.EncodeToString(sum[:])
}
type logEntryChecksumBody struct {
Seq uint64 `json:"seq"`
Timestamp string `json:"timestamp"`
platform/validator/log_ingest_test.go
+15
View File
@@ -1,6 +1,8 @@
package validator
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
@@ -22,3 +24,16 @@ func TestValidateLogBatchIngestAcceptsVerbatimBlankAndLongLines(t *testing.T) {
t.Fatalf("verbatim log batch was rejected: %v", err)
}
}
func TestValidateLogBatchIngestRejectsLineOnlyChecksumCompatibility(t *testing.T) {
entry := domain.LogEntry{Seq: 1, Timestamp: time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC), Line: "password=opaque /Users/operator/game.log"}
batch := domain.LogBatchIngest{RunEndpointID: "run-1", SessionToken: "session-1", LogStreamID: "run.run-1.server-1.stdout", ServerInstanceID: "server-1", StreamKey: "stdout", Source: domain.LogStreamSourceProcess, FirstSeq: 1, LastSeq: 1, Compression: "none", Checksum: lineOnlyChecksum(entry.Line), Entries: []domain.LogEntry{entry}}
if err := ValidateLogBatchIngest(batch); err == nil || !strings.Contains(err.Error(), "checksum") {
t.Fatalf("expected full-entry checksum rejection, got %v", err)
}
}
func lineOnlyChecksum(value string) string {
sum := sha256.Sum256([]byte(value))
return "sha256:" + hex.EncodeToString(sum[:])
}