admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restore durable log ingest and typed plugin projections

Browse Source
This commit is contained in:
npc0-hue
2026-09-02 10:20:30 +08:00
parent 40ac46ba17
commit 6018d8f0fc
61 changed files with 809 additions and 3157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/service/log_ingest.go
+2 -88
View File
@@ -12,65 +12,6 @@ import (
const defaultLogQueryLimit = 100
// RelayLiveLogBatch forwards output observed by Run to live subscribers. It
// intentionally updates only stream metadata; the log body is not written to
// the platform log store. Game plugins own durable log storage and analysis.
func (svc *CoreService) RelayLiveLogBatch(batch domain.LogBatchIngest) (domain.LogBatchIngestResult, error) {
batch = domain.CopyLogBatchIngest(batch)
if err := validator.ValidateLogBatchIngest(batch); err != nil {
return domain.LogBatchIngestResult{}, err
}
if err := svc.validateRunSession(batch.RunEndpointID, batch.SessionToken); err != nil {
return domain.LogBatchIngestResult{}, err
}
lock := svc.logIngestLock(batch.ServerInstanceID)
lock.Lock()
stamp := svc.now()
stream, err := svc.store.LogStreams().Get(batch.LogStreamID)
if errors.Is(err, repo.ErrNotFound) {
if repairErr := svc.ensureLogStreamForBatch(batch, stamp); repairErr != nil {
lock.Unlock()
return domain.LogBatchIngestResult{}, repairErr
}
stream, err = svc.store.LogStreams().Get(batch.LogStreamID)
}
if err != nil {
lock.Unlock()
return domain.LogBatchIngestResult{}, err
}
if err := validateLogBatchStream(batch, stream); err != nil {
lock.Unlock()
return domain.LogBatchIngestResult{}, err
}
if batch.LastSeq > stream.LatestSeq {
stream.LatestSeq = batch.LastSeq
}
stream.UpdatedAt = stamp
if err := svc.store.LogStreams().Update(stream); err != nil {
lock.Unlock()
return domain.LogBatchIngestResult{}, err
}
lock.Unlock()
entries := domain.CopyLogEntries(batch.Entries)
// Run may be on a machine whose wall clock is skewed. Relay time is the
// authoritative observation time for this best-effort live event; using it
// keeps the SSE live boundary from treating current output as old history.
for index := range entries {
entries[index].Timestamp = stamp.Add(time.Duration(index) * time.Nanosecond)
}
svc.publishLiveLogEvents(stream, entries)
return domain.LogBatchIngestResult{
Accepted: true,
LogStreamID: batch.LogStreamID,
AcceptedFrom: batch.FirstSeq,
AcceptedTo: batch.LastSeq,
LatestSeq: stream.LatestSeq,
ServerTime: stamp,
}, nil
}
func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogBatchIngestResult, error) {
batch = domain.CopyLogBatchIngest(batch)
if err := validator.ValidateLogBatchIngest(batch); err != nil {
@@ -110,9 +51,6 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
if exists && record.LastSeq == batch.LastSeq && logBatchRecordMatches(record, batch) {
locked = false
lock.Unlock()
if err := svc.projectPluginLogBatch(stream, storedLogEntries(batch.Entries)); err != nil {
return domain.LogBatchIngestResult{}, err
}
return domain.LogBatchIngestResult{
Accepted: true,
LogStreamID: batch.LogStreamID,
@@ -130,7 +68,6 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
}
storedBatch := domain.CopyLogBatchIngest(batch)
sanitizeLogNetworkFields(&storedBatch)
record := domain.CopyLogBatchRecord(domain.LogBatchRecord{
Checksum: batch.Checksum,
FirstSeq: batch.FirstSeq,
@@ -147,9 +84,6 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
}
locked = false
lock.Unlock()
if err := svc.projectPluginLogBatch(stream, storedBatch.Entries); err != nil {
return domain.LogBatchIngestResult{}, err
}
svc.publishLogEvents(stream, storedBatch.Entries)
return domain.LogBatchIngestResult{
Accepted: true,
@@ -162,10 +96,7 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
}
func storedLogEntries(entries []domain.LogEntry) []domain.LogEntry {
stored := domain.CopyLogEntries(entries)
batch := domain.LogBatchIngest{Entries: stored}
sanitizeLogNetworkFields(&batch)
return batch.Entries
return domain.CopyLogEntries(entries)
}
func (svc *CoreService) ensureJobLogStreamForBatch(batch domain.LogBatchIngest, stamp time.Time) error {
@@ -209,7 +140,7 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
expectedStreamID = runSessionLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.LogSessionID, batch.StreamKey)
}
if batch.LogStreamID != expectedStreamID {
if !legacySessionRunLogStream(batch) && !legacyAutonomousLogStream(batch) {
if batch.LogSessionID != "" || !legacyAutonomousLogStream(batch) {
return repo.ErrNotFound
}
}
@@ -227,10 +158,6 @@ func (svc *CoreService) ensureRunLogStreamForBatch(batch domain.LogBatchIngest,
return err
}
func legacySessionRunLogStream(batch domain.LogBatchIngest) bool {
return batch.LogSessionID != "" && batch.LogStreamID == runLogStreamID(batch.RunEndpointID, batch.ServerInstanceID, batch.StreamKey)
}
func legacyAutonomousLogStream(batch domain.LogBatchIngest) bool {
jobID, ok := jobIDFromLogBatch(batch)
return ok && strings.HasPrefix(jobID, "autonomous-")
@@ -260,19 +187,6 @@ func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIn
return false
}
// sanitizeLogNetworkFields removes raw network material before the durable log body is written.
func sanitizeLogNetworkFields(batch *domain.LogBatchIngest) {
for index := range batch.Entries {
fields := batch.Entries[index].Fields
if fields == nil {
continue
}
delete(fields, "networkFingerprint")
delete(fields, "ip")
delete(fields, "ipAddress")
}
}
func (svc *CoreService) QueryLogStream(query domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error) {
if err := validator.ValidateLogStreamCursorQuery(query); err != nil {
return domain.LogStreamCursorResult{}, err