Stream live server logs over SSE

This commit is contained in:
npc0-hue
2026-08-03 22:28:54 +08:00
parent 5d4fca14f9
commit 7eac1926dd
48 changed files with 1526 additions and 263 deletions
+21 -1
View File
@@ -745,6 +745,11 @@ func (svc *CoreService) QueueLogBackfillForSession(sessionID string, request dom
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: plugin capability is not declared")
return domain.Job{}, ErrForbidden
}
source, err := declaredFileLogSource(plugin, request.SourceKey)
if err != nil {
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: log source is not declared")
return domain.Job{}, err
}
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "logs.backfill.denied"); err != nil {
return domain.Job{}, err
}
@@ -753,10 +758,11 @@ func (svc *CoreService) QueueLogBackfillForSession(sessionID string, request dom
ServerInstanceID: instance.ID,
RunEndpointID: instance.RunEndpointID,
Capability: domain.JobCapabilityLogsBackfill,
TargetKey: "logs/" + request.SourceKey,
TargetKey: "logs/" + source.Key,
InputRef: request.CheckpointRef,
IdempotencyKey: request.IdempotencyKey,
Progress: domain.JobProgress{Percent: 0, Message: "historical log backfill queued"},
ExecutionInput: domain.JobExecutionInput{LogSource: &source},
})
if err != nil {
_ = svc.recordAuditEvent(user.ID, "logs.backfill.denied", "server-instance", instance.ID, domain.AuditResultDenied, "log backfill denied: endpoint unsupported or offline")
@@ -768,6 +774,20 @@ func (svc *CoreService) QueueLogBackfillForSession(sessionID string, request dom
return domain.CopyJob(job), nil
}
func declaredFileLogSource(plugin domain.GamePlugin, sourceKey string) (domain.RuntimeLogSource, error) {
for _, source := range plugin.RuntimeProfiles.LogSources {
if source.Key != sourceKey {
continue
}
if source.Kind != "file.tail" || strings.TrimSpace(source.TargetKey) == "" || strings.TrimSpace(source.StreamKey) == "" {
return domain.RuntimeLogSource{}, validationError("log source must be a file.tail source with target and stream keys")
}
copy := source
return copy, nil
}
return domain.RuntimeLogSource{}, validationError("log source is not declared by plugin")
}
func (svc *CoreService) validateDistributionPluginPermission(actorID string, plugin domain.GamePlugin, serverInstanceID string, permission string, deniedAction string) error {
if plugin.Status != domain.GamePluginStatusInstalled {
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "distribution operation denied: plugin is not installed")
+27 -1
View File
@@ -627,6 +627,7 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
)
plugin.RuntimeProfiles.DependencyProbes = []domain.RuntimeDependencyProbe{{Key: "java-runtime", Kind: "command.version", TargetKey: "java", Platforms: []string{"linux"}}}
plugin.RuntimeProfiles.InstallPlans = []domain.RuntimeInstallPlan{{Key: "java-install", Title: "Install Java", Platforms: []string{"linux"}, Steps: []domain.RuntimeInstallStep{{Type: "package", TargetKey: "java", PackageManager: "apt", PackageName: "openjdk-21-jre"}}}}
plugin.RuntimeProfiles.LogSources = []domain.RuntimeLogSource{{Key: "latest", Kind: "file.tail", TargetKey: "logs/latest", StreamKey: "latest-log", CursorKind: "offset", RetentionDays: 30}}
plugin.RuntimeProfiles.ClientManagers = []domain.RuntimeClientManagerProfile{{Key: "scum-client-manager", DisplayName: "SCUM Client Manager", Version: "1.0.0", RepositoryURL: "https://github.com/F88888/scum_client.git", RevisionPolicy: "branch", Branch: "main", SupportedTargets: []domain.RuntimeTarget{{OS: "windows", Arch: "amd64"}, {OS: "linux", Arch: "amd64"}}, BuildSystem: "go", EntryRef: "main.go", OutputArtifacts: []string{"scum_client.exe"}, Deployment: domain.RuntimeClientManagerDeployment{Mode: "run-supervised", ExecutableRef: "scum_client.exe", RequiredRunCapabilities: []string{domain.JobCapabilityClientManagerDeploy, domain.JobCapabilityClientManagerControl, domain.JobCapabilityClientManagerUpdate, domain.JobCapabilityClientManagerRollback, domain.JobCapabilityClientManagerUninstall}}, Lifecycle: domain.RuntimeClientManagerLifecycle{Actions: []string{"start", "stop", "restart", "status", "update", "rollback", "uninstall"}, StartupTimeoutSeconds: 60, StopTimeoutSeconds: 30}, Health: domain.RuntimeClientManagerHealth{Mode: "component-heartbeat", IntervalSeconds: 15, DegradedAfterSeconds: 45, OfflineAfterSeconds: 120, RequiredCapabilities: []string{"component.register", "component.heartbeat", "component.health"}}, Compatibility: domain.RuntimeClientManagerCompatibility{MinimumVersion: "1.0.0"}, UpdatePolicy: domain.RuntimeClientManagerUpdatePolicy{Strategy: "manual-staged", RequireApproval: true, HealthConfirmationSeconds: 60, RetainPrevious: true}}}
if err := svc.store.GamePlugins().Update(plugin); err != nil {
t.Fatalf("update plugin fixture: %v", err)
@@ -664,10 +665,35 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
if err != nil {
t.Fatalf("create distribution server: %v", err)
}
createCompleteRuntimeBinding(t, svc, instance, "local")
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"logs/latest": "runtime.logs.latest"}}, true)
if err != nil {
t.Fatalf("build complete runtime binding: %v", err)
}
if err := svc.store.RuntimeBindings().Create(binding); err != nil {
t.Fatalf("create runtime binding: %v", err)
}
return svc, session, instance
}
func TestQueueLogBackfillFreezesDeclaredFileLogSource(t *testing.T) {
svc, session, instance := newDistributionTestFixture(t)
job, err := svc.QueueLogBackfillForSession(session, domain.LogBackfillRequest{ServerInstanceID: instance.ID, SourceKey: "latest", CheckpointRef: "artifact://logs/checkpoint/1", IdempotencyKey: "log-source-freeze"})
if err != nil {
t.Fatalf("queue log backfill: %v", err)
}
if job.ExecutionInput.LogSource == nil || job.ExecutionInput.LogSource.Key != "latest" || job.ExecutionInput.LogSource.StreamKey != "latest-log" || job.ExecutionInput.LogSource.TargetKey != "logs/latest" {
t.Fatalf("expected frozen declared log source, got %+v", job.ExecutionInput.LogSource)
}
stream, err := svc.GetLogStream(jobLogStreamID(job.ID, "latest-log"))
if err != nil || stream.Source != domain.LogStreamSourceFile || stream.StreamKey != "latest-log" {
t.Fatalf("expected file log stream for backfill job, stream=%+v err=%v", stream, err)
}
if _, err := svc.QueueLogBackfillForSession(session, domain.LogBackfillRequest{ServerInstanceID: instance.ID, SourceKey: "missing", IdempotencyKey: "log-source-missing"}); err == nil || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("expected undeclared source rejection, got %v", err)
}
}
func readGeneratedPackageConfig(t *testing.T, svc *CoreService, session string, artifactID string) generatedPackageConfig {
t.Helper()
_ = session
+1 -1
View File
@@ -639,7 +639,7 @@ func assignmentFromJob(job domain.Job, leaseToken string) domain.RunJobAssignmen
State: job.State,
Progress: domain.RunJobProgressReport{Percent: job.Progress.Percent, Phase: job.Progress.Phase, Message: job.Progress.Message},
ResultRef: job.ResultRef,
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: job.ExecutionInput.WorkspaceScope, Content: job.ExecutionInput.Content, ExpectedVersion: job.ExecutionInput.ExpectedVersion, ExpectedChecksum: job.ExecutionInput.ExpectedChecksum, MaxReadBytes: job.ExecutionInput.MaxReadBytes, RemoteAdapterKey: job.ExecutionInput.RemoteAdapterKey, RemoteAdapterKind: job.ExecutionInput.RemoteAdapterKind, TimeoutSeconds: job.ExecutionInput.TimeoutSeconds, PluginID: job.ExecutionInput.PluginID, LifecycleOperation: job.ExecutionInput.LifecycleOperation, TargetVersion: job.ExecutionInput.TargetVersion, Inputs: domain.CopyStringMap(job.ExecutionInput.Inputs), DLLExtensions: append([]domain.RuntimeDLLExtensionPlan(nil), job.ExecutionInput.DLLExtensions...), SourceRCON: domain.CopyRuntimeSourceRCONPlan(job.ExecutionInput.SourceRCON), Deployment: deploymentPlanForDispatchValue(job.ExecutionInput.Deployment), ServerDeploymentPlan: domain.CopyServerDeploymentPlan(job.ExecutionInput.ServerDeploymentPlan)},
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: job.ExecutionInput.WorkspaceScope, Content: job.ExecutionInput.Content, ExpectedVersion: job.ExecutionInput.ExpectedVersion, ExpectedChecksum: job.ExecutionInput.ExpectedChecksum, MaxReadBytes: job.ExecutionInput.MaxReadBytes, RemoteAdapterKey: job.ExecutionInput.RemoteAdapterKey, RemoteAdapterKind: job.ExecutionInput.RemoteAdapterKind, TimeoutSeconds: job.ExecutionInput.TimeoutSeconds, PluginID: job.ExecutionInput.PluginID, LifecycleOperation: job.ExecutionInput.LifecycleOperation, TargetVersion: job.ExecutionInput.TargetVersion, Inputs: domain.CopyStringMap(job.ExecutionInput.Inputs), LogSource: domain.CopyRuntimeLogSourcePtr(job.ExecutionInput.LogSource), LogSources: domain.CopyRuntimeLogSources(job.ExecutionInput.LogSources), DLLExtensions: append([]domain.RuntimeDLLExtensionPlan(nil), job.ExecutionInput.DLLExtensions...), SourceRCON: domain.CopyRuntimeSourceRCONPlan(job.ExecutionInput.SourceRCON), Deployment: deploymentPlanForDispatchValue(job.ExecutionInput.Deployment), ServerDeploymentPlan: domain.CopyServerDeploymentPlan(job.ExecutionInput.ServerDeploymentPlan)},
LeaseToken: leaseToken,
Attempt: job.Attempt,
FencingToken: fencingToken,
+87
View File
@@ -0,0 +1,87 @@
package service
import (
"strings"
"browser.local/platform/domain"
)
const logEventSubscriberBuffer = 512
type LogEventSubscription struct {
Events <-chan domain.LogStreamEvent
Close func()
}
type logEventSubscriber struct {
serverInstanceID string
events chan domain.LogStreamEvent
}
func (svc *CoreService) SubscribeLogEvents(serverInstanceID string) (LogEventSubscription, error) {
serverInstanceID = strings.TrimSpace(serverInstanceID)
if serverInstanceID == "" {
return LogEventSubscription{}, validationError("serverInstanceId is required")
}
if _, err := svc.store.ServerInstances().Get(serverInstanceID); err != nil {
return LogEventSubscription{}, err
}
events := make(chan domain.LogStreamEvent, logEventSubscriberBuffer)
svc.logEventMu.Lock()
svc.logEventSubscriberSeq++
id := svc.logEventSubscriberSeq
svc.logEventSubscribers[id] = logEventSubscriber{serverInstanceID: serverInstanceID, events: events}
svc.logEventMu.Unlock()
closeOnce := func() {
svc.logEventMu.Lock()
if subscriber, ok := svc.logEventSubscribers[id]; ok {
delete(svc.logEventSubscribers, id)
close(subscriber.events)
}
svc.logEventMu.Unlock()
}
return LogEventSubscription{Events: events, Close: closeOnce}, nil
}
func (svc *CoreService) SubscribeLogEventsForSession(sessionID string, serverInstanceID string) (LogEventSubscription, error) {
instance, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID)
if err != nil {
return LogEventSubscription{}, err
}
return svc.SubscribeLogEvents(instance.ID)
}
func (svc *CoreService) publishLogEvents(stream domain.LogStream, entries []domain.LogEntry) {
if len(entries) == 0 {
return
}
events := make([]domain.LogStreamEvent, len(entries))
for index, entry := range entries {
events[index] = domain.CopyLogStreamEvent(domain.LogStreamEvent{
ServerInstanceID: stream.ServerInstanceID,
Stream: stream,
Entry: entry,
LatestSeq: stream.LatestSeq,
})
}
svc.logEventMu.Lock()
for id, subscriber := range svc.logEventSubscribers {
if subscriber.serverInstanceID != stream.ServerInstanceID {
continue
}
dropped := false
for _, event := range events {
select {
case subscriber.events <- event:
default:
delete(svc.logEventSubscribers, id)
close(subscriber.events)
dropped = true
}
if dropped {
break
}
}
}
svc.logEventMu.Unlock()
}
+40
View File
@@ -1,7 +1,12 @@
package service
import (
"errors"
"strings"
"time"
"browser.local/platform/domain"
"browser.local/platform/repo"
"browser.local/platform/validator"
)
@@ -19,6 +24,11 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
stamp := svc.now()
stream, err := svc.store.LogStreams().Get(batch.LogStreamID)
if errors.Is(err, repo.ErrNotFound) {
if repairErr := svc.ensureJobLogStreamForBatch(batch, stamp); repairErr == nil {
stream, err = svc.store.LogStreams().Get(batch.LogStreamID)
}
}
if err != nil {
return domain.LogBatchIngestResult{}, err
}
@@ -76,6 +86,7 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
if err := svc.projectGameMapTrajectoryEvents(projectionBatch); err != nil {
return domain.LogBatchIngestResult{}, err
}
svc.publishLogEvents(stream, storedBatch.Entries)
return domain.LogBatchIngestResult{
Accepted: true,
LogStreamID: batch.LogStreamID,
@@ -86,6 +97,35 @@ func (svc *CoreService) IngestLogBatch(batch domain.LogBatchIngest) (domain.LogB
}, nil
}
func (svc *CoreService) ensureJobLogStreamForBatch(batch domain.LogBatchIngest, stamp time.Time) error {
jobID, ok := jobIDFromLogBatch(batch)
if !ok {
return repo.ErrNotFound
}
job, err := svc.store.Jobs().Get(jobID)
if err != nil {
return err
}
if job.ServerInstanceID != batch.ServerInstanceID || job.RunEndpointID != batch.RunEndpointID {
return validationError("log batch job scope does not match stream")
}
return svc.ensureJobLogStreams(job, stamp)
}
func jobIDFromLogBatch(batch domain.LogBatchIngest) (string, bool) {
streamKey := strings.TrimSpace(batch.StreamKey)
if streamKey == "" || !strings.HasPrefix(batch.LogStreamID, "job.") {
return "", false
}
suffix := "." + streamKey
body := strings.TrimPrefix(batch.LogStreamID, "job.")
if !strings.HasSuffix(body, suffix) {
return "", false
}
jobID := strings.TrimSuffix(body, suffix)
return jobID, strings.TrimSpace(jobID) != ""
}
func logBatchRecordMatches(record domain.LogBatchRecord, batch domain.LogBatchIngest) bool {
if record.Checksum == batch.Checksum {
return true
+119
View File
@@ -39,6 +39,38 @@ func TestCoreServiceIngestsLogBatchAndQueriesCursor(t *testing.T) {
}
}
func TestCoreServicePublishesLogEventsForAcceptedBatch(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
subscription, err := svc.SubscribeLogEvents("server-1")
if err != nil {
t.Fatalf("subscribe log events: %v", err)
}
defer subscription.Close()
batch := validLogBatch(t, sessionToken, 1, 1)
if _, err := svc.IngestLogBatch(batch); err != nil {
t.Fatalf("ingest log batch: %v", err)
}
select {
case event := <-subscription.Events:
if event.Stream.ID != "log-1" || event.Entry.Seq != 1 || event.LatestSeq != 1 {
t.Fatalf("unexpected log event: %+v", event)
}
case <-time.After(time.Second):
t.Fatal("expected log event after accepted batch")
}
if _, err := svc.IngestLogBatch(batch); err != nil {
t.Fatalf("ingest duplicate batch: %v", err)
}
select {
case event := <-subscription.Events:
t.Fatalf("duplicate batch should not publish a second event: %+v", event)
default:
}
}
func TestCoreServiceLogBatchDuplicateAck(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
@@ -114,6 +146,93 @@ func TestCoreServiceAcceptsAutoCreatedRunJobLogStreams(t *testing.T) {
}
}
func TestCoreServiceAcceptsPluginDeclaredProcessLogStreams(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
job, err := svc.CreateJob(domain.Job{
ID: "job-declared-process-logs",
ServerInstanceID: "server-1",
RunEndpointID: "run-local",
Capability: domain.LifecycleCapabilityStart,
IdempotencyKey: "job-declared-process-logs",
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: "run-local", LifecycleOperation: "start", LogSources: []domain.RuntimeLogSource{
{Key: "console-out", Kind: "process.stdout", StreamKey: "scum.console.stdout", CursorKind: "sequence", RetentionDays: 30},
{Key: "console-err", Kind: "process.stderr", StreamKey: "scum.console.stderr", CursorKind: "sequence", RetentionDays: 30},
}},
})
if err != nil {
t.Fatalf("create job: %v", err)
}
streamID := jobLogStreamID(job.ID, "scum.console.stdout")
stream, err := svc.GetLogStream(streamID)
if err != nil {
t.Fatalf("get declared process log stream: %v", err)
}
if stream.StreamKey != "scum.console.stdout" || stream.Source != domain.LogStreamSourceProcess {
t.Fatalf("unexpected declared stream metadata: %+v", stream)
}
entry := domain.LogEntry{Seq: 1, Timestamp: time.Date(2026, 7, 3, 12, 0, 1, 0, time.UTC), Level: "info", Line: "LogStreaming: Display: server ready"}
ack, err := svc.IngestLogBatch(domain.LogBatchIngest{
RunEndpointID: "run-local",
SessionToken: sessionToken,
LogStreamID: streamID,
ServerInstanceID: "server-1",
StreamKey: "scum.console.stdout",
Source: domain.LogStreamSourceProcess,
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Entries: []domain.LogEntry{entry},
})
if err != nil {
t.Fatalf("ingest declared process log batch: %v", err)
}
if !ack.Accepted || ack.LatestSeq != entry.Seq {
t.Fatalf("unexpected declared stream ack: %+v", ack)
}
}
func TestCoreServiceRepairsMissingDeclaredProcessLogStreamOnIngest(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
job := domain.Job{
ID: "job-repaired-process-logs",
ServerInstanceID: "server-1",
RunEndpointID: "run-local",
Capability: domain.LifecycleCapabilityStart,
IdempotencyKey: "job-repaired-process-logs",
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: "run-local", LifecycleOperation: "start", LogSources: []domain.RuntimeLogSource{
{Key: "console-out", Kind: "process.stdout", StreamKey: "scum.console.stdout", CursorKind: "sequence", RetentionDays: 30},
}},
}
if err := svc.store.Jobs().Create(job); err != nil {
t.Fatalf("seed legacy job without streams: %v", err)
}
streamID := jobLogStreamID(job.ID, "scum.console.stdout")
if _, err := svc.GetLogStream(streamID); err == nil {
t.Fatal("expected declared stream to be missing before ingest repair")
}
entry := domain.LogEntry{Seq: 1, Timestamp: time.Date(2026, 7, 3, 12, 0, 1, 0, time.UTC), Level: "info", Line: "LogStreaming: Display: recovered from spool"}
ack, err := svc.IngestLogBatch(domain.LogBatchIngest{
RunEndpointID: "run-local",
SessionToken: sessionToken,
LogStreamID: streamID,
ServerInstanceID: "server-1",
StreamKey: "scum.console.stdout",
Source: domain.LogStreamSourceProcess,
FirstSeq: entry.Seq,
LastSeq: entry.Seq,
Compression: "none",
Checksum: validator.LogLineChecksum(entry.Line),
Entries: []domain.LogEntry{entry},
})
if err != nil {
t.Fatalf("ingest repaired declared process log batch: %v", err)
}
if !ack.Accepted || ack.LatestSeq != entry.Seq {
t.Fatalf("unexpected repaired stream ack: %+v", ack)
}
}
func TestCoreServiceRejectsOutOfOrderAndConflictingLogBatches(t *testing.T) {
svc, sessionToken := newRegisteredLogIngestService(t)
createLogStreamFixture(t, svc)
+14 -7
View File
@@ -100,6 +100,19 @@ func (svc *CoreService) dispatchProtectedRequest(command domain.GameClientBridge
if jobID == "" {
return validationError("protected request job binding is missing")
}
executionInput := domain.JobExecutionInput{
WorkspaceScope: command.ProfileKey,
RemoteAdapterKey: declaration.ProtectedRequest.TransportKey,
RemoteAdapterKind: adapterKind,
TimeoutSeconds: declaration.TimeoutSeconds,
PluginID: command.PluginID,
}
if declaration.ProtectedRequest.Kind == "rcon" {
if resolution, resolveErr := svc.resolveProtectedSourceRCONDispatch(command.ServerInstanceID, declaration.ProtectedRequest); resolveErr == nil {
executionInput.WorkspaceScope = resolution.binding.ProfileKey
executionInput.SourceRCON = resolution.plan
}
}
if err := svc.protectedRequests.Put(jobID, protectedRequestPayload{commandID: command.ID, kind: declaration.ProtectedRequest.Kind, transportKey: declaration.ProtectedRequest.TransportKey, targetKey: declaration.ProtectedRequest.TargetKey, requestText: requestText, expiresAt: command.ExpiresAt}); err != nil {
return err
}
@@ -113,13 +126,7 @@ func (svc *CoreService) dispatchProtectedRequest(command domain.GameClientBridge
IdempotencyKey: "protected-request:" + command.ID,
Progress: domain.JobProgress{Percent: 0, Message: "protected request queued"},
RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 1, InitialBackoffSeconds: 1, MaxBackoffSeconds: 1},
ExecutionInput: domain.JobExecutionInput{
WorkspaceScope: command.ProfileKey,
RemoteAdapterKey: declaration.ProtectedRequest.TransportKey,
RemoteAdapterKind: adapterKind,
TimeoutSeconds: declaration.TimeoutSeconds,
PluginID: command.PluginID,
},
ExecutionInput: executionInput,
}
if job.RunEndpointID == "" {
svc.protectedRequests.Delete(jobID)
+34 -13
View File
@@ -202,6 +202,8 @@ type Core interface {
GetLogStreamForSession(string, string) (domain.LogStream, error)
ListLogStreamsForSession(string, domain.LogStreamFilter) ([]domain.LogStream, error)
QueryLogStreamForSession(string, domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
SubscribeLogEvents(string) (LogEventSubscription, error)
SubscribeLogEventsForSession(string, string) (LogEventSubscription, error)
IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error)
QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
ListGamePlayersForSession(string, domain.GamePlayerFilter) ([]domain.GamePlayer, error)
@@ -236,6 +238,9 @@ type CoreService struct {
bridgeMu sync.Mutex
bridgeSeq uint64
logStore LogBodyStore
logEventMu sync.Mutex
logEventSubscribers map[uint64]logEventSubscriber
logEventSubscriberSeq uint64
artifactStore ArtifactBodyStore
artifactMu sync.Mutex
artifactTransfers map[string]domain.ArtifactTransferSession
@@ -279,6 +284,7 @@ func newCoreServiceWithLogStore(store repo.Store, logStore LogBodyStore, now fun
authSessions: map[string]string{},
runSessions: map[string]domain.RunControlSession{},
logStore: logStore,
logEventSubscribers: map[uint64]logEventSubscriber{},
artifactStore: artifactStore,
artifactTransfers: map[string]domain.ArtifactTransferSession{},
artifactPayloads: map[string][]byte{},
@@ -2329,21 +2335,36 @@ func (svc *CoreService) ensureJobLogStreams(job domain.Job, stamp time.Time) err
streams := []struct {
key string
source domain.LogStreamSource
}{
{key: "stdout", source: domain.LogStreamSourceProcess},
{key: "stderr", source: domain.LogStreamSourceProcess},
}{}
addStream := func(key string, source domain.LogStreamSource) {
key = strings.TrimSpace(key)
if key == "" {
return
}
for _, stream := range streams {
if stream.key == key {
return
}
}
streams = append(streams, struct {
key string
source domain.LogStreamSource
}{key: key, source: source})
}
addStream("stdout", domain.LogStreamSourceProcess)
addStream("stderr", domain.LogStreamSourceProcess)
for _, source := range job.ExecutionInput.LogSources {
if source.Kind != "process.stdout" && source.Kind != "process.stderr" {
continue
}
addStream(source.StreamKey, domain.LogStreamSourceProcess)
}
if job.Capability == domain.JobCapabilityRemoteRunProgram {
streams = append(streams,
struct {
key string
source domain.LogStreamSource
}{key: "management-program.stdout", source: domain.LogStreamSourceManagementProgram},
struct {
key string
source domain.LogStreamSource
}{key: "management-program.stderr", source: domain.LogStreamSourceManagementProgram},
)
addStream("management-program.stdout", domain.LogStreamSourceManagementProgram)
addStream("management-program.stderr", domain.LogStreamSourceManagementProgram)
}
if job.Capability == domain.JobCapabilityLogsBackfill && job.ExecutionInput.LogSource != nil && strings.TrimSpace(job.ExecutionInput.LogSource.StreamKey) != "" {
addStream(job.ExecutionInput.LogSource.StreamKey, domain.LogStreamSourceFile)
}
for _, item := range streams {
stream := domain.LogStream{
+19 -1
View File
@@ -1695,7 +1695,20 @@ func createCompleteRuntimeBinding(t *testing.T, svc *CoreService, instance domai
if err != nil {
t.Fatalf("get plugin for runtime binding: %v", err)
}
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: profileKey, Bindings: map[string]string{}}, true)
profile, ok := runtimeLifecycleProfile(plugin.RuntimeProfiles, profileKey)
if !ok {
t.Fatalf("runtime profile %s missing", profileKey)
}
required, _ := runtimeBindingKeys(plugin.RuntimeProfiles, profile)
bindings := map[string]string{}
for _, key := range required {
if runtimeBindingTestKeyIsSensitive(key) {
bindings[key] = "secret://" + instance.ID + "/" + strings.ReplaceAll(key, "/", "-")
} else {
bindings[key] = "runtime." + strings.ReplaceAll(key, "/", ".")
}
}
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: profileKey, Bindings: bindings}, true)
if err != nil {
t.Fatalf("build runtime binding: %v", err)
}
@@ -1705,6 +1718,11 @@ func createCompleteRuntimeBinding(t *testing.T, svc *CoreService, instance domai
return binding
}
func runtimeBindingTestKeyIsSensitive(key string) bool {
normalized := strings.ToLower(key)
return strings.Contains(normalized, "password") || strings.Contains(normalized, "credential") || strings.Contains(normalized, "secret") || strings.Contains(normalized, "token") || strings.Contains(normalized, "dsn")
}
func validPluginManifestRegistration() domain.GamePluginManifestRegistration {
return domain.GamePluginManifestRegistration{
ManifestRef: "artifact://manifests/game.example/0.1.0",
+3 -3
View File
@@ -39,8 +39,8 @@ func (svc *CoreService) UpdateServerRuntimeBindingForSession(sessionID, serverIn
if existingErr != nil && !errors.Is(existingErr, repo.ErrNotFound) {
return domain.RuntimeBindingView{}, existingErr
}
if (instance.State == domain.ServerInstanceStateInstalling || instance.State == domain.ServerInstanceStateRunning) && existingErr == nil || instance.State == domain.ServerInstanceStateDeleted {
return domain.RuntimeBindingView{}, validationError("runtime binding cannot be changed while the server is active")
if instance.State == domain.ServerInstanceStateDeleted {
return domain.RuntimeBindingView{}, validationError("runtime binding cannot be changed after the server is deleted")
}
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
if err != nil {
@@ -124,7 +124,7 @@ func runtimeBindingKeys(profiles domain.GamePluginRuntimeProfiles, profile domai
add(probe.TargetKey, probe.Required)
}
for _, source := range profiles.LogSources {
add(source.TargetKey, source.TargetKey != "")
add(source.TargetKey, false)
}
for _, plan := range profiles.InstallPlans {
for _, step := range plan.Steps {
+46
View File
@@ -98,6 +98,52 @@ func TestRuntimeBindingValidationAndLifecycleGating(t *testing.T) {
if err != nil || result.Job.TargetKey != "actions/start.json" || result.Job.ExecutionInput.WorkspaceScope != "local" {
t.Fatalf("expected complete binding to permit start, result=%+v err=%v", result, err)
}
view, err = svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"server-root": "runtime.server-root-updated"}})
if err != nil || view.Status != domain.RuntimeBindingStatusComplete {
t.Fatalf("expected active server runtime binding edits to remain available, view=%+v err=%v", view, err)
}
instance.State = domain.ServerInstanceStateDeleted
if err := svc.store.ServerInstances().Update(instance); err != nil {
t.Fatalf("mark deleted: %v", err)
}
if _, err := svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"server-root": "runtime.server-root"}}); err == nil || !strings.Contains(err.Error(), "deleted") {
t.Fatalf("expected deleted server runtime binding edit rejection, got %v", err)
}
}
func TestRuntimeBindingLogSourcesAreConfigurableButNotRequired(t *testing.T) {
svc := newTestCoreService()
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
plugin.RuntimeProfiles = requiredRuntimeProfilesFixture()
plugin.RuntimeProfiles.LogSources = []domain.RuntimeLogSource{
{Key: "console-stdout", Kind: "process.stdout", TargetKey: "process/server", StreamKey: "console.stdout", CursorKind: "sequence", RetentionDays: 30},
{Key: "latest", Kind: "file.tail", TargetKey: "logs/latest", StreamKey: "latest-log", CursorKind: "offset", RetentionDays: 30},
}
if err := svc.store.GamePlugins().Update(plugin); err != nil {
t.Fatalf("update plugin profiles: %v", err)
}
ownerSession := createServiceUserAndLogin(t, svc, domain.User{ID: "runtime-logs-owner", DisplayName: "Runtime Logs Owner", Email: "runtime-logs-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
instance, err := svc.CreateServerInstanceForSession(ownerSession, domain.ServerInstance{ID: "runtime-log-sources", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Runtime Log Sources", State: domain.ServerInstanceStateRunning})
if err != nil {
t.Fatalf("create server: %v", err)
}
view, err := svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"server-root": "runtime.server-root", "rcon.password": "secret://runtime-log-sources/rcon"}})
if err != nil || view.Status != domain.RuntimeBindingStatusComplete || len(view.MissingKeys) != 0 {
t.Fatalf("log sources should not block runtime readiness: view=%+v err=%v", view, err)
}
seenLogs := map[string]domain.RuntimeBindingKeyView{}
for _, key := range view.Keys {
if strings.HasPrefix(key.Key, "logs/") || strings.HasPrefix(key.Key, "process/") {
seenLogs[key.Key] = key
}
}
for _, key := range []string{"logs/latest", "process/server"} {
item, ok := seenLogs[key]
if !ok || item.Required || item.Configured {
t.Fatalf("expected optional unconfigured log key %s, seen=%+v view=%+v", key, seenLogs, view)
}
}
}
func requiredRuntimeProfilesFixture() domain.GamePluginRuntimeProfiles {
+13
View File
@@ -310,6 +310,7 @@ func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, act
return domain.Job{}, validationError(fmt.Sprintf("plugin %s lifecycle action is required", action))
}
var dllExtensions []domain.RuntimeDLLExtensionPlan
var logSources []domain.RuntimeLogSource
if action == domain.ServerLifecycleActionStart && hasProfile {
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
if err != nil {
@@ -319,6 +320,7 @@ func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, act
if err != nil {
return domain.Job{}, err
}
logSources = lifecycleProcessLogSources(plugin.RuntimeProfiles)
}
job, err := svc.CreateJob(domain.Job{
ID: lifecycleJobID(instance.ID, action, idempotencyKey),
@@ -332,6 +334,7 @@ func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, act
WorkspaceScope: profileKey,
PluginID: plugin.ID,
LifecycleOperation: lifecycleExecutionOperation(action),
LogSources: logSources,
DLLExtensions: dllExtensions,
Deployment: deploymentPlanForDispatch(instance.Deployment),
},
@@ -345,6 +348,16 @@ func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, act
return job, nil
}
func lifecycleProcessLogSources(profiles domain.GamePluginRuntimeProfiles) []domain.RuntimeLogSource {
sources := []domain.RuntimeLogSource{}
for _, source := range profiles.LogSources {
if source.Kind == "process.stdout" || source.Kind == "process.stderr" {
sources = append(sources, source)
}
}
return sources
}
func lifecycleJobProgress(deployment domain.ServerDeploymentDefinition) domain.JobProgress {
if deployment.Mode != "" {
return domain.JobProgress{Percent: 0, Phase: "queued", Message: "deployment queued; awaiting Run claim"}
+8 -2
View File
@@ -52,6 +52,9 @@ func TestCoreServiceServerLifecycleWorkflows(t *testing.T) {
if started.Job.ExecutionInput.PluginID != "server.scum" || started.Job.ExecutionInput.WorkspaceScope != "local" || started.Job.ExecutionInput.LifecycleOperation != "start" {
t.Fatalf("expected start job to carry plugin/profile metadata, got %+v", started.Job.ExecutionInput)
}
if len(started.Job.ExecutionInput.LogSources) != 2 || started.Job.ExecutionInput.LogSources[0].StreamKey != "scum.console.stdout" || started.Job.ExecutionInput.LogSources[1].StreamKey != "scum.console.stderr" {
t.Fatalf("expected start job to carry plugin-declared process log sources, got %+v", started.Job.ExecutionInput.LogSources)
}
claimAndCompleteLifecycleJob(t, svc, sessionToken, domain.LifecycleCapabilityStart, domain.JobStateSucceeded)
running, err := svc.GetServerInstance("server-1")
if err != nil {
@@ -362,8 +365,11 @@ func createLifecyclePlugin(t *testing.T, svc *CoreService) domain.GamePlugin {
Start: "actions/start.json",
Stop: "actions/stop.json",
},
Permissions: domain.PluginPermissions{Jobs: true, Logs: true},
RuntimeProfiles: domain.GamePluginRuntimeProfiles{LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop}}}},
Permissions: domain.PluginPermissions{Jobs: true, Logs: true},
RuntimeProfiles: domain.GamePluginRuntimeProfiles{
LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop}}},
LogSources: []domain.RuntimeLogSource{{Key: "scum-console-stdout", Kind: "process.stdout", TargetKey: "scum/server-process", StreamKey: "scum.console.stdout", CursorKind: "sequence", RetentionDays: 30}, {Key: "scum-console-stderr", Kind: "process.stderr", TargetKey: "scum/server-process", StreamKey: "scum.console.stderr", CursorKind: "sequence", RetentionDays: 30}},
},
})
if err != nil {
t.Fatalf("create lifecycle plugin: %v", err)
+64 -1
View File
@@ -212,9 +212,16 @@ func (svc *CoreService) resolveSourceRCONDispatch(instance domain.ServerInstance
}
func sourceRCONTransport(profiles domain.GamePluginRuntimeProfiles, profile domain.RuntimeLifecycleProfile) (domain.RuntimeTransportProfile, error) {
return sourceRCONTransportForCapability(profiles, profile, "", domain.JobCapabilityRemoteRunRCONCommand)
}
func sourceRCONTransportForCapability(profiles domain.GamePluginRuntimeProfiles, profile domain.RuntimeLifecycleProfile, requiredKey string, capability string) (domain.RuntimeTransportProfile, error) {
var selected domain.RuntimeTransportProfile
for _, candidate := range profiles.TransportProfiles {
if !containsString(profile.TransportKeys, candidate.Key) || candidate.Kind != "rcon" || !containsString(candidate.Capabilities, domain.JobCapabilityRemoteRunRCONCommand) {
if requiredKey != "" && candidate.Key != requiredKey {
continue
}
if !containsString(profile.TransportKeys, candidate.Key) || candidate.Kind != "rcon" || !containsString(candidate.Capabilities, capability) {
continue
}
if selected.Key != "" {
@@ -228,6 +235,62 @@ func sourceRCONTransport(profiles domain.GamePluginRuntimeProfiles, profile doma
return selected, nil
}
func (svc *CoreService) resolveProtectedSourceRCONDispatch(serverInstanceID string, request *domain.GameClientBridgeProtectedRequestDeclaration) (sourceRCONDispatchResolution, error) {
if request == nil || request.Kind != "rcon" {
return sourceRCONDispatchResolution{}, validationError("protected request is not RCON")
}
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
capability := domain.JobCapabilityRemoteRunProtectedRCON
if plugin.Status != domain.GamePluginStatusInstalled || plugin.Version != instance.PluginVersion || !plugin.Permissions.RemoteAccess || !plugin.RemoteAccess.RCON || !containsString(plugin.RequiredRunCapabilities, capability) || !containsString(plugin.RemoteAccess.RunCapabilities, capability) {
return sourceRCONDispatchResolution{}, forbiddenError("plugin does not declare protected SCUM RCON access")
}
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
if err := svc.validateRunnableEndpoint(endpoint, capability); err != nil {
return sourceRCONDispatchResolution{}, err
}
if !strings.EqualFold(endpoint.Platform, "windows") || !strings.EqualFold(endpoint.Architecture, "amd64") {
return sourceRCONDispatchResolution{}, validationError("unsupported_extension_platform: SCUM Source RCON requires windows/amd64")
}
binding, err := svc.runtimeBindingForServer(instance.ID)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
binding, err = normalizeRuntimeBinding(plugin, binding)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
if binding.Status != domain.RuntimeBindingStatusComplete || binding.PluginVersion != plugin.Version {
return sourceRCONDispatchResolution{}, validationError("runtime binding is incomplete or stale")
}
profile, exists := runtimeLifecycleProfileForKey(plugin.RuntimeProfiles, binding.ProfileKey)
if !exists || !containsString(profile.Capabilities, capability) || !runtimePlatformsContain(profile.Platforms, "windows") {
return sourceRCONDispatchResolution{}, validationError("selected runtime profile does not support protected SCUM RCON")
}
transport, err := sourceRCONTransportForCapability(plugin.RuntimeProfiles, profile, request.TransportKey, capability)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
if transport.TargetKey != request.TargetKey {
return sourceRCONDispatchResolution{}, validationError("protected RCON transport target is invalid")
}
extension, err := sourceRCONExtension(plugin.RuntimeProfiles, profile, endpoint)
if err != nil {
return sourceRCONDispatchResolution{}, err
}
plan := &domain.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: extension.Key, ModKey: extension.ModKey, ConfigRef: "ue4ss/Mods/" + extension.ModKey + "/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/" + extension.TargetKey + "/release.json", Port: extension.RCONPort}
return sourceRCONDispatchResolution{plugin: plugin, binding: binding, transport: transport, plan: plan}, nil
}
func sourceRCONExtension(profiles domain.GamePluginRuntimeProfiles, profile domain.RuntimeLifecycleProfile, endpoint domain.RunEndpoint) (domain.RuntimeDLLExtensionProfile, error) {
byKey := make(map[string]domain.RuntimeDLLExtensionProfile, len(profiles.DLLExtensions))
for _, extension := range profiles.DLLExtensions {
+59
View File
@@ -85,6 +85,65 @@ func TestSourceRCONDispatchUsesOneTimeRedactedInput(t *testing.T) {
}
}
func TestProtectedRCONBridgeDispatchCarriesSourceRCONPlan(t *testing.T) {
svc, _, _, instance := newSourceRCONFixture(t)
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
if err != nil {
t.Fatal(err)
}
protectedCapability := domain.JobCapabilityRemoteRunProtectedRCON
plugin.RequiredRunCapabilities = append(plugin.RequiredRunCapabilities, protectedCapability)
plugin.RemoteAccess.RunCapabilities = append(plugin.RemoteAccess.RunCapabilities, protectedCapability)
plugin.RuntimeProfiles.ClientManagers = []domain.RuntimeClientManagerProfile{{Key: "scum-client-manager", Health: domain.RuntimeClientManagerHealth{RequiredCapabilities: []string{gameClientBridgeCapability}}}}
plugin.RuntimeProfiles.LifecycleProfiles[0].Capabilities = append(plugin.RuntimeProfiles.LifecycleProfiles[0].Capabilities, protectedCapability)
plugin.RuntimeProfiles.LifecycleProfiles[0].TransportKeys = append(plugin.RuntimeProfiles.LifecycleProfiles[0].TransportKeys, "scum-management")
plugin.RuntimeProfiles.TransportProfiles = append(plugin.RuntimeProfiles.TransportProfiles, domain.RuntimeTransportProfile{Key: "scum-management", Kind: "rcon", TargetKey: "scum-management", Capabilities: []string{protectedCapability}})
plugin.GameClientBridge.Commands = append(plugin.GameClientBridge.Commands, domain.GameClientBridgeCommandDeclaration{Type: "management.rcon.request", ApprovalLevel: domain.GameClientBridgeApprovalLevelOperator, TimeoutSeconds: 120, MaxPayloadBytes: 8192, ProtectedRequest: &domain.GameClientBridgeProtectedRequestDeclaration{Kind: "rcon", TransportKey: "scum-management", TargetKey: "scum-management", TextField: "requestText", MaxTextBytes: 8192}})
if err := svc.store.GamePlugins().Update(plugin); err != nil {
t.Fatal(err)
}
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"rcon": "runtime-rcon", "scum-management": "runtime-rcon"}}, true)
if err != nil {
t.Fatalf("refresh protected RCON binding: %v", err)
}
if err := svc.store.RuntimeBindings().Update(binding); err != nil {
t.Fatalf("store protected RCON binding: %v", err)
}
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
if err != nil {
t.Fatal(err)
}
endpoint.Capabilities = append(endpoint.Capabilities, protectedCapability)
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
t.Fatal(err)
}
if _, err := svc.resolveProtectedSourceRCONDispatch(instance.ID, plugin.GameClientBridge.Commands[len(plugin.GameClientBridge.Commands)-1].ProtectedRequest); err != nil {
t.Fatalf("resolve protected Source RCON plan: %v", err)
}
command, err := svc.queueGameClientBridgeCommand("user-rcon-owner", domain.GameClientBridgeQueueRequest{ServerInstanceID: instance.ID, PluginID: plugin.ID, ProfileKey: "scum-client-manager", CommandType: "management.rcon.request", Payload: map[string]any{"requestText": "#ListPlayers"}, IdempotencyKey: "protected-rcon-1", ExpiresAt: fixedTime.Add(time.Minute)})
if err != nil {
t.Fatalf("queue protected RCON: %v", err)
}
job, err := svc.store.Jobs().Get(command.RunJobID)
if err != nil {
t.Fatalf("get protected RCON job: %v", err)
}
if job.Capability != protectedCapability || job.InputRef == "" || !strings.HasPrefix(job.InputRef, "input://protected-request/") || job.ExecutionInput.SourceRCON == nil {
t.Fatalf("expected protected RCON job with frozen Source RCON plan, got %+v", job)
}
if job.ExecutionInput.WorkspaceScope != "local" || job.ExecutionInput.RemoteAdapterKey != "scum-management" || job.ExecutionInput.RemoteAdapterKind != "protected-rcon" || job.ExecutionInput.SourceRCON.Port != 27015 {
t.Fatalf("protected RCON plan did not preserve logical runtime binding: %+v", job.ExecutionInput)
}
serialized, err := json.Marshal(job)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(serialized), "#ListPlayers") || strings.Contains(string(serialized), "password=") {
t.Fatalf("protected RCON job leaked transient input: %s", serialized)
}
}
func TestSourceRCONDispatchRejectsUnsafeOrIncompatibleState(t *testing.T) {
svc, session, _, instance := newSourceRCONFixture(t)
unsafe := domain.SourceRCONCommandRequest{ServerInstanceID: instance.ID, Kind: domain.SourceRCONCommandKindCommand, Command: "SetTime 12\nSpawnItem", IdempotencyKey: "rcon-unsafe"}