admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add SCUM trade catalog projections

Browse Source
This commit is contained in:
npc0-hue
2026-08-27 16:14:59 +08:00
parent 316efbe780
commit a0e7ae362b
16 changed files with 267 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/domain/game_client_bridge.go
+1
View File
@@ -56,6 +56,7 @@ type GameClientBridgeQueryProjectionDeclaration struct {
FieldMappings map[string]string
FixedValues map[string]string
ObservedAtField string
MergeExisting bool
}
type GameClientBridgeLogProjectionStepDeclaration struct {
platform/domain/game_client_bridge_test.go
+2 -2
View File
@@ -4,7 +4,7 @@ import "testing"
func TestCopyGameClientBridgeDeclarationsCopiesQueryTemplateSlices(t *testing.T) {
manifest := GameClientBridgeManifest{
QueryTemplates: []GameClientBridgeQueryTemplateDeclaration{{Key: "player.lookup", PollIntervalSeconds: 3, SQLRef: "sql/player-lookup.sql", Projections: []GameClientBridgeQueryProjectionDeclaration{{Collection: "users", RowPath: "rows", MatchField: "kind", MatchValue: "player", UpsertKeys: []string{"steamId"}, FieldMappings: map[string]string{"steamId": "steamId"}, FixedValues: map[string]string{"source": "sqlite"}, ObservedAtField: "sampledAt"}}}},
QueryTemplates: []GameClientBridgeQueryTemplateDeclaration{{Key: "player.lookup", PollIntervalSeconds: 3, SQLRef: "sql/player-lookup.sql", Projections: []GameClientBridgeQueryProjectionDeclaration{{Collection: "users", RowPath: "rows", MatchField: "kind", MatchValue: "player", UpsertKeys: []string{"steamId"}, FieldMappings: map[string]string{"steamId": "steamId"}, FixedValues: map[string]string{"source": "sqlite"}, ObservedAtField: "sampledAt", MergeExisting: true}}}},
LogProjections: []GameClientBridgeLogProjectionDeclaration{{
Key: "player.login", StreamKeys: []string{"process.stdout"}, Steps: []GameClientBridgeLogProjectionStepDeclaration{{Pattern: `Player (?<slot>\d+)`}}, CorrelationFields: []string{"slot"}, MaxInterveningLines: 8,
Target: GameClientBridgeLogProjectionTargetDeclaration{Collection: "users", UpsertKeys: []string{"steamId"}, CaptureMappings: map[string]string{"steamId": "steamId"}, HashMappings: map[string]string{"networkCorrelation": "ip"}, FixedValues: map[string]string{"source": "stdout"}, ObservedAtField: "lastLoginAt"},
@@ -26,7 +26,7 @@ func TestCopyGameClientBridgeDeclarationsCopiesQueryTemplateSlices(t *testing.T)
manifestCopy.DataPacks[0].LogParserRefs[0] = "mutated"
manifestCopy.DataPacks[0].DataRefs[0] = "mutated"
manifestCopy.Pages[0].QueryTemplateKeys[0] = "mutated"
if manifest.QueryTemplates[0].Key != "player.lookup" || manifest.QueryTemplates[0].SQLRef != "sql/player-lookup.sql" || manifest.QueryTemplates[0].Projections[0].FieldMappings["steamId"] != "steamId" || manifest.LogProjections[0].StreamKeys[0] != "process.stdout" || manifest.LogProjections[0].Target.CaptureMappings["steamId"] != "steamId" || manifest.LogProjections[0].Target.HashMappings["networkCorrelation"] != "ip" || manifest.LogProjections[0].Presence.ActivityTarget.CaptureMappings["steamId"] != "steamId" || manifest.LifecycleProjections[0].Capabilities[0] != "process.stop" || manifest.LifecycleProjections[0].Target.ActivityTarget.RowMappings["steamId"] != "steamId" || manifest.DataPacks[0].LogParserRefs[0] != "logs.json" || manifest.DataPacks[0].DataRefs[0] != "data.json" || manifest.Pages[0].QueryTemplateKeys[0] != "player.lookup" {
if manifest.QueryTemplates[0].Key != "player.lookup" || manifest.QueryTemplates[0].SQLRef != "sql/player-lookup.sql" || manifest.QueryTemplates[0].Projections[0].FieldMappings["steamId"] != "steamId" || !manifest.QueryTemplates[0].Projections[0].MergeExisting || manifest.LogProjections[0].StreamKeys[0] != "process.stdout" || manifest.LogProjections[0].Target.CaptureMappings["steamId"] != "steamId" || manifest.LogProjections[0].Target.HashMappings["networkCorrelation"] != "ip" || manifest.LogProjections[0].Presence.ActivityTarget.CaptureMappings["steamId"] != "steamId" || manifest.LifecycleProjections[0].Capabilities[0] != "process.stop" || manifest.LifecycleProjections[0].Target.ActivityTarget.RowMappings["steamId"] != "steamId" || manifest.DataPacks[0].LogParserRefs[0] != "logs.json" || manifest.DataPacks[0].DataRefs[0] != "data.json" || manifest.Pages[0].QueryTemplateKeys[0] != "player.lookup" {
t.Fatalf("manifest copy aliases query template declarations: source=%#v copy=%#v", manifest, manifestCopy)
}
platform/dto/resources.go
+3 -2
View File
@@ -303,6 +303,7 @@ type GameClientBridgeQueryProjectionDeclarationBody struct {
FieldMappings map[string]string `json:"fieldMappings,omitempty"`
FixedValues map[string]string `json:"fixedValues,omitempty"`
ObservedAtField string `json:"observedAtField,omitempty"`
MergeExisting bool `json:"mergeExisting,omitempty"`
}
type GameClientBridgeLogProjectionStepDeclarationBody struct {
@@ -1339,7 +1340,7 @@ func gameClientBridgeQueryProjectionsToDomain(values []GameClientBridgeQueryProj
}
out := make([]domain.GameClientBridgeQueryProjectionDeclaration, len(values))
for index, value := range values {
out[index] = domain.GameClientBridgeQueryProjectionDeclaration{Collection: value.Collection, RowPath: value.RowPath, MatchField: value.MatchField, MatchValue: value.MatchValue, UpsertKeys: domain.CopyStringSlice(value.UpsertKeys), FieldMappings: domain.CopyStringMap(value.FieldMappings), FixedValues: domain.CopyStringMap(value.FixedValues), ObservedAtField: value.ObservedAtField}
out[index] = domain.GameClientBridgeQueryProjectionDeclaration{Collection: value.Collection, RowPath: value.RowPath, MatchField: value.MatchField, MatchValue: value.MatchValue, UpsertKeys: domain.CopyStringSlice(value.UpsertKeys), FieldMappings: domain.CopyStringMap(value.FieldMappings), FixedValues: domain.CopyStringMap(value.FixedValues), ObservedAtField: value.ObservedAtField, MergeExisting: value.MergeExisting}
}
return out
}
@@ -1838,7 +1839,7 @@ func gameClientBridgeQueryProjectionsFromDomain(values []domain.GameClientBridge
}
out := make([]GameClientBridgeQueryProjectionDeclarationBody, len(values))
for index, value := range values {
out[index] = GameClientBridgeQueryProjectionDeclarationBody{Collection: value.Collection, RowPath: value.RowPath, MatchField: value.MatchField, MatchValue: value.MatchValue, UpsertKeys: domain.CopyStringSlice(value.UpsertKeys), FieldMappings: domain.CopyStringMap(value.FieldMappings), FixedValues: domain.CopyStringMap(value.FixedValues), ObservedAtField: value.ObservedAtField}
out[index] = GameClientBridgeQueryProjectionDeclarationBody{Collection: value.Collection, RowPath: value.RowPath, MatchField: value.MatchField, MatchValue: value.MatchValue, UpsertKeys: domain.CopyStringSlice(value.UpsertKeys), FieldMappings: domain.CopyStringMap(value.FieldMappings), FixedValues: domain.CopyStringMap(value.FixedValues), ObservedAtField: value.ObservedAtField, MergeExisting: value.MergeExisting}
}
return out
}
platform/dto/resources_test.go
+5 -2
View File
@@ -137,7 +137,7 @@ func TestGameClientBridgeQueryTemplateDeclarationRoundTripIsSafe(t *testing.T) {
QueryTemplates: []GameClientBridgeQueryTemplateDeclarationBody{{
Key: "player.lookup", Title: "Player lookup", Permission: "server.game-client.read", Engine: "sqlite", TransportKey: "sqlite-db", TargetKey: "db/sqlite",
ParameterSchemaRef: "schemas/bridge/query/player-lookup.parameters.schema.json", ResultSchemaRef: "schemas/bridge/query/player-lookup.result.schema.json", SQLRef: "sql/player-lookup.sql", MaxRows: 50, TimeoutSeconds: 10, PollIntervalSeconds: 3,
Projections: []GameClientBridgeQueryProjectionDeclarationBody{{Collection: "users", RowPath: "rows", MatchField: "kind", MatchValue: "player", UpsertKeys: []string{"steamId"}, FieldMappings: map[string]string{"steamId": "steamId"}, FixedValues: map[string]string{"source": "sqlite"}, ObservedAtField: "sampledAt"}},
Projections: []GameClientBridgeQueryProjectionDeclarationBody{{Collection: "users", RowPath: "rows", MatchField: "kind", MatchValue: "player", UpsertKeys: []string{"steamId"}, FieldMappings: map[string]string{"steamId": "steamId"}, FixedValues: map[string]string{"source": "sqlite"}, ObservedAtField: "sampledAt", MergeExisting: true}},
}},
LogProjections: []GameClientBridgeLogProjectionDeclarationBody{{
Key: "player.login", StreamKeys: []string{"process.stdout"}, Steps: []GameClientBridgeLogProjectionStepDeclarationBody{{Pattern: `Player (?<slot>\d+)`}}, CorrelationFields: []string{"slot"}, MaxInterveningLines: 8,
@@ -152,7 +152,7 @@ func TestGameClientBridgeQueryTemplateDeclarationRoundTripIsSafe(t *testing.T) {
}
domainManifest := body.ToDomain()
if len(domainManifest.QueryTemplates) != 1 || domainManifest.QueryTemplates[0].SQLRef != "sql/player-lookup.sql" || domainManifest.QueryTemplates[0].PollIntervalSeconds != 3 || len(domainManifest.QueryTemplates[0].Projections) != 1 || domainManifest.QueryTemplates[0].Projections[0].MatchValue != "player" || len(domainManifest.LogProjections) != 1 || domainManifest.LogProjections[0].Presence.ActiveWindowSeconds != 600 || domainManifest.LogProjections[0].Target.HashMappings["networkCorrelation"] != "ip" || len(domainManifest.LifecycleProjections) != 1 || domainManifest.LifecycleProjections[0].Target.ActivityTarget.FixedValues["eventType"] != "logout" || len(domainManifest.DataPacks) != 1 || domainManifest.DataPacks[0].DataRefs[0] != "data/items.json" || domainManifest.Pages[0].QueryTemplateKeys[0] != "player.lookup" {
if len(domainManifest.QueryTemplates) != 1 || domainManifest.QueryTemplates[0].SQLRef != "sql/player-lookup.sql" || domainManifest.QueryTemplates[0].PollIntervalSeconds != 3 || len(domainManifest.QueryTemplates[0].Projections) != 1 || domainManifest.QueryTemplates[0].Projections[0].MatchValue != "player" || !domainManifest.QueryTemplates[0].Projections[0].MergeExisting || len(domainManifest.LogProjections) != 1 || domainManifest.LogProjections[0].Presence.ActiveWindowSeconds != 600 || domainManifest.LogProjections[0].Target.HashMappings["networkCorrelation"] != "ip" || len(domainManifest.LifecycleProjections) != 1 || domainManifest.LifecycleProjections[0].Target.ActivityTarget.FixedValues["eventType"] != "logout" || len(domainManifest.DataPacks) != 1 || domainManifest.DataPacks[0].DataRefs[0] != "data/items.json" || domainManifest.Pages[0].QueryTemplateKeys[0] != "player.lookup" {
t.Fatalf("query template conversion lost declaration fields: %#v", domainManifest)
}
domainManifest.QueryTemplates[0].Projections[0].FieldMappings["steamId"] = "mutated"
@@ -182,6 +182,9 @@ func TestGameClientBridgeQueryTemplateDeclarationRoundTripIsSafe(t *testing.T) {
domainManifest.DataPacks[0].DataRefs[0] = "data/items.json"
response := gameClientBridgeManifestFromDomain(domainManifest)
if !response.QueryTemplates[0].Projections[0].MergeExisting {
t.Fatal("query projection mergeExisting was not preserved")
}
response.QueryTemplates[0].Projections[0].FixedValues["source"] = "mutated"
if domainManifest.QueryTemplates[0].Projections[0].FixedValues["source"] != "sqlite" {
t.Fatal("query projection target aliases domain data")
platform/service/plugin_data_test.go
+10
View File
@@ -163,10 +163,16 @@ func TestRunPollSchedulesAndProjectsDeclaredSQLiteQuery(t *testing.T) {
plugin.GameClientBridge.QueryTemplates[0].Projections = []domain.GameClientBridgeQueryProjectionDeclaration{{
Collection: "scum_users", RowPath: "rows", MatchField: "kind", MatchValue: "player", UpsertKeys: []string{"steamId"},
FieldMappings: map[string]string{"steamId": "steamId", "displayName": "displayName"}, FixedValues: map[string]string{"source": "sqlite"}, ObservedAtField: "sampledAt",
}, {
Collection: "scum_trade_goods", RowPath: "rows", MatchField: "kind", MatchValue: "vehicle", UpsertKeys: []string{"code"},
FieldMappings: map[string]string{"className": "displayName"}, FixedValues: map[string]string{"code": "#spawnvehicle {{displayName}}", "spawnCommand": "#spawnvehicle {{displayName}}", "catalogType": "vehicle", "type": "21", "typeName": "其他载具", "imagePath": "/original/{{displayName}}.webp", "source": "sqlite"}, ObservedAtField: "lastSeenAt", MergeExisting: true,
}}
if err := svc.store.GamePlugins().Update(plugin); err != nil {
t.Fatalf("enable query projection polling: %v", err)
}
if _, err := svc.applyPluginDataTransaction(domain.PluginDataTransaction{PluginID: plugin.ID, ServerInstanceID: instance.ID, Collection: "scum_trade_goods", Mutations: []domain.PluginDataMutation{{Operation: domain.PluginDataMutationPut, Key: "#spawnvehicle Truck", Value: map[string]any{"code": "#spawnvehicle Truck", "name": "Named Truck"}}}}); err != nil {
t.Fatalf("seed vehicle catalog: %v", err)
}
helloRequest := validRunControlHello()
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, domain.JobCapabilityRemoteRunDBSQLiteQuery)
helloRequest.CapabilityReport.Fingerprint = "cap-plugin-query-scheduler"
@@ -189,6 +195,10 @@ func TestRunPollSchedulesAndProjectsDeclaredSQLiteQuery(t *testing.T) {
if err != nil || len(items) != 1 || items[0].Key != "steam-1" || items[0].Value["displayName"] != "Ada" || items[0].Value["source"] != "sqlite" || items[0].Value["sampledAt"] == nil {
t.Fatalf("declared projection did not write scoped plugin data=%+v err=%v", items, err)
}
goods, err := svc.ListPluginDataForSession(session, domain.PluginDataFilter{PluginID: plugin.ID, ServerInstanceID: instance.ID, Collection: "scum_trade_goods"})
if err != nil || len(goods) != 1 || goods[0].Key != "#spawnvehicle Truck" || goods[0].Value["name"] != "Named Truck" || goods[0].Value["className"] != "Truck" || goods[0].Value["type"] != "21" || goods[0].Value["lastSeenAt"] == nil {
t.Fatalf("declared vehicle catalog projection did not merge scoped plugin data=%+v err=%v", goods, err)
}
second, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, Capabilities: []string{domain.JobCapabilityRemoteRunDBSQLiteQuery}, Capacity: domain.RunCapacity{MaxJobs: 1}})
if err != nil || second.HasJob {
t.Fatalf("fresh projection poll should not reschedule immediately: %+v err=%v", second, err)
platform/service/plugin_log_projection_test.go
+53
View File
@@ -71,6 +71,46 @@ func TestDurableStdoutProjectionCreatesUsersAndSuppressesRapidDuplicates(t *test
assertPresenceProjectionCounts(t, svc, plugin.ID, instance.ID, 1, 2, 0)
}
func TestTradeLogProjectionsCreateCatalogAndTradeEvents(t *testing.T) {
svc := newTestCoreService()
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
pattern := `^\d{4}\.\d{2}\.\d{2}-\d{2}\.\d{2}\.\d{2}: \[Trade\] Tradeable \((?P<itemCode>[A-Za-z0-9_.-]{1,128}) \(x(?P<quantity>\d{1,9})\)\) (?P<tradeVerb>purchased|sold) by .*?\((?P<steamId>\d{1,50})\) for (?P<price>-?\d{1,12})$`
plugin.GameClientBridge.LogProjections = []domain.GameClientBridgeLogProjectionDeclaration{{
Key: "scum.trade.catalog", StreamKeys: []string{"scum.trade"}, Steps: []domain.GameClientBridgeLogProjectionStepDeclaration{{Pattern: pattern}}, CorrelationFields: []string{"itemCode"}, MaxInterveningLines: 0,
Target: domain.GameClientBridgeLogProjectionTargetDeclaration{Collection: "scum_trade_goods", UpsertKeys: []string{"code"}, CaptureMappings: map[string]string{"code": "itemCode"}, FixedValues: map[string]string{"catalogType": "item", "source": "scum.trade"}, ObservedAtField: "lastSeenAt"},
}, {
Key: "scum.trade.events", StreamKeys: []string{"scum.trade"}, Steps: []domain.GameClientBridgeLogProjectionStepDeclaration{{Pattern: pattern}}, CorrelationFields: []string{"steamId", "itemCode", "tradeVerb"}, MaxInterveningLines: 0,
Target: domain.GameClientBridgeLogProjectionTargetDeclaration{Collection: "scum_trade_events", UpsertKeys: []string{"steamId", "itemCode", "tradeVerb", "quantity", "price", "observedAt"}, CaptureMappings: map[string]string{"steamId": "steamId", "itemCode": "itemCode", "tradeVerb": "tradeVerb", "quantity": "quantity", "price": "price"}, FixedValues: map[string]string{"eventType": "trade", "source": "scum.trade"}, ObservedAtField: "observedAt"},
}}
if err := svc.store.GamePlugins().Update(plugin); err != nil {
t.Fatalf("update trade projections: %v", err)
}
instance, err := svc.CreateServerInstance(domain.ServerInstance{ID: "server-trade-projection", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "SCUM trade projection", State: domain.ServerInstanceStateRunning})
if err != nil {
t.Fatalf("create server: %v", err)
}
helloRequest := validRunControlHello()
helloRequest.CapabilityReport.Fingerprint = "cap-trade-log-projection"
hello, err := svc.RegisterRunHello(helloRequest)
if err != nil {
t.Fatalf("register Run: %v", err)
}
stream, err := svc.CreateLogStream(domain.LogStream{ID: "trade-log-projection", ServerInstanceID: instance.ID, Source: domain.LogStreamSourceFile, StreamKey: "scum.trade", StorageBackend: domain.LogStorageBackendLocalSegments, RetentionPolicy: "default"})
if err != nil {
t.Fatalf("create trade stream: %v", err)
}
base := time.Date(2026, 8, 27, 12, 34, 56, 0, time.UTC)
ingestTradeProjectionLines(t, svc, hello.SessionToken, endpoint.ID, instance.ID, stream.ID, 1, base, []string{`2026.08.27-12.34.56: [Trade] Tradeable (BPC_Apple (x2)) purchased by Mira(76561198000000001) for 120`})
goods, err := svc.store.PluginDataRecords().List(domain.PluginDataFilter{PluginID: plugin.ID, ServerInstanceID: instance.ID, Collection: "scum_trade_goods"})
if err != nil || len(goods) != 1 || goods[0].Key != "BPC_Apple" || goods[0].Value["catalogType"] != "item" || goods[0].Value["lastSeenAt"] == nil {
t.Fatalf("trade item catalog was not projected: %+v err=%v", goods, err)
}
trades, err := svc.store.PluginDataRecords().List(domain.PluginDataFilter{PluginID: plugin.ID, ServerInstanceID: instance.ID, Collection: "scum_trade_events"})
if err != nil || len(trades) != 1 || trades[0].Value["itemCode"] != "BPC_Apple" || trades[0].Value["quantity"] != "2" || trades[0].Value["tradeVerb"] != "purchased" || trades[0].Value["steamId"] != "76561198000000001" || trades[0].Value["price"] != "120" {
t.Fatalf("trade event was not projected: %+v err=%v", trades, err)
}
}
func TestLifecycleProjectionMarksOnlineUsersOffline(t *testing.T) {
svc := newTestCoreService()
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
@@ -167,6 +207,19 @@ func ingestProjectionLines(t *testing.T, svc *CoreService, sessionToken, endpoin
}
}
func ingestTradeProjectionLines(t *testing.T, svc *CoreService, sessionToken, endpointID, serverID, streamID string, firstSeq uint64, observedAt time.Time, lines []string) {
t.Helper()
entries := make([]domain.LogEntry, len(lines))
for index, line := range lines {
entries[index] = domain.LogEntry{Seq: firstSeq + uint64(index), Timestamp: observedAt.Add(time.Duration(index) * time.Second), Level: "info", Line: line}
}
lastSeq := firstSeq + uint64(len(entries)) - 1
batch := domain.LogBatchIngest{RunEndpointID: endpointID, SessionToken: sessionToken, LogStreamID: streamID, ServerInstanceID: serverID, StreamKey: "scum.trade", Source: domain.LogStreamSourceFile, FirstSeq: firstSeq, LastSeq: lastSeq, Compression: "none", Checksum: checksumForEntries(t, entries), Entries: entries}
if result, err := svc.IngestLogBatch(batch); err != nil || !result.Accepted {
t.Fatalf("ingest trade projection lines result=%+v err=%v", result, err)
}
}
func assertPresenceProjectionCounts(t *testing.T, svc *CoreService, pluginID, serverID string, users, activities, commands int) {
t.Helper()
userRows, err := svc.store.PluginDataRecords().List(domain.PluginDataFilter{PluginID: pluginID, ServerInstanceID: serverID, Collection: "scum_users"})
platform/service/plugin_query_projection.go
+9
View File
@@ -3,11 +3,13 @@ package service
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"browser.local/platform/domain"
"browser.local/platform/repo"
)
func (svc *CoreService) scheduleDuePluginQueryProjectionJobs(claim domain.RunJobClaim, stamp time.Time) error {
@@ -147,6 +149,13 @@ func (svc *CoreService) projectPluginQueryJobResult(job domain.Job, stamp time.T
if keyErr != nil {
return keyErr
}
if projection.MergeExisting {
if existing, existingErr := svc.store.PluginDataRecords().Get(pluginDataID(instance.ID, plugin.ID, projection.Collection, key)); existingErr == nil {
value = mergePluginDataValues(existing.Value, value)
} else if !errors.Is(existingErr, repo.ErrNotFound) {
return existingErr
}
}
collectionMutations[key] = domain.PluginDataMutation{Operation: domain.PluginDataMutationPut, Key: key, Value: value}
}
}