Implement SCUM login log parser
This commit is contained in:
npc0-hue
@@ -0,0 +1,232 @@
|
||||
package companion
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
const (
|
||||
SCUMLoginLogParserKey = "scum-login-log-parser-v1"
|
||||
SCUMLoginLogParserVersion = "scum-login-log-v1"
|
||||
SCUMLoginLogMaxLineBytes = 4096
|
||||
)
|
||||
|
||||
var scumLoginLogPattern = regexp.MustCompile(`^(\d{4}\.\d{2}\.\d{2}-\d{2}\.\d{2}\.\d{2}): '(.+)\s+(\d{17}):(.+)\(([^()]*)\)' logged (in|out) at: X=([+-]?\d+(?:\.\d+)?) Y=([+-]?\d+(?:\.\d+)?) Z=([+-]?\d+(?:\.\d+)?)$`)
|
||||
|
||||
// SCUMLoginLogRecord is a single complete or buffered line from a plugin-declared
|
||||
// UTF-16LE SCUM login log source. It carries transport identity, not host paths.
|
||||
type SCUMLoginLogRecord struct {
|
||||
ServerID string
|
||||
SourceIdentity string
|
||||
StreamGeneration string
|
||||
Sequence uint64
|
||||
RawLine []byte
|
||||
Partial bool
|
||||
}
|
||||
|
||||
type SCUMLoginLogParseOptions struct {
|
||||
MaxLineBytes int
|
||||
AcknowledgedTransportCursors map[SCUMTransportCursor]struct{}
|
||||
AcknowledgedLogicalEventIdentities map[string]struct{}
|
||||
}
|
||||
|
||||
type SCUMTransportCursor struct {
|
||||
SourceIdentity string
|
||||
StreamGeneration string
|
||||
Sequence uint64
|
||||
}
|
||||
|
||||
type SCUMLoginLogEvent struct {
|
||||
ServerID string
|
||||
ParserKey string
|
||||
ParserVersion string
|
||||
TransportCursor SCUMTransportCursor
|
||||
Type string
|
||||
ExternalPlayerID string
|
||||
DisplayName string
|
||||
ProfileLocalID string
|
||||
OccurredAt time.Time
|
||||
OccurredAtSourceText string
|
||||
LogicalEventIdentity string
|
||||
}
|
||||
|
||||
type SCUMLoginLogDiagnostic struct {
|
||||
ServerID string
|
||||
TransportCursor SCUMTransportCursor
|
||||
Code string
|
||||
}
|
||||
|
||||
type SCUMLoginLogBatch struct {
|
||||
ServerID string
|
||||
ParserKey string
|
||||
ParserVersion string
|
||||
Events []SCUMLoginLogEvent
|
||||
Diagnostics []SCUMLoginLogDiagnostic
|
||||
}
|
||||
|
||||
type SCUMLoginLogParserAvailability struct {
|
||||
Available bool
|
||||
ParserKey string
|
||||
ParserVersion string
|
||||
MaxLineBytes int
|
||||
Reason string
|
||||
}
|
||||
|
||||
func VerifiedSCUMLoginLogParser() SCUMLoginLogParserAvailability {
|
||||
return SCUMLoginLogParserAvailability{Available: true, ParserKey: SCUMLoginLogParserKey, ParserVersion: SCUMLoginLogParserVersion, MaxLineBytes: SCUMLoginLogMaxLineBytes, Reason: "SCUM UTF-16LE login log parser is available"}
|
||||
}
|
||||
|
||||
func ParseSCUMLoginLogRecords(serverID string, records []SCUMLoginLogRecord, options SCUMLoginLogParseOptions) SCUMLoginLogBatch {
|
||||
maxLineBytes := options.MaxLineBytes
|
||||
if maxLineBytes <= 0 || maxLineBytes > SCUMLoginLogMaxLineBytes {
|
||||
maxLineBytes = SCUMLoginLogMaxLineBytes
|
||||
}
|
||||
batch := SCUMLoginLogBatch{ServerID: serverID, ParserKey: SCUMLoginLogParserKey, ParserVersion: SCUMLoginLogParserVersion}
|
||||
seenTransport := cloneTransportSet(options.AcknowledgedTransportCursors)
|
||||
seenLogical := cloneStringSet(options.AcknowledgedLogicalEventIdentities)
|
||||
lastSequenceByGeneration := map[string]uint64{}
|
||||
if len(records) > 100 {
|
||||
records = records[:100]
|
||||
}
|
||||
for _, record := range records {
|
||||
cursor := SCUMTransportCursor{SourceIdentity: record.SourceIdentity, StreamGeneration: record.StreamGeneration, Sequence: record.Sequence}
|
||||
if record.ServerID != serverID || cursor.SourceIdentity == "" || cursor.StreamGeneration == "" || cursor.Sequence == 0 {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "invalid-transport-cursor")
|
||||
continue
|
||||
}
|
||||
generationKey := cursor.SourceIdentity + "\x00" + cursor.StreamGeneration
|
||||
if last := lastSequenceByGeneration[generationKey]; last != 0 && cursor.Sequence < last {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "out-of-order-transport")
|
||||
}
|
||||
if cursor.Sequence > lastSequenceByGeneration[generationKey] {
|
||||
lastSequenceByGeneration[generationKey] = cursor.Sequence
|
||||
}
|
||||
if _, ok := seenTransport[cursor]; ok {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "duplicate-transport-cursor")
|
||||
continue
|
||||
}
|
||||
seenTransport[cursor] = struct{}{}
|
||||
if record.Partial {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "partial-line")
|
||||
continue
|
||||
}
|
||||
if len(record.RawLine) == 0 || len(record.RawLine) > maxLineBytes {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "oversized-line")
|
||||
continue
|
||||
}
|
||||
line, ok := decodeUTF16LELine(record.RawLine)
|
||||
if !ok {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "undecodable-line")
|
||||
continue
|
||||
}
|
||||
event, diagnosticCode := parseSCUMLoginLogLine(serverID, cursor, line)
|
||||
if diagnosticCode != "" {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, diagnosticCode)
|
||||
continue
|
||||
}
|
||||
if _, ok := seenLogical[event.LogicalEventIdentity]; ok {
|
||||
batch.Diagnostics = appendSCUMLoginDiagnostic(batch.Diagnostics, serverID, cursor, "duplicate-logical-event")
|
||||
continue
|
||||
}
|
||||
seenLogical[event.LogicalEventIdentity] = struct{}{}
|
||||
batch.Events = append(batch.Events, event)
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
func parseSCUMLoginLogLine(serverID string, cursor SCUMTransportCursor, rawLine string) (SCUMLoginLogEvent, string) {
|
||||
line := strings.TrimPrefix(strings.TrimRight(rawLine, "\r\n"), "\ufeff")
|
||||
lower := strings.ToLower(line)
|
||||
if strings.Contains(lower, "failed") || strings.Contains(lower, "rejected") {
|
||||
return SCUMLoginLogEvent{}, "failed-login"
|
||||
}
|
||||
match := scumLoginLogPattern.FindStringSubmatch(line)
|
||||
if match == nil {
|
||||
return SCUMLoginLogEvent{}, "malformed-line"
|
||||
}
|
||||
occurredAt, err := time.ParseInLocation("2006.01.02-15.04.05", match[1], time.UTC)
|
||||
if err != nil {
|
||||
return SCUMLoginLogEvent{}, "malformed-line"
|
||||
}
|
||||
eventType := "scum.login"
|
||||
if match[6] == "out" {
|
||||
eventType = "scum.logout"
|
||||
}
|
||||
displayName := strings.TrimSpace(match[4])
|
||||
profileLocalID := strings.TrimSpace(match[5])
|
||||
if displayName == "" || profileLocalID == "" {
|
||||
return SCUMLoginLogEvent{}, "malformed-line"
|
||||
}
|
||||
event := SCUMLoginLogEvent{
|
||||
ServerID: serverID,
|
||||
ParserKey: SCUMLoginLogParserKey,
|
||||
ParserVersion: SCUMLoginLogParserVersion,
|
||||
TransportCursor: cursor,
|
||||
Type: eventType,
|
||||
ExternalPlayerID: match[3],
|
||||
DisplayName: displayName,
|
||||
ProfileLocalID: profileLocalID,
|
||||
OccurredAt: occurredAt,
|
||||
OccurredAtSourceText: match[1],
|
||||
}
|
||||
event.LogicalEventIdentity = scumLoginLogicalIdentity(serverID, eventType, match[1], event.ExternalPlayerID, displayName, profileLocalID)
|
||||
return event, ""
|
||||
}
|
||||
|
||||
func scumLoginLogicalIdentity(serverID, eventType, occurrenceText, externalPlayerID, displayName, profileLocalID string) string {
|
||||
normalized := strings.Join([]string{SCUMLoginLogParserVersion, serverID, eventType, occurrenceText, externalPlayerID, displayName, profileLocalID}, "\x00")
|
||||
digest := sha256.Sum256([]byte(normalized))
|
||||
return "sha256:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func decodeUTF16LELine(raw []byte) (string, bool) {
|
||||
if len(raw)%2 != 0 {
|
||||
return "", false
|
||||
}
|
||||
codeUnits := make([]uint16, len(raw)/2)
|
||||
for index := range codeUnits {
|
||||
codeUnits[index] = binary.LittleEndian.Uint16(raw[index*2:])
|
||||
}
|
||||
for index := 0; index < len(codeUnits); index++ {
|
||||
unit := codeUnits[index]
|
||||
if 0xD800 <= unit && unit <= 0xDBFF {
|
||||
if index+1 >= len(codeUnits) || codeUnits[index+1] < 0xDC00 || codeUnits[index+1] > 0xDFFF {
|
||||
return "", false
|
||||
}
|
||||
index++
|
||||
continue
|
||||
}
|
||||
if 0xDC00 <= unit && unit <= 0xDFFF {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return string(utf16.Decode(codeUnits)), true
|
||||
}
|
||||
|
||||
func appendSCUMLoginDiagnostic(existing []SCUMLoginLogDiagnostic, serverID string, cursor SCUMTransportCursor, code string) []SCUMLoginLogDiagnostic {
|
||||
if len(existing) >= 64 {
|
||||
return existing
|
||||
}
|
||||
return append(existing, SCUMLoginLogDiagnostic{ServerID: serverID, TransportCursor: cursor, Code: code})
|
||||
}
|
||||
|
||||
func cloneTransportSet(source map[SCUMTransportCursor]struct{}) map[SCUMTransportCursor]struct{} {
|
||||
target := make(map[SCUMTransportCursor]struct{}, len(source))
|
||||
for key, value := range source {
|
||||
target[key] = value
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
func cloneStringSet(source map[string]struct{}) map[string]struct{} {
|
||||
target := make(map[string]struct{}, len(source))
|
||||
for key, value := range source {
|
||||
target[key] = value
|
||||
}
|
||||
return target
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package companion
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
func TestSCUMLoginLogParserParsesAuthenticUTF16LELoginAndLogout(t *testing.T) {
|
||||
availability := VerifiedSCUMLoginLogParser()
|
||||
if !availability.Available || availability.ParserKey != SCUMLoginLogParserKey || availability.ParserVersion != SCUMLoginLogParserVersion {
|
||||
t.Fatalf("login log parser should be versioned and available: %+v", availability)
|
||||
}
|
||||
records := []SCUMLoginLogRecord{
|
||||
scumLoginFixtureRecord(1, "gen-1", "2026.07.05-05.39.10: '203.0.113.10:7777 76561198000000001:Fixture One(profile-1)' logged in at: X=123.45 Y=-456.78 Z=90"),
|
||||
scumLoginFixtureRecord(2, "gen-1", "2026.07.05-07.21.15: '203.0.113.10:7777 76561198000000001:Fixture One(profile-1)' logged out at: X=124.45 Y=-455.78 Z=91"),
|
||||
}
|
||||
batch := ParseSCUMLoginLogRecords("server-1", records, SCUMLoginLogParseOptions{})
|
||||
if len(batch.Diagnostics) != 0 {
|
||||
t.Fatalf("unexpected diagnostics: %+v", batch.Diagnostics)
|
||||
}
|
||||
if len(batch.Events) != 2 {
|
||||
t.Fatalf("expected login and logout events, got %+v", batch.Events)
|
||||
}
|
||||
login := batch.Events[0]
|
||||
if login.Type != "scum.login" || login.ExternalPlayerID != "76561198000000001" || login.DisplayName != "Fixture One" || login.ProfileLocalID != "profile-1" || login.OccurredAtSourceText != "2026.07.05-05.39.10" {
|
||||
t.Fatalf("unexpected login event: %+v", login)
|
||||
}
|
||||
logout := batch.Events[1]
|
||||
if logout.Type != "scum.logout" || logout.LogicalEventIdentity == login.LogicalEventIdentity || logout.TransportCursor.Sequence != 2 {
|
||||
t.Fatalf("unexpected logout event: %+v", logout)
|
||||
}
|
||||
serialized, err := json.Marshal(batch.Events)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serializedText := string(serialized)
|
||||
for _, forbidden := range []string{"203.0.113.10", "7777", "123.45", "-456.78", "124.45", "-455.78"} {
|
||||
if strings.Contains(serializedText, forbidden) {
|
||||
t.Fatalf("parser leaked network or coordinate material %q in %s", forbidden, serializedText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSCUMLoginLogParserRejectsFailedPartialUndecodableOversizedAndMalformedLines(t *testing.T) {
|
||||
records := []SCUMLoginLogRecord{
|
||||
scumLoginFixtureRecord(1, "gen-1", "2026.07.05-05.39.10: '203.0.113.20 76561198000000002:Rejected(profile-2)' login failed at: X=1 Y=2 Z=3"),
|
||||
{ServerID: "server-1", SourceIdentity: "source-1", StreamGeneration: "gen-1", Sequence: 2, RawLine: scumUTF16LE("2026.07.05-05.39.10: '203.0.113.20 76561198000000002:Partial(profile-2)' logged in at: X=1 Y=2 Z=3"), Partial: true},
|
||||
{ServerID: "server-1", SourceIdentity: "source-1", StreamGeneration: "gen-1", Sequence: 3, RawLine: []byte{0x00, 0xD8}},
|
||||
{ServerID: "server-1", SourceIdentity: "source-1", StreamGeneration: "gen-1", Sequence: 4, RawLine: make([]byte, SCUMLoginLogMaxLineBytes+2)},
|
||||
scumLoginFixtureRecord(5, "gen-1", "not a real login line"),
|
||||
}
|
||||
batch := ParseSCUMLoginLogRecords("server-1", records, SCUMLoginLogParseOptions{})
|
||||
if len(batch.Events) != 0 {
|
||||
t.Fatalf("invalid lines must not create events: %+v", batch.Events)
|
||||
}
|
||||
assertDiagnosticCodes(t, batch.Diagnostics, []string{"failed-login", "partial-line", "undecodable-line", "oversized-line", "malformed-line"})
|
||||
}
|
||||
|
||||
func TestSCUMLoginLogParserDeduplicatesRotationOverlapAcrossNewGeneration(t *testing.T) {
|
||||
line := "2026.06.20-06.36.46: '198.51.100.44 76561198000000003:Overlap(profile-3)' logged in at: X=10 Y=20 Z=30"
|
||||
first := scumLoginFixtureRecord(5, "gen-old", line)
|
||||
second := scumLoginFixtureRecord(1, "gen-new", line)
|
||||
batch := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{first, second}, SCUMLoginLogParseOptions{})
|
||||
if len(batch.Events) != 1 {
|
||||
t.Fatalf("rotation overlap should emit one logical event, got %+v", batch.Events)
|
||||
}
|
||||
assertDiagnosticCodes(t, batch.Diagnostics, []string{"duplicate-logical-event"})
|
||||
oldGeneration := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{first}, SCUMLoginLogParseOptions{})
|
||||
newGeneration := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{second}, SCUMLoginLogParseOptions{})
|
||||
if oldGeneration.Events[0].LogicalEventIdentity != newGeneration.Events[0].LogicalEventIdentity {
|
||||
t.Fatalf("logical identity must be generation-independent: %s != %s", oldGeneration.Events[0].LogicalEventIdentity, newGeneration.Events[0].LogicalEventIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSCUMLoginLogParserLogicalIdentityExcludesNetworkAndCoordinates(t *testing.T) {
|
||||
first := scumLoginFixtureRecord(1, "gen-1", "2026.06.20-06.36.46: '198.51.100.44 76561198000000003:Private(profile-3)' logged in at: X=10 Y=20 Z=30")
|
||||
second := scumLoginFixtureRecord(2, "gen-1", "2026.06.20-06.36.46: '203.0.113.44:7777 76561198000000003:Private(profile-3)' logged in at: X=-999 Y=888 Z=777")
|
||||
firstBatch := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{first}, SCUMLoginLogParseOptions{})
|
||||
secondBatch := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{second}, SCUMLoginLogParseOptions{})
|
||||
if len(firstBatch.Events) != 1 || len(secondBatch.Events) != 1 {
|
||||
t.Fatalf("expected both privacy variants to parse: %+v %+v", firstBatch, secondBatch)
|
||||
}
|
||||
if firstBatch.Events[0].LogicalEventIdentity != secondBatch.Events[0].LogicalEventIdentity {
|
||||
t.Fatalf("logical identity must exclude network and coordinates: %s != %s", firstBatch.Events[0].LogicalEventIdentity, secondBatch.Events[0].LogicalEventIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSCUMLoginLogParserHonorsRestartResumeAndDuplicateDelivery(t *testing.T) {
|
||||
record := scumLoginFixtureRecord(9, "gen-1", "2026.06.20-12.30.26: '198.51.100.60 76561198000000004:Resume(profile-4)' logged in at: X=10 Y=20 Z=30")
|
||||
first := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{record}, SCUMLoginLogParseOptions{})
|
||||
if len(first.Events) != 1 {
|
||||
t.Fatalf("expected first delivery to emit one event: %+v", first)
|
||||
}
|
||||
acknowledgedTransport := map[SCUMTransportCursor]struct{}{first.Events[0].TransportCursor: {}}
|
||||
acknowledgedLogical := map[string]struct{}{first.Events[0].LogicalEventIdentity: {}}
|
||||
duplicateCursor := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{record}, SCUMLoginLogParseOptions{AcknowledgedTransportCursors: acknowledgedTransport, AcknowledgedLogicalEventIdentities: acknowledgedLogical})
|
||||
if len(duplicateCursor.Events) != 0 {
|
||||
t.Fatalf("duplicate acknowledged cursor must not emit events: %+v", duplicateCursor.Events)
|
||||
}
|
||||
assertDiagnosticCodes(t, duplicateCursor.Diagnostics, []string{"duplicate-transport-cursor"})
|
||||
replayedAfterRestart := record
|
||||
replayedAfterRestart.StreamGeneration = "gen-after-restart"
|
||||
replayedAfterRestart.Sequence = 1
|
||||
duplicateLogical := ParseSCUMLoginLogRecords("server-1", []SCUMLoginLogRecord{replayedAfterRestart}, SCUMLoginLogParseOptions{AcknowledgedLogicalEventIdentities: acknowledgedLogical})
|
||||
if len(duplicateLogical.Events) != 0 {
|
||||
t.Fatalf("restart replay must be idempotent by logical identity: %+v", duplicateLogical.Events)
|
||||
}
|
||||
assertDiagnosticCodes(t, duplicateLogical.Diagnostics, []string{"duplicate-logical-event"})
|
||||
}
|
||||
|
||||
func TestSCUMLoginLogParserAcceptsOutOfOrderNewEventsWithDiagnostic(t *testing.T) {
|
||||
records := []SCUMLoginLogRecord{
|
||||
scumLoginFixtureRecord(8, "gen-1", "2026.06.20-12.32.40: '198.51.100.70 76561198000000005:Late(profile-5)' logged out at: X=10 Y=20 Z=30"),
|
||||
scumLoginFixtureRecord(7, "gen-1", "2026.06.20-12.30.26: '198.51.100.70 76561198000000005:Late(profile-5)' logged in at: X=10 Y=20 Z=30"),
|
||||
}
|
||||
batch := ParseSCUMLoginLogRecords("server-1", records, SCUMLoginLogParseOptions{})
|
||||
if len(batch.Events) != 2 {
|
||||
t.Fatalf("out-of-order unique events should still be parsed: %+v", batch.Events)
|
||||
}
|
||||
assertDiagnosticCodes(t, batch.Diagnostics, []string{"out-of-order-transport"})
|
||||
}
|
||||
|
||||
func scumLoginFixtureRecord(sequence uint64, generation string, line string) SCUMLoginLogRecord {
|
||||
return SCUMLoginLogRecord{ServerID: "server-1", SourceIdentity: "source-1", StreamGeneration: generation, Sequence: sequence, RawLine: scumUTF16LE(line)}
|
||||
}
|
||||
|
||||
func scumUTF16LE(value string) []byte {
|
||||
encoded := utf16.Encode([]rune(value))
|
||||
bytes := make([]byte, len(encoded)*2)
|
||||
for index, codeUnit := range encoded {
|
||||
binary.LittleEndian.PutUint16(bytes[index*2:], codeUnit)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
func assertDiagnosticCodes(t *testing.T, diagnostics []SCUMLoginLogDiagnostic, expected []string) {
|
||||
t.Helper()
|
||||
if len(diagnostics) != len(expected) {
|
||||
t.Fatalf("expected diagnostics %v, got %+v", expected, diagnostics)
|
||||
}
|
||||
for index, code := range expected {
|
||||
if diagnostics[index].Code != code {
|
||||
t.Fatalf("expected diagnostic %d to be %q, got %+v", index, code, diagnostics[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user