185 lines
9.7 KiB
Go
185 lines
9.7 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
"browser.local/run/spool"
|
|
)
|
|
|
|
func TestWorkerExecutesFencedProtectedProgramAndSpoolsDedicatedLogs(t *testing.T) {
|
|
registry := NewProtectedRequestRegistry()
|
|
requestText := `{"operation":"status"}`
|
|
handlerCalled := false
|
|
if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) {
|
|
handlerCalled = true
|
|
if request.ServerInstanceID != "server-worker" || request.FencingToken != 12 || request.TargetKey != "scum-program" || request.RequestText != requestText {
|
|
t.Fatalf("unexpected protected request: %+v", request)
|
|
}
|
|
return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded, Stdout: "SCUM ready\npassword=hidden\nconfig /Users/private/scum.ini\ndsn mysql://private", Stderr: "bounded warning"}, nil
|
|
})); err != nil {
|
|
t.Fatalf("register protected handler: %v", err)
|
|
}
|
|
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 12)
|
|
client := newFakeWorkerClient()
|
|
client.claimJob = assignment
|
|
client.protectedInput = protectedWorkerInput(assignment, "program", requestText)
|
|
logSpool, err := spool.NewLogSpool(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("log spool: %v", err)
|
|
}
|
|
worker, err := NewWorker(workerTestConfig(t), client, WithProtectedRequestRegistry(registry), WithProcessLogSink(&SpoolLogSink{Spool: logSpool}))
|
|
if err != nil {
|
|
t.Fatalf("new worker: %v", err)
|
|
}
|
|
if err := worker.Register(context.Background()); err != nil {
|
|
t.Fatalf("register worker: %v", err)
|
|
}
|
|
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
|
t.Fatalf("claim protected request handled=%v err=%v", handled, err)
|
|
}
|
|
if !handlerCalled || len(client.protectedRequests) != 1 || client.protectedRequests[0].FencingToken != assignment.FencingToken {
|
|
t.Fatalf("expected one fenced protected input read: %+v", client.protectedRequests)
|
|
}
|
|
if len(client.resultRequests) != 1 || client.resultRequests[0].State != lifecycleResultStateSucceeded || client.resultRequests[0].ExecutionResult.Kind != "protected.program" {
|
|
t.Fatalf("unexpected protected result: %+v", client.resultRequests)
|
|
}
|
|
batches, err := logSpool.Pending()
|
|
entryCount := 0
|
|
for _, batch := range batches {
|
|
entryCount += len(batch.Entries)
|
|
}
|
|
if err != nil || entryCount != 5 {
|
|
t.Fatalf("expected five program log lines: batches=%+v err=%v", batches, err)
|
|
}
|
|
logLines := []string{}
|
|
for _, batch := range batches {
|
|
if batch.Source != "management-program" || batch.Source == "file" || batch.Source == "process" {
|
|
t.Fatalf("program output used wrong log source: %+v", batch)
|
|
}
|
|
for _, entry := range batch.Entries {
|
|
logLines = append(logLines, entry.Line)
|
|
}
|
|
}
|
|
joinedLogs := strings.Join(logLines, "\n")
|
|
for _, expected := range []string{"password=hidden", "/Users/private/scum.ini", "mysql://private"} {
|
|
if !strings.Contains(joinedLogs, expected) {
|
|
t.Fatalf("program log did not preserve %q: %q", expected, joinedLogs)
|
|
}
|
|
}
|
|
serialized, err := json.Marshal([]any{client.resultRequests, client.protectedRequests, worker.journal.ActiveJobs()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, private := range []string{requestText, "password=hidden", "/Users/private/scum.ini", "mysql://private", "bounded warning"} {
|
|
if strings.Contains(string(serialized), private) {
|
|
t.Fatalf("protected text or output leaked into control projection %q: %s", private, serialized)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProtectedRequestUnknownAndBindingFailureAreIsolated(t *testing.T) {
|
|
registry := NewProtectedRequestRegistry()
|
|
called := 0
|
|
if err := registry.Register("rcon", "scum-management", ProtectedRequestHandlerFunc(func(_ context.Context, request ProtectedRequest) (ProtectedRequestOutcome, error) {
|
|
called++
|
|
if request.RequestText == "unknown.command" {
|
|
return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown}, ErrProtectedRequestUnknown
|
|
}
|
|
return ProtectedRequestOutcome{Status: ProtectedRequestStatusSucceeded}, nil
|
|
})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry))
|
|
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 21)
|
|
unknown := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "unknown.command"))
|
|
if unknown.State != lifecycleResultStateFailed || unknown.ErrorCode != "protected_request_unknown" || unknown.Retryable {
|
|
t.Fatalf("unexpected unknown outcome: %+v", unknown)
|
|
}
|
|
succeeded := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "status"))
|
|
if succeeded.State != lifecycleResultStateSucceeded || called != 2 {
|
|
t.Fatalf("unknown request affected later request: result=%+v called=%d", succeeded, called)
|
|
}
|
|
mismatched := protectedWorkerInput(assignment, "rcon", "status")
|
|
mismatched.FencingToken++
|
|
failed := executor.ExecuteProtectedRequest(context.Background(), assignment, mismatched)
|
|
if failed.ErrorCode != "protected_request_binding_invalid" || called != 2 {
|
|
t.Fatalf("binding failure reached handler: result=%+v called=%d", failed, called)
|
|
}
|
|
encoded, _ := json.Marshal([]LifecycleExecutionResult{unknown, failed})
|
|
if strings.Contains(string(encoded), "unknown.command") || strings.Contains(string(encoded), "status") {
|
|
t.Fatalf("safe failure leaked request text: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestProtectedRCONFallsBackToSourceRCONPlan(t *testing.T) {
|
|
listener, port := newSourceRCONListener(t)
|
|
defer listener.Close()
|
|
password := strings.Repeat("f", 64)
|
|
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProtectedRCON, "rcon", "scum-management", 41)
|
|
assignment.ExecutionInput.WorkspaceScope = "run-local"
|
|
assignment.ExecutionInput.TimeoutSeconds = 5
|
|
assignment.ExecutionInput.SourceRCON = &protocol.RuntimeSourceRCONPlan{Protocol: "source-rcon", ExtensionKey: "scum-simple-rcon", ModKey: "scum_simple_rcon", ConfigRef: "ue4ss/Mods/scum_simple_rcon/config.ini", DeploymentStateRef: "runtime/ue4ss-dll/ue4ss/scum-simple-rcon/release.json", Port: port}
|
|
root := t.TempDir()
|
|
writeSourceRCONConfig(t, root, assignment, "127.0.0.1", password)
|
|
commands := make(chan string, 1)
|
|
serverDone := serveSourceRCONSession(listener, password, func(connection net.Conn, packet sourceRCONPacket) error {
|
|
commands <- packet.body
|
|
return writeSourceRCONPacket(context.Background(), connection, sourceRCONPacket{id: packet.id, typeCode: sourceRCONResponseValue})
|
|
})
|
|
|
|
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithDLLExtensionRuntimeTarget("windows", "amd64")).ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "rcon", "#ListPlayers"))
|
|
if result.State != lifecycleResultStateSucceeded || result.ExecutionResult.Kind != "protected.rcon" {
|
|
t.Fatalf("expected protected RCON delivery through Source RCON, got %+v", result)
|
|
}
|
|
if got := <-commands; got != "#ListPlayers" {
|
|
t.Fatalf("expected SCUM command delivery, got %q", got)
|
|
}
|
|
awaitSourceRCONServer(t, serverDone)
|
|
serialized, err := json.Marshal(result)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(serialized), "#ListPlayers") || strings.Contains(string(serialized), password) {
|
|
t.Fatalf("protected Source RCON result leaked private input: %s", serialized)
|
|
}
|
|
}
|
|
|
|
func TestUnknownProtectedProgramKeepsSafeDiagnosticInProgramLogOnly(t *testing.T) {
|
|
registry := NewProtectedRequestRegistry()
|
|
if err := registry.Register("program", "scum-program", ProtectedRequestHandlerFunc(func(context.Context, ProtectedRequest) (ProtectedRequestOutcome, error) {
|
|
return ProtectedRequestOutcome{Status: ProtectedRequestStatusUnknown, Stderr: "unknown field database=/private/scum.db"}, ErrProtectedRequestUnknown
|
|
})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sink := &recordingLogSink{}
|
|
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(t.TempDir()), WithProtectedRequestRegistry(registry), WithProcessLogSink(sink))
|
|
assignment := protectedWorkerAssignment(protocol.RunCapabilityRemoteRunProgram, "program", "scum-program", 31)
|
|
result := executor.ExecuteProtectedRequest(context.Background(), assignment, protectedWorkerInput(assignment, "program", `{"unexpected":true}`))
|
|
if result.ErrorCode != "protected_request_unknown" || result.State != lifecycleResultStateFailed {
|
|
t.Fatalf("unexpected unknown program result: %+v", result)
|
|
}
|
|
if len(sink.lines) != 1 || sink.lines[0] != "management-program.stderr:unknown field database=/private/scum.db" {
|
|
t.Fatalf("unknown program diagnostic was not faithfully channelized: %+v", sink.lines)
|
|
}
|
|
}
|
|
|
|
func protectedWorkerAssignment(capability string, kind string, key string, fence uint64) protocol.RunJobAssignment {
|
|
assignment := workerJobAssignment(capability)
|
|
assignment.TargetKey = key
|
|
assignment.InputRef = "input://protected-request/" + assignment.JobID
|
|
assignment.FencingToken = fence
|
|
assignment.MaxAttempts = 1
|
|
assignment.ExecutionInput = protocol.RunJobExecutionInput{RemoteAdapterKey: key, RemoteAdapterKind: "protected-" + kind, TimeoutSeconds: 5}
|
|
return assignment
|
|
}
|
|
|
|
func protectedWorkerInput(assignment protocol.RunJobAssignment, kind string, text string) protocol.ProtectedRequestExecutionInputResponse {
|
|
return protocol.ProtectedRequestExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, FencingToken: assignment.FencingToken, Authorized: true, ApprovalState: "approved", QueueState: "claimed", ExpiresAt: time.Now().UTC().Add(time.Minute), Kind: kind, TransportKey: assignment.ExecutionInput.RemoteAdapterKey, TargetKey: assignment.TargetKey, RequestText: text}
|
|
}
|