first commit
This commit is contained in:
npc0-hue
@@ -0,0 +1,454 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
lifecycleResultStateSucceeded = "succeeded"
|
||||
lifecycleResultStateFailed = "failed"
|
||||
lifecycleResultStateCancelled = "cancelled"
|
||||
|
||||
defaultLifecycleTimeout = 30 * time.Second
|
||||
maxLifecycleOutputBytes = 4096
|
||||
)
|
||||
|
||||
var (
|
||||
commandNamePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||
envNamePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,63}$`)
|
||||
disallowedExecutables = map[string]struct{}{
|
||||
"bash": {},
|
||||
"cmd": {},
|
||||
"fish": {},
|
||||
"powershell": {},
|
||||
"pwsh": {},
|
||||
"sh": {},
|
||||
"zsh": {},
|
||||
}
|
||||
)
|
||||
|
||||
type LifecycleExecutor struct {
|
||||
workspaceRoot string
|
||||
supervisor ProcessSupervisor
|
||||
logSink ProcessLogSink
|
||||
artifactHook LifecycleArtifactHook
|
||||
}
|
||||
|
||||
type LifecycleExecutionResult struct {
|
||||
State string
|
||||
Progress protocol.RunJobProgressReport
|
||||
ResultRef string
|
||||
Message string
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
type LifecycleExecutorOption func(*LifecycleExecutor)
|
||||
|
||||
func NewLifecycleExecutor(options ...LifecycleExecutorOption) LifecycleExecutor {
|
||||
executor := LifecycleExecutor{
|
||||
workspaceRoot: filepath.Join(".", ".run-workspace"),
|
||||
supervisor: OSProcessSupervisor{},
|
||||
logSink: NoopProcessLogSink{},
|
||||
artifactHook: StaticLifecycleArtifactHook{},
|
||||
}
|
||||
for _, option := range options {
|
||||
option(&executor)
|
||||
}
|
||||
return executor
|
||||
}
|
||||
|
||||
func WithLifecycleWorkspaceRoot(root string) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if strings.TrimSpace(root) != "" {
|
||||
executor.workspaceRoot = root
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithProcessSupervisor(supervisor ProcessSupervisor) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if supervisor != nil {
|
||||
executor.supervisor = supervisor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithProcessLogSink(sink ProcessLogSink) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if sink != nil {
|
||||
executor.logSink = sink
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithLifecycleArtifactHook(hook LifecycleArtifactHook) LifecycleExecutorOption {
|
||||
return func(executor *LifecycleExecutor) {
|
||||
if hook != nil {
|
||||
executor.artifactHook = hook
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SupportedLifecycleCapabilities() []string {
|
||||
return []string{
|
||||
protocol.RunCapabilityProcessInstall,
|
||||
protocol.RunCapabilityProcessStart,
|
||||
protocol.RunCapabilityProcessStop,
|
||||
}
|
||||
}
|
||||
|
||||
func SupportedRunCapabilities() []string {
|
||||
capabilities := append([]string(nil), SupportedLifecycleCapabilities()...)
|
||||
capabilities = append(capabilities, protocol.RunCapabilityLogsRead)
|
||||
return capabilities
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) SupportedCapabilities() []string {
|
||||
return SupportedLifecycleCapabilities()
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) Execute(assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
return executor.ExecuteContext(context.Background(), assignment)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
||||
if !isSupportedLifecycleCapability(assignment.Capability) {
|
||||
return lifecycleFailure("unsupported_lifecycle_capability", "unsupported lifecycle capability")
|
||||
}
|
||||
command, err := executor.ResolveCommand(assignment)
|
||||
if err != nil {
|
||||
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
||||
}
|
||||
result, err := executor.supervisor.Run(ctx, command)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action cancelled"},
|
||||
Message: "lifecycle action cancelled",
|
||||
ErrorCode: "lifecycle_cancelled",
|
||||
}
|
||||
}
|
||||
executor.writeProcessLogs(ctx, assignment, result)
|
||||
if err != nil {
|
||||
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode))
|
||||
}
|
||||
artifactRef, err := executor.artifactHook.QueueLifecycleResult(ctx, assignment, result)
|
||||
if err != nil {
|
||||
return lifecycleFailure("lifecycle_artifact_hook_failed", err.Error())
|
||||
}
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateSucceeded,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action completed"},
|
||||
ResultRef: artifactRef,
|
||||
Message: fmt.Sprintf("%s completed", assignment.Capability),
|
||||
}
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) ResolveCommand(assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
||||
workdir, err := scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, err
|
||||
}
|
||||
if err := os.MkdirAll(workdir, 0o755); err != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("create scoped workspace: %w", err)
|
||||
}
|
||||
template := LifecycleActionTemplate{
|
||||
Command: []string{"true"},
|
||||
TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond),
|
||||
}
|
||||
if assignment.TargetKey != "" {
|
||||
path, err := scopedPath(workdir, assignment.TargetKey)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("open lifecycle action template: %w", err)
|
||||
}
|
||||
decodeErr := json.NewDecoder(file).Decode(&template)
|
||||
closeErr := file.Close()
|
||||
if decodeErr != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("decode lifecycle action template: %w", decodeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return ProcessCommand{}, fmt.Errorf("close lifecycle action template: %w", closeErr)
|
||||
}
|
||||
}
|
||||
return template.ToProcessCommand(workdir)
|
||||
}
|
||||
|
||||
func (executor LifecycleExecutor) writeProcessLogs(ctx context.Context, assignment protocol.RunJobAssignment, result ProcessResult) {
|
||||
for _, item := range []struct {
|
||||
stream string
|
||||
body string
|
||||
}{
|
||||
{stream: "stdout", body: result.Stdout},
|
||||
{stream: "stderr", body: result.Stderr},
|
||||
} {
|
||||
for _, line := range splitBoundedLines(item.body) {
|
||||
_ = executor.logSink.Append(ctx, assignment, item.stream, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type LifecycleActionTemplate struct {
|
||||
Command []string `json:"command"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
TimeoutMS int `json:"timeoutMs,omitempty"`
|
||||
}
|
||||
|
||||
func (template LifecycleActionTemplate) ToProcessCommand(workdir string) (ProcessCommand, error) {
|
||||
if len(template.Command) == 0 {
|
||||
return ProcessCommand{}, fmt.Errorf("command is required")
|
||||
}
|
||||
for i, part := range template.Command {
|
||||
if strings.TrimSpace(part) == "" {
|
||||
return ProcessCommand{}, fmt.Errorf("command part is required")
|
||||
}
|
||||
if containsUnsafeRuntimeText(part) {
|
||||
return ProcessCommand{}, fmt.Errorf("command contains unsafe content")
|
||||
}
|
||||
if i == 0 {
|
||||
if !commandNamePattern.MatchString(part) || strings.Contains(part, "/") || filepath.IsAbs(part) {
|
||||
return ProcessCommand{}, fmt.Errorf("command executable must be an allowlisted name")
|
||||
}
|
||||
if _, disallowed := disallowedExecutables[strings.ToLower(part)]; disallowed {
|
||||
return ProcessCommand{}, fmt.Errorf("command executable must not be a shell")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.ContainsAny(part, "|;&`$<>") {
|
||||
return ProcessCommand{}, fmt.Errorf("command arguments must not contain shell metacharacters")
|
||||
}
|
||||
}
|
||||
env := make(map[string]string, len(template.Env))
|
||||
for key, value := range template.Env {
|
||||
if !envNamePattern.MatchString(key) || !strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_") {
|
||||
return ProcessCommand{}, fmt.Errorf("env key is not allowlisted")
|
||||
}
|
||||
if containsUnsafeRuntimeText(value) {
|
||||
return ProcessCommand{}, fmt.Errorf("env value contains unsafe content")
|
||||
}
|
||||
env[key] = value
|
||||
}
|
||||
timeout := defaultLifecycleTimeout
|
||||
if template.TimeoutMS > 0 {
|
||||
timeout = time.Duration(template.TimeoutMS) * time.Millisecond
|
||||
}
|
||||
if timeout > 5*time.Minute {
|
||||
return ProcessCommand{}, fmt.Errorf("timeout is too large")
|
||||
}
|
||||
return ProcessCommand{WorkDir: workdir, Args: append([]string(nil), template.Command...), Env: env, Timeout: timeout}, nil
|
||||
}
|
||||
|
||||
type ProcessCommand struct {
|
||||
WorkDir string
|
||||
Args []string
|
||||
Env map[string]string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type ProcessResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
}
|
||||
|
||||
type ProcessSupervisor interface {
|
||||
Run(context.Context, ProcessCommand) (ProcessResult, error)
|
||||
}
|
||||
|
||||
type OSProcessSupervisor struct{}
|
||||
|
||||
func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) {
|
||||
if len(command.Args) == 0 {
|
||||
return ProcessResult{ExitCode: -1}, fmt.Errorf("command is required")
|
||||
}
|
||||
if command.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, command.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, command.Args[0], command.Args[1:]...)
|
||||
cmd.Dir = command.WorkDir
|
||||
cmd.Env = os.Environ()
|
||||
for key, value := range command.Env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = ioLimitWriter{Writer: &stdout, Limit: maxLifecycleOutputBytes}
|
||||
cmd.Stderr = ioLimitWriter{Writer: &stderr, Limit: maxLifecycleOutputBytes}
|
||||
err := cmd.Run()
|
||||
result := ProcessResult{Stdout: RedactText(stdout.String()), Stderr: RedactText(stderr.String())}
|
||||
if cmd.ProcessState != nil {
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type ioLimitWriter struct {
|
||||
Writer *bytes.Buffer
|
||||
Limit int
|
||||
}
|
||||
|
||||
func (writer ioLimitWriter) Write(p []byte) (int, error) {
|
||||
remaining := writer.Limit - writer.Writer.Len()
|
||||
if remaining > 0 {
|
||||
if len(p) > remaining {
|
||||
_, _ = writer.Writer.Write(p[:remaining])
|
||||
} else {
|
||||
_, _ = writer.Writer.Write(p)
|
||||
}
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
type ProcessLogSink interface {
|
||||
Append(context.Context, protocol.RunJobAssignment, string, string) error
|
||||
}
|
||||
|
||||
type NoopProcessLogSink struct{}
|
||||
|
||||
func (NoopProcessLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type LifecycleArtifactHook interface {
|
||||
QueueLifecycleResult(context.Context, protocol.RunJobAssignment, ProcessResult) (string, error)
|
||||
}
|
||||
|
||||
type StaticLifecycleArtifactHook struct{}
|
||||
|
||||
func (StaticLifecycleArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
||||
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", url.PathEscape(assignment.JobID)), nil
|
||||
}
|
||||
|
||||
func LifecycleResultRequest(assignment protocol.RunJobAssignment, sessionToken string, result LifecycleExecutionResult) protocol.RunJobResultRequest {
|
||||
return protocol.RunJobResultRequest{
|
||||
RunEndpointID: assignment.RunEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
State: result.State,
|
||||
Progress: result.Progress,
|
||||
ResultRef: result.ResultRef,
|
||||
Message: result.Message,
|
||||
ErrorCode: result.ErrorCode,
|
||||
}
|
||||
}
|
||||
|
||||
func isSupportedLifecycleCapability(capability string) bool {
|
||||
for _, supported := range SupportedLifecycleCapabilities() {
|
||||
if capability == supported {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lifecycleFailure(code string, message string) LifecycleExecutionResult {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateFailed,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)},
|
||||
Message: RedactText(message),
|
||||
ErrorCode: code,
|
||||
}
|
||||
}
|
||||
|
||||
func scopedServerWorkspace(root string, serverInstanceID string) (string, error) {
|
||||
if strings.TrimSpace(serverInstanceID) == "" {
|
||||
return "", fmt.Errorf("server instance id is required")
|
||||
}
|
||||
if containsUnsafeRuntimeText(serverInstanceID) || strings.ContainsAny(serverInstanceID, `/\`) || serverInstanceID == "." || serverInstanceID == ".." {
|
||||
return "", fmt.Errorf("server instance id is unsafe")
|
||||
}
|
||||
return scopedPath(root, serverInstanceID)
|
||||
}
|
||||
|
||||
func scopedPath(root string, key string) (string, error) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return "", fmt.Errorf("workspace root is required")
|
||||
}
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return "", fmt.Errorf("logical key is required")
|
||||
}
|
||||
if filepath.IsAbs(key) || strings.Contains(key, "..") || strings.Contains(key, `\`) || containsUnsafeRuntimeText(key) {
|
||||
return "", fmt.Errorf("logical key is unsafe")
|
||||
}
|
||||
cleanRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
candidate := filepath.Clean(filepath.Join(cleanRoot, filepath.FromSlash(key)))
|
||||
rel, err := filepath.Rel(cleanRoot, candidate)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("logical key escapes workspace")
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func containsUnsafeRuntimeText(value string) bool {
|
||||
normalized := strings.ToLower(value)
|
||||
for _, marker := range []string{"/users/", "/.ssh/", "password=", "apikey", "api_key", "secret=", "bearer ", "sk-", "unix://", "tcp://", "://"} {
|
||||
if strings.Contains(normalized, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func RedactText(value string) string {
|
||||
redacted := value
|
||||
replacements := []string{"/Users/", "[host]/", "Bearer ", "Bearer [redacted] ", "sk-", "sk-[redacted]", "password=", "password=[redacted]", "api_key=", "api_key=[redacted]", "secret=", "secret=[redacted]", "unix://", "socket://"}
|
||||
for i := 0; i+1 < len(replacements); i += 2 {
|
||||
redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1])
|
||||
}
|
||||
if len(redacted) > maxLifecycleOutputBytes {
|
||||
return redacted[:maxLifecycleOutputBytes]
|
||||
}
|
||||
return redacted
|
||||
}
|
||||
|
||||
func splitBoundedLines(value string) []string {
|
||||
value = RedactText(value)
|
||||
lines := strings.Split(value, "\n")
|
||||
out := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checksumForText(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
func TestLifecycleExecutorHandlesSupportedJobs(t *testing.T) {
|
||||
executor := NewLifecycleExecutor()
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
assignment := lifecycleAssignment(capability)
|
||||
result := executor.Execute(assignment)
|
||||
if result.State != "succeeded" || result.Progress.Percent != 100 || result.ResultRef == "" {
|
||||
t.Fatalf("expected successful bounded result for %s, got %+v", capability, result)
|
||||
}
|
||||
for _, forbidden := range []string{"host path", "/Users/", "run socket", "api_key", "sk-"} {
|
||||
if strings.Contains(result.Message, forbidden) || strings.Contains(result.ResultRef, forbidden) {
|
||||
t.Fatalf("lifecycle result exposed forbidden content %q: %+v", forbidden, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsupportedJobs(t *testing.T) {
|
||||
result := NewLifecycleExecutor().Execute(lifecycleAssignment("files.write"))
|
||||
if result.State != "failed" || result.ErrorCode != "unsupported_lifecycle_capability" || result.ResultRef != "" {
|
||||
t.Fatalf("expected unsupported lifecycle failure, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleResultRequestUsesAssignmentLease(t *testing.T) {
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
execution := NewLifecycleExecutor().Execute(assignment)
|
||||
request := LifecycleResultRequest(assignment, "session-token", execution)
|
||||
|
||||
if request.RunEndpointID != assignment.RunEndpointID || request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken || request.Attempt != assignment.Attempt {
|
||||
t.Fatalf("expected result request to use assignment lease, got %+v", request)
|
||||
}
|
||||
if request.SessionToken != "session-token" || request.State != "succeeded" {
|
||||
t.Fatalf("unexpected result request: %+v", request)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRunsScopedCommandTemplateAndHooks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
template := LifecycleActionTemplate{
|
||||
Command: []string{"echo", "server-ready"},
|
||||
Env: map[string]string{"GAME_MODE": "test"},
|
||||
}
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "start.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template: %v", err)
|
||||
}
|
||||
assignment.TargetKey = "actions/start.json"
|
||||
logSink := &recordingLogSink{}
|
||||
artifactHook := &recordingArtifactHook{}
|
||||
|
||||
result := NewLifecycleExecutor(
|
||||
WithLifecycleWorkspaceRoot(root),
|
||||
WithProcessLogSink(logSink),
|
||||
WithLifecycleArtifactHook(artifactHook),
|
||||
).Execute(assignment)
|
||||
|
||||
if result.State != "succeeded" || result.ResultRef != "artifact://jobs/job-1/lifecycle-result" {
|
||||
t.Fatalf("expected scoped lifecycle success, got %+v", result)
|
||||
}
|
||||
if len(logSink.lines) != 1 || logSink.lines[0] != "stdout:server-ready" {
|
||||
t.Fatalf("expected process stdout to be logged, got %+v", logSink.lines)
|
||||
}
|
||||
if !artifactHook.called {
|
||||
t.Fatal("expected artifact hook to be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsUnsafeTemplates(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir: %v", err)
|
||||
}
|
||||
for name, template := range map[string]LifecycleActionTemplate{
|
||||
"absolute": {Command: []string{"/bin/echo", "nope"}},
|
||||
"shell": {Command: []string{"sh", "-c", "echo nope"}},
|
||||
"secret": {Command: []string{"echo", "sk-secret"}},
|
||||
"env": {Command: []string{"echo", "ok"}, Env: map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}},
|
||||
} {
|
||||
body, err := json.Marshal(template)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
actionPath := filepath.Join(serverRoot, "actions", name+".json")
|
||||
if err := os.WriteFile(actionPath, body, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
unsafeAssignment := assignment
|
||||
unsafeAssignment.TargetKey = "actions/" + name + ".json"
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(unsafeAssignment)
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected unsafe command rejection for %s, got %+v", name, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorRejectsWorkspaceEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
assignment := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
assignment.TargetKey = "../outside.json"
|
||||
|
||||
result := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root)).Execute(assignment)
|
||||
|
||||
if result.State != "failed" || result.ErrorCode != "unsafe_lifecycle_command" {
|
||||
t.Fatalf("expected workspace escape rejection, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorKeepsSiblingInstanceWorkspacesIsolated(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
first := lifecycleAssignment(protocol.RunCapabilityProcessStart)
|
||||
first.ServerInstanceID = "server-alpha"
|
||||
second := lifecycleAssignment(protocol.RunCapabilityProcessStop)
|
||||
second.JobID = "job-2"
|
||||
second.ServerInstanceID = "server-beta"
|
||||
for _, assignment := range []protocol.RunJobAssignment{first, second} {
|
||||
serverRoot := filepath.Join(root, assignment.ServerInstanceID)
|
||||
if err := os.MkdirAll(filepath.Join(serverRoot, "actions"), 0o755); err != nil {
|
||||
t.Fatalf("create action dir for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
body, err := json.Marshal(LifecycleActionTemplate{Command: []string{"echo", assignment.ServerInstanceID}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(serverRoot, "actions", "lifecycle.json"), body, 0o644); err != nil {
|
||||
t.Fatalf("write template for %s: %v", assignment.ServerInstanceID, err)
|
||||
}
|
||||
}
|
||||
first.TargetKey = "actions/lifecycle.json"
|
||||
second.TargetKey = "actions/lifecycle.json"
|
||||
logSink := &recordingLogSink{}
|
||||
executor := NewLifecycleExecutor(WithLifecycleWorkspaceRoot(root), WithProcessLogSink(logSink))
|
||||
|
||||
firstResult := executor.Execute(first)
|
||||
secondResult := executor.Execute(second)
|
||||
|
||||
if firstResult.State != "succeeded" || secondResult.State != "succeeded" {
|
||||
t.Fatalf("expected both lifecycle jobs to succeed, got first=%+v second=%+v", firstResult, secondResult)
|
||||
}
|
||||
joined := strings.Join(logSink.lines, "\n")
|
||||
if !strings.Contains(joined, "stdout:server-alpha") || !strings.Contains(joined, "stdout:server-beta") {
|
||||
t.Fatalf("expected instance-specific output, got %q", joined)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-alpha", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected alpha template to remain scoped: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "server-beta", "actions", "lifecycle.json")); err != nil {
|
||||
t.Fatalf("expected beta template to remain scoped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleExecutorCancelsRunningCommand(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
result := NewLifecycleExecutor(WithProcessSupervisor(blockingSupervisor{})).ExecuteContext(ctx, lifecycleAssignment(protocol.RunCapabilityProcessStart))
|
||||
|
||||
if result.State != "cancelled" || result.ErrorCode != "lifecycle_cancelled" {
|
||||
t.Fatalf("expected cancelled lifecycle result, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLifecycleCapabilities(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
for _, capability := range SupportedLifecycleCapabilities() {
|
||||
if !containsCapability(summary.Capabilities, capability) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", capability, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeSummaryReportsLogReadCapability(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{Mode: "smoke", PlatformURL: "http://platform.test"})
|
||||
if !containsCapability(summary.Capabilities, protocol.RunCapabilityLogsRead) {
|
||||
t.Fatalf("expected smoke capabilities to include %s, got %+v", protocol.RunCapabilityLogsRead, summary.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingLogSink struct {
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (sink *recordingLogSink) Append(_ context.Context, _ protocol.RunJobAssignment, stream string, line string) error {
|
||||
sink.lines = append(sink.lines, stream+":"+line)
|
||||
return nil
|
||||
}
|
||||
|
||||
type recordingArtifactHook struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
func (hook *recordingArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
||||
hook.called = true
|
||||
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID), nil
|
||||
}
|
||||
|
||||
type blockingSupervisor struct{}
|
||||
|
||||
func (blockingSupervisor) Run(ctx context.Context, _ ProcessCommand) (ProcessResult, error) {
|
||||
<-ctx.Done()
|
||||
return ProcessResult{ExitCode: -1}, ctx.Err()
|
||||
}
|
||||
|
||||
func lifecycleAssignment(capability string) protocol.RunJobAssignment {
|
||||
now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
|
||||
return protocol.RunJobAssignment{
|
||||
JobID: "job-1",
|
||||
ServerInstanceID: "server-1",
|
||||
RunEndpointID: "run-local",
|
||||
Capability: capability,
|
||||
IdempotencyKey: "idem-1",
|
||||
State: "accepted",
|
||||
LeaseToken: "lease-1",
|
||||
Attempt: 1,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func containsCapability(capabilities []string, capability string) bool {
|
||||
for _, item := range capabilities {
|
||||
if item == capability {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/domain"
|
||||
)
|
||||
|
||||
func SmokeSummary(cfg config.Config) domain.ExecutorStatus {
|
||||
return domain.ExecutorStatus{
|
||||
Mode: cfg.Mode,
|
||||
PlatformURL: cfg.PlatformURL,
|
||||
Status: "ok",
|
||||
ExposedHostPath: false,
|
||||
Capabilities: append([]string{
|
||||
"control.hello",
|
||||
"control.heartbeat",
|
||||
}, SupportedRunCapabilities()...),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"browser.local/run/config"
|
||||
)
|
||||
|
||||
func TestSmokeSummaryDoesNotExposeHostPaths(t *testing.T) {
|
||||
summary := SmokeSummary(config.Config{
|
||||
Mode: "smoke",
|
||||
PlatformURL: "http://platform.test",
|
||||
})
|
||||
|
||||
if summary.Status != "ok" {
|
||||
t.Fatalf("expected ok status, got %q", summary.Status)
|
||||
}
|
||||
if summary.ExposedHostPath {
|
||||
t.Fatal("smoke summary must not expose host paths")
|
||||
}
|
||||
if len(summary.Capabilities) == 0 {
|
||||
t.Fatal("expected baseline capabilities")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
type WorkerClient interface {
|
||||
Hello(context.Context, protocol.RunHelloRequest) (protocol.RunHelloResponse, error)
|
||||
Heartbeat(context.Context, protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error)
|
||||
ClaimJob(context.Context, protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error)
|
||||
AckJob(context.Context, protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error)
|
||||
UpdateJobProgress(context.Context, protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error)
|
||||
CompleteJob(context.Context, protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error)
|
||||
PollJobCancel(context.Context, protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error)
|
||||
ReconcileJobs(context.Context, protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error)
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
cfg config.Config
|
||||
client WorkerClient
|
||||
executor LifecycleExecutor
|
||||
state WorkerState
|
||||
journal *JobJournal
|
||||
}
|
||||
|
||||
type WorkerState struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Capabilities []string
|
||||
Capacity protocol.RunCapacityReport
|
||||
LastHeartbeat time.Time
|
||||
Sequence uint64
|
||||
}
|
||||
|
||||
func NewWorker(cfg config.Config, client WorkerClient, options ...LifecycleExecutorOption) (*Worker, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("worker client is required")
|
||||
}
|
||||
if cfg.RunEndpointID == "" {
|
||||
cfg.RunEndpointID = config.DefaultEndpointID
|
||||
}
|
||||
if cfg.DisplayName == "" {
|
||||
cfg.DisplayName = config.DefaultDisplayName
|
||||
}
|
||||
if cfg.Version == "" {
|
||||
cfg.Version = config.DefaultVersion
|
||||
}
|
||||
if cfg.MaxJobs <= 0 {
|
||||
cfg.MaxJobs = 1
|
||||
}
|
||||
executorOptions := append([]LifecycleExecutorOption{
|
||||
WithLifecycleWorkspaceRoot(cfg.WorkspaceRoot),
|
||||
}, options...)
|
||||
return &Worker{
|
||||
cfg: cfg,
|
||||
client: client,
|
||||
executor: NewLifecycleExecutor(executorOptions...),
|
||||
state: WorkerState{
|
||||
RunEndpointID: cfg.RunEndpointID,
|
||||
Capabilities: SupportedRunCapabilities(),
|
||||
Capacity: protocol.RunCapacityReport{MaxJobs: cfg.MaxJobs},
|
||||
},
|
||||
journal: NewJobJournal(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (worker *Worker) Register(ctx context.Context) error {
|
||||
response, err := worker.client.Hello(ctx, protocol.RunHelloRequest{
|
||||
RegistrationToken: worker.cfg.RegistrationToken,
|
||||
RunEndpointID: worker.cfg.RunEndpointID,
|
||||
DisplayName: worker.cfg.DisplayName,
|
||||
Version: worker.cfg.Version,
|
||||
Status: "online",
|
||||
Platform: runtime.GOOS,
|
||||
CapabilityReport: protocol.RunCapabilityReport{
|
||||
Capabilities: worker.state.Capabilities,
|
||||
Fingerprint: capabilityFingerprint(worker.state.Capabilities),
|
||||
},
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted || response.SessionToken == "" {
|
||||
return fmt.Errorf("run hello was not accepted")
|
||||
}
|
||||
worker.state.SessionToken = response.SessionToken
|
||||
if sink, ok := worker.executor.logSink.(*SpoolLogSink); ok {
|
||||
sink.RunEndpointID = worker.state.RunEndpointID
|
||||
sink.SessionToken = worker.state.SessionToken
|
||||
}
|
||||
if hook, ok := worker.executor.artifactHook.(*QueueArtifactHook); ok {
|
||||
hook.RunEndpointID = worker.state.RunEndpointID
|
||||
hook.SessionToken = worker.state.SessionToken
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) HeartbeatOnce(ctx context.Context) error {
|
||||
if worker.state.SessionToken == "" {
|
||||
return fmt.Errorf("worker is not registered")
|
||||
}
|
||||
response, err := worker.client.Heartbeat(ctx, protocol.RunHeartbeatRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
Version: worker.cfg.Version,
|
||||
Status: "online",
|
||||
CapabilityFingerprint: capabilityFingerprint(worker.state.Capabilities),
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !response.Accepted {
|
||||
return fmt.Errorf("heartbeat was not accepted")
|
||||
}
|
||||
worker.state.LastHeartbeat = response.ServerTime
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) ClaimAndRunOnce(ctx context.Context) (bool, error) {
|
||||
if worker.state.SessionToken == "" {
|
||||
return false, fmt.Errorf("worker is not registered")
|
||||
}
|
||||
claim, err := worker.client.ClaimJob(ctx, protocol.RunJobClaimRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
Capabilities: worker.state.Capabilities,
|
||||
Capacity: worker.capacityReport(),
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !claim.Accepted || !claim.HasJob || claim.Job == nil {
|
||||
return false, nil
|
||||
}
|
||||
assignment := *claim.Job
|
||||
worker.journal.MarkActive(assignment)
|
||||
ack, err := worker.client.AckJob(ctx, protocol.RunJobAckRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
Message: "job accepted by run worker",
|
||||
})
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
assignment = ack.Job
|
||||
worker.journal.MarkActive(assignment)
|
||||
worker.state.Sequence++
|
||||
if _, err := worker.client.UpdateJobProgress(ctx, protocol.RunJobProgressRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
Attempt: assignment.Attempt,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 10, Message: "lifecycle execution started"},
|
||||
Sequence: worker.state.Sequence,
|
||||
}); err != nil {
|
||||
return true, err
|
||||
}
|
||||
jobCtx, cancel := context.WithCancel(ctx)
|
||||
cancelled := make(chan protocol.RunJobCancelPollResponse, 1)
|
||||
go func() {
|
||||
cancelPoll, pollErr := worker.client.PollJobCancel(ctx, protocol.RunJobCancelPollRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
JobID: assignment.JobID,
|
||||
LeaseToken: assignment.LeaseToken,
|
||||
})
|
||||
if pollErr == nil && cancelPoll.HasCancel {
|
||||
cancel()
|
||||
cancelled <- cancelPoll
|
||||
return
|
||||
}
|
||||
cancelled <- protocol.RunJobCancelPollResponse{Accepted: true}
|
||||
}()
|
||||
execution := worker.executor.ExecuteContext(jobCtx, assignment)
|
||||
cancel()
|
||||
select {
|
||||
case poll := <-cancelled:
|
||||
if poll.HasCancel && execution.State == lifecycleResultStateSucceeded {
|
||||
execution = LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "cancelled by platform"},
|
||||
Message: "cancelled by platform",
|
||||
ErrorCode: "lifecycle_cancelled",
|
||||
}
|
||||
}
|
||||
default:
|
||||
}
|
||||
if _, err := worker.client.CompleteJob(ctx, LifecycleResultRequest(assignment, worker.state.SessionToken, execution)); err != nil {
|
||||
return true, err
|
||||
}
|
||||
worker.journal.MarkTerminal(assignment.JobID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (worker *Worker) ReconcileOnce(ctx context.Context) error {
|
||||
if worker.state.SessionToken == "" {
|
||||
return fmt.Errorf("worker is not registered")
|
||||
}
|
||||
response, err := worker.client.ReconcileJobs(ctx, protocol.RunJobReconcileRequest{
|
||||
RunEndpointID: worker.state.RunEndpointID,
|
||||
SessionToken: worker.state.SessionToken,
|
||||
ActiveJobIDs: worker.journal.ActiveJobIDs(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range response.ActiveJobs {
|
||||
worker.journal.MarkActive(job)
|
||||
}
|
||||
for _, unknown := range response.UnknownJobIDs {
|
||||
worker.journal.MarkTerminal(unknown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (worker *Worker) Run(ctx context.Context) error {
|
||||
if err := worker.Register(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
heartbeatInterval := durationOrDefault(worker.cfg.HeartbeatInterval, 15*time.Second)
|
||||
jobInterval := durationOrDefault(worker.cfg.PollInterval, 2*time.Second)
|
||||
heartbeatTicker := time.NewTicker(heartbeatInterval)
|
||||
jobTicker := time.NewTicker(jobInterval)
|
||||
defer heartbeatTicker.Stop()
|
||||
defer jobTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-heartbeatTicker.C:
|
||||
if err := worker.HeartbeatOnce(ctx); err != nil {
|
||||
heartbeatTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
|
||||
continue
|
||||
}
|
||||
heartbeatTicker.Reset(heartbeatInterval)
|
||||
case <-jobTicker.C:
|
||||
if _, err := worker.ClaimAndRunOnce(ctx); err != nil {
|
||||
jobTicker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
|
||||
continue
|
||||
}
|
||||
jobTicker.Reset(jobInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) capacityReport() protocol.RunCapacityReport {
|
||||
return protocol.RunCapacityReport{
|
||||
MaxJobs: worker.state.Capacity.MaxJobs,
|
||||
RunningJobs: worker.journal.ActiveCount(),
|
||||
QueuedJobs: 0,
|
||||
Summary: "worker control active; job capacity reported separately",
|
||||
}
|
||||
}
|
||||
|
||||
func (worker *Worker) State() WorkerState {
|
||||
state := worker.state
|
||||
state.Capabilities = append([]string(nil), state.Capabilities...)
|
||||
return state
|
||||
}
|
||||
|
||||
type JobJournal struct {
|
||||
mu sync.Mutex
|
||||
active map[string]protocol.RunJobAssignment
|
||||
}
|
||||
|
||||
func NewJobJournal() *JobJournal {
|
||||
return &JobJournal{active: map[string]protocol.RunJobAssignment{}}
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkActive(job protocol.RunJobAssignment) {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
journal.active[job.JobID] = job
|
||||
}
|
||||
|
||||
func (journal *JobJournal) MarkTerminal(jobID string) {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
delete(journal.active, jobID)
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveJobIDs() []string {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
ids := make([]string, 0, len(journal.active))
|
||||
for id := range journal.active {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (journal *JobJournal) ActiveCount() int {
|
||||
journal.mu.Lock()
|
||||
defer journal.mu.Unlock()
|
||||
return len(journal.active)
|
||||
}
|
||||
|
||||
type SpoolLogSink struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Spool spool.LogSpool
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (sink *SpoolLogSink) Append(_ context.Context, assignment protocol.RunJobAssignment, stream string, line string) error {
|
||||
sink.seq++
|
||||
entry := protocol.LogEntry{Seq: sink.seq, Timestamp: time.Now().UTC(), Level: "info", Line: RedactText(line), Redacted: line != RedactText(line)}
|
||||
logStreamID := fmt.Sprintf("job.%s.%s", assignment.JobID, stream)
|
||||
return sink.Spool.Enqueue(protocol.LogBatchIngestRequest{
|
||||
RunEndpointID: sink.RunEndpointID,
|
||||
SessionToken: sink.SessionToken,
|
||||
LogStreamID: logStreamID,
|
||||
ServerInstanceID: assignment.ServerInstanceID,
|
||||
StreamKey: stream,
|
||||
Source: "process",
|
||||
FirstSeq: sink.seq,
|
||||
LastSeq: sink.seq,
|
||||
Checksum: checksumForText(entry.Line),
|
||||
Entries: []protocol.LogEntry{entry},
|
||||
})
|
||||
}
|
||||
|
||||
type QueueArtifactHook struct {
|
||||
RunEndpointID string
|
||||
SessionToken string
|
||||
Queue spool.ArtifactQueue
|
||||
}
|
||||
|
||||
func (hook QueueArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, result ProcessResult) (string, error) {
|
||||
ref := fmt.Sprintf("artifact://jobs/%s/lifecycle-result", assignment.JobID)
|
||||
payload := []byte(RedactText(result.Stdout + result.Stderr))
|
||||
if len(payload) == 0 {
|
||||
payload = []byte("lifecycle result metadata")
|
||||
}
|
||||
artifactID := "artifact-" + assignment.JobID + "-lifecycle"
|
||||
if err := hook.Queue.Enqueue(protocol.ArtifactChunkUploadRequest{
|
||||
RunEndpointID: hook.RunEndpointID,
|
||||
SessionToken: hook.SessionToken,
|
||||
TransferID: "transfer-" + assignment.JobID,
|
||||
ArtifactID: artifactID,
|
||||
ChunkIndex: 0,
|
||||
Offset: 0,
|
||||
SizeBytes: len(payload),
|
||||
Checksum: checksumForText(string(payload)),
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func capabilityFingerprint(capabilities []string) string {
|
||||
return checksumForText(strings.Join(capabilities, ","))
|
||||
}
|
||||
|
||||
func durationOrDefault(value time.Duration, fallback time.Duration) time.Duration {
|
||||
if value <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func boundedRetryBackoff(value time.Duration) time.Duration {
|
||||
value = durationOrDefault(value, time.Second)
|
||||
if value > 30*time.Second {
|
||||
return 30 * time.Second
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/run/api"
|
||||
"browser.local/run/config"
|
||||
"browser.local/run/protocol"
|
||||
"browser.local/run/spool"
|
||||
)
|
||||
|
||||
func TestWorkerRegistersHeartbeatsAndStoresSession(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
worker, err := NewWorker(workerTestConfig(t), client)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if worker.State().SessionToken != "session-token" {
|
||||
t.Fatalf("expected session token stored, got %+v", worker.State())
|
||||
}
|
||||
if len(client.helloRequests) != 1 || client.helloRequests[0].RegistrationToken != "registration-token" || len(client.helloRequests[0].CapabilityReport.Capabilities) == 0 {
|
||||
t.Fatalf("unexpected hello request: %+v", client.helloRequests)
|
||||
}
|
||||
|
||||
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if len(client.heartbeatRequests) != 1 {
|
||||
t.Fatalf("expected heartbeat request")
|
||||
}
|
||||
heartbeat := client.heartbeatRequests[0]
|
||||
if heartbeat.SessionToken != "session-token" || heartbeat.Capacity.MaxJobs != 2 || heartbeat.Capacity.RunningJobs != 0 {
|
||||
t.Fatalf("unexpected heartbeat request: %+v", heartbeat)
|
||||
}
|
||||
for _, forbidden := range []string{"/Users/", "unix://", "Bearer ", "sk-", "password=", "artifact", "log"} {
|
||||
if containsText(heartbeat.Capacity.Summary, forbidden) {
|
||||
t.Fatalf("heartbeat summary exposed forbidden fragment %q: %+v", forbidden, heartbeat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerClaimsAcksProgressAndCompletesJob(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(staticSupervisor{stdout: "server ready\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
handled, err := worker.ClaimAndRunOnce(context.Background())
|
||||
if err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
if len(client.ackRequests) != 1 || len(client.progressRequests) != 1 || len(client.resultRequests) != 1 || len(client.cancelPollRequests) != 1 {
|
||||
t.Fatalf("expected ack/progress/result/cancel calls, got ack=%d progress=%d result=%d cancel=%d", len(client.ackRequests), len(client.progressRequests), len(client.resultRequests), len(client.cancelPollRequests))
|
||||
}
|
||||
if client.progressRequests[0].Progress.Percent != 10 || client.resultRequests[0].State != "succeeded" || client.resultRequests[0].ResultRef == "" {
|
||||
t.Fatalf("unexpected job channel payloads: progress=%+v result=%+v", client.progressRequests[0], client.resultRequests[0])
|
||||
}
|
||||
if worker.journal.ActiveCount() != 0 {
|
||||
t.Fatalf("expected terminal job removed from journal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlesCancellationAndReconcile(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
client.cancelResponse = protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", HasCancel: true, JobID: "job-worker", Reason: "operator requested", ServerTime: workerTestTime()}
|
||||
worker, err := NewWorker(workerTestConfig(t), client, WithProcessSupervisor(blockingSupervisor{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
handled, err := worker.ClaimAndRunOnce(context.Background())
|
||||
if err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
if len(client.resultRequests) != 1 || client.resultRequests[0].State != "cancelled" || client.resultRequests[0].ErrorCode != "lifecycle_cancelled" {
|
||||
t.Fatalf("expected cancelled terminal result, got %+v", client.resultRequests)
|
||||
}
|
||||
|
||||
worker.journal.MarkActive(workerJobAssignment(protocol.RunCapabilityProcessStart))
|
||||
client.reconcileResponse = protocol.RunJobReconcileResponse{
|
||||
Accepted: true,
|
||||
RunEndpointID: "run-test",
|
||||
ActiveJobs: []protocol.RunJobAssignment{workerJobAssignment(protocol.RunCapabilityProcessStop)},
|
||||
UnknownJobIDs: []string{"job-worker"},
|
||||
ServerTime: workerTestTime(),
|
||||
}
|
||||
if err := worker.ReconcileOnce(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if ids := worker.journal.ActiveJobIDs(); !reflect.DeepEqual(ids, []string{"job-worker-stop"}) {
|
||||
t.Fatalf("expected reconcile to replace active job ids, got %+v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSpoolHooksUseRegisteredSession(t *testing.T) {
|
||||
client := newFakeWorkerClient()
|
||||
client.claimJob = workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
logSpool, err := spool.NewLogSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("log spool: %v", err)
|
||||
}
|
||||
artifactQueue, err := spool.NewArtifactQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("artifact queue: %v", err)
|
||||
}
|
||||
worker, err := NewWorker(
|
||||
workerTestConfig(t),
|
||||
client,
|
||||
WithProcessSupervisor(staticSupervisor{stdout: "started password=hidden\n"}),
|
||||
WithProcessLogSink(&SpoolLogSink{Spool: logSpool}),
|
||||
WithLifecycleArtifactHook(&QueueArtifactHook{Queue: artifactQueue}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if _, err := worker.ClaimAndRunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("claim/run: %v", err)
|
||||
}
|
||||
logs, err := logSpool.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending logs: %v", err)
|
||||
}
|
||||
if len(logs) != 1 || logs[0].RunEndpointID != "run-test" || logs[0].SessionToken != "session-token" || containsText(logs[0].Entries[0].Line, "password=hidden") {
|
||||
t.Fatalf("unexpected spooled logs: %+v", logs)
|
||||
}
|
||||
chunks, err := artifactQueue.Pending()
|
||||
if err != nil {
|
||||
t.Fatalf("pending artifact chunks: %v", err)
|
||||
}
|
||||
if len(chunks) != 1 || chunks[0].RunEndpointID != "run-test" || chunks[0].SessionToken != "session-token" {
|
||||
t.Fatalf("unexpected artifact chunks: %+v", chunks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerRetryBackoffIsBounded(t *testing.T) {
|
||||
if got := boundedRetryBackoff(75 * time.Millisecond); got != 75*time.Millisecond {
|
||||
t.Fatalf("expected configured backoff, got %s", got)
|
||||
}
|
||||
if got := boundedRetryBackoff(time.Minute); got != 30*time.Second {
|
||||
t.Fatalf("expected capped backoff, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerIntegrationWithPlatformLikeServer(t *testing.T) {
|
||||
assignment := workerJobAssignment(protocol.RunCapabilityProcessStart)
|
||||
seen := []string{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seen = append(seen, r.URL.Path)
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/run/control/hello":
|
||||
var request protocol.RunHelloRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.RunEndpointID != "run-test" || request.Capacity.MaxJobs != 2 {
|
||||
t.Fatalf("unexpected hello: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", HeartbeatIntervalSeconds: 15, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/control/heartbeat":
|
||||
var request protocol.RunHeartbeatRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.SessionToken != "session-token" || request.Capacity.RunningJobs != 0 {
|
||||
t.Fatalf("unexpected heartbeat: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/claim":
|
||||
var request protocol.RunJobClaimRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.SessionToken != "session-token" || len(request.Capabilities) == 0 {
|
||||
t.Fatalf("unexpected claim: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &assignment, NextPollSeconds: 2, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/ack":
|
||||
var request protocol.RunJobAckRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.JobID != assignment.JobID || request.LeaseToken != assignment.LeaseToken {
|
||||
t.Fatalf("unexpected ack: %+v", request)
|
||||
}
|
||||
assignment.State = "running"
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobAckResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/progress":
|
||||
var request protocol.RunJobProgressRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.Progress.Percent != 10 {
|
||||
t.Fatalf("unexpected progress: %+v", request)
|
||||
}
|
||||
assignment.Progress = request.Progress
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobProgressResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/cancel":
|
||||
var request protocol.RunJobCancelPollRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.JobID != assignment.JobID {
|
||||
t.Fatalf("unexpected cancel poll: %+v", request)
|
||||
}
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: request.RunEndpointID, ServerTime: workerTestTime()})
|
||||
case "/api/v1/run/jobs/result":
|
||||
var request protocol.RunJobResultRequest
|
||||
decodeWorkerTestJSON(t, r, &request)
|
||||
if request.State != "succeeded" || request.ResultRef == "" {
|
||||
t.Fatalf("unexpected result: %+v", request)
|
||||
}
|
||||
assignment.State = request.State
|
||||
assignment.ResultRef = request.ResultRef
|
||||
writeWorkerTestJSON(t, w, protocol.RunJobResultResponse{Accepted: true, Job: assignment, ServerTime: workerTestTime()})
|
||||
default:
|
||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := api.NewPlatformClient(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("platform client: %v", err)
|
||||
}
|
||||
cfg := workerTestConfig(t)
|
||||
cfg.PlatformURL = server.URL
|
||||
worker, err := NewWorker(cfg, client, WithProcessSupervisor(staticSupervisor{stdout: "integration ok\n"}))
|
||||
if err != nil {
|
||||
t.Fatalf("new worker: %v", err)
|
||||
}
|
||||
if err := worker.Register(context.Background()); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if err := worker.HeartbeatOnce(context.Background()); err != nil {
|
||||
t.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
if handled, err := worker.ClaimAndRunOnce(context.Background()); err != nil || !handled {
|
||||
t.Fatalf("claim/run handled=%v err=%v", handled, err)
|
||||
}
|
||||
expected := []string{
|
||||
"/api/v1/run/control/hello",
|
||||
"/api/v1/run/control/heartbeat",
|
||||
"/api/v1/run/jobs/claim",
|
||||
"/api/v1/run/jobs/ack",
|
||||
"/api/v1/run/jobs/progress",
|
||||
"/api/v1/run/jobs/cancel",
|
||||
"/api/v1/run/jobs/result",
|
||||
}
|
||||
if !reflect.DeepEqual(seen, expected) {
|
||||
t.Fatalf("unexpected platform flow: %+v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeWorkerClient struct {
|
||||
helloRequests []protocol.RunHelloRequest
|
||||
heartbeatRequests []protocol.RunHeartbeatRequest
|
||||
claimRequests []protocol.RunJobClaimRequest
|
||||
ackRequests []protocol.RunJobAckRequest
|
||||
progressRequests []protocol.RunJobProgressRequest
|
||||
resultRequests []protocol.RunJobResultRequest
|
||||
cancelPollRequests []protocol.RunJobCancelPollRequest
|
||||
reconcileRequests []protocol.RunJobReconcileRequest
|
||||
claimJob protocol.RunJobAssignment
|
||||
cancelResponse protocol.RunJobCancelPollResponse
|
||||
reconcileResponse protocol.RunJobReconcileResponse
|
||||
}
|
||||
|
||||
func newFakeWorkerClient() *fakeWorkerClient {
|
||||
return &fakeWorkerClient{
|
||||
cancelResponse: protocol.RunJobCancelPollResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
||||
reconcileResponse: protocol.RunJobReconcileResponse{Accepted: true, RunEndpointID: "run-test", ServerTime: workerTestTime()},
|
||||
}
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) Hello(_ context.Context, request protocol.RunHelloRequest) (protocol.RunHelloResponse, error) {
|
||||
client.helloRequests = append(client.helloRequests, request)
|
||||
return protocol.RunHelloResponse{Accepted: true, RunEndpointID: request.RunEndpointID, SessionToken: "session-token", ServerTime: workerTestTime(), HeartbeatIntervalSeconds: 15}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) Heartbeat(_ context.Context, request protocol.RunHeartbeatRequest) (protocol.RunHeartbeatResponse, error) {
|
||||
client.heartbeatRequests = append(client.heartbeatRequests, request)
|
||||
return protocol.RunHeartbeatResponse{Accepted: true, RunEndpointID: request.RunEndpointID, NextHeartbeatSeconds: 15, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) ClaimJob(_ context.Context, request protocol.RunJobClaimRequest) (protocol.RunJobClaimResponse, error) {
|
||||
client.claimRequests = append(client.claimRequests, request)
|
||||
if client.claimJob.JobID == "" {
|
||||
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: false, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
job := client.claimJob
|
||||
return protocol.RunJobClaimResponse{Accepted: true, RunEndpointID: request.RunEndpointID, HasJob: true, Job: &job, NextPollSeconds: 2, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) AckJob(_ context.Context, request protocol.RunJobAckRequest) (protocol.RunJobAckResponse, error) {
|
||||
client.ackRequests = append(client.ackRequests, request)
|
||||
job := client.claimJob
|
||||
job.State = "running"
|
||||
return protocol.RunJobAckResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) UpdateJobProgress(_ context.Context, request protocol.RunJobProgressRequest) (protocol.RunJobProgressResponse, error) {
|
||||
client.progressRequests = append(client.progressRequests, request)
|
||||
job := client.claimJob
|
||||
job.Progress = request.Progress
|
||||
return protocol.RunJobProgressResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) CompleteJob(_ context.Context, request protocol.RunJobResultRequest) (protocol.RunJobResultResponse, error) {
|
||||
client.resultRequests = append(client.resultRequests, request)
|
||||
job := client.claimJob
|
||||
job.State = request.State
|
||||
job.Progress = request.Progress
|
||||
job.ResultRef = request.ResultRef
|
||||
return protocol.RunJobResultResponse{Accepted: true, Job: job, ServerTime: workerTestTime()}, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) PollJobCancel(_ context.Context, request protocol.RunJobCancelPollRequest) (protocol.RunJobCancelPollResponse, error) {
|
||||
client.cancelPollRequests = append(client.cancelPollRequests, request)
|
||||
return client.cancelResponse, nil
|
||||
}
|
||||
|
||||
func (client *fakeWorkerClient) ReconcileJobs(_ context.Context, request protocol.RunJobReconcileRequest) (protocol.RunJobReconcileResponse, error) {
|
||||
client.reconcileRequests = append(client.reconcileRequests, request)
|
||||
return client.reconcileResponse, nil
|
||||
}
|
||||
|
||||
type staticSupervisor struct {
|
||||
stdout string
|
||||
stderr string
|
||||
err error
|
||||
}
|
||||
|
||||
func (supervisor staticSupervisor) Run(context.Context, ProcessCommand) (ProcessResult, error) {
|
||||
return ProcessResult{ExitCode: 0, Stdout: supervisor.stdout, Stderr: supervisor.stderr}, supervisor.err
|
||||
}
|
||||
|
||||
func workerTestConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
return config.Config{
|
||||
Mode: "worker",
|
||||
PlatformURL: "http://platform.test",
|
||||
RunEndpointID: "run-test",
|
||||
DisplayName: "Run Test",
|
||||
Version: "0.1.0-test",
|
||||
RegistrationToken: "registration-token",
|
||||
WorkspaceRoot: t.TempDir(),
|
||||
SpoolRoot: t.TempDir(),
|
||||
MaxJobs: 2,
|
||||
HeartbeatInterval: time.Second,
|
||||
PollInterval: time.Second,
|
||||
RetryBackoff: time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func workerJobAssignment(capability string) protocol.RunJobAssignment {
|
||||
job := lifecycleAssignment(capability)
|
||||
job.JobID = "job-worker"
|
||||
if capability == protocol.RunCapabilityProcessStop {
|
||||
job.JobID = "job-worker-stop"
|
||||
}
|
||||
job.RunEndpointID = "run-test"
|
||||
job.ServerInstanceID = "server-worker"
|
||||
return job
|
||||
}
|
||||
|
||||
func workerTestTime() time.Time {
|
||||
return time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func containsText(value string, needle string) bool {
|
||||
return strings.Contains(value, needle)
|
||||
}
|
||||
|
||||
func decodeWorkerTestJSON(t *testing.T, r *http.Request, target any) {
|
||||
t.Helper()
|
||||
if r.Method != http.MethodPost {
|
||||
t.Fatalf("expected POST, got %s", r.Method)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(target); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeWorkerTestJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Fatalf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user