1262 lines
52 KiB
Go
1262 lines
52 KiB
Go
package runtime
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
const (
|
|
lifecycleResultStateSucceeded = "succeeded"
|
|
lifecycleResultStateFailed = "failed"
|
|
lifecycleResultStateCancelled = "cancelled"
|
|
|
|
defaultLifecycleTimeout = 30 * time.Second
|
|
maxLifecycleTimeout = 2 * time.Hour
|
|
maxLifecycleOutputBytes = 4096
|
|
)
|
|
|
|
var (
|
|
commandNamePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
|
envNamePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,63}$`)
|
|
disallowedExecutables = map[string]struct{}{
|
|
"bash": {},
|
|
"cmd": {},
|
|
"fish": {},
|
|
"powershell": {},
|
|
"pwsh": {},
|
|
"sh": {},
|
|
"zsh": {},
|
|
}
|
|
)
|
|
|
|
type LifecycleExecutor struct {
|
|
workspaceRoot string
|
|
managedProcessStateRoot string
|
|
managedProcessOutputRoot string
|
|
supervisor ProcessSupervisor
|
|
managed ManagedProcessSupervisor
|
|
fileExecutor *FileExecutor
|
|
logSink ProcessLogSink
|
|
artifactHook LifecycleArtifactHook
|
|
dependencyRunner ProcessSupervisor
|
|
dependencyDownloader DependencyDownloader
|
|
selfUpdateActivator SelfUpdateActivator
|
|
logCheckpointStore LogCheckpointStore
|
|
runtimeTargetOS string
|
|
runtimeTargetArch string
|
|
runtimeFileWriter runtimeFileWriter
|
|
localStartupDiagnostics bool
|
|
protectedRequests *ProtectedRequestRegistry
|
|
sqliteSchemaProbe *SQLiteSchemaProbeExecutor
|
|
sqliteQuery *SQLiteQueryExecutor
|
|
metricCollector MetricCollector
|
|
}
|
|
|
|
type runtimeFileWriter func(string, []byte, os.FileMode) error
|
|
|
|
type LifecycleExecutionResult struct {
|
|
State string
|
|
Progress protocol.RunJobProgressReport
|
|
ResultRef string
|
|
Message string
|
|
ErrorCode string
|
|
Retryable bool
|
|
ExecutionResult protocol.RunJobExecutionResult
|
|
ActivationManifest string
|
|
}
|
|
|
|
type LifecycleExecutorOption func(*LifecycleExecutor)
|
|
|
|
func NewLifecycleExecutor(options ...LifecycleExecutorOption) LifecycleExecutor {
|
|
executor := LifecycleExecutor{
|
|
workspaceRoot: filepath.Join(".", ".run-workspace"),
|
|
supervisor: OSProcessSupervisor{},
|
|
logSink: NoopProcessLogSink{},
|
|
artifactHook: StaticLifecycleArtifactHook{},
|
|
dependencyRunner: OSProcessSupervisor{},
|
|
dependencyDownloader: HTTPDependencyDownloader{},
|
|
selfUpdateActivator: ProcessSelfUpdateActivator{},
|
|
logCheckpointStore: NewMemoryLogCheckpointStore(),
|
|
runtimeTargetOS: runtime.GOOS,
|
|
runtimeTargetArch: runtime.GOARCH,
|
|
runtimeFileWriter: writeRuntimeAtomicFile,
|
|
protectedRequests: NewProtectedRequestRegistry(),
|
|
}
|
|
for _, option := range options {
|
|
option(&executor)
|
|
}
|
|
if executor.managedProcessStateRoot == "" {
|
|
executor.managedProcessStateRoot = executor.workspaceRoot
|
|
}
|
|
if executor.managedProcessOutputRoot == "" {
|
|
executor.managedProcessOutputRoot = executor.workspaceRoot
|
|
}
|
|
if executor.managed == nil {
|
|
if managed, err := NewOSManagedProcessSupervisorWithOutputRoot(executor.managedProcessStateRoot, executor.managedProcessOutputRoot); err == nil {
|
|
executor.managed = managed
|
|
}
|
|
}
|
|
if files, err := NewFileExecutor(executor.workspaceRoot); err == nil {
|
|
executor.fileExecutor = files
|
|
}
|
|
executor.sqliteSchemaProbe = NewSQLiteSchemaProbeExecutor(executor.workspaceRoot)
|
|
executor.sqliteQuery = NewSQLiteQueryExecutor(executor.workspaceRoot)
|
|
return executor
|
|
}
|
|
|
|
func (executor LifecycleExecutor) ExecuteLogBackfill(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
if assignment.ExecutionInput.LogSource == nil {
|
|
return ExecuteLogBackfillJob(ctx, assignment)
|
|
}
|
|
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
|
return lifecycleFailure("unsafe_log_backfill_job", err.Error())
|
|
}
|
|
source := RuntimeLogSource{
|
|
Key: assignment.ExecutionInput.LogSource.Key,
|
|
Kind: assignment.ExecutionInput.LogSource.Kind,
|
|
TargetKey: assignment.ExecutionInput.LogSource.TargetKey,
|
|
StreamKey: assignment.ExecutionInput.LogSource.StreamKey,
|
|
CursorKind: assignment.ExecutionInput.LogSource.CursorKind,
|
|
RetentionDays: assignment.ExecutionInput.LogSource.RetentionDays,
|
|
}
|
|
return TailDeclaredFileLogSource(ctx, executor.workspaceRoot, assignment, source, executor.logSink, executor.logCheckpointStore)
|
|
}
|
|
|
|
func (executor LifecycleExecutor) writeRuntimeFile(path string, body []byte, mode os.FileMode) error {
|
|
writer := executor.runtimeFileWriter
|
|
if writer == nil {
|
|
writer = writeRuntimeAtomicFile
|
|
}
|
|
return writer(path, body, mode)
|
|
}
|
|
|
|
func WithLifecycleWorkspaceRoot(root string) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if strings.TrimSpace(root) != "" {
|
|
executor.workspaceRoot = root
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithManagedProcessStateRoot(root string) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if strings.TrimSpace(root) != "" {
|
|
executor.managedProcessStateRoot = root
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithManagedProcessOutputRoot(root string) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if strings.TrimSpace(root) != "" {
|
|
executor.managedProcessOutputRoot = root
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithLocalStartupDiagnostics(enabled bool) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) { executor.localStartupDiagnostics = enabled }
|
|
}
|
|
|
|
// WithProtectedRequestRegistry supplies Run-owned handlers for logical
|
|
// transports. It does not expose handler configuration through any protocol.
|
|
func WithProtectedRequestRegistry(registry *ProtectedRequestRegistry) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if registry != nil {
|
|
executor.protectedRequests = registry
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithProcessSupervisor(supervisor ProcessSupervisor) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if supervisor != nil {
|
|
executor.supervisor = supervisor
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithManagedProcessSupervisor(supervisor ManagedProcessSupervisor) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if supervisor != nil {
|
|
executor.managed = supervisor
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithProcessLogSink(sink ProcessLogSink) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if sink != nil {
|
|
executor.logSink = sink
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithLifecycleArtifactHook(hook LifecycleArtifactHook) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if hook != nil {
|
|
executor.artifactHook = hook
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithDependencyCommandRunner(runner ProcessSupervisor) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if runner != nil {
|
|
executor.dependencyRunner = runner
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithDependencyDownloader(downloader DependencyDownloader) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if downloader != nil {
|
|
executor.dependencyDownloader = downloader
|
|
}
|
|
}
|
|
}
|
|
|
|
func WithSelfUpdateActivator(activator SelfUpdateActivator) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if activator != nil {
|
|
executor.selfUpdateActivator = activator
|
|
}
|
|
}
|
|
}
|
|
|
|
// WithDLLExtensionRuntimeTarget is primarily useful for exercising the
|
|
// Windows-only extension path in cross-platform tests. Production workers use
|
|
// the current Go runtime target.
|
|
func WithDLLExtensionRuntimeTarget(targetOS string, targetArch string) LifecycleExecutorOption {
|
|
return func(executor *LifecycleExecutor) {
|
|
if strings.TrimSpace(targetOS) != "" {
|
|
executor.runtimeTargetOS = strings.ToLower(strings.TrimSpace(targetOS))
|
|
}
|
|
if strings.TrimSpace(targetArch) != "" {
|
|
executor.runtimeTargetArch = strings.ToLower(strings.TrimSpace(targetArch))
|
|
}
|
|
}
|
|
}
|
|
|
|
func SupportedLifecycleCapabilities() []string {
|
|
return []string{
|
|
protocol.RunCapabilityProcessInstall,
|
|
protocol.RunCapabilityProcessStart,
|
|
protocol.RunCapabilityProcessStop,
|
|
protocol.RunCapabilityProcessStatus,
|
|
}
|
|
}
|
|
|
|
func SupportedFileCapabilities() []string {
|
|
return []string{protocol.RunCapabilityConfigWrite, protocol.RunCapabilityFilesList, protocol.RunCapabilityFilesRead, protocol.RunCapabilityFilesWrite}
|
|
}
|
|
|
|
func SupportedRunCapabilities() []string {
|
|
return SupportedRunCapabilitiesForComponent("")
|
|
}
|
|
|
|
// SupportedRunCapabilitiesForComponent keeps a generic worker capable of
|
|
// building distributions while ensuring a generated server Run cannot claim
|
|
// a shared build-worker endpoint after it is deployed.
|
|
func SupportedRunCapabilitiesForComponent(componentKind string) []string {
|
|
capabilities := append([]string(nil), SupportedLifecycleCapabilities()...)
|
|
capabilities = append(capabilities, SupportedFileCapabilities()...)
|
|
capabilities = append(capabilities, protocol.RunCapabilityLogsRead)
|
|
capabilities = append(capabilities, protocol.RunCapabilityDeploymentPlan)
|
|
for _, capability := range SupportedDistributionCapabilities() {
|
|
if componentKind == "run" && capability == protocol.RunCapabilityDistributionBuild {
|
|
continue
|
|
}
|
|
capabilities = append(capabilities, capability)
|
|
}
|
|
capabilities = append(capabilities, SupportedRemoteCapabilities()...)
|
|
return capabilities
|
|
}
|
|
|
|
func SupportedRemoteCapabilities() []string {
|
|
return []string{
|
|
protocol.RunCapabilityRemoteFTPRead,
|
|
protocol.RunCapabilityRemoteFTPWrite,
|
|
protocol.RunCapabilityRemoteRsyncRead,
|
|
protocol.RunCapabilityRemoteRsyncWrite,
|
|
protocol.RunCapabilityRemoteRunFilesRead,
|
|
protocol.RunCapabilityRemoteRunFilesWrite,
|
|
protocol.RunCapabilityRemoteRunProcessStart,
|
|
protocol.RunCapabilityRemoteRunProcessStop,
|
|
protocol.RunCapabilityRemoteRunDBMySQLQuery,
|
|
protocol.RunCapabilityRemoteRunDBSQLiteProbe,
|
|
protocol.RunCapabilityRemoteRunDBSQLiteQuery,
|
|
protocol.RunCapabilityRemoteRunLogsTransfer,
|
|
protocol.RunCapabilityRemoteRunRCONCommand,
|
|
protocol.RunCapabilityRemoteRunProtectedSQL,
|
|
protocol.RunCapabilityRemoteRunProtectedRCON,
|
|
protocol.RunCapabilityRemoteRunProgram,
|
|
}
|
|
}
|
|
|
|
func (executor LifecycleExecutor) SupportedCapabilities() []string {
|
|
return SupportedLifecycleCapabilities()
|
|
}
|
|
|
|
func (executor LifecycleExecutor) Execute(assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
return executor.ExecuteContext(context.Background(), assignment)
|
|
}
|
|
|
|
func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
log.Printf("RUN phase=lifecycle status=starting job=%s capability=%s target=%s server=%s workspaceScope=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), assignment.ServerInstanceID, safeOptional(assignment.ExecutionInput.WorkspaceScope))
|
|
if (assignment.Capability == protocol.RunCapabilityConfigWrite || assignment.Capability == protocol.RunCapabilityFilesList || assignment.Capability == protocol.RunCapabilityFilesRead || assignment.Capability == protocol.RunCapabilityFilesWrite) && assignment.ExecutionInput.WorkspaceScope != "" {
|
|
if executor.fileExecutor == nil {
|
|
log.Printf("RUN phase=lifecycle status=file_executor_unavailable job=%s", assignment.JobID)
|
|
return lifecycleFailure("file_executor_unavailable", "file executor is unavailable")
|
|
}
|
|
log.Printf("RUN phase=lifecycle status=file_executor_start job=%s capability=%s", assignment.JobID, assignment.Capability)
|
|
return executor.fileExecutor.Execute(ctx, assignment)
|
|
}
|
|
if !isSupportedLifecycleCapability(assignment.Capability) {
|
|
log.Printf("RUN phase=lifecycle status=unsupported_capability job=%s capability=%s", assignment.JobID, assignment.Capability)
|
|
return lifecycleFailure("unsupported_lifecycle_capability", "unsupported lifecycle capability")
|
|
}
|
|
if len(assignment.ExecutionInput.DLLExtensions) > 0 {
|
|
log.Printf("RUN phase=lifecycle.dll status=validating job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions))
|
|
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
|
log.Printf("RUN phase=lifecycle.dll status=invalid job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("unsafe_dll_extension_plan", "DLL extension plan is invalid")
|
|
}
|
|
}
|
|
if assignment.ExecutionInput.ServerDeploymentPlan != nil {
|
|
log.Printf("RUN phase=lifecycle status=legacy_deployment_rejected job=%s", assignment.JobID)
|
|
return lifecycleFailure("unsupported_legacy_deployment_plan", "game-specific deployment plans must be implemented by plugin lifecycle actions")
|
|
}
|
|
if assignment.ExecutionInput.Deployment != nil && assignment.ExecutionInput.Deployment.Mode == "custom-command" {
|
|
log.Printf("RUN phase=lifecycle status=custom_deployment job=%s", assignment.JobID)
|
|
return executor.executeDeployment(ctx, assignment)
|
|
}
|
|
log.Printf("RUN phase=lifecycle.template status=loading job=%s target=%s", assignment.JobID, safeOptional(assignment.TargetKey))
|
|
template, scope, err := executor.loadLifecycleTemplate(assignment)
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.template status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
|
}
|
|
log.Printf("RUN phase=lifecycle.template status=loaded job=%s action=%s mode=%s scope=%s commandArgs=%d envKeys=%s", assignment.JobID, safeOptional(template.Action), safeOptional(template.Mode), scope, len(template.Command)+len(template.Arguments), envKeysSummary(template.Env, template.Environment))
|
|
if template.Action != "" {
|
|
expectedAction := map[string]string{protocol.RunCapabilityProcessInstall: "install", protocol.RunCapabilityProcessStart: "start", protocol.RunCapabilityProcessStop: "stop", protocol.RunCapabilityProcessStatus: "status"}[assignment.Capability]
|
|
if template.Action != expectedAction {
|
|
log.Printf("RUN phase=lifecycle.template status=action_mismatch job=%s expected=%s actual=%s", assignment.JobID, expectedAction, template.Action)
|
|
return lifecycleFailure("unsafe_lifecycle_command", "typed lifecycle action does not match capability")
|
|
}
|
|
if assignment.Capability == protocol.RunCapabilityProcessStart && template.Mode != "supervised" {
|
|
log.Printf("RUN phase=lifecycle.template status=mode_mismatch job=%s expected=supervised actual=%s", assignment.JobID, template.Mode)
|
|
return lifecycleFailure("unsafe_lifecycle_command", "typed start action must be supervised")
|
|
}
|
|
if (assignment.Capability == protocol.RunCapabilityProcessStop || assignment.Capability == protocol.RunCapabilityProcessStatus) && template.Mode != "control" {
|
|
log.Printf("RUN phase=lifecycle.template status=mode_mismatch job=%s expected=control actual=%s", assignment.JobID, template.Mode)
|
|
return lifecycleFailure("unsafe_lifecycle_command", "typed control action must use control mode")
|
|
}
|
|
}
|
|
if assignment.Capability == protocol.RunCapabilityProcessStart && len(assignment.ExecutionInput.DLLExtensions) > 0 {
|
|
log.Printf("RUN phase=lifecycle.dll status=synchronizing job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions))
|
|
if err := executor.synchronizeUE4SSDLLExtensions(ctx, assignment, template, scope); err != nil {
|
|
log.Printf("RUN phase=lifecycle.dll status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return dllExtensionLifecycleFailure(err)
|
|
}
|
|
log.Printf("RUN phase=lifecycle.dll status=complete job=%s", assignment.JobID)
|
|
}
|
|
if executor.managed != nil && template.Action != "" && (assignment.Capability == protocol.RunCapabilityProcessStart || assignment.Capability == protocol.RunCapabilityProcessStop || assignment.Capability == protocol.RunCapabilityProcessStatus) {
|
|
log.Printf("RUN phase=lifecycle.managed status=dispatch job=%s action=%s mode=%s", assignment.JobID, template.Action, template.Mode)
|
|
return executor.executeManaged(ctx, assignment, template, scope)
|
|
}
|
|
log.Printf("RUN phase=lifecycle.command status=building job=%s", assignment.JobID)
|
|
command, err := template.ToProcessCommand(scope, NewWorkspaceResolver(executor.workspaceRoot), assignment)
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.command status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
|
}
|
|
log.Printf("RUN phase=lifecycle.command status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
|
command.OutputLine = func(stream string, line string) {
|
|
_ = executor.logSink.Append(ctx, assignment, stream, line)
|
|
}
|
|
result, err := executor.supervisor.Run(ctx, command)
|
|
if err != nil && ctx.Err() != nil {
|
|
log.Printf("RUN phase=lifecycle.command status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error()))
|
|
return LifecycleExecutionResult{
|
|
State: lifecycleResultStateCancelled,
|
|
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action cancelled"},
|
|
Message: "lifecycle action cancelled",
|
|
ErrorCode: "lifecycle_cancelled",
|
|
}
|
|
}
|
|
if !result.OutputRelayed {
|
|
executor.writeProcessLogs(ctx, assignment, result)
|
|
}
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error()))
|
|
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
|
}
|
|
if result.ExitCode != 0 {
|
|
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d", assignment.JobID, result.ExitCode)
|
|
return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode))
|
|
}
|
|
log.Printf("RUN phase=lifecycle.command status=exited job=%s exitCode=%d stdoutBytes=%d stderrBytes=%d", assignment.JobID, result.ExitCode, len(result.Stdout), len(result.Stderr))
|
|
artifactRef, err := executor.artifactHook.QueueLifecycleResult(ctx, assignment, result)
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.artifact status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("lifecycle_artifact_hook_failed", err.Error())
|
|
}
|
|
log.Printf("RUN phase=lifecycle status=succeeded job=%s resultRef=%s", assignment.JobID, safeOptional(artifactRef))
|
|
return LifecycleExecutionResult{
|
|
State: lifecycleResultStateSucceeded,
|
|
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action completed"},
|
|
ResultRef: artifactRef,
|
|
Message: fmt.Sprintf("%s completed", assignment.Capability),
|
|
}
|
|
}
|
|
|
|
func (executor LifecycleExecutor) executeDeployment(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
|
|
definition := assignment.ExecutionInput.Deployment
|
|
log.Printf("RUN phase=deployment status=starting job=%s mode=%s revision=%d", assignment.JobID, definition.Mode, definition.Revision)
|
|
if definition.SchemaVersion != "1" || definition.Revision < 1 || definition.Mode == "" {
|
|
log.Printf("RUN phase=deployment status=invalid job=%s", assignment.JobID)
|
|
return lifecycleFailure("invalid_deployment_definition", "deployment definition is invalid")
|
|
}
|
|
action := map[string]string{protocol.RunCapabilityProcessInstall: "install", protocol.RunCapabilityProcessStart: "start", protocol.RunCapabilityProcessStop: "stop", protocol.RunCapabilityProcessStatus: "status"}[assignment.Capability]
|
|
if action == "" {
|
|
log.Printf("RUN phase=deployment status=invalid_action job=%s capability=%s", assignment.JobID, assignment.Capability)
|
|
return lifecycleFailure("invalid_deployment_action", "deployment action is invalid")
|
|
}
|
|
commandText := map[string]string{"start": definition.StartCommand, "stop": definition.StopCommand, "status": definition.StatusCommand}[action]
|
|
if action == "install" {
|
|
commandText = definition.StartCommand
|
|
}
|
|
if commandText == "" {
|
|
log.Printf("RUN phase=deployment status=missing_command job=%s action=%s", assignment.JobID, action)
|
|
return lifecycleFailure("deployment_command_missing", "deployment command is not configured")
|
|
}
|
|
workdir := definition.WorkingDirectory
|
|
if workdir == "" {
|
|
workdir = definition.ServerRoot
|
|
}
|
|
if workdir == "" {
|
|
log.Printf("RUN phase=deployment status=missing_workdir job=%s action=%s", assignment.JobID, action)
|
|
return lifecycleFailure("deployment_workdir_missing", "deployment working directory is not configured")
|
|
}
|
|
args := strings.Fields(commandText)
|
|
if len(args) == 0 {
|
|
log.Printf("RUN phase=deployment status=invalid_command job=%s", assignment.JobID)
|
|
return lifecycleFailure("deployment_command_invalid", "deployment command is invalid")
|
|
}
|
|
if definition.Shell != "" {
|
|
log.Printf("RUN phase=deployment status=unsupported_shell job=%s shell=%s", assignment.JobID, definition.Shell)
|
|
return lifecycleFailure("deployment_shell_unsupported", "deployment shell is not supported by this Run")
|
|
}
|
|
command := ProcessCommand{Args: args, WorkDir: workdir, JobID: assignment.JobID, Capability: assignment.Capability, Action: action}
|
|
log.Printf("RUN phase=deployment.command status=starting job=%s action=%s revision=%d root=%s workdir=%s command=%s", assignment.JobID, action, definition.Revision, safeOptional(definition.ServerRoot), safeOptional(workdir), redactedCommandLine(command.Args))
|
|
result, err := executor.supervisor.Run(ctx, command)
|
|
if err != nil {
|
|
log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error()))
|
|
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
|
}
|
|
executor.writeProcessLogs(ctx, assignment, result)
|
|
if result.ExitCode != 0 {
|
|
log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d", assignment.JobID, result.ExitCode)
|
|
return lifecycleFailure("lifecycle_process_failed", fmt.Sprintf("lifecycle command exited with code %d", result.ExitCode))
|
|
}
|
|
log.Printf("RUN phase=deployment.command status=exited job=%s exitCode=%d stdoutBytes=%d stderrBytes=%d", assignment.JobID, result.ExitCode, len(result.Stdout), len(result.Stderr))
|
|
receipt := &protocol.ServerDeploymentExecutionReceipt{SchemaVersion: "1", Revision: definition.Revision, Action: action, Mode: definition.Mode, Shell: definition.Shell, UsedServerRoot: definition.ServerRoot != ""}
|
|
log.Printf("RUN phase=deployment status=succeeded job=%s action=%s revision=%d", assignment.JobID, action, definition.Revision)
|
|
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "deployment lifecycle action completed"}, Message: fmt.Sprintf("%s completed", assignment.Capability), ExecutionResult: protocol.RunJobExecutionResult{Kind: "deployment.lifecycle", Summary: "deployment revision confirmed", DeploymentReceipt: receipt}}
|
|
}
|
|
|
|
func (executor LifecycleExecutor) ResolveCommand(assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
|
workdir, err := scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
if err := os.MkdirAll(workdir, 0o755); err != nil {
|
|
return ProcessCommand{}, fmt.Errorf("create scoped workspace: %w", err)
|
|
}
|
|
template := LifecycleActionTemplate{
|
|
Command: []string{"true"},
|
|
TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond),
|
|
}
|
|
if assignment.TargetKey != "" {
|
|
path, err := scopedPath(workdir, assignment.TargetKey)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return ProcessCommand{}, fmt.Errorf("open lifecycle action template: %w", err)
|
|
}
|
|
decodeErr := json.NewDecoder(file).Decode(&template)
|
|
closeErr := file.Close()
|
|
if decodeErr != nil {
|
|
return ProcessCommand{}, fmt.Errorf("decode lifecycle action template: %w", decodeErr)
|
|
}
|
|
if closeErr != nil {
|
|
return ProcessCommand{}, fmt.Errorf("close lifecycle action template: %w", closeErr)
|
|
}
|
|
}
|
|
return template.ToProcessCommand(workdir, NewWorkspaceResolver(executor.workspaceRoot), assignment)
|
|
}
|
|
|
|
func (executor LifecycleExecutor) writeProcessLogs(ctx context.Context, assignment protocol.RunJobAssignment, result ProcessResult) {
|
|
for _, item := range []struct {
|
|
stream string
|
|
body string
|
|
}{
|
|
{stream: "stdout", body: result.Stdout},
|
|
{stream: "stderr", body: result.Stderr},
|
|
} {
|
|
for _, line := range splitRawLogLines(item.body) {
|
|
_ = executor.logSink.Append(ctx, assignment, item.stream, line)
|
|
}
|
|
}
|
|
}
|
|
|
|
type LifecycleActionTemplate struct {
|
|
Version int `json:"version,omitempty"`
|
|
Action string `json:"action,omitempty"`
|
|
Mode string `json:"mode,omitempty"`
|
|
ExecutableKey string `json:"executableKey,omitempty"`
|
|
TargetExecutableKey string `json:"targetExecutableKey,omitempty"`
|
|
Arguments []string `json:"arguments,omitempty"`
|
|
Environment map[string]string `json:"environment,omitempty"`
|
|
OutputMode string `json:"outputMode,omitempty"`
|
|
StopTimeoutMS int `json:"stopTimeoutMs,omitempty"`
|
|
Command []string `json:"command"`
|
|
Env map[string]string `json:"env,omitempty"`
|
|
TimeoutMS int `json:"timeoutMs,omitempty"`
|
|
}
|
|
|
|
func (executor LifecycleExecutor) loadLifecycleTemplate(assignment protocol.RunJobAssignment) (LifecycleActionTemplate, string, error) {
|
|
scope, err := executor.lifecycleScope(assignment)
|
|
if err != nil {
|
|
return LifecycleActionTemplate{}, "", err
|
|
}
|
|
template := LifecycleActionTemplate{Command: []string{"true"}, TimeoutMS: int(defaultLifecycleTimeout / time.Millisecond)}
|
|
if assignment.TargetKey == "" {
|
|
if err := os.MkdirAll(scope, 0o755); err != nil {
|
|
return LifecycleActionTemplate{}, "", fmt.Errorf("create lifecycle workspace: %w", err)
|
|
}
|
|
return template, scope, nil
|
|
}
|
|
path, err := NewWorkspaceResolver(executor.workspaceRoot).ExistingTarget(scope, assignment.TargetKey)
|
|
if err != nil {
|
|
return LifecycleActionTemplate{}, "", fmt.Errorf("open lifecycle action template: %w", err)
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return LifecycleActionTemplate{}, "", fmt.Errorf("open lifecycle action template: %w", err)
|
|
}
|
|
defer file.Close()
|
|
decoder := json.NewDecoder(io.LimitReader(file, 16*1024))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&template); err != nil {
|
|
return LifecycleActionTemplate{}, "", fmt.Errorf("decode lifecycle action template: %w", err)
|
|
}
|
|
return template, scope, nil
|
|
}
|
|
|
|
func (executor LifecycleExecutor) lifecycleScope(assignment protocol.RunJobAssignment) (string, error) {
|
|
if assignment.ExecutionInput.WorkspaceScope != "" {
|
|
return NewWorkspaceResolver(executor.workspaceRoot).Scope(assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope)
|
|
}
|
|
return scopedServerWorkspace(executor.workspaceRoot, assignment.ServerInstanceID)
|
|
}
|
|
|
|
func (executor LifecycleExecutor) executeManaged(ctx context.Context, assignment protocol.RunJobAssignment, template LifecycleActionTemplate, scope string) LifecycleExecutionResult {
|
|
resolver := NewWorkspaceResolver(executor.workspaceRoot)
|
|
identity := ProcessIdentity{Scope: scope, ServerInstanceID: assignment.ServerInstanceID, RunEndpointID: assignment.RunEndpointID, JobID: assignment.JobID, Capability: assignment.Capability, ProfileKey: assignment.ExecutionInput.WorkspaceScope, Attempt: assignment.Attempt, StdoutStreamKey: processLogStreamKey(assignment, "process.stdout", "stdout"), StderrStreamKey: processLogStreamKey(assignment, "process.stderr", "stderr")}
|
|
if assignment.Capability == protocol.RunCapabilityProcessStart {
|
|
log.Printf("RUN phase=lifecycle.managed status=building_start_command job=%s scope=%s", assignment.JobID, scope)
|
|
command, err := template.ToManagedProcessCommand(resolver, scope, assignment)
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.managed status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
|
}
|
|
log.Printf("RUN phase=lifecycle.managed status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
|
item, err := executor.managed.Start(ctx, command, identity, executor.managedProcessOutput(ctx, assignment))
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
log.Printf("RUN phase=lifecycle.managed status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error()))
|
|
return lifecycleExecutionFailure("lifecycle_cancelled", "lifecycle action cancelled", false)
|
|
}
|
|
log.Printf("RUN phase=lifecycle.managed status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
|
}
|
|
log.Printf("RUN phase=lifecycle.managed status=started job=%s pid=%d state=%s stdoutRef=%s stderrRef=%s", assignment.JobID, item.PID, item.State, safeOptional(item.StdoutLogRef), safeOptional(item.StderrLogRef))
|
|
return processExecutionResult(item, "process started")
|
|
}
|
|
if assignment.Capability == protocol.RunCapabilityProcessStop {
|
|
log.Printf("RUN phase=lifecycle.managed status=stopping job=%s scope=%s", assignment.JobID, scope)
|
|
item, err := executor.managed.Stop(ctx, identity)
|
|
if err != nil {
|
|
log.Printf("RUN phase=lifecycle.managed status=stop_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
|
return lifecycleFailure("lifecycle_stop_failed", RedactText(err.Error()))
|
|
}
|
|
log.Printf("RUN phase=lifecycle.managed status=stopped job=%s pid=%d state=%s classification=%s", assignment.JobID, item.PID, item.State, safeOptional(item.ExitClassification))
|
|
return processExecutionResult(item, "process stopped")
|
|
}
|
|
log.Printf("RUN phase=lifecycle.managed status=querying job=%s scope=%s", assignment.JobID, scope)
|
|
item := executor.managed.Status(identity)
|
|
log.Printf("RUN phase=lifecycle.managed status=queried job=%s pid=%d state=%s classification=%s", assignment.JobID, item.PID, item.State, safeOptional(item.ExitClassification))
|
|
return processExecutionResult(item, "process status queried")
|
|
}
|
|
|
|
func (executor LifecycleExecutor) ResumeManagedProcessLogs(ctx context.Context) {
|
|
if executor.managed == nil {
|
|
return
|
|
}
|
|
executor.managed.ResumeOutput(executor.managedProcessOutput(ctx, protocol.RunJobAssignment{}))
|
|
}
|
|
|
|
func (executor LifecycleExecutor) managedProcessOutput(ctx context.Context, assignment protocol.RunJobAssignment) ManagedProcessOutput {
|
|
return ManagedProcessOutput{
|
|
Stdout: func(identity ProcessIdentity, line ManagedProcessLine) error {
|
|
return executor.appendManagedProcessLog(ctx, assignment, identity, "stdout", line)
|
|
},
|
|
Stderr: func(identity ProcessIdentity, line ManagedProcessLine) error {
|
|
return executor.appendManagedProcessLog(ctx, assignment, identity, "stderr", line)
|
|
},
|
|
}
|
|
}
|
|
|
|
func (executor LifecycleExecutor) appendManagedProcessLog(ctx context.Context, assignment protocol.RunJobAssignment, identity ProcessIdentity, stream string, line ManagedProcessLine) error {
|
|
if executor.logSink == nil {
|
|
return nil
|
|
}
|
|
if assignment.JobID == "" {
|
|
assignment = assignmentFromProcessIdentity(identity)
|
|
}
|
|
assignment.LogSessionID = identity.LogSessionID
|
|
assignment.SessionStartedAt = identity.StartedAt
|
|
if assignment.JobID == "" || assignment.ServerInstanceID == "" {
|
|
return nil
|
|
}
|
|
if sink, ok := executor.logSink.(ProcessLogCursorSink); ok {
|
|
return sink.AppendWithCursor(ctx, assignment, stream, line.Text, ProcessLogCursor{StartOffset: line.StartOffset, EndOffset: line.EndOffset})
|
|
}
|
|
return executor.logSink.Append(ctx, assignment, stream, line.Text)
|
|
}
|
|
|
|
func assignmentFromProcessIdentity(identity ProcessIdentity) protocol.RunJobAssignment {
|
|
assignment := protocol.RunJobAssignment{
|
|
JobID: identity.JobID,
|
|
ServerInstanceID: identity.ServerInstanceID,
|
|
RunEndpointID: identity.RunEndpointID,
|
|
Capability: identity.Capability,
|
|
Attempt: identity.Attempt,
|
|
LogSessionID: identity.LogSessionID,
|
|
SessionStartedAt: identity.StartedAt,
|
|
}
|
|
if identity.StdoutStreamKey != "" {
|
|
assignment.ExecutionInput.LogSources = append(assignment.ExecutionInput.LogSources, protocol.RuntimeLogSourcePlan{Key: "process-stdout", Kind: "process.stdout", StreamKey: identity.StdoutStreamKey, CursorKind: "sequence"})
|
|
}
|
|
if identity.StderrStreamKey != "" {
|
|
assignment.ExecutionInput.LogSources = append(assignment.ExecutionInput.LogSources, protocol.RuntimeLogSourcePlan{Key: "process-stderr", Kind: "process.stderr", StreamKey: identity.StderrStreamKey, CursorKind: "sequence"})
|
|
}
|
|
return assignment
|
|
}
|
|
|
|
func processLogStreamKey(assignment protocol.RunJobAssignment, kind string, fallback string) string {
|
|
for _, source := range assignment.ExecutionInput.LogSources {
|
|
if source.Kind == kind && strings.TrimSpace(source.StreamKey) != "" {
|
|
return source.StreamKey
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func processExecutionResult(item ProcessIdentity, message string) LifecycleExecutionResult {
|
|
state := item.State
|
|
if state == "" {
|
|
state = "stopped"
|
|
}
|
|
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, Message: message, ExecutionResult: protocol.RunJobExecutionResult{Kind: "process", ProcessState: state, ExitClassification: item.ExitClassification, ExitCode: item.ExitCode, Summary: "private supervised process identity"}}
|
|
}
|
|
|
|
func lifecycleExecutionFailure(code string, message string, retryable bool) LifecycleExecutionResult {
|
|
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)}, Message: RedactText(message), ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "file", Summary: code}}
|
|
}
|
|
|
|
func (template LifecycleActionTemplate) ToProcessCommand(workdir string, resolver WorkspaceResolver, assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
|
if template.ExecutableKey != "" {
|
|
return template.ToExecutableProcessCommand(resolver, workdir, assignment)
|
|
}
|
|
args := append([]string(nil), template.Command...)
|
|
if len(args) == 0 {
|
|
return ProcessCommand{}, fmt.Errorf("command is required")
|
|
}
|
|
for i, part := range args {
|
|
if strings.TrimSpace(part) == "" {
|
|
return ProcessCommand{}, fmt.Errorf("command part is required")
|
|
}
|
|
if containsUnsafeRuntimeText(part) {
|
|
return ProcessCommand{}, fmt.Errorf("command contains unsafe content")
|
|
}
|
|
if i == 0 {
|
|
if !commandNamePattern.MatchString(part) || strings.Contains(part, "/") || filepath.IsAbs(part) {
|
|
return ProcessCommand{}, fmt.Errorf("command executable must be an allowlisted name")
|
|
}
|
|
if _, disallowed := disallowedExecutables[strings.ToLower(part)]; disallowed {
|
|
return ProcessCommand{}, fmt.Errorf("command executable must not be a shell")
|
|
}
|
|
continue
|
|
}
|
|
if strings.ContainsAny(part, "|;&`$<>") {
|
|
return ProcessCommand{}, fmt.Errorf("command arguments must not contain shell metacharacters")
|
|
}
|
|
}
|
|
env, err := template.executionEnvironment(assignment)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
timeout := defaultLifecycleTimeout
|
|
if template.TimeoutMS > 0 {
|
|
timeout = time.Duration(template.TimeoutMS) * time.Millisecond
|
|
}
|
|
if timeout > maxLifecycleTimeout {
|
|
return ProcessCommand{}, fmt.Errorf("timeout is too large")
|
|
}
|
|
return ProcessCommand{WorkDir: workdir, Args: args, Env: env, Timeout: timeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: template.Action}, nil
|
|
}
|
|
|
|
func (template LifecycleActionTemplate) ToExecutableProcessCommand(resolver WorkspaceResolver, scope string, assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
|
if template.ExecutableKey == "" {
|
|
return ProcessCommand{}, fmt.Errorf("typed executableKey is required")
|
|
}
|
|
executable, err := resolver.ExistingTarget(scope, template.ExecutableKey)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
info, err := os.Stat(executable)
|
|
if err != nil || !info.Mode().IsRegular() || runtime.GOOS != "windows" && info.Mode().Perm()&0o111 == 0 {
|
|
return ProcessCommand{}, fmt.Errorf("typed executable is not executable")
|
|
}
|
|
args := append([]string{executable}, template.Arguments...)
|
|
for _, part := range args[1:] {
|
|
if strings.TrimSpace(part) == "" || containsUnsafeRuntimeText(part) || strings.ContainsAny(part, "|;&`$<>") {
|
|
return ProcessCommand{}, fmt.Errorf("typed argument is unsafe")
|
|
}
|
|
}
|
|
if template.OutputMode != "" && template.OutputMode != "pipes" && template.OutputMode != "console" {
|
|
return ProcessCommand{}, fmt.Errorf("typed output mode is unsupported")
|
|
}
|
|
env, err := template.executionEnvironment(assignment)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
timeout := defaultLifecycleTimeout
|
|
if template.TimeoutMS > 0 {
|
|
timeout = time.Duration(template.TimeoutMS) * time.Millisecond
|
|
}
|
|
if timeout > maxLifecycleTimeout {
|
|
return ProcessCommand{}, fmt.Errorf("timeout is too large")
|
|
}
|
|
args = managedExecutableArgs(runtime.GOOS, args)
|
|
return ProcessCommand{WorkDir: scope, Args: args, Env: env, OutputMode: template.OutputMode, Timeout: timeout, JobID: assignment.JobID, Capability: assignment.Capability, Action: template.Action}, nil
|
|
}
|
|
|
|
func managedExecutableArgs(targetOS string, args []string) []string {
|
|
if targetOS != "windows" || len(args) == 0 {
|
|
return args
|
|
}
|
|
ext := strings.ToLower(filepath.Ext(args[0]))
|
|
if ext != ".cmd" && ext != ".bat" {
|
|
return args
|
|
}
|
|
// Keep the batch path as a normal /c argument. Passing a pre-quoted
|
|
// command string here makes ComposeCommandLine escape the inner quotes
|
|
// for CreateProcess; cmd.exe does not treat those backslashes as quote
|
|
// escapes and consequently fails before it can run the script.
|
|
return append([]string{"cmd.exe", "/d", "/c", "call"}, args...)
|
|
}
|
|
|
|
func quoteWindowsCommandArg(value string) string {
|
|
if value != "" && !strings.ContainsAny(value, " \t\"") {
|
|
return value
|
|
}
|
|
var builder strings.Builder
|
|
builder.WriteByte('"')
|
|
backslashes := 0
|
|
for _, char := range value {
|
|
if char == '\\' {
|
|
backslashes++
|
|
continue
|
|
}
|
|
if char == '"' {
|
|
builder.WriteString(strings.Repeat("\\", backslashes*2+1))
|
|
builder.WriteRune(char)
|
|
backslashes = 0
|
|
continue
|
|
}
|
|
if backslashes > 0 {
|
|
builder.WriteString(strings.Repeat("\\", backslashes))
|
|
backslashes = 0
|
|
}
|
|
builder.WriteRune(char)
|
|
}
|
|
if backslashes > 0 {
|
|
builder.WriteString(strings.Repeat("\\", backslashes*2))
|
|
}
|
|
builder.WriteByte('"')
|
|
return builder.String()
|
|
}
|
|
|
|
func (template LifecycleActionTemplate) ToManagedProcessCommand(resolver WorkspaceResolver, scope string, assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
|
command, err := template.ToExecutableProcessCommand(resolver, scope, assignment)
|
|
if err != nil {
|
|
return ProcessCommand{}, err
|
|
}
|
|
return command, nil
|
|
}
|
|
|
|
func (template LifecycleActionTemplate) executionEnvironment(assignment protocol.RunJobAssignment) (map[string]string, error) {
|
|
env := template.Environment
|
|
if env == nil {
|
|
env = template.Env
|
|
}
|
|
validated := make(map[string]string, len(env)+16)
|
|
for key, value := range env {
|
|
if !envNamePattern.MatchString(key) || (!strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_")) || containsUnsafeRuntimeText(value) {
|
|
return nil, fmt.Errorf("typed environment is unsafe")
|
|
}
|
|
validated[key] = value
|
|
}
|
|
add := func(key string, value string) error {
|
|
if strings.TrimSpace(value) == "" {
|
|
return nil
|
|
}
|
|
if !envNamePattern.MatchString(key) || (!strings.HasPrefix(key, "GAME_") && !strings.HasPrefix(key, "SERVER_") && !strings.HasPrefix(key, "RUN_")) || containsUnsafeRuntimeText(value) {
|
|
return fmt.Errorf("typed environment is unsafe")
|
|
}
|
|
validated[key] = value
|
|
return nil
|
|
}
|
|
if err := add("SERVER_PLUGIN_ID", assignment.ExecutionInput.PluginID); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := add("SERVER_LIFECYCLE_OPERATION", assignment.ExecutionInput.LifecycleOperation); err != nil {
|
|
return nil, err
|
|
}
|
|
if deployment := assignment.ExecutionInput.Deployment; deployment != nil {
|
|
if err := add("SERVER_DEPLOYMENT_MODE", deployment.Mode); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := add("SERVER_PROFILE_KEY", deployment.ProfileKey); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := add("SERVER_ROOT", deployment.ServerRoot); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := add("SERVER_WORKING_DIRECTORY", deployment.WorkingDirectory); err != nil {
|
|
return nil, err
|
|
}
|
|
if deployment.Revision > 0 {
|
|
if err := add("SERVER_REVISION", fmt.Sprint(deployment.Revision)); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for key, value := range deployment.CreateInputs {
|
|
suffix := envKeySuffix(key)
|
|
if suffix == "" {
|
|
return nil, fmt.Errorf("typed environment is unsafe")
|
|
}
|
|
if err := add("SERVER_CREATE_"+suffix, value); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
for key, value := range assignment.ExecutionInput.Inputs {
|
|
suffix := envKeySuffix(key)
|
|
if suffix == "" {
|
|
return nil, fmt.Errorf("typed environment is unsafe")
|
|
}
|
|
if err := add("SERVER_INPUT_"+suffix, value); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return validated, nil
|
|
}
|
|
|
|
func envKeySuffix(key string) string {
|
|
var builder strings.Builder
|
|
for _, char := range key {
|
|
switch {
|
|
case char >= 'a' && char <= 'z':
|
|
builder.WriteRune(char - 'a' + 'A')
|
|
case char >= 'A' && char <= 'Z':
|
|
builder.WriteRune(char)
|
|
case char >= '0' && char <= '9':
|
|
builder.WriteRune(char)
|
|
case char == '_' || char == '-' || char == '.' || char == '/':
|
|
builder.WriteByte('_')
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
return strings.Trim(builder.String(), "_")
|
|
}
|
|
|
|
type ProcessCommand struct {
|
|
WorkDir string
|
|
Args []string
|
|
Env map[string]string
|
|
OutputMode string
|
|
Timeout time.Duration
|
|
JobID string
|
|
Capability string
|
|
Action string
|
|
OutputLine func(string, string) `json:"-"`
|
|
}
|
|
|
|
type ProcessResult struct {
|
|
ExitCode int
|
|
Stdout string
|
|
Stderr string
|
|
OutputRelayed bool
|
|
}
|
|
|
|
type ProcessSupervisor interface {
|
|
Run(context.Context, ProcessCommand) (ProcessResult, error)
|
|
}
|
|
|
|
type OSProcessSupervisor struct{}
|
|
|
|
func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCommand) (ProcessResult, error) {
|
|
if len(command.Args) == 0 {
|
|
log.Printf("RUN phase=process.command status=missing_executable job=%s capability=%s action=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action))
|
|
return ProcessResult{ExitCode: -1}, fmt.Errorf("command is required")
|
|
}
|
|
startedAt := time.Now()
|
|
log.Printf("RUN phase=process.command status=starting job=%s capability=%s action=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
|
if command.Timeout > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, command.Timeout)
|
|
defer cancel()
|
|
}
|
|
cmd := exec.CommandContext(ctx, command.Args[0], command.Args[1:]...)
|
|
cmd.Dir = command.WorkDir
|
|
cmd.Env = os.Environ()
|
|
for key, value := range command.Env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
stdoutWriter := newLifecycleOutputWriter(&stdout, maxLifecycleOutputBytes, command, "stdout")
|
|
stderrWriter := newLifecycleOutputWriter(&stderr, maxLifecycleOutputBytes, command, "stderr")
|
|
cmd.Stdout = stdoutWriter
|
|
cmd.Stderr = stderrWriter
|
|
if err := cmd.Start(); err != nil {
|
|
log.Printf("RUN phase=process.command status=start_failed job=%s capability=%s action=%s command=%s durationMs=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), redactedCommandLine(command.Args), time.Since(startedAt).Milliseconds(), RedactText(err.Error()))
|
|
return ProcessResult{ExitCode: -1}, err
|
|
}
|
|
pid := 0
|
|
if cmd.Process != nil {
|
|
pid = cmd.Process.Pid
|
|
}
|
|
log.Printf("RUN phase=process.command status=started job=%s capability=%s action=%s pid=%d command=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args))
|
|
err := cmd.Wait()
|
|
stdoutWriter.Flush()
|
|
stderrWriter.Flush()
|
|
result := ProcessResult{Stdout: stdout.String(), Stderr: stderr.String(), OutputRelayed: command.OutputLine != nil}
|
|
if cmd.ProcessState != nil {
|
|
result.ExitCode = cmd.ProcessState.ExitCode()
|
|
}
|
|
if err != nil {
|
|
log.Printf("RUN phase=process.command status=failed job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr), RedactText(err.Error()))
|
|
return result, err
|
|
}
|
|
log.Printf("RUN phase=process.command status=exited job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr))
|
|
return result, nil
|
|
}
|
|
|
|
type lifecycleOutputWriter struct {
|
|
mu sync.Mutex
|
|
buffer *bytes.Buffer
|
|
limit int
|
|
command ProcessCommand
|
|
stream string
|
|
pending string
|
|
}
|
|
|
|
func newLifecycleOutputWriter(buffer *bytes.Buffer, limit int, command ProcessCommand, stream string) *lifecycleOutputWriter {
|
|
return &lifecycleOutputWriter{buffer: buffer, limit: limit, command: command, stream: stream}
|
|
}
|
|
|
|
func (writer *lifecycleOutputWriter) Write(p []byte) (int, error) {
|
|
writer.mu.Lock()
|
|
defer writer.mu.Unlock()
|
|
remaining := writer.limit - writer.buffer.Len()
|
|
if remaining > 0 {
|
|
if len(p) > remaining {
|
|
_, _ = writer.buffer.Write(p[:remaining])
|
|
} else {
|
|
_, _ = writer.buffer.Write(p)
|
|
}
|
|
}
|
|
writer.pending += string(p)
|
|
for {
|
|
index := strings.IndexByte(writer.pending, '\n')
|
|
if index < 0 {
|
|
break
|
|
}
|
|
line := writer.pending[:index]
|
|
writer.pending = writer.pending[index+1:]
|
|
writer.logLine(line)
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
func (writer *lifecycleOutputWriter) Flush() {
|
|
writer.mu.Lock()
|
|
defer writer.mu.Unlock()
|
|
if writer.pending == "" {
|
|
return
|
|
}
|
|
writer.logLine(writer.pending)
|
|
writer.pending = ""
|
|
}
|
|
|
|
func (writer *lifecycleOutputWriter) logLine(line string) {
|
|
if writer.command.OutputLine != nil {
|
|
writer.command.OutputLine(writer.stream, line)
|
|
}
|
|
}
|
|
|
|
type ioLimitWriter struct {
|
|
Writer *bytes.Buffer
|
|
Limit int
|
|
}
|
|
|
|
func (writer ioLimitWriter) Write(p []byte) (int, error) {
|
|
remaining := writer.Limit - writer.Writer.Len()
|
|
if remaining > 0 {
|
|
if len(p) > remaining {
|
|
_, _ = writer.Writer.Write(p[:remaining])
|
|
} else {
|
|
_, _ = writer.Writer.Write(p)
|
|
}
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
type ProcessLogSink interface {
|
|
Append(context.Context, protocol.RunJobAssignment, string, string) error
|
|
}
|
|
|
|
type ProcessLogCursor struct {
|
|
StartOffset int64
|
|
EndOffset int64
|
|
}
|
|
|
|
type ProcessLogCursorSink interface {
|
|
AppendWithCursor(context.Context, protocol.RunJobAssignment, string, string, ProcessLogCursor) error
|
|
}
|
|
|
|
type NoopProcessLogSink struct{}
|
|
|
|
func (NoopProcessLogSink) Append(context.Context, protocol.RunJobAssignment, string, string) error {
|
|
return nil
|
|
}
|
|
|
|
type LifecycleArtifactHook interface {
|
|
QueueLifecycleResult(context.Context, protocol.RunJobAssignment, ProcessResult) (string, error)
|
|
}
|
|
|
|
type StaticLifecycleArtifactHook struct{}
|
|
|
|
func (StaticLifecycleArtifactHook) QueueLifecycleResult(_ context.Context, assignment protocol.RunJobAssignment, _ ProcessResult) (string, error) {
|
|
return fmt.Sprintf("artifact://jobs/%s/lifecycle-result", url.PathEscape(assignment.JobID)), nil
|
|
}
|
|
|
|
func LifecycleResultRequest(assignment protocol.RunJobAssignment, sessionToken string, result LifecycleExecutionResult) protocol.RunJobResultRequest {
|
|
return protocol.RunJobResultRequest{
|
|
RunEndpointID: assignment.RunEndpointID,
|
|
SessionToken: sessionToken,
|
|
JobID: assignment.JobID,
|
|
LeaseToken: assignment.LeaseToken,
|
|
Attempt: assignment.Attempt,
|
|
State: result.State,
|
|
Progress: result.Progress,
|
|
ResultRef: result.ResultRef,
|
|
Message: result.Message,
|
|
ErrorCode: result.ErrorCode,
|
|
Retryable: result.Retryable,
|
|
ExecutionResult: protocol.RunJobExecutionResult{Kind: result.ExecutionResult.Kind, ProcessState: result.ExecutionResult.ProcessState, ExitClassification: result.ExecutionResult.ExitClassification, ExitCode: result.ExecutionResult.ExitCode, Version: result.ExecutionResult.Version, Checksum: result.ExecutionResult.Checksum, SizeBytes: result.ExecutionResult.SizeBytes, Summary: result.ExecutionResult.Summary, Content: result.ExecutionResult.Content, SQLiteSchemaProbe: result.ExecutionResult.SQLiteSchemaProbe, DeploymentReceipt: result.ExecutionResult.DeploymentReceipt, ServerDeploymentEvidence: result.ExecutionResult.ServerDeploymentEvidence},
|
|
}
|
|
}
|
|
|
|
func isSupportedLifecycleCapability(capability string) bool {
|
|
for _, supported := range SupportedLifecycleCapabilities() {
|
|
if capability == supported {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isSupportedRemoteCapability(capability string) bool {
|
|
for _, supported := range SupportedRemoteCapabilities() {
|
|
if capability == supported {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func lifecycleFailure(code string, message string) LifecycleExecutionResult {
|
|
return LifecycleExecutionResult{
|
|
State: lifecycleResultStateFailed,
|
|
Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)},
|
|
Message: RedactText(message),
|
|
ErrorCode: code,
|
|
}
|
|
}
|
|
|
|
func scopedServerWorkspace(root string, serverInstanceID string) (string, error) {
|
|
if strings.TrimSpace(serverInstanceID) == "" {
|
|
return "", fmt.Errorf("server instance id is required")
|
|
}
|
|
if containsUnsafeRuntimeText(serverInstanceID) || strings.ContainsAny(serverInstanceID, `/\`) || serverInstanceID == "." || serverInstanceID == ".." {
|
|
return "", fmt.Errorf("server instance id is unsafe")
|
|
}
|
|
return scopedPath(root, serverInstanceID)
|
|
}
|
|
|
|
func scopedPath(root string, key string) (string, error) {
|
|
if strings.TrimSpace(root) == "" {
|
|
return "", fmt.Errorf("workspace root is required")
|
|
}
|
|
if strings.TrimSpace(key) == "" {
|
|
return "", fmt.Errorf("logical key is required")
|
|
}
|
|
if filepath.IsAbs(key) || strings.Contains(key, "..") || strings.Contains(key, `\`) || containsUnsafeRuntimeText(key) {
|
|
return "", fmt.Errorf("logical key is unsafe")
|
|
}
|
|
cleanRoot, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
candidate := filepath.Clean(filepath.Join(cleanRoot, filepath.FromSlash(key)))
|
|
rel, err := filepath.Rel(cleanRoot, candidate)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
|
return "", fmt.Errorf("logical key escapes workspace")
|
|
}
|
|
return candidate, nil
|
|
}
|
|
|
|
func containsUnsafeRuntimeText(value string) bool {
|
|
normalized := strings.ToLower(value)
|
|
for _, marker := range []string{"/users/", "/.ssh/", "password=", "apikey", "api_key", "secret=", "bearer ", "sk-", "unix://", "tcp://", "://"} {
|
|
if strings.Contains(normalized, marker) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func safeOptional(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "-"
|
|
}
|
|
return RedactText(value)
|
|
}
|
|
|
|
func errorSummary(err error) string {
|
|
if err == nil {
|
|
return "-"
|
|
}
|
|
return RedactText(err.Error())
|
|
}
|
|
|
|
func redactedCommandLine(args []string) string {
|
|
if len(args) == 0 {
|
|
return "-"
|
|
}
|
|
parts := make([]string, len(args))
|
|
for i, arg := range args {
|
|
parts[i] = strconv.Quote(RedactText(arg))
|
|
}
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
func envKeysSummary(first map[string]string, second map[string]string) string {
|
|
seen := map[string]struct{}{}
|
|
for key := range first {
|
|
seen[key] = struct{}{}
|
|
}
|
|
for key := range second {
|
|
seen[key] = struct{}{}
|
|
}
|
|
if len(seen) == 0 {
|
|
return "-"
|
|
}
|
|
keys := make([]string, 0, len(seen))
|
|
for key := range seen {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
return strings.Join(keys, ",")
|
|
}
|
|
|
|
func RedactText(value string) string {
|
|
redacted := value
|
|
replacements := []string{"/Users/", "[host]/", "Bearer ", "Bearer [redacted] ", "sk-", "sk-[redacted]", "password=", "password=[redacted]", "api_key=", "api_key=[redacted]", "secret=", "secret=[redacted]", "unix://", "socket://"}
|
|
for i := 0; i+1 < len(replacements); i += 2 {
|
|
redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1])
|
|
}
|
|
if len(redacted) > maxLifecycleOutputBytes {
|
|
return redacted[:maxLifecycleOutputBytes]
|
|
}
|
|
return redacted
|
|
}
|
|
|
|
// splitRawLogLines only removes the newline framing used by LogEntry. It
|
|
// deliberately preserves every other byte, including blank lines and spaces.
|
|
func splitRawLogLines(value string) []string {
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
lines := make([]string, 0, strings.Count(value, "\n")+1)
|
|
for start := 0; start < len(value); {
|
|
end := strings.IndexByte(value[start:], '\n')
|
|
if end < 0 {
|
|
lines = append(lines, value[start:])
|
|
break
|
|
}
|
|
end += start
|
|
lines = append(lines, value[start:end])
|
|
start = end + 1
|
|
}
|
|
return lines
|
|
}
|
|
|
|
func checksumForText(value string) string {
|
|
sum := sha256.Sum256([]byte(value))
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|