Keep run logs opaque and streamline transfers
This commit is contained in:
npc0-hue
+31
-43
@@ -338,7 +338,7 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
if len(assignment.ExecutionInput.DLLExtensions) > 0 {
|
||||
log.Printf("RUN phase=lifecycle.dll status=validating job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions))
|
||||
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
|
||||
log.Printf("RUN phase=lifecycle.dll status=invalid job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.dll status=invalid job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("unsafe_dll_extension_plan", "DLL extension plan is invalid")
|
||||
}
|
||||
}
|
||||
@@ -353,7 +353,7 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
log.Printf("RUN phase=lifecycle.template status=loading job=%s target=%s", assignment.JobID, safeOptional(assignment.TargetKey))
|
||||
template, scope, err := executor.loadLifecycleTemplate(assignment)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.template status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.template status=failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.template status=loaded job=%s action=%s mode=%s scope=%s commandArgs=%d envKeys=%s", assignment.JobID, safeOptional(template.Action), safeOptional(template.Mode), scope, len(template.Command)+len(template.Arguments), envKeysSummary(template.Env, template.Environment))
|
||||
@@ -375,7 +375,7 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
if assignment.Capability == protocol.RunCapabilityProcessStart && len(assignment.ExecutionInput.DLLExtensions) > 0 {
|
||||
log.Printf("RUN phase=lifecycle.dll status=synchronizing job=%s extensions=%d", assignment.JobID, len(assignment.ExecutionInput.DLLExtensions))
|
||||
if err := executor.synchronizeUE4SSDLLExtensions(ctx, assignment, template, scope); err != nil {
|
||||
log.Printf("RUN phase=lifecycle.dll status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.dll status=failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return dllExtensionLifecycleFailure(err)
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.dll status=complete job=%s", assignment.JobID)
|
||||
@@ -387,16 +387,16 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
log.Printf("RUN phase=lifecycle.command status=building job=%s", assignment.JobID)
|
||||
command, err := template.ToProcessCommand(scope, NewWorkspaceResolver(executor.workspaceRoot), assignment)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.command status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.command status=build_failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.command status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
log.Printf("RUN phase=lifecycle.command status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), quotedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
command.OutputLine = func(stream string, line string) {
|
||||
_ = executor.logSink.Append(ctx, assignment, stream, line)
|
||||
}
|
||||
result, err := executor.supervisor.Run(ctx, command)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
log.Printf("RUN phase=lifecycle.command status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error()))
|
||||
log.Printf("RUN phase=lifecycle.command status=cancelled job=%s error=%s", assignment.JobID, ctx.Err().Error())
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateCancelled,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: "lifecycle action cancelled"},
|
||||
@@ -408,8 +408,8 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
executor.writeProcessLogs(ctx, assignment, result)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error()))
|
||||
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, err.Error())
|
||||
return lifecycleFailure("lifecycle_process_failed", err.Error())
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
log.Printf("RUN phase=lifecycle.command status=failed job=%s exitCode=%d", assignment.JobID, result.ExitCode)
|
||||
@@ -418,7 +418,7 @@ func (executor LifecycleExecutor) ExecuteContext(ctx context.Context, assignment
|
||||
log.Printf("RUN phase=lifecycle.command status=exited job=%s exitCode=%d stdoutBytes=%d stderrBytes=%d", assignment.JobID, result.ExitCode, len(result.Stdout), len(result.Stderr))
|
||||
artifactRef, err := executor.artifactHook.QueueLifecycleResult(ctx, assignment, result)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.artifact status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.artifact status=failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("lifecycle_artifact_hook_failed", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle status=succeeded job=%s resultRef=%s", assignment.JobID, safeOptional(artifactRef))
|
||||
@@ -468,11 +468,11 @@ func (executor LifecycleExecutor) executeDeployment(ctx context.Context, assignm
|
||||
return lifecycleFailure("deployment_shell_unsupported", "deployment shell is not supported by this Run")
|
||||
}
|
||||
command := ProcessCommand{Args: args, WorkDir: workdir, JobID: assignment.JobID, Capability: assignment.Capability, Action: action}
|
||||
log.Printf("RUN phase=deployment.command status=starting job=%s action=%s revision=%d root=%s workdir=%s command=%s", assignment.JobID, action, definition.Revision, safeOptional(definition.ServerRoot), safeOptional(workdir), redactedCommandLine(command.Args))
|
||||
log.Printf("RUN phase=deployment.command status=starting job=%s action=%s revision=%d root=%s workdir=%s command=%s", assignment.JobID, action, definition.Revision, safeOptional(definition.ServerRoot), safeOptional(workdir), quotedCommandLine(command.Args))
|
||||
result, err := executor.supervisor.Run(ctx, command)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, RedactText(err.Error()))
|
||||
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
||||
log.Printf("RUN phase=deployment.command status=failed job=%s exitCode=%d error=%s", assignment.JobID, result.ExitCode, err.Error())
|
||||
return lifecycleFailure("lifecycle_process_failed", err.Error())
|
||||
}
|
||||
executor.writeProcessLogs(ctx, assignment, result)
|
||||
if result.ExitCode != 0 {
|
||||
@@ -590,18 +590,18 @@ func (executor LifecycleExecutor) executeManaged(ctx context.Context, assignment
|
||||
log.Printf("RUN phase=lifecycle.managed status=building_start_command job=%s scope=%s", assignment.JobID, scope)
|
||||
command, err := template.ToManagedProcessCommand(resolver, scope, assignment)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.managed status=build_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.managed status=build_failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("unsafe_lifecycle_command", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.managed status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
log.Printf("RUN phase=lifecycle.managed status=starting job=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", assignment.JobID, safeOptional(command.WorkDir), quotedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
item, err := executor.managed.Start(ctx, command, identity, executor.managedProcessOutput(ctx, assignment))
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
log.Printf("RUN phase=lifecycle.managed status=cancelled job=%s error=%s", assignment.JobID, RedactText(ctx.Err().Error()))
|
||||
log.Printf("RUN phase=lifecycle.managed status=cancelled job=%s error=%s", assignment.JobID, ctx.Err().Error())
|
||||
return lifecycleExecutionFailure("lifecycle_cancelled", "lifecycle action cancelled", false)
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.managed status=failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
return lifecycleFailure("lifecycle_process_failed", RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.managed status=failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("lifecycle_process_failed", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.managed status=started job=%s pid=%d state=%s stdoutRef=%s stderrRef=%s", assignment.JobID, item.PID, item.State, safeOptional(item.StdoutLogRef), safeOptional(item.StderrLogRef))
|
||||
return processExecutionResult(item, "process started")
|
||||
@@ -610,8 +610,8 @@ func (executor LifecycleExecutor) executeManaged(ctx context.Context, assignment
|
||||
log.Printf("RUN phase=lifecycle.managed status=stopping job=%s scope=%s", assignment.JobID, scope)
|
||||
item, err := executor.managed.Stop(ctx, identity)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=lifecycle.managed status=stop_failed job=%s error=%s", assignment.JobID, RedactText(err.Error()))
|
||||
return lifecycleFailure("lifecycle_stop_failed", RedactText(err.Error()))
|
||||
log.Printf("RUN phase=lifecycle.managed status=stop_failed job=%s error=%s", assignment.JobID, err.Error())
|
||||
return lifecycleFailure("lifecycle_stop_failed", err.Error())
|
||||
}
|
||||
log.Printf("RUN phase=lifecycle.managed status=stopped job=%s pid=%d state=%s classification=%s", assignment.JobID, item.PID, item.State, safeOptional(item.ExitClassification))
|
||||
return processExecutionResult(item, "process stopped")
|
||||
@@ -695,7 +695,7 @@ func processExecutionResult(item ProcessIdentity, message string) LifecycleExecu
|
||||
}
|
||||
|
||||
func lifecycleExecutionFailure(code string, message string, retryable bool) LifecycleExecutionResult {
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)}, Message: RedactText(message), ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "file", Summary: code}}
|
||||
return LifecycleExecutionResult{State: lifecycleResultStateFailed, Progress: protocol.RunJobProgressReport{Percent: 100, Message: message}, Message: message, ErrorCode: code, Retryable: retryable, ExecutionResult: protocol.RunJobExecutionResult{Kind: "file", Summary: code}}
|
||||
}
|
||||
|
||||
func (template LifecycleActionTemplate) ToProcessCommand(workdir string, resolver WorkspaceResolver, assignment protocol.RunJobAssignment) (ProcessCommand, error) {
|
||||
@@ -948,7 +948,7 @@ func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCo
|
||||
return ProcessResult{ExitCode: -1}, fmt.Errorf("command is required")
|
||||
}
|
||||
startedAt := time.Now()
|
||||
log.Printf("RUN phase=process.command status=starting job=%s capability=%s action=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), safeOptional(command.WorkDir), redactedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
log.Printf("RUN phase=process.command status=starting job=%s capability=%s action=%s workdir=%s command=%s timeoutMs=%d envKeys=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), safeOptional(command.WorkDir), quotedCommandLine(command.Args), command.Timeout.Milliseconds(), envKeysSummary(command.Env, nil))
|
||||
if command.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, command.Timeout)
|
||||
@@ -967,14 +967,14 @@ func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCo
|
||||
cmd.Stdout = stdoutWriter
|
||||
cmd.Stderr = stderrWriter
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Printf("RUN phase=process.command status=start_failed job=%s capability=%s action=%s command=%s durationMs=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), redactedCommandLine(command.Args), time.Since(startedAt).Milliseconds(), RedactText(err.Error()))
|
||||
log.Printf("RUN phase=process.command status=start_failed job=%s capability=%s action=%s command=%s durationMs=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), quotedCommandLine(command.Args), time.Since(startedAt).Milliseconds(), err.Error())
|
||||
return ProcessResult{ExitCode: -1}, err
|
||||
}
|
||||
pid := 0
|
||||
if cmd.Process != nil {
|
||||
pid = cmd.Process.Pid
|
||||
}
|
||||
log.Printf("RUN phase=process.command status=started job=%s capability=%s action=%s pid=%d command=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args))
|
||||
log.Printf("RUN phase=process.command status=started job=%s capability=%s action=%s pid=%d command=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, quotedCommandLine(command.Args))
|
||||
err := cmd.Wait()
|
||||
stdoutWriter.Flush()
|
||||
stderrWriter.Flush()
|
||||
@@ -983,10 +983,10 @@ func (supervisor OSProcessSupervisor) Run(ctx context.Context, command ProcessCo
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.command status=failed job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr), RedactText(err.Error()))
|
||||
log.Printf("RUN phase=process.command status=failed job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d error=%s", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, quotedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr), err.Error())
|
||||
return result, err
|
||||
}
|
||||
log.Printf("RUN phase=process.command status=exited job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, redactedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr))
|
||||
log.Printf("RUN phase=process.command status=exited job=%s capability=%s action=%s pid=%d command=%s exitCode=%d durationMs=%d stdoutBytes=%d stderrBytes=%d", safeOptional(command.JobID), safeOptional(command.Capability), safeOptional(command.Action), pid, quotedCommandLine(command.Args), result.ExitCode, time.Since(startedAt).Milliseconds(), len(result.Stdout), len(result.Stderr))
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -1127,8 +1127,8 @@ func isSupportedRemoteCapability(capability string) bool {
|
||||
func lifecycleFailure(code string, message string) LifecycleExecutionResult {
|
||||
return LifecycleExecutionResult{
|
||||
State: lifecycleResultStateFailed,
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: RedactText(message)},
|
||||
Message: RedactText(message),
|
||||
Progress: protocol.RunJobProgressReport{Percent: 100, Message: message},
|
||||
Message: message,
|
||||
ErrorCode: code,
|
||||
}
|
||||
}
|
||||
@@ -1183,23 +1183,23 @@ func safeOptional(value string) string {
|
||||
if value == "" {
|
||||
return "-"
|
||||
}
|
||||
return RedactText(value)
|
||||
return value
|
||||
}
|
||||
|
||||
func errorSummary(err error) string {
|
||||
if err == nil {
|
||||
return "-"
|
||||
}
|
||||
return RedactText(err.Error())
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func redactedCommandLine(args []string) string {
|
||||
func quotedCommandLine(args []string) string {
|
||||
if len(args) == 0 {
|
||||
return "-"
|
||||
}
|
||||
parts := make([]string, len(args))
|
||||
for i, arg := range args {
|
||||
parts[i] = strconv.Quote(RedactText(arg))
|
||||
parts[i] = strconv.Quote(arg)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
@@ -1223,18 +1223,6 @@ func envKeysSummary(first map[string]string, second map[string]string) string {
|
||||
return strings.Join(keys, ",")
|
||||
}
|
||||
|
||||
func RedactText(value string) string {
|
||||
redacted := value
|
||||
replacements := []string{"/Users/", "[host]/", "Bearer ", "Bearer [redacted] ", "sk-", "sk-[redacted]", "password=", "password=[redacted]", "api_key=", "api_key=[redacted]", "secret=", "secret=[redacted]", "unix://", "socket://"}
|
||||
for i := 0; i+1 < len(replacements); i += 2 {
|
||||
redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1])
|
||||
}
|
||||
if len(redacted) > maxLifecycleOutputBytes {
|
||||
return redacted[:maxLifecycleOutputBytes]
|
||||
}
|
||||
return redacted
|
||||
}
|
||||
|
||||
// splitRawLogLines only removes the newline framing used by LogEntry. It
|
||||
// deliberately preserves every other byte, including blank lines and spaces.
|
||||
func splitRawLogLines(value string) []string {
|
||||
|
||||
Reference in New Issue
Block a user