init
This commit is contained in:
npc0-hue
@@ -0,0 +1,811 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
managedProcessJournalVersion = 2
|
||||
managedProcessOutputPollInterval = 50 * time.Millisecond
|
||||
managedProcessOutputDrainDelay = 750 * time.Millisecond
|
||||
managedProcessOutputRetryDelay = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
type ProcessIdentity struct {
|
||||
Scope string `json:"scope"`
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
RunEndpointID string `json:"runEndpointId,omitempty"`
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
Capability string `json:"capability,omitempty"`
|
||||
ProfileKey string `json:"profileKey"`
|
||||
LogSessionID string `json:"logSessionId,omitempty"`
|
||||
PID int `json:"pid"`
|
||||
SupervisorPID int `json:"supervisorPid,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CommandFingerprint string `json:"commandFingerprint"`
|
||||
State string `json:"state"`
|
||||
ExitCode int `json:"exitCode,omitempty"`
|
||||
ExitClassification string `json:"exitClassification,omitempty"`
|
||||
ObservationSeq uint64 `json:"observationSeq,omitempty"`
|
||||
Attempt int `json:"attempt"`
|
||||
LeaseTokenHash string `json:"leaseTokenHash,omitempty"`
|
||||
StdoutLogRef string `json:"stdoutLogRef,omitempty"`
|
||||
StderrLogRef string `json:"stderrLogRef,omitempty"`
|
||||
StdoutStreamKey string `json:"stdoutStreamKey,omitempty"`
|
||||
StderrStreamKey string `json:"stderrStreamKey,omitempty"`
|
||||
StopEventName string `json:"stopEventName,omitempty"`
|
||||
StdoutOffset int64 `json:"stdoutOffset,omitempty"`
|
||||
StderrOffset int64 `json:"stderrOffset,omitempty"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type processJournal struct {
|
||||
Version int `json:"version"`
|
||||
Items map[string]ProcessIdentity `json:"items"`
|
||||
Retired map[string]ProcessIdentity `json:"retired,omitempty"`
|
||||
}
|
||||
|
||||
type ManagedProcessSupervisor interface {
|
||||
Start(context.Context, ProcessCommand, ProcessIdentity, ManagedProcessOutput) (ProcessIdentity, error)
|
||||
Stop(context.Context, ProcessIdentity) (ProcessIdentity, error)
|
||||
Status(ProcessIdentity) ProcessIdentity
|
||||
ResumeOutput(ManagedProcessOutput)
|
||||
}
|
||||
|
||||
// ManagedProcessObservationSource exposes only generic supervised-process
|
||||
// facts. Worker uses it to report persisted transitions after registration.
|
||||
type ManagedProcessObservationSource interface {
|
||||
ManagedProcessObservations() []ProcessIdentity
|
||||
}
|
||||
|
||||
type ManagedProcessLine struct {
|
||||
Text string
|
||||
StartOffset int64
|
||||
EndOffset int64
|
||||
}
|
||||
|
||||
type ManagedProcessLineSink func(ProcessIdentity, ManagedProcessLine) error
|
||||
|
||||
type ManagedProcessOutput struct {
|
||||
Stdout ManagedProcessLineSink
|
||||
Stderr ManagedProcessLineSink
|
||||
}
|
||||
|
||||
type managedProcessFiles struct {
|
||||
stdout *os.File
|
||||
stderr *os.File
|
||||
}
|
||||
|
||||
type managedProcess interface {
|
||||
PID() int
|
||||
TargetPID() int
|
||||
Wait() (int, error)
|
||||
Kill() error
|
||||
}
|
||||
|
||||
type managedProcessTailer struct {
|
||||
cancel context.CancelFunc
|
||||
drain chan struct{}
|
||||
}
|
||||
|
||||
type OSManagedProcessSupervisor struct {
|
||||
root string
|
||||
path string
|
||||
outputRoot string
|
||||
mu sync.Mutex
|
||||
items map[string]ProcessIdentity
|
||||
retired map[string]ProcessIdentity
|
||||
tailers map[string]*managedProcessTailer
|
||||
}
|
||||
|
||||
func NewOSManagedProcessSupervisor(root string) (*OSManagedProcessSupervisor, error) {
|
||||
return NewOSManagedProcessSupervisorWithOutputRoot(root, root)
|
||||
}
|
||||
|
||||
func NewOSManagedProcessSupervisorWithOutputRoot(root string, outputRoot string) (*OSManagedProcessSupervisor, error) {
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputRootAbs, err := filepath.Abs(outputRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stateDir := filepath.Join(rootAbs, "state")
|
||||
if err := ensureDirectory(stateDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
supervisor := &OSManagedProcessSupervisor{root: rootAbs, path: filepath.Join(stateDir, "processes.json"), outputRoot: outputRootAbs, items: map[string]ProcessIdentity{}, retired: map[string]ProcessIdentity{}, tailers: map[string]*managedProcessTailer{}}
|
||||
if err := supervisor.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := supervisor.migrateLegacySessions(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
supervisor.Reconcile()
|
||||
return supervisor, nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Start(ctx context.Context, command ProcessCommand, identity ProcessIdentity, output ManagedProcessOutput) (ProcessIdentity, error) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
key := identity.Scope
|
||||
log.Printf("RUN phase=process.managed status=start_requested job=%s server=%s scope=%s command=%s workdir=%s", safeOptional(identity.JobID), identity.ServerInstanceID, safeOptional(identity.Scope), redactedCommandLine(command.Args), safeOptional(command.WorkDir))
|
||||
if existing, ok := supervisor.items[key]; ok && existing.State == "running" && supervisor.isAlive(existing) {
|
||||
if existing.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err)
|
||||
}
|
||||
existing.LogSessionID = logSessionID
|
||||
}
|
||||
if existing.StopEventName == "" {
|
||||
existing.StopEventName = managedProcessStopEventName(existing)
|
||||
}
|
||||
existing.State = "running"
|
||||
existing.ObservationSeq++
|
||||
existing.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[key] = existing
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("persist managed process session: %w", err)
|
||||
}
|
||||
supervisor.startTailersLocked(existing, output)
|
||||
log.Printf("RUN phase=process.managed status=reusing_existing job=%s pid=%d state=%s", safeOptional(identity.JobID), existing.PID, existing.State)
|
||||
return existing, nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=context_done job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
if len(command.Args) == 0 {
|
||||
log.Printf("RUN phase=process.managed status=missing_executable job=%s", safeOptional(identity.JobID))
|
||||
return ProcessIdentity{}, fmt.Errorf("process executable is required")
|
||||
}
|
||||
startedAt := time.Now().UTC()
|
||||
if identity.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return ProcessIdentity{}, fmt.Errorf("generate managed process log session: %w", err)
|
||||
}
|
||||
identity.LogSessionID = logSessionID
|
||||
}
|
||||
if identity.StopEventName == "" {
|
||||
identity.StopEventName = managedProcessStopEventName(identity)
|
||||
}
|
||||
files, identity, err := supervisor.prepareOutputFilesLocked(identity, startedAt)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed status=prepare_output_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=output_ready job=%s stdoutRef=%s stderrRef=%s", safeOptional(identity.JobID), safeOptional(identity.StdoutLogRef), safeOptional(identity.StderrLogRef))
|
||||
process, err := startManagedProcess(command, files, identity.StopEventName)
|
||||
if err != nil {
|
||||
files.close()
|
||||
log.Printf("RUN phase=process.managed status=start_failed job=%s error=%s", safeOptional(identity.JobID), RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
identity.SupervisorPID = process.PID()
|
||||
identity.PID = process.TargetPID()
|
||||
identity.StartedAt = startedAt
|
||||
identity.State = "running"
|
||||
identity.ObservationSeq = 1
|
||||
identity.UpdatedAt = identity.StartedAt
|
||||
identity.CommandFingerprint = fingerprintArgs(command.Args)
|
||||
previous, hadPrevious := supervisor.items[key]
|
||||
retiredKey := ""
|
||||
if hadPrevious && supervisor.hasPendingOutput(previous) {
|
||||
retiredKey = managedProcessGenerationKey(previous)
|
||||
supervisor.retired[retiredKey] = previous
|
||||
}
|
||||
supervisor.items[key] = identity
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
_ = forceManagedProcessStop(identity)
|
||||
files.close()
|
||||
if hadPrevious {
|
||||
supervisor.items[key] = previous
|
||||
} else {
|
||||
delete(supervisor.items, key)
|
||||
}
|
||||
if retiredKey != "" {
|
||||
delete(supervisor.retired, retiredKey)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=persist_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error()))
|
||||
return ProcessIdentity{}, err
|
||||
}
|
||||
supervisor.startTailersLocked(identity, output)
|
||||
go supervisor.wait(key, process, identity.PID, files)
|
||||
log.Printf("RUN phase=process.managed status=started job=%s pid=%d fingerprint=%s", safeOptional(identity.JobID), identity.PID, safeOptional(identity.CommandFingerprint))
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Stop(ctx context.Context, identity ProcessIdentity) (ProcessIdentity, error) {
|
||||
supervisor.mu.Lock()
|
||||
current, ok := supervisor.items[identity.Scope]
|
||||
if !ok || current.State != "running" || !supervisor.isAlive(current) {
|
||||
if ok {
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "already-stopped"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[identity.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
supervisor.mu.Unlock()
|
||||
log.Printf("RUN phase=process.managed status=already_stopped job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope))
|
||||
return current, nil
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=stop_requested job=%s pid=%d scope=%s", safeOptional(identity.JobID), current.PID, safeOptional(identity.Scope))
|
||||
if err := requestManagedProcessStop(current); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error()))
|
||||
}
|
||||
supervisor.mu.Unlock()
|
||||
deadline := time.NewTimer(2 * time.Second)
|
||||
ticker := time.NewTicker(20 * time.Millisecond)
|
||||
defer deadline.Stop()
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if !supervisor.isAlive(current) {
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "requested-stop"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.mu.Lock()
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.mu.Unlock()
|
||||
supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay)
|
||||
log.Printf("RUN phase=process.managed status=stopped job=%s pid=%d classification=%s", safeOptional(identity.JobID), current.PID, current.ExitClassification)
|
||||
return current, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("RUN phase=process.managed status=stop_context_done job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(ctx.Err().Error()))
|
||||
return ProcessIdentity{}, ctx.Err()
|
||||
case <-deadline.C:
|
||||
if err := forceManagedProcessStop(current); err != nil {
|
||||
log.Printf("RUN phase=process.managed status=forced_stop_signal_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), current.PID, RedactText(err.Error()))
|
||||
}
|
||||
current.State = "stopped"
|
||||
current.ExitClassification = "forced-stop"
|
||||
current.ObservationSeq++
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
supervisor.mu.Lock()
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.mu.Unlock()
|
||||
supervisor.drainTailersAfter(current, managedProcessOutputDrainDelay)
|
||||
log.Printf("RUN phase=process.managed status=forced_stop job=%s pid=%d", safeOptional(identity.JobID), current.PID)
|
||||
return current, nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Status(identity ProcessIdentity) ProcessIdentity {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
current, ok := supervisor.items[identity.Scope]
|
||||
if !ok {
|
||||
log.Printf("RUN phase=process.managed status=not_started job=%s scope=%s", safeOptional(identity.JobID), safeOptional(identity.Scope))
|
||||
return ProcessIdentity{Scope: identity.Scope, State: "stopped", ExitClassification: "not-started"}
|
||||
}
|
||||
if current.State == "running" && !supervisor.isAlive(current) {
|
||||
current.State = "exited"
|
||||
if current.ExitClassification == "" {
|
||||
current.ExitClassification = "unexpected-exit"
|
||||
}
|
||||
current.UpdatedAt = time.Now().UTC()
|
||||
current.ObservationSeq++
|
||||
supervisor.items[current.Scope] = current
|
||||
_ = supervisor.persistLocked()
|
||||
supervisor.drainTailersLocked(current)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=current job=%s pid=%d state=%s classification=%s", safeOptional(identity.JobID), current.PID, current.State, safeOptional(current.ExitClassification))
|
||||
return current
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) ResumeOutput(output ManagedProcessOutput) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
for _, item := range supervisor.items {
|
||||
if item.State == "running" && supervisor.isAlive(item) {
|
||||
supervisor.startTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
} else if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
}
|
||||
}
|
||||
changed := false
|
||||
for key, item := range supervisor.retired {
|
||||
if supervisor.hasPendingOutput(item) {
|
||||
supervisor.startDrainTailersLocked(item, output)
|
||||
log.Printf("RUN phase=process.managed status=resume_retired_output_drain pid=%d server=%s scope=%s", item.PID, item.ServerInstanceID, safeOptional(item.Scope))
|
||||
continue
|
||||
}
|
||||
delete(supervisor.retired, key)
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) Reconcile() {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
changed := false
|
||||
for key, item := range supervisor.items {
|
||||
if item.State == "running" && !supervisor.isAlive(item) {
|
||||
item.State = "exited"
|
||||
item.ExitClassification = "unexpected-exit"
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
item.ObservationSeq++
|
||||
supervisor.items[key] = item
|
||||
supervisor.drainTailersLocked(item)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
_ = supervisor.persistLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) wait(key string, process managedProcess, pid int, files managedProcessFiles) {
|
||||
exitCode, err := process.Wait()
|
||||
files.close()
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
item, ok := supervisor.items[key]
|
||||
if !ok || item.PID != pid {
|
||||
return
|
||||
}
|
||||
// A durable helper may exit independently of the target process (legacy
|
||||
// shell wrappers can detach their child). Preserve a live target so the
|
||||
// next Run can continue monitoring it by PID.
|
||||
if processAlivePID(item.PID) {
|
||||
item.State = "running"
|
||||
item.SupervisorPID = 0
|
||||
item.ExitClassification = "supervisor-exited-target-alive"
|
||||
} else {
|
||||
item.State = "exited"
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
item.ObservationSeq++
|
||||
item.ExitCode = exitCode
|
||||
if item.State == "running" {
|
||||
// Keep the classification assigned above.
|
||||
} else if err == nil {
|
||||
item.ExitClassification = "clean-exit"
|
||||
} else {
|
||||
item.ExitClassification = "unexpected-exit"
|
||||
}
|
||||
supervisor.items[key] = item
|
||||
_ = supervisor.persistLocked()
|
||||
if item.State != "running" {
|
||||
supervisor.drainTailersAfter(item, managedProcessOutputDrainDelay)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s error=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification, RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
log.Printf("RUN phase=process.managed status=%s job=%s pid=%d state=%s exitCode=%d classification=%s", processTerminalStatus(item.State), safeOptional(item.JobID), pid, item.State, item.ExitCode, item.ExitClassification)
|
||||
}
|
||||
|
||||
func processTerminalStatus(state string) string {
|
||||
if state == "running" {
|
||||
return "supervisor_exited_target_alive"
|
||||
}
|
||||
return "exited"
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) ManagedProcessObservations() []ProcessIdentity {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
items := make([]ProcessIdentity, 0, len(supervisor.items))
|
||||
for _, item := range supervisor.items {
|
||||
items = append(items, item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (files managedProcessFiles) close() {
|
||||
if files.stdout != nil {
|
||||
_ = files.stdout.Close()
|
||||
}
|
||||
if files.stderr != nil {
|
||||
_ = files.stderr.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) prepareOutputFilesLocked(identity ProcessIdentity, startedAt time.Time) (managedProcessFiles, ProcessIdentity, error) {
|
||||
outputDir := filepath.Join(supervisor.outputRoot, "state", "process-output")
|
||||
if err := ensureDirectory(outputDir); err != nil {
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
base := processOutputBase(identity.Scope+"\x00"+identity.LogSessionID, startedAt)
|
||||
identity.StdoutLogRef = base + ".stdout.log"
|
||||
identity.StderrLogRef = base + ".stderr.log"
|
||||
stdout, stdoutOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StdoutLogRef))
|
||||
if err != nil {
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
stderr, stderrOffset, err := openManagedOutputFile(filepath.Join(outputDir, identity.StderrLogRef))
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
return managedProcessFiles{}, ProcessIdentity{}, err
|
||||
}
|
||||
identity.StdoutOffset = stdoutOffset
|
||||
identity.StderrOffset = stderrOffset
|
||||
return managedProcessFiles{stdout: stdout, stderr: stderr}, identity, nil
|
||||
}
|
||||
|
||||
func openManagedOutputFile(path string) (*os.File, int64, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return file, info.Size(), nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) {
|
||||
if output.Stdout != nil && identity.StdoutLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, true)
|
||||
}
|
||||
if output.Stderr != nil && identity.StderrLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, true)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startDrainTailersLocked(identity ProcessIdentity, output ManagedProcessOutput) {
|
||||
if output.Stdout != nil && identity.StdoutLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stdout", identity.StdoutLogRef, identity.StdoutOffset, output.Stdout, false)
|
||||
}
|
||||
if output.Stderr != nil && identity.StderrLogRef != "" {
|
||||
supervisor.startTailerLocked(identity, "stderr", identity.StderrLogRef, identity.StderrOffset, output.Stderr, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) startTailerLocked(identity ProcessIdentity, stream string, ref string, offset int64, sink ManagedProcessLineSink, follow bool) {
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
tailerID := managedProcessTailerID(identity, stream)
|
||||
if tailer, exists := supervisor.tailers[tailerID]; exists {
|
||||
if !follow {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
log.Printf("RUN phase=process.managed.output status=tail_reuse job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset)
|
||||
return
|
||||
}
|
||||
path := filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(ref))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
tailer := &managedProcessTailer{cancel: cancel, drain: make(chan struct{})}
|
||||
if !follow {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
supervisor.tailers[tailerID] = tailer
|
||||
log.Printf("RUN phase=process.managed.output status=tail_start job=%s pid=%d stream=%s ref=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, safeOptional(ref), offset)
|
||||
go supervisor.tailOutput(ctx, tailerID, tailer, identity, stream, path, offset, sink)
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) tailOutput(ctx context.Context, tailerID string, tailer *managedProcessTailer, identity ProcessIdentity, stream string, path string, offset int64, sink ManagedProcessLineSink) {
|
||||
defer supervisor.removeTailer(tailerID, tailer, identity)
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_open_failed job=%s pid=%d stream=%s path=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
if offset > 0 {
|
||||
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_seek_failed job=%s pid=%d stream=%s path=%s offset=%d error=%s", safeOptional(identity.JobID), identity.PID, stream, safeOptional(path), offset, RedactText(err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
reader := bufio.NewReader(file)
|
||||
defer func() {
|
||||
log.Printf("RUN phase=process.managed.output status=tail_stop job=%s pid=%d stream=%s offset=%d", safeOptional(identity.JobID), identity.PID, stream, offset)
|
||||
}()
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
startOffset := offset
|
||||
endOffset := offset + int64(len(line))
|
||||
text := strings.TrimSpace(line)
|
||||
if text != "" {
|
||||
for {
|
||||
if sinkErr := sink(identity, ManagedProcessLine{Text: text, StartOffset: startOffset, EndOffset: endOffset}); sinkErr == nil {
|
||||
log.Printf("RUN phase=process.managed.output status=line job=%s pid=%d stream=%s line=%q", safeOptional(identity.JobID), identity.PID, stream, RedactText(text))
|
||||
break
|
||||
} else {
|
||||
log.Printf("RUN phase=process.managed.output status=spool_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(sinkErr.Error()))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(managedProcessOutputRetryDelay):
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
if offsetErr := supervisor.updateOutputOffset(identity, stream, endOffset); offsetErr == nil {
|
||||
break
|
||||
} else {
|
||||
log.Printf("RUN phase=process.managed.output status=offset_retry job=%s pid=%d stream=%s error=%s", safeOptional(identity.JobID), identity.PID, stream, RedactText(offsetErr.Error()))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(managedProcessOutputRetryDelay):
|
||||
}
|
||||
}
|
||||
offset = endOffset
|
||||
}
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if err != io.EOF {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tailer.drain:
|
||||
return
|
||||
case <-time.After(managedProcessOutputPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) removeTailer(tailerID string, tailer *managedProcessTailer, identity ProcessIdentity) {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
if current, ok := supervisor.tailers[tailerID]; ok && current == tailer {
|
||||
delete(supervisor.tailers, tailerID)
|
||||
}
|
||||
key := managedProcessGenerationKey(identity)
|
||||
if retired, ok := supervisor.retired[key]; ok && !supervisor.hasPendingOutput(retired) {
|
||||
delete(supervisor.retired, key)
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
supervisor.retired[key] = retired
|
||||
log.Printf("RUN phase=process.managed.output status=retired_prune_failed job=%s pid=%d error=%s", safeOptional(identity.JobID), identity.PID, RedactText(err.Error()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) updateOutputOffset(identity ProcessIdentity, stream string, offset int64) error {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
item, ok := supervisor.items[identity.Scope]
|
||||
retiredKey := ""
|
||||
if !ok || !sameManagedProcessGeneration(item, identity) {
|
||||
retiredKey = managedProcessGenerationKey(identity)
|
||||
item, ok = supervisor.retired[retiredKey]
|
||||
if !ok || !sameManagedProcessGeneration(item, identity) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if stream == "stdout" {
|
||||
item.StdoutOffset = offset
|
||||
} else {
|
||||
item.StderrOffset = offset
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
if retiredKey == "" {
|
||||
supervisor.items[identity.Scope] = item
|
||||
} else {
|
||||
supervisor.retired[retiredKey] = item
|
||||
}
|
||||
return supervisor.persistLocked()
|
||||
}
|
||||
|
||||
func sameManagedProcessGeneration(current ProcessIdentity, expected ProcessIdentity) bool {
|
||||
if current.LogSessionID != "" || expected.LogSessionID != "" {
|
||||
if current.LogSessionID == "" || current.LogSessionID != expected.LogSessionID {
|
||||
return false
|
||||
}
|
||||
return sameManagedProcessOutputFiles(current, expected)
|
||||
}
|
||||
return current.PID == expected.PID && current.StdoutLogRef == expected.StdoutLogRef && current.StderrLogRef == expected.StderrLogRef
|
||||
}
|
||||
|
||||
func sameManagedProcessOutputFiles(current ProcessIdentity, expected ProcessIdentity) bool {
|
||||
if current.StdoutLogRef != "" && expected.StdoutLogRef != "" && current.StdoutLogRef != expected.StdoutLogRef {
|
||||
return false
|
||||
}
|
||||
if current.StderrLogRef != "" && expected.StderrLogRef != "" && current.StderrLogRef != expected.StderrLogRef {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) drainTailersAfter(identity ProcessIdentity, delay time.Duration) {
|
||||
go func() {
|
||||
time.Sleep(delay)
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
supervisor.drainTailersLocked(identity)
|
||||
}()
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) drainTailersLocked(identity ProcessIdentity) {
|
||||
for _, stream := range []string{"stdout", "stderr"} {
|
||||
if tailer, ok := supervisor.tailers[managedProcessTailerID(identity, stream)]; ok {
|
||||
beginManagedProcessTailerDrain(tailer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func beginManagedProcessTailerDrain(tailer *managedProcessTailer) {
|
||||
select {
|
||||
case <-tailer.drain:
|
||||
default:
|
||||
close(tailer.drain)
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) stopTailersLocked(identity ProcessIdentity) {
|
||||
for _, stream := range []string{"stdout", "stderr"} {
|
||||
tailerID := managedProcessTailerID(identity, stream)
|
||||
if tailer, ok := supervisor.tailers[tailerID]; ok {
|
||||
tailer.cancel()
|
||||
delete(supervisor.tailers, tailerID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func managedProcessTailerID(identity ProcessIdentity, stream string) string {
|
||||
generation := identity.LogSessionID
|
||||
if generation == "" {
|
||||
generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef)
|
||||
} else {
|
||||
generation += "\x00" + identity.StdoutLogRef + "\x00" + identity.StderrLogRef
|
||||
}
|
||||
return identity.Scope + "\x00" + generation + "\x00" + stream
|
||||
}
|
||||
|
||||
func managedProcessGenerationKey(identity ProcessIdentity) string {
|
||||
generation := identity.LogSessionID
|
||||
if generation == "" {
|
||||
generation = fmt.Sprintf("legacy:%d:%s:%s", identity.PID, identity.StdoutLogRef, identity.StderrLogRef)
|
||||
}
|
||||
return identity.Scope + "\x00" + generation
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) hasPendingOutput(identity ProcessIdentity) bool {
|
||||
for _, item := range []struct {
|
||||
ref string
|
||||
offset int64
|
||||
}{{identity.StdoutLogRef, identity.StdoutOffset}, {identity.StderrLogRef, identity.StderrOffset}} {
|
||||
if item.ref == "" {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(supervisor.outputRoot, "state", "process-output", filepath.Base(item.ref)))
|
||||
if err == nil && info.Size() > item.offset {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) isAlive(item ProcessIdentity) bool {
|
||||
return processAlivePID(item.PID)
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) load() error {
|
||||
body, err := os.ReadFile(supervisor.path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var file processJournal
|
||||
if err := json.Unmarshal(body, &file); err != nil {
|
||||
return err
|
||||
}
|
||||
for key, item := range file.Items {
|
||||
supervisor.items[key] = item
|
||||
}
|
||||
for key, item := range file.Retired {
|
||||
supervisor.retired[key] = item
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) migrateLegacySessions() error {
|
||||
supervisor.mu.Lock()
|
||||
defer supervisor.mu.Unlock()
|
||||
changed := false
|
||||
for key, item := range supervisor.items {
|
||||
if item.State != "running" || !supervisor.isAlive(item) {
|
||||
continue
|
||||
}
|
||||
if item.LogSessionID == "" {
|
||||
logSessionID, err := newManagedProcessLogSessionID()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate legacy managed process log session: %w", err)
|
||||
}
|
||||
item.LogSessionID = logSessionID
|
||||
changed = true
|
||||
}
|
||||
if item.StartedAt.IsZero() {
|
||||
item.StartedAt = time.Now().UTC()
|
||||
changed = true
|
||||
}
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
supervisor.items[key] = item
|
||||
}
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
if err := supervisor.persistLocked(); err != nil {
|
||||
return fmt.Errorf("persist legacy managed process log session: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *OSManagedProcessSupervisor) persistLocked() error {
|
||||
body, err := json.Marshal(processJournal{Version: managedProcessJournalVersion, Items: supervisor.items, Retired: supervisor.retired})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporary := supervisor.path + ".tmp"
|
||||
if err := os.WriteFile(temporary, body, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporary, supervisor.path); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processOutputBase(scope string, startedAt time.Time) string {
|
||||
sum := sha256.Sum256([]byte(scope + "\x00" + startedAt.Format(time.RFC3339Nano)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func managedProcessStopEventName(identity ProcessIdentity) string {
|
||||
seed := strings.Join([]string{
|
||||
"run-managed-stop-v1",
|
||||
identity.RunEndpointID,
|
||||
identity.ServerInstanceID,
|
||||
identity.Scope,
|
||||
identity.LogSessionID,
|
||||
}, "\x00")
|
||||
sum := sha256.Sum256([]byte(seed))
|
||||
return "Local\\run-managed-stop-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func newManagedProcessLogSessionID() (string, error) {
|
||||
var value [16]byte
|
||||
if _, err := rand.Read(value[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(value[:]), nil
|
||||
}
|
||||
|
||||
func fingerprintArgs(args []string) string {
|
||||
sum := sha256.Sum256([]byte(strings.Join(args, "\x00")))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user