320 lines
16 KiB
Go
320 lines
16 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"browser.local/run/config"
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
const autonomousLifecyclePlanKey = ".platform/autonomous-lifecycle-plan.json"
|
|
|
|
func LoadAutonomousLifecyclePlan(cfg config.Config) (*protocol.RunAutonomousLifecyclePlan, string, bool, error) {
|
|
if strings.TrimSpace(cfg.ComponentKind) != config.PackageComponentRun || strings.TrimSpace(cfg.ServerInstanceID) == "" {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=not_generated_run server=%s component=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKind))
|
|
return nil, "", false, nil
|
|
}
|
|
scope, err := seededWorkspaceScope(cfg)
|
|
if err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=scope_failed server=%s componentKey=%s error=%s", safeOptional(cfg.ServerInstanceID), safeOptional(cfg.ComponentKey), err.Error())
|
|
return nil, "", false, err
|
|
}
|
|
path, err := NewWorkspaceResolver(cfg.WorkspaceRoot).ExistingTarget(scope, autonomousLifecyclePlanKey)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=plan_missing scope=%s", safeOptional(scope))
|
|
return nil, scope, false, nil
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle status=load_failed scope=%s error=%s", safeOptional(scope), err.Error())
|
|
return nil, scope, false, err
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=open_failed scope=%s error=%s", safeOptional(scope), err.Error())
|
|
return nil, scope, false, err
|
|
}
|
|
defer file.Close()
|
|
var plan protocol.RunAutonomousLifecyclePlan
|
|
decoder := json.NewDecoder(io.LimitReader(file, 64*1024))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&plan); err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=decode_failed scope=%s error=%s", safeOptional(scope), err.Error())
|
|
return nil, scope, false, err
|
|
}
|
|
if err := protocol.ValidateRunAutonomousLifecyclePlan(plan); err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=invalid scope=%s error=%s", safeOptional(scope), err.Error())
|
|
return nil, scope, false, err
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle status=loaded server=%s plugin=%s endpoint=%s profile=%s bootstrap=%t actions=%d dependencies=%d installs=%d", plan.ServerInstanceID, plan.PluginID, plan.RunEndpointID, safeOptional(plan.ProfileKey), plan.Bootstrap != nil, len(plan.Actions), len(plan.DependencyProbes), len(plan.InstallPlans))
|
|
return &plan, scope, true, nil
|
|
}
|
|
|
|
func (worker *Worker) RunAutonomousLifecycleOnce(ctx context.Context) error {
|
|
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
|
|
if err != nil || !ok {
|
|
return err
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := validateAutonomousLifecycleScope(worker.cfg, state, *plan); err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=scope_mismatch error=%s", err.Error())
|
|
return err
|
|
}
|
|
if err := worker.runAutonomousDependencies(ctx, *plan); err != nil {
|
|
return err
|
|
}
|
|
if plan.Bootstrap == nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle status=skipped reason=no_bootstrap server=%s", plan.ServerInstanceID)
|
|
return nil
|
|
}
|
|
assignment := autonomousLifecycleAssignment(state, worker.cfg, *plan, *plan.Bootstrap)
|
|
log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=starting job=%s capability=%s target=%s operation=%s", assignment.JobID, assignment.Capability, safeOptional(assignment.TargetKey), safeOptional(assignment.ExecutionInput.LifecycleOperation))
|
|
executor := worker.executor
|
|
executor.artifactHook = StaticLifecycleArtifactHook{}
|
|
execution := executor.ExecuteContext(ctx, assignment)
|
|
log.Printf("RUN phase=autonomous_lifecycle.bootstrap status=complete job=%s state=%s processState=%s errorCode=%s message=%s", assignment.JobID, execution.State, safeOptional(execution.ExecutionResult.ProcessState), safeOptional(execution.ErrorCode), safeOptional(execution.Message))
|
|
if err := worker.reportAutonomousLifecycle(ctx, assignment, execution); err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle.report status=degraded server=%s action=continue_worker reason=projection_report_failed", assignment.ServerInstanceID)
|
|
}
|
|
if execution.State != lifecycleResultStateSucceeded {
|
|
return fmt.Errorf("autonomous lifecycle bootstrap failed: %s", errorSummaryFromResult(execution))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (worker *Worker) reportAutonomousLifecycle(ctx context.Context, assignment protocol.RunJobAssignment, execution LifecycleExecutionResult) error {
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request := protocol.RunLifecycleReportRequest{
|
|
RunEndpointID: state.RunEndpointID,
|
|
SessionToken: state.SessionToken,
|
|
ServerInstanceID: assignment.ServerInstanceID,
|
|
Capability: assignment.Capability,
|
|
State: execution.State,
|
|
Progress: execution.Progress,
|
|
Message: execution.Message,
|
|
ErrorCode: execution.ErrorCode,
|
|
ExecutionResult: execution.ExecutionResult,
|
|
}
|
|
if source, ok := worker.executor.managed.(ManagedProcessObservationSource); ok {
|
|
for _, identity := range source.ManagedProcessObservations() {
|
|
if identity.ServerInstanceID == assignment.ServerInstanceID && identity.RunEndpointID == state.RunEndpointID && identity.ObservationSeq > 0 && identity.State == execution.ExecutionResult.ProcessState {
|
|
request.ManagedProcessID = managedProcessObservationID(identity)
|
|
request.ObservationSeq = identity.ObservationSeq
|
|
request.ObservedAt = identity.UpdatedAt
|
|
break
|
|
}
|
|
}
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle.report status=starting server=%s capability=%s state=%s processState=%s", assignment.ServerInstanceID, assignment.Capability, execution.State, safeOptional(execution.ExecutionResult.ProcessState))
|
|
response, err := worker.client.ReportLifecycle(ctx, request)
|
|
if err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle.report status=failed server=%s error=%s", assignment.ServerInstanceID, err.Error())
|
|
return err
|
|
}
|
|
if !response.Accepted || response.RunEndpointID != state.RunEndpointID || response.ServerInstanceID != assignment.ServerInstanceID {
|
|
log.Printf("RUN phase=autonomous_lifecycle.report status=rejected server=%s accepted=%t", assignment.ServerInstanceID, response.Accepted)
|
|
return fmt.Errorf("autonomous lifecycle report was not accepted")
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle.report status=accepted server=%s projectedState=%s", assignment.ServerInstanceID, safeOptional(response.ProjectedState))
|
|
return nil
|
|
}
|
|
|
|
func (worker *Worker) reportAutonomousProcessObservations(ctx context.Context) error {
|
|
source, ok := worker.executor.managed.(ManagedProcessObservationSource)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
state, err := worker.registeredState()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, identity := range source.ManagedProcessObservations() {
|
|
if identity.ServerInstanceID == "" || identity.RunEndpointID != state.RunEndpointID || identity.ObservationSeq == 0 {
|
|
continue
|
|
}
|
|
processID := managedProcessObservationID(identity)
|
|
worker.observationMu.Lock()
|
|
alreadyReported := worker.reportedObservations[processID] >= identity.ObservationSeq
|
|
worker.observationMu.Unlock()
|
|
if alreadyReported {
|
|
continue
|
|
}
|
|
request := protocol.RunLifecycleReportRequest{RunEndpointID: state.RunEndpointID, SessionToken: state.SessionToken, ServerInstanceID: identity.ServerInstanceID, Capability: protocol.RunCapabilityProcessStatus, State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "supervised process observation"}, ManagedProcessID: processID, ObservationSeq: identity.ObservationSeq, ObservedAt: identity.UpdatedAt, ExecutionResult: protocol.RunJobExecutionResult{Kind: "process", ProcessState: identity.State, ExitClassification: identity.ExitClassification, ExitCode: identity.ExitCode, Summary: "supervised process observation"}}
|
|
if _, err := worker.client.ReportLifecycle(ctx, request); err != nil {
|
|
return err
|
|
}
|
|
worker.observationMu.Lock()
|
|
worker.reportedObservations[processID] = identity.ObservationSeq
|
|
worker.observationMu.Unlock()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func managedProcessObservationID(identity ProcessIdentity) string {
|
|
if identity.LogSessionID != "" {
|
|
return "log-session:" + identity.LogSessionID
|
|
}
|
|
sum := sha256.Sum256([]byte(identity.RunEndpointID + "\x00" + identity.ServerInstanceID + "\x00" + identity.CommandFingerprint + "\x00" + identity.StartedAt.UTC().Format(time.RFC3339Nano)))
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func validateAutonomousLifecycleScope(cfg config.Config, state WorkerState, plan protocol.RunAutonomousLifecyclePlan) error {
|
|
if plan.RunEndpointID != state.RunEndpointID || plan.ServerInstanceID != cfg.ServerInstanceID || plan.PluginID != cfg.PluginID {
|
|
return fmt.Errorf("autonomous lifecycle plan identity does not match this Run")
|
|
}
|
|
if plan.TargetOS != runtime.GOOS || plan.TargetArch != runtime.GOARCH {
|
|
return fmt.Errorf("autonomous lifecycle target does not match this Run")
|
|
}
|
|
if cfg.ComponentKey != "" && plan.ProfileKey != "" && plan.ProfileKey != cfg.ComponentKey {
|
|
return fmt.Errorf("autonomous lifecycle profile does not match this Run")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (worker *Worker) runAutonomousDependencies(ctx context.Context, plan protocol.RunAutonomousLifecyclePlan) error {
|
|
if len(plan.DependencyProbes) == 0 {
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=skipped reason=no_probes")
|
|
return nil
|
|
}
|
|
for _, probe := range plan.DependencyProbes {
|
|
state, evidence, err := worker.executor.runDependencyProbe(ctx, probe, plan.RuntimeBindings)
|
|
if err != nil {
|
|
if probe.Required {
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=failed probe=%s error=%s", safeOptional(probe.Key), err.Error())
|
|
return err
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=optional_probe_failed probe=%s error=%s", safeOptional(probe.Key), err.Error())
|
|
continue
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=probed probe=%s state=%s evidence=%s required=%t", safeOptional(probe.Key), safeOptional(state), safeOptional(evidence), probe.Required)
|
|
if state == "present" || !probe.Required {
|
|
continue
|
|
}
|
|
installPlan, found := autonomousInstallPlanForProbe(plan.InstallPlans, probe)
|
|
if !found {
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=missing_without_install_plan probe=%s action=defer_to_bootstrap", safeOptional(probe.Key))
|
|
continue
|
|
}
|
|
assignment := autonomousDependencyAssignment(worker.State(), worker.cfg, plan, installPlan)
|
|
input := protocol.DependencyExecutionInputResponse{JobID: assignment.JobID, ServerInstanceID: plan.ServerInstanceID, RunEndpointID: plan.RunEndpointID, PluginID: plan.PluginID, PluginVersion: plan.PluginVersion, ProfileKey: autonomousProfileKey(worker.cfg, plan), TargetOS: plan.TargetOS, TargetArch: plan.TargetArch, PlanDigest: autonomousInstallPlanDigest(installPlan), Probe: probe, Plan: installPlan, Bindings: plan.RuntimeBindings}
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_start probe=%s plan=%s steps=%d", safeOptional(probe.Key), safeOptional(installPlan.Key), len(installPlan.Steps))
|
|
if err := worker.runAutonomousInstallPlan(ctx, assignment, input); err != nil {
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_failed probe=%s plan=%s error=%s", safeOptional(probe.Key), safeOptional(installPlan.Key), err.Error())
|
|
return err
|
|
}
|
|
log.Printf("RUN phase=autonomous_lifecycle.dependencies status=install_complete probe=%s plan=%s", safeOptional(probe.Key), safeOptional(installPlan.Key))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (worker *Worker) runAutonomousInstallPlan(ctx context.Context, assignment protocol.RunJobAssignment, input protocol.DependencyExecutionInputResponse) error {
|
|
for index, step := range input.Plan.Steps {
|
|
if err := worker.executor.runDependencyInstallStep(ctx, assignment, input, step, index); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func autonomousInstallPlanForProbe(plans []protocol.DependencyInstallPlan, probe protocol.DependencyProbe) (protocol.DependencyInstallPlan, bool) {
|
|
for _, plan := range plans {
|
|
for _, step := range plan.Steps {
|
|
if step.TargetKey == probe.TargetKey {
|
|
return plan, true
|
|
}
|
|
}
|
|
}
|
|
return protocol.DependencyInstallPlan{}, false
|
|
}
|
|
|
|
func autonomousLifecycleAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, action protocol.RunAutonomousLifecycleAction) protocol.RunJobAssignment {
|
|
now := time.Now().UTC()
|
|
profileKey := autonomousProfileKey(cfg, plan)
|
|
assignment := protocol.RunJobAssignment{
|
|
JobID: "autonomous-bootstrap-" + safeWorkspaceName(action.Operation),
|
|
ServerInstanceID: plan.ServerInstanceID,
|
|
RunEndpointID: state.RunEndpointID,
|
|
Capability: action.Capability,
|
|
TargetKey: action.TargetKey,
|
|
IdempotencyKey: "autonomous:" + safeWorkspaceName(plan.TargetRelease) + ":" + safeWorkspaceName(action.Operation),
|
|
State: "running",
|
|
LeaseToken: "local-autonomous-bootstrap",
|
|
Attempt: 1,
|
|
MaxAttempts: 1,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
ExecutionInput: protocol.RunJobExecutionInput{
|
|
WorkspaceScope: profileKey,
|
|
PluginID: plan.PluginID,
|
|
LifecycleOperation: action.Operation,
|
|
Deployment: autonomousDeploymentExecution(plan.Deployment, action.Operation),
|
|
},
|
|
}
|
|
if action.Capability == protocol.RunCapabilityProcessStart {
|
|
assignment.ExecutionInput.LogSources = append([]protocol.RuntimeLogSourcePlan(nil), plan.LogSources...)
|
|
assignment.ExecutionInput.DLLExtensions = append([]protocol.RuntimeDLLExtensionPlan(nil), plan.DLLExtensions...)
|
|
}
|
|
return assignment
|
|
}
|
|
|
|
func autonomousDependencyAssignment(state WorkerState, cfg config.Config, plan protocol.RunAutonomousLifecyclePlan, installPlan protocol.DependencyInstallPlan) protocol.RunJobAssignment {
|
|
now := time.Now().UTC()
|
|
return protocol.RunJobAssignment{JobID: "autonomous-dependencies-" + safeWorkspaceName(installPlan.Key), ServerInstanceID: plan.ServerInstanceID, RunEndpointID: state.RunEndpointID, Capability: protocol.RunCapabilityDependenciesInstall, TargetKey: "dependencies/install/" + installPlan.Key, IdempotencyKey: "autonomous:dependencies:" + safeWorkspaceName(installPlan.Key), State: "running", LeaseToken: "local-autonomous-dependencies", Attempt: 1, MaxAttempts: 1, CreatedAt: now, UpdatedAt: now, ExecutionInput: protocol.RunJobExecutionInput{WorkspaceScope: autonomousProfileKey(cfg, plan), PluginID: plan.PluginID, LifecycleOperation: "install"}}
|
|
}
|
|
|
|
func autonomousDeploymentExecution(deployment *protocol.RunAutonomousDeployment, operation string) *protocol.ServerDeploymentExecution {
|
|
if deployment == nil {
|
|
return nil
|
|
}
|
|
startCommand := deployment.StartCommand
|
|
if operation == "install" && deployment.InstallCommand != "" {
|
|
startCommand = deployment.InstallCommand
|
|
}
|
|
return &protocol.ServerDeploymentExecution{SchemaVersion: deployment.SchemaVersion, Mode: deployment.Mode, ProfileKey: deployment.ProfileKey, CreateInputs: copyStringMap(deployment.CreateInputs), ServerRoot: deployment.ServerRoot, WorkingDirectory: deployment.WorkingDirectory, StartCommand: startCommand, StopCommand: deployment.StopCommand, StatusCommand: deployment.StatusCommand, Shell: deployment.Shell, Revision: deployment.Revision}
|
|
}
|
|
|
|
func autonomousProfileKey(cfg config.Config, plan protocol.RunAutonomousLifecyclePlan) string {
|
|
if strings.TrimSpace(cfg.ComponentKey) != "" {
|
|
return cfg.ComponentKey
|
|
}
|
|
return plan.ProfileKey
|
|
}
|
|
|
|
func autonomousInstallPlanDigest(plan protocol.DependencyInstallPlan) string {
|
|
body, err := json.Marshal(plan)
|
|
if err != nil {
|
|
return "sha256:" + strings.Repeat("0", 64)
|
|
}
|
|
sum := sha256.Sum256(body)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func errorSummaryFromResult(result LifecycleExecutionResult) string {
|
|
if result.ErrorCode != "" && result.Message != "" {
|
|
return result.ErrorCode + ": " + result.Message
|
|
}
|
|
if result.ErrorCode != "" {
|
|
return result.ErrorCode
|
|
}
|
|
if result.Message != "" {
|
|
return result.Message
|
|
}
|
|
return result.State
|
|
}
|