feat: 完整游戏运维功能
This commit is contained in:
npc0-hue
@@ -0,0 +1,615 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const runUpdateChunkSize = 1024 * 1024
|
||||
|
||||
type dependencyResolution struct {
|
||||
instance domain.ServerInstance
|
||||
plugin domain.GamePlugin
|
||||
binding domain.RuntimeBinding
|
||||
endpoint domain.RunEndpoint
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetDependencyCatalogForSession(sessionID, serverInstanceID string) (domain.DependencyCatalog, error) {
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID); err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
resolution, err := svc.resolveDependencyContext(serverInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
statuses, err := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
statusByProbe := map[string]domain.DependencyStatus{}
|
||||
for _, status := range statuses {
|
||||
statusByProbe[status.ProbeKey] = status
|
||||
}
|
||||
|
||||
plans := make([]domain.DependencyPlanView, 0, len(resolution.plugin.RuntimeProfiles.InstallPlans))
|
||||
for _, plan := range resolution.plugin.RuntimeProfiles.InstallPlans {
|
||||
if !runtimePlatformsContain(plan.Platforms, resolution.endpoint.Platform) {
|
||||
continue
|
||||
}
|
||||
steps := make([]domain.DependencyPlanStepView, len(plan.Steps))
|
||||
for i, step := range plan.Steps {
|
||||
host := ""
|
||||
if parsed, parseErr := url.Parse(step.DownloadRef); parseErr == nil {
|
||||
host = parsed.Hostname()
|
||||
}
|
||||
steps[i] = domain.DependencyPlanStepView{Type: step.Type, TargetKey: step.TargetKey, PackageManager: step.PackageManager, PackageName: step.PackageName, Version: step.Version, DownloadHost: host}
|
||||
}
|
||||
var planProbe domain.RuntimeDependencyProbe
|
||||
for _, candidate := range resolution.plugin.RuntimeProfiles.DependencyProbes {
|
||||
if runtimePlatformsContain(candidate.Platforms, resolution.endpoint.Platform) && planTargetsProbe(plan, candidate) {
|
||||
planProbe = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
plans = append(plans, domain.DependencyPlanView{Key: plan.Key, Title: plan.Title, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, Digest: dependencyPlanDigest(resolution, planProbe, plan), Steps: steps})
|
||||
}
|
||||
sort.Slice(plans, func(i, j int) bool { return plans[i].Key < plans[j].Key })
|
||||
|
||||
probes := make([]domain.DependencyProbeView, 0, len(resolution.plugin.RuntimeProfiles.DependencyProbes))
|
||||
for _, probe := range resolution.plugin.RuntimeProfiles.DependencyProbes {
|
||||
if !runtimePlatformsContain(probe.Platforms, resolution.endpoint.Platform) {
|
||||
continue
|
||||
}
|
||||
status := statusByProbe[probe.Key]
|
||||
planKey := ""
|
||||
for _, plan := range resolution.plugin.RuntimeProfiles.InstallPlans {
|
||||
if runtimePlatformsContain(plan.Platforms, resolution.endpoint.Platform) && planTargetsProbe(plan, probe) {
|
||||
planKey = plan.Key
|
||||
break
|
||||
}
|
||||
}
|
||||
state := status.State
|
||||
if state == "" {
|
||||
state = domain.DependencyStateUnknown
|
||||
}
|
||||
probes = append(probes, domain.DependencyProbeView{Key: probe.Key, Kind: probe.Kind, Required: probe.Required, MinimumVersion: probe.MinimumVersion, State: state, Evidence: status.Evidence, InstallPlanKey: planKey})
|
||||
}
|
||||
sort.Slice(probes, func(i, j int) bool { return probes[i].Key < probes[j].Key })
|
||||
|
||||
return domain.CopyDependencyCatalog(domain.DependencyCatalog{ServerInstanceID: resolution.instance.ID, PluginID: resolution.plugin.ID, PluginVersion: resolution.plugin.Version, ProfileKey: resolution.binding.ProfileKey, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, Probes: probes, Plans: plans, UpdatedAt: svc.now()}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListRunUpdateJobsForSession(sessionID, serverInstanceID string) ([]domain.RunUpdateJob, error) {
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].UpdatedAt.After(items[j].UpdatedAt) })
|
||||
for i := range items {
|
||||
items[i] = domain.CopyRunUpdateJob(items[i])
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetDependencyExecutionInput(request domain.DependencyExecutionInputRequest) (domain.DependencyExecutionInput, error) {
|
||||
if err := validator.ValidateDependencyExecutionInputRequest(request); err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityDependenciesCheck && job.Capability != domain.JobCapabilityDependenciesInstall {
|
||||
return domain.DependencyExecutionInput{}, validationError("job is not a dependency operation")
|
||||
}
|
||||
resolution, err := svc.resolveDependencyContext(job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if resolution.endpoint.ID != job.RunEndpointID {
|
||||
return domain.DependencyExecutionInput{}, validationError("dependency endpoint no longer matches")
|
||||
}
|
||||
probeKey := strings.TrimPrefix(job.TargetKey, "dependencies/")
|
||||
if job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
probeKey = ""
|
||||
}
|
||||
var probe domain.RuntimeDependencyProbe
|
||||
if probeKey != "" {
|
||||
probe, err = declaredDependencyProbe(resolution.plugin, probeKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
}
|
||||
var plan domain.RuntimeInstallPlan
|
||||
if job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
planKey := strings.TrimPrefix(job.TargetKey, "dependencies/install/")
|
||||
plan, err = declaredInstallPlan(resolution.plugin, planKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
statuses, listErr := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if listErr != nil {
|
||||
return domain.DependencyExecutionInput{}, listErr
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.JobID == job.ID {
|
||||
probe, err = declaredDependencyProbe(resolution.plugin, status.ProbeKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
digest := dependencyPlanDigest(resolution, probe, plan)
|
||||
status, err := svc.dependencyStatusForJob(job.ID, job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if status.PlanDigest != digest {
|
||||
return domain.DependencyExecutionInput{}, validationError("dependency declaration changed after dispatch")
|
||||
}
|
||||
bindings, err := dependencyBindings(resolution.binding, probe, plan)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
return domain.CopyDependencyExecutionInput(domain.DependencyExecutionInput{JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, PluginID: resolution.plugin.ID, PluginVersion: resolution.plugin.Version, ProfileKey: resolution.binding.ProfileKey, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, PlanDigest: digest, Probe: probe, Plan: plan, Bindings: bindings}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetRunUpdateInput(request domain.RunUpdateInputRequest) (domain.RunUpdateInput, error) {
|
||||
if err := validator.ValidateRunUpdateInputRequest(request); err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return domain.RunUpdateInput{}, validationError("job is not a Run self-update")
|
||||
}
|
||||
update, distribution, artifact, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
return domain.RunUpdateInput{JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, ArtifactID: artifact.ID, Checksum: artifact.Checksum, SizeBytes: artifact.SizeBytes, TargetOS: distribution.TargetOS, TargetArch: distribution.TargetArch, PackageFormat: distribution.PackageFormat, ExecutableName: executableFilename("run", distribution.TargetOS), TargetRelease: update.TargetRelease, ChunkSizeBytes: runUpdateChunkSize}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ReadRunUpdateChunk(request domain.RunUpdateChunkRequest) (domain.RunUpdateChunk, error) {
|
||||
if err := validator.ValidateRunUpdateChunkRequest(request); err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return domain.RunUpdateChunk{}, validationError("job is not a Run self-update")
|
||||
}
|
||||
_, _, artifact, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
payload, err := svc.artifactPayload(artifact.ID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
if int64(len(payload)) != artifact.SizeBytes || validator.BytesChecksum(payload) != artifact.Checksum {
|
||||
return domain.RunUpdateChunk{}, validationError("update artifact content does not match metadata")
|
||||
}
|
||||
if request.Offset >= artifact.SizeBytes {
|
||||
return domain.RunUpdateChunk{}, validationError("offset must be inside update artifact")
|
||||
}
|
||||
length := request.Length
|
||||
remaining := artifact.SizeBytes - request.Offset
|
||||
if int64(length) > remaining {
|
||||
length = int(remaining)
|
||||
}
|
||||
end := request.Offset + int64(length)
|
||||
return domain.CopyRunUpdateChunk(domain.RunUpdateChunk{JobID: job.ID, ArtifactID: artifact.ID, Offset: request.Offset, TotalBytes: artifact.SizeBytes, Checksum: artifact.Checksum, Payload: payload[int(request.Offset):int(end)], Complete: end == artifact.SizeBytes}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) activeFencedInputJob(endpointID, sessionToken, jobID, leaseToken string, attempt int) (domain.Job, error) {
|
||||
session, err := svc.validatedRunSession(endpointID, sessionToken)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
job, err := svc.fencedJob(session, jobID, leaseToken, attempt)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.Job{}, validationError("job input is not active")
|
||||
}
|
||||
if !job.CancelRequestedAt.IsZero() {
|
||||
return domain.Job{}, validationError("job input is cancelled")
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveDependencyContext(serverInstanceID string) (dependencyResolution, error) {
|
||||
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if plugin.Status != domain.GamePluginStatusInstalled || plugin.Version != instance.PluginVersion {
|
||||
return dependencyResolution{}, validationError("installed plugin version does not match server")
|
||||
}
|
||||
binding, err := svc.runtimeBindingForServer(instance.ID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
binding, err = normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if binding.Status != domain.RuntimeBindingStatusComplete || binding.PluginVersion != plugin.Version {
|
||||
return dependencyResolution{}, validationError("runtime binding is incomplete or stale")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if endpoint.Platform == "" || endpoint.Architecture == "" {
|
||||
return dependencyResolution{}, validationError("Run endpoint target is not registered")
|
||||
}
|
||||
return dependencyResolution{instance: instance, plugin: plugin, binding: binding, endpoint: endpoint}, nil
|
||||
}
|
||||
|
||||
func declaredDependencyProbe(plugin domain.GamePlugin, key, targetOS string) (domain.RuntimeDependencyProbe, error) {
|
||||
for _, probe := range plugin.RuntimeProfiles.DependencyProbes {
|
||||
if probe.Key == key && runtimePlatformsContain(probe.Platforms, targetOS) {
|
||||
return probe, nil
|
||||
}
|
||||
}
|
||||
return domain.RuntimeDependencyProbe{}, validationError("dependency probe is not declared for endpoint target")
|
||||
}
|
||||
|
||||
func declaredInstallPlan(plugin domain.GamePlugin, key, targetOS string) (domain.RuntimeInstallPlan, error) {
|
||||
for _, plan := range plugin.RuntimeProfiles.InstallPlans {
|
||||
if plan.Key == key && runtimePlatformsContain(plan.Platforms, targetOS) {
|
||||
return plan, nil
|
||||
}
|
||||
}
|
||||
return domain.RuntimeInstallPlan{}, validationError("dependency install plan is not declared for endpoint target")
|
||||
}
|
||||
|
||||
func runtimePlatformsContain(platforms []string, target string) bool {
|
||||
if len(platforms) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, platform := range platforms {
|
||||
if platform == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func planTargetsProbe(plan domain.RuntimeInstallPlan, probe domain.RuntimeDependencyProbe) bool {
|
||||
for _, step := range plan.Steps {
|
||||
if step.TargetKey == probe.TargetKey {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dependencyPlanDigest(resolution dependencyResolution, probe domain.RuntimeDependencyProbe, plan domain.RuntimeInstallPlan) string {
|
||||
keys := make([]string, 0, len(resolution.binding.Bindings))
|
||||
for key := range resolution.binding.Bindings {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
bindingEvidence := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
bindingEvidence = append(bindingEvidence, key+"="+validator.BytesChecksum([]byte(resolution.binding.Bindings[key])))
|
||||
}
|
||||
payload := struct {
|
||||
PluginID string `json:"pluginId"`
|
||||
PluginVersion string `json:"pluginVersion"`
|
||||
ProfileKey string `json:"profileKey"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
Binding []string `json:"binding"`
|
||||
Probe domain.RuntimeDependencyProbe `json:"probe"`
|
||||
Plan domain.RuntimeInstallPlan `json:"plan"`
|
||||
}{resolution.plugin.ID, resolution.plugin.Version, resolution.binding.ProfileKey, resolution.endpoint.Platform, resolution.endpoint.Architecture, bindingEvidence, probe, plan}
|
||||
body, _ := json.Marshal(payload)
|
||||
return validator.BytesChecksum(body)
|
||||
}
|
||||
|
||||
func dependencyBindings(binding domain.RuntimeBinding, probe domain.RuntimeDependencyProbe, plan domain.RuntimeInstallPlan) (map[string]string, error) {
|
||||
keys := map[string]struct{}{}
|
||||
if probe.TargetKey != "" {
|
||||
keys[probe.TargetKey] = struct{}{}
|
||||
}
|
||||
for _, step := range plan.Steps {
|
||||
if step.TargetKey != "" {
|
||||
keys[step.TargetKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make(map[string]string, len(keys))
|
||||
for key := range keys {
|
||||
value := strings.TrimSpace(binding.Bindings[key])
|
||||
if value == "" {
|
||||
value = key
|
||||
}
|
||||
lower := strings.ToLower(value)
|
||||
if strings.HasPrefix(lower, "secret://") || strings.Contains(lower, "password=") || strings.Contains(lower, "token=") {
|
||||
return nil, validationError("dependency target binding cannot be a secret")
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) dependencyStatusForJob(jobID, serverInstanceID string) (domain.DependencyStatus, error) {
|
||||
statuses, err := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return domain.DependencyStatus{}, err
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.JobID == jobID {
|
||||
return status, nil
|
||||
}
|
||||
}
|
||||
return domain.DependencyStatus{}, repo.ErrNotFound
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveRunUpdate(job domain.Job) (domain.RunUpdateJob, domain.RunDistribution, domain.Artifact, error) {
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
var update domain.RunUpdateJob
|
||||
for _, candidate := range updates {
|
||||
if candidate.JobID == job.ID {
|
||||
update = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if update.ID == "" || update.RunEndpointID != job.RunEndpointID {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update record does not match active job")
|
||||
}
|
||||
distributions, err := svc.store.RunDistributions().List(domain.RunDistributionFilter{ServerInstanceID: job.ServerInstanceID, Status: domain.DistributionStatusAvailable})
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
var distribution domain.RunDistribution
|
||||
for _, candidate := range distributions {
|
||||
if candidate.ArtifactID == update.ArtifactID {
|
||||
distribution = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if distribution.ID == "" || distribution.RunEndpointID != job.RunEndpointID || distribution.TargetOS != update.TargetOS || distribution.TargetArch != update.TargetArch || distribution.Checksum != update.Checksum {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run distribution no longer matches update")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(job.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
if endpoint.Platform != distribution.TargetOS || endpoint.Architecture != distribution.TargetArch {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update target no longer matches endpoint")
|
||||
}
|
||||
artifact, err := svc.store.Artifacts().Get(update.ArtifactID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
if artifact.State != domain.ArtifactStateAvailable || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != distribution.BuildJobID || artifact.Checksum != update.Checksum {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update artifact is unavailable or outside distribution scope")
|
||||
}
|
||||
return update, distribution, artifact, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectDependencyAndRunUpdateResult(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityDependenciesCheck || job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
status, err := svc.dependencyStatusForJob(job.ID, job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status.UpdatedAt = stamp
|
||||
status.CheckedAt = stamp
|
||||
status.JobID = job.ID
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
var evidence domain.DependencyExecutionEvidence
|
||||
if err := json.Unmarshal([]byte(job.ExecutionResult.Content), &evidence); err != nil {
|
||||
return validationError("dependency result evidence is invalid")
|
||||
}
|
||||
if evidence.ProbeKey != status.ProbeKey || evidence.PlanDigest != status.PlanDigest || job.ExecutionResult.Checksum != status.PlanDigest {
|
||||
return validationError("dependency result evidence does not match approved plan")
|
||||
}
|
||||
status.State = domain.DependencyState(evidence.State)
|
||||
status.Evidence = evidence.Evidence
|
||||
status.CompletedSteps = evidence.CompletedSteps
|
||||
status.Message = "dependency execution completed"
|
||||
} else if job.State == domain.JobStateCancelled {
|
||||
status.State = domain.DependencyStateFailed
|
||||
status.Message = "dependency execution cancelled"
|
||||
} else {
|
||||
status.State = domain.DependencyStateFailed
|
||||
status.Message = "dependency execution failed"
|
||||
}
|
||||
if err := validator.ValidateDependencyStatus(status); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.DependencyStatuses().Update(status); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "dependency.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), status.Message)
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return nil
|
||||
}
|
||||
update, _, _, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
update.UpdatedAt = stamp
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
var evidence domain.RunUpdateExecutionEvidence
|
||||
if err := json.Unmarshal([]byte(job.ExecutionResult.Content), &evidence); err != nil || evidence.TargetRelease != update.TargetRelease || evidence.Phase != "staged" || job.ExecutionResult.Checksum != update.Checksum {
|
||||
return validationError("Run update staged evidence is invalid")
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusRunning
|
||||
update.Phase = domain.RunUpdatePhaseRestartRequested
|
||||
update.Message = "verified update staged; restart requested"
|
||||
} else if job.State == domain.JobStateCancelled {
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseFailed
|
||||
update.Message = "Run update cancelled before activation"
|
||||
} else {
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseFailed
|
||||
update.Message = "Run update verification or staging failed"
|
||||
}
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "run.update.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), update.Message)
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectDependencyAndRunUpdateProgress(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityRunSelfUpdate {
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, update := range updates {
|
||||
if update.JobID != job.ID || update.Status != domain.DistributionJobStatusQueued {
|
||||
continue
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusRunning
|
||||
update.Phase = domain.RunUpdatePhaseDownloading
|
||||
update.Message = "Run is downloading and verifying the update"
|
||||
update.UpdatedAt = stamp
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.store.RunUpdateJobs().Update(update)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ReportRunUpdateHealth(report domain.RunUpdateHealthReport) (domain.RunUpdateHealthResult, error) {
|
||||
if err := validator.ValidateRunUpdateHealthReport(report); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if _, err := svc.validatedRunSession(report.RunEndpointID, report.SessionToken); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
job, err := svc.store.Jobs().Get(report.JobID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if job.RunEndpointID != report.RunEndpointID || job.Capability != domain.JobCapabilityRunSelfUpdate || job.State != domain.JobStateSucceeded || job.Attempt != report.Attempt || !leaseTokenMatches(job.LeaseTokenHash, report.LeaseToken) {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health report does not match terminal attempt")
|
||||
}
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
var update domain.RunUpdateJob
|
||||
for _, candidate := range updates {
|
||||
if candidate.JobID == job.ID && candidate.RunEndpointID == report.RunEndpointID {
|
||||
update = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if update.ID == "" || job.ExecutionResult.Checksum != update.Checksum {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health report does not match staged update")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(report.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if endpoint.Version != report.Version || endpoint.Status != domain.RunEndpointStatusOnline {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health version does not match online endpoint")
|
||||
}
|
||||
stamp := svc.now()
|
||||
if report.Outcome == "succeeded" {
|
||||
if report.Version != update.TargetRelease || update.Phase == domain.RunUpdatePhaseRolledBack || update.Phase == domain.RunUpdatePhaseFailed {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health version does not match target release")
|
||||
}
|
||||
if update.Phase == domain.RunUpdatePhaseSucceeded {
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusSucceeded
|
||||
update.Phase = domain.RunUpdatePhaseSucceeded
|
||||
update.Rollback = false
|
||||
update.Message = "updated Run registered, reconciled, and reported healthy"
|
||||
} else {
|
||||
if update.PreviousVersion != "" && report.Version != update.PreviousVersion {
|
||||
return domain.RunUpdateHealthResult{}, validationError("rolled-back Run version does not match previous release")
|
||||
}
|
||||
if update.Phase == domain.RunUpdatePhaseRolledBack {
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseRolledBack
|
||||
update.Rollback = true
|
||||
update.Message = "Run update activation failed and previous executable was restored"
|
||||
}
|
||||
update.UpdatedAt = stamp
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if report.Outcome == "rolled-back" {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
if err := svc.recordAuditEvent("run", "run.update.health", "server-instance", update.ServerInstanceID, auditResult, update.Message); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func auditResultForJob(job domain.Job) domain.AuditResult {
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
return domain.AuditResultSuccess
|
||||
}
|
||||
if job.State == domain.JobStateCancelled {
|
||||
return domain.AuditResultDenied
|
||||
}
|
||||
return domain.AuditResultFailed
|
||||
}
|
||||
|
||||
func sameRunUpdateTarget(existing, expected domain.RunUpdateJob) bool {
|
||||
return existing.ServerInstanceID == expected.ServerInstanceID && existing.RunEndpointID == expected.RunEndpointID && existing.ArtifactID == expected.ArtifactID && existing.Checksum == expected.Checksum && existing.TargetOS == expected.TargetOS && existing.TargetArch == expected.TargetArch && existing.TargetRelease == expected.TargetRelease && existing.JobID == expected.JobID && existing.IdempotencyKey == expected.IdempotencyKey
|
||||
}
|
||||
|
||||
func findRunDistributionForArtifact(distributions []domain.RunDistribution, artifactID string) (domain.RunDistribution, error) {
|
||||
for _, distribution := range distributions {
|
||||
if distribution.ArtifactID == artifactID && distribution.Status == domain.DistributionStatusAvailable {
|
||||
return distribution, nil
|
||||
}
|
||||
}
|
||||
return domain.RunDistribution{}, errors.New("available Run distribution not found")
|
||||
}
|
||||
Reference in New Issue
Block a user