first commit
This commit is contained in:
npc0-hue
@@ -0,0 +1,408 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const defaultJobPollSeconds = 2
|
||||
|
||||
func (svc *CoreService) ClaimRunJob(claim domain.RunJobClaim) (domain.RunJobClaimResult, error) {
|
||||
claim = domain.CopyRunJobClaim(claim)
|
||||
if err := validator.ValidateRunJobClaim(claim); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(claim.RunEndpointID, claim.SessionToken); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: claim.RunEndpointID, State: domain.JobStateQueued})
|
||||
if err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
job, ok := firstSupportedJob(jobs, claim.Capabilities)
|
||||
if !ok {
|
||||
return domain.RunJobClaimResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: claim.RunEndpointID,
|
||||
NextPollSeconds: defaultJobPollSeconds,
|
||||
ServerTime: stamp,
|
||||
}, nil
|
||||
}
|
||||
|
||||
lease := svc.newJobLease(job.ID, claim.RunEndpointID, claim.SessionToken, stamp)
|
||||
svc.jobLeases[job.ID] = lease
|
||||
job.State = domain.JobStateAccepted
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
assignment := assignmentFromJob(job, lease)
|
||||
return domain.CopyRunJobClaimResult(domain.RunJobClaimResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: claim.RunEndpointID,
|
||||
HasJob: true,
|
||||
Job: &assignment,
|
||||
NextPollSeconds: defaultJobPollSeconds,
|
||||
ServerTime: stamp,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) AckRunJob(ack domain.RunJobAck) (domain.RunJobAckResult, error) {
|
||||
if err := validator.ValidateRunJobAck(ack); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(ack.RunEndpointID, ack.SessionToken); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(ack.RunEndpointID, ack.SessionToken, ack.JobID, ack.LeaseToken, ack.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if isTerminalJobState(job.State) {
|
||||
return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.RunJobAckResult{}, validationError("job is not claimable for ack")
|
||||
}
|
||||
job.State = domain.JobStateRunning
|
||||
if strings.TrimSpace(ack.Message) != "" {
|
||||
job.Progress.Message = ack.Message
|
||||
}
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) UpdateRunJobProgress(progress domain.RunJobProgress) (domain.RunJobProgressResult, error) {
|
||||
if err := validator.ValidateRunJobProgress(progress); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(progress.RunEndpointID, progress.SessionToken); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(progress.RunEndpointID, progress.SessionToken, progress.JobID, progress.LeaseToken, progress.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.RunJobProgressResult{}, validationError("job is not active")
|
||||
}
|
||||
job.State = domain.JobStateRunning
|
||||
job.Progress = domain.JobProgress{Percent: progress.Progress.Percent, Message: progress.Progress.Message}
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobProgressResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CompleteRunJob(result domain.RunJobResult) (domain.RunJobResultResult, error) {
|
||||
if err := validator.ValidateRunJobResult(result); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(result.RunEndpointID, result.SessionToken); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(result.RunEndpointID, result.SessionToken, result.JobID, result.LeaseToken, result.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
fingerprint := terminalFingerprint(result)
|
||||
if isTerminalJobState(job.State) {
|
||||
if lease.TerminalFingerprint != "" && lease.TerminalFingerprint == fingerprint {
|
||||
if err := svc.projectLifecycleJobResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
}
|
||||
return domain.RunJobResultResult{}, validationError("terminal result conflicts with existing job result")
|
||||
}
|
||||
|
||||
job.State = result.State
|
||||
job.Progress = domain.JobProgress{Percent: result.Progress.Percent, Message: terminalMessage(result)}
|
||||
job.ResultRef = result.ResultRef
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectLifecycleJobResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
lease.TerminalFingerprint = fingerprint
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RequestRunJobCancel(request domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error) {
|
||||
if err := validator.ValidateRunJobCancelRequest(request); err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, err := svc.store.Jobs().Get(request.JobID)
|
||||
if err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
if !isActiveJobState(job.State) {
|
||||
return domain.RunJobCancelRequestResult{}, validationError("job is not active")
|
||||
}
|
||||
lease, exists := svc.jobLeases[job.ID]
|
||||
if !exists {
|
||||
return domain.RunJobCancelRequestResult{}, validationError("job lease is missing")
|
||||
}
|
||||
lease.CancelReason = request.Reason
|
||||
lease.CancelRequestedAt = stamp
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobCancelRequestResult{Accepted: true, JobID: job.ID, Reason: request.Reason, RequestedAt: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) PollRunJobCancel(poll domain.RunJobCancelPoll) (domain.RunJobCancelPollResult, error) {
|
||||
if err := validator.ValidateRunJobCancelPoll(poll); err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(poll.RunEndpointID, poll.SessionToken); err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
lease, ok := svc.findCancelLease(poll)
|
||||
if !ok {
|
||||
return domain.RunJobCancelPollResult{Accepted: true, RunEndpointID: poll.RunEndpointID, ServerTime: stamp}, nil
|
||||
}
|
||||
return domain.RunJobCancelPollResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: poll.RunEndpointID,
|
||||
HasCancel: true,
|
||||
JobID: lease.JobID,
|
||||
Reason: lease.CancelReason,
|
||||
RequestedAt: lease.CancelRequestedAt,
|
||||
ServerTime: stamp,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ReconcileRunJobs(reconcile domain.RunJobReconcile) (domain.RunJobReconcileResult, error) {
|
||||
reconcile = domain.CopyRunJobReconcile(reconcile)
|
||||
if err := validator.ValidateRunJobReconcile(reconcile); err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(reconcile.RunEndpointID, reconcile.SessionToken); err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: reconcile.RunEndpointID})
|
||||
if err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
activeByID := map[string]domain.Job{}
|
||||
for _, job := range jobs {
|
||||
if isActiveJobState(job.State) {
|
||||
activeByID[job.ID] = job
|
||||
}
|
||||
}
|
||||
|
||||
activeJobs := make([]domain.RunJobAssignment, 0, len(activeByID))
|
||||
ids := make([]string, 0, len(activeByID))
|
||||
for id := range activeByID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
job := activeByID[id]
|
||||
lease := svc.jobLeases[job.ID]
|
||||
if lease.JobID == "" || lease.SessionToken != reconcile.SessionToken {
|
||||
lease = svc.newJobLease(job.ID, reconcile.RunEndpointID, reconcile.SessionToken, stamp)
|
||||
} else {
|
||||
lease.UpdatedAt = stamp
|
||||
}
|
||||
svc.jobLeases[job.ID] = lease
|
||||
activeJobs = append(activeJobs, assignmentFromJob(job, lease))
|
||||
}
|
||||
|
||||
unknown := make([]string, 0)
|
||||
for _, reportedID := range reconcile.ActiveJobIDs {
|
||||
if _, exists := activeByID[reportedID]; !exists {
|
||||
unknown = append(unknown, reportedID)
|
||||
}
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return domain.CopyRunJobReconcileResult(domain.RunJobReconcileResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: reconcile.RunEndpointID,
|
||||
ActiveJobs: activeJobs,
|
||||
UnknownJobIDs: unknown,
|
||||
ServerTime: stamp,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateRunSession(runEndpointID string, sessionToken string) error {
|
||||
svc.controlMu.Lock()
|
||||
defer svc.controlMu.Unlock()
|
||||
session, exists := svc.runSessions[runEndpointID]
|
||||
if !exists || session.SessionToken != sessionToken {
|
||||
return validationError("sessionToken is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) newJobLease(jobID string, runEndpointID string, sessionToken string, stamp time.Time) domain.RunJobLease {
|
||||
svc.jobLeaseSeq++
|
||||
return domain.RunJobLease{
|
||||
JobID: jobID,
|
||||
RunEndpointID: runEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
LeaseToken: fmt.Sprintf("job-lease:%s:%d:%d", jobID, stamp.UnixNano(), svc.jobLeaseSeq),
|
||||
Attempt: int(svc.jobLeaseSeq),
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *CoreService) activeLeasedJob(runEndpointID string, sessionToken string, jobID string, leaseToken string, attempt int) (domain.Job, domain.RunJobLease, error) {
|
||||
job, err := svc.store.Jobs().Get(jobID)
|
||||
if err != nil {
|
||||
return domain.Job{}, domain.RunJobLease{}, err
|
||||
}
|
||||
if job.RunEndpointID != runEndpointID {
|
||||
return domain.Job{}, domain.RunJobLease{}, validationError("job runEndpointId does not match request")
|
||||
}
|
||||
lease, exists := svc.jobLeases[jobID]
|
||||
if !exists || lease.SessionToken != sessionToken || lease.LeaseToken != leaseToken || lease.Attempt != attempt {
|
||||
return domain.Job{}, domain.RunJobLease{}, validationError("leaseToken is invalid")
|
||||
}
|
||||
return job, lease, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) findCancelLease(poll domain.RunJobCancelPoll) (domain.RunJobLease, bool) {
|
||||
if poll.JobID != "" {
|
||||
lease, exists := svc.jobLeases[poll.JobID]
|
||||
if !exists || lease.RunEndpointID != poll.RunEndpointID || lease.SessionToken != poll.SessionToken {
|
||||
return domain.RunJobLease{}, false
|
||||
}
|
||||
if poll.LeaseToken != "" && lease.LeaseToken != poll.LeaseToken {
|
||||
return domain.RunJobLease{}, false
|
||||
}
|
||||
return lease, lease.CancelReason != ""
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(svc.jobLeases))
|
||||
for id := range svc.jobLeases {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
lease := svc.jobLeases[id]
|
||||
if lease.RunEndpointID == poll.RunEndpointID && lease.SessionToken == poll.SessionToken && lease.CancelReason != "" {
|
||||
return lease, true
|
||||
}
|
||||
}
|
||||
return domain.RunJobLease{}, false
|
||||
}
|
||||
|
||||
func firstSupportedJob(jobs []domain.Job, capabilities []string) (domain.Job, bool) {
|
||||
capabilitySet := map[string]struct{}{}
|
||||
for _, capability := range capabilities {
|
||||
capabilitySet[capability] = struct{}{}
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if len(capabilitySet) == 0 {
|
||||
return job, true
|
||||
}
|
||||
if _, supported := capabilitySet[job.Capability]; supported {
|
||||
return job, true
|
||||
}
|
||||
}
|
||||
return domain.Job{}, false
|
||||
}
|
||||
|
||||
func assignmentFromJob(job domain.Job, lease domain.RunJobLease) domain.RunJobAssignment {
|
||||
return domain.RunJobAssignment{
|
||||
JobID: job.ID,
|
||||
ServerInstanceID: job.ServerInstanceID,
|
||||
RunEndpointID: job.RunEndpointID,
|
||||
Capability: job.Capability,
|
||||
TargetKey: job.TargetKey,
|
||||
InputRef: job.InputRef,
|
||||
IdempotencyKey: job.IdempotencyKey,
|
||||
State: job.State,
|
||||
Progress: domain.RunJobProgressReport{Percent: job.Progress.Percent, Message: job.Progress.Message},
|
||||
ResultRef: job.ResultRef,
|
||||
LeaseToken: lease.LeaseToken,
|
||||
Attempt: lease.Attempt,
|
||||
CreatedAt: job.CreatedAt,
|
||||
UpdatedAt: job.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func terminalFingerprint(result domain.RunJobResult) string {
|
||||
return fmt.Sprintf("%s|%d|%s|%s|%s|%s", result.State, result.Progress.Percent, result.ResultRef, result.Message, result.ErrorCode, result.Progress.Message)
|
||||
}
|
||||
|
||||
func terminalMessage(result domain.RunJobResult) string {
|
||||
if strings.TrimSpace(result.Message) != "" {
|
||||
return result.Message
|
||||
}
|
||||
return result.Progress.Message
|
||||
}
|
||||
|
||||
func isActiveJobState(state domain.JobState) bool {
|
||||
return state == domain.JobStateAccepted || state == domain.JobStateRunning
|
||||
}
|
||||
|
||||
func isTerminalJobState(state domain.JobState) bool {
|
||||
return state == domain.JobStateSucceeded || state == domain.JobStateFailed || state == domain.JobStateCancelled
|
||||
}
|
||||
Reference in New Issue
Block a user