package service import ( "fmt" "sort" "strings" "time" "browser.local/platform/domain" "browser.local/platform/validator" ) const defaultJobPollSeconds = 2 func (svc *CoreService) ClaimRunJob(claim domain.RunJobClaim) (domain.RunJobClaimResult, error) { claim = domain.CopyRunJobClaim(claim) if err := validator.ValidateRunJobClaim(claim); err != nil { return domain.RunJobClaimResult{}, err } if err := svc.validateRunSession(claim.RunEndpointID, claim.SessionToken); err != nil { return domain.RunJobClaimResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: claim.RunEndpointID, State: domain.JobStateQueued}) if err != nil { return domain.RunJobClaimResult{}, err } job, ok := firstSupportedJob(jobs, claim.Capabilities) if !ok { return domain.RunJobClaimResult{ Accepted: true, RunEndpointID: claim.RunEndpointID, NextPollSeconds: defaultJobPollSeconds, ServerTime: stamp, }, nil } lease := svc.newJobLease(job.ID, claim.RunEndpointID, claim.SessionToken, stamp) svc.jobLeases[job.ID] = lease job.State = domain.JobStateAccepted job.UpdatedAt = stamp if err := validator.ValidateJob(job); err != nil { return domain.RunJobClaimResult{}, err } if err := svc.store.Jobs().Update(job); err != nil { return domain.RunJobClaimResult{}, err } assignment := assignmentFromJob(job, lease) return domain.CopyRunJobClaimResult(domain.RunJobClaimResult{ Accepted: true, RunEndpointID: claim.RunEndpointID, HasJob: true, Job: &assignment, NextPollSeconds: defaultJobPollSeconds, ServerTime: stamp, }), nil } func (svc *CoreService) AckRunJob(ack domain.RunJobAck) (domain.RunJobAckResult, error) { if err := validator.ValidateRunJobAck(ack); err != nil { return domain.RunJobAckResult{}, err } if err := svc.validateRunSession(ack.RunEndpointID, ack.SessionToken); err != nil { return domain.RunJobAckResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() job, lease, err := svc.activeLeasedJob(ack.RunEndpointID, ack.SessionToken, ack.JobID, ack.LeaseToken, ack.Attempt) if err != nil { return domain.RunJobAckResult{}, err } if isTerminalJobState(job.State) { return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil } if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning { return domain.RunJobAckResult{}, validationError("job is not claimable for ack") } job.State = domain.JobStateRunning if strings.TrimSpace(ack.Message) != "" { job.Progress.Message = ack.Message } job.UpdatedAt = stamp if err := validator.ValidateJob(job); err != nil { return domain.RunJobAckResult{}, err } if err := svc.store.Jobs().Update(job); err != nil { return domain.RunJobAckResult{}, err } lease.UpdatedAt = stamp svc.jobLeases[job.ID] = lease return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil } func (svc *CoreService) UpdateRunJobProgress(progress domain.RunJobProgress) (domain.RunJobProgressResult, error) { if err := validator.ValidateRunJobProgress(progress); err != nil { return domain.RunJobProgressResult{}, err } if err := svc.validateRunSession(progress.RunEndpointID, progress.SessionToken); err != nil { return domain.RunJobProgressResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() job, lease, err := svc.activeLeasedJob(progress.RunEndpointID, progress.SessionToken, progress.JobID, progress.LeaseToken, progress.Attempt) if err != nil { return domain.RunJobProgressResult{}, err } if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning { return domain.RunJobProgressResult{}, validationError("job is not active") } job.State = domain.JobStateRunning job.Progress = domain.JobProgress{Percent: progress.Progress.Percent, Message: progress.Progress.Message} job.UpdatedAt = stamp if err := validator.ValidateJob(job); err != nil { return domain.RunJobProgressResult{}, err } if err := svc.store.Jobs().Update(job); err != nil { return domain.RunJobProgressResult{}, err } lease.UpdatedAt = stamp svc.jobLeases[job.ID] = lease return domain.RunJobProgressResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil } func (svc *CoreService) CompleteRunJob(result domain.RunJobResult) (domain.RunJobResultResult, error) { if err := validator.ValidateRunJobResult(result); err != nil { return domain.RunJobResultResult{}, err } if err := svc.validateRunSession(result.RunEndpointID, result.SessionToken); err != nil { return domain.RunJobResultResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() job, lease, err := svc.activeLeasedJob(result.RunEndpointID, result.SessionToken, result.JobID, result.LeaseToken, result.Attempt) if err != nil { return domain.RunJobResultResult{}, err } fingerprint := terminalFingerprint(result) if isTerminalJobState(job.State) { if lease.TerminalFingerprint != "" && lease.TerminalFingerprint == fingerprint { if err := svc.projectLifecycleJobResult(job, stamp); err != nil { return domain.RunJobResultResult{}, err } return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil } return domain.RunJobResultResult{}, validationError("terminal result conflicts with existing job result") } job.State = result.State job.Progress = domain.JobProgress{Percent: result.Progress.Percent, Message: terminalMessage(result)} job.ResultRef = result.ResultRef job.UpdatedAt = stamp if err := validator.ValidateJob(job); err != nil { return domain.RunJobResultResult{}, err } if err := svc.store.Jobs().Update(job); err != nil { return domain.RunJobResultResult{}, err } if err := svc.projectLifecycleJobResult(job, stamp); err != nil { return domain.RunJobResultResult{}, err } lease.TerminalFingerprint = fingerprint lease.UpdatedAt = stamp svc.jobLeases[job.ID] = lease return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil } func (svc *CoreService) RequestRunJobCancel(request domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error) { if err := validator.ValidateRunJobCancelRequest(request); err != nil { return domain.RunJobCancelRequestResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() job, err := svc.store.Jobs().Get(request.JobID) if err != nil { return domain.RunJobCancelRequestResult{}, err } if !isActiveJobState(job.State) { return domain.RunJobCancelRequestResult{}, validationError("job is not active") } lease, exists := svc.jobLeases[job.ID] if !exists { return domain.RunJobCancelRequestResult{}, validationError("job lease is missing") } lease.CancelReason = request.Reason lease.CancelRequestedAt = stamp lease.UpdatedAt = stamp svc.jobLeases[job.ID] = lease return domain.RunJobCancelRequestResult{Accepted: true, JobID: job.ID, Reason: request.Reason, RequestedAt: stamp}, nil } func (svc *CoreService) PollRunJobCancel(poll domain.RunJobCancelPoll) (domain.RunJobCancelPollResult, error) { if err := validator.ValidateRunJobCancelPoll(poll); err != nil { return domain.RunJobCancelPollResult{}, err } if err := svc.validateRunSession(poll.RunEndpointID, poll.SessionToken); err != nil { return domain.RunJobCancelPollResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() lease, ok := svc.findCancelLease(poll) if !ok { return domain.RunJobCancelPollResult{Accepted: true, RunEndpointID: poll.RunEndpointID, ServerTime: stamp}, nil } return domain.RunJobCancelPollResult{ Accepted: true, RunEndpointID: poll.RunEndpointID, HasCancel: true, JobID: lease.JobID, Reason: lease.CancelReason, RequestedAt: lease.CancelRequestedAt, ServerTime: stamp, }, nil } func (svc *CoreService) ReconcileRunJobs(reconcile domain.RunJobReconcile) (domain.RunJobReconcileResult, error) { reconcile = domain.CopyRunJobReconcile(reconcile) if err := validator.ValidateRunJobReconcile(reconcile); err != nil { return domain.RunJobReconcileResult{}, err } if err := svc.validateRunSession(reconcile.RunEndpointID, reconcile.SessionToken); err != nil { return domain.RunJobReconcileResult{}, err } stamp := svc.now() svc.jobMu.Lock() defer svc.jobMu.Unlock() jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: reconcile.RunEndpointID}) if err != nil { return domain.RunJobReconcileResult{}, err } activeByID := map[string]domain.Job{} for _, job := range jobs { if isActiveJobState(job.State) { activeByID[job.ID] = job } } activeJobs := make([]domain.RunJobAssignment, 0, len(activeByID)) ids := make([]string, 0, len(activeByID)) for id := range activeByID { ids = append(ids, id) } sort.Strings(ids) for _, id := range ids { job := activeByID[id] lease := svc.jobLeases[job.ID] if lease.JobID == "" || lease.SessionToken != reconcile.SessionToken { lease = svc.newJobLease(job.ID, reconcile.RunEndpointID, reconcile.SessionToken, stamp) } else { lease.UpdatedAt = stamp } svc.jobLeases[job.ID] = lease activeJobs = append(activeJobs, assignmentFromJob(job, lease)) } unknown := make([]string, 0) for _, reportedID := range reconcile.ActiveJobIDs { if _, exists := activeByID[reportedID]; !exists { unknown = append(unknown, reportedID) } } sort.Strings(unknown) return domain.CopyRunJobReconcileResult(domain.RunJobReconcileResult{ Accepted: true, RunEndpointID: reconcile.RunEndpointID, ActiveJobs: activeJobs, UnknownJobIDs: unknown, ServerTime: stamp, }), nil } func (svc *CoreService) validateRunSession(runEndpointID string, sessionToken string) error { svc.controlMu.Lock() defer svc.controlMu.Unlock() session, exists := svc.runSessions[runEndpointID] if !exists || session.SessionToken != sessionToken { return validationError("sessionToken is invalid") } return nil } func (svc *CoreService) newJobLease(jobID string, runEndpointID string, sessionToken string, stamp time.Time) domain.RunJobLease { svc.jobLeaseSeq++ return domain.RunJobLease{ JobID: jobID, RunEndpointID: runEndpointID, SessionToken: sessionToken, LeaseToken: fmt.Sprintf("job-lease:%s:%d:%d", jobID, stamp.UnixNano(), svc.jobLeaseSeq), Attempt: int(svc.jobLeaseSeq), CreatedAt: stamp, UpdatedAt: stamp, } } func (svc *CoreService) activeLeasedJob(runEndpointID string, sessionToken string, jobID string, leaseToken string, attempt int) (domain.Job, domain.RunJobLease, error) { job, err := svc.store.Jobs().Get(jobID) if err != nil { return domain.Job{}, domain.RunJobLease{}, err } if job.RunEndpointID != runEndpointID { return domain.Job{}, domain.RunJobLease{}, validationError("job runEndpointId does not match request") } lease, exists := svc.jobLeases[jobID] if !exists || lease.SessionToken != sessionToken || lease.LeaseToken != leaseToken || lease.Attempt != attempt { return domain.Job{}, domain.RunJobLease{}, validationError("leaseToken is invalid") } return job, lease, nil } func (svc *CoreService) findCancelLease(poll domain.RunJobCancelPoll) (domain.RunJobLease, bool) { if poll.JobID != "" { lease, exists := svc.jobLeases[poll.JobID] if !exists || lease.RunEndpointID != poll.RunEndpointID || lease.SessionToken != poll.SessionToken { return domain.RunJobLease{}, false } if poll.LeaseToken != "" && lease.LeaseToken != poll.LeaseToken { return domain.RunJobLease{}, false } return lease, lease.CancelReason != "" } ids := make([]string, 0, len(svc.jobLeases)) for id := range svc.jobLeases { ids = append(ids, id) } sort.Strings(ids) for _, id := range ids { lease := svc.jobLeases[id] if lease.RunEndpointID == poll.RunEndpointID && lease.SessionToken == poll.SessionToken && lease.CancelReason != "" { return lease, true } } return domain.RunJobLease{}, false } func firstSupportedJob(jobs []domain.Job, capabilities []string) (domain.Job, bool) { capabilitySet := map[string]struct{}{} for _, capability := range capabilities { capabilitySet[capability] = struct{}{} } for _, job := range jobs { if len(capabilitySet) == 0 { return job, true } if _, supported := capabilitySet[job.Capability]; supported { return job, true } } return domain.Job{}, false } func assignmentFromJob(job domain.Job, lease domain.RunJobLease) domain.RunJobAssignment { return domain.RunJobAssignment{ JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, Capability: job.Capability, TargetKey: job.TargetKey, InputRef: job.InputRef, IdempotencyKey: job.IdempotencyKey, State: job.State, Progress: domain.RunJobProgressReport{Percent: job.Progress.Percent, Message: job.Progress.Message}, ResultRef: job.ResultRef, LeaseToken: lease.LeaseToken, Attempt: lease.Attempt, CreatedAt: job.CreatedAt, UpdatedAt: job.UpdatedAt, } } func terminalFingerprint(result domain.RunJobResult) string { return fmt.Sprintf("%s|%d|%s|%s|%s|%s", result.State, result.Progress.Percent, result.ResultRef, result.Message, result.ErrorCode, result.Progress.Message) } func terminalMessage(result domain.RunJobResult) string { if strings.TrimSpace(result.Message) != "" { return result.Message } return result.Progress.Message } func isActiveJobState(state domain.JobState) bool { return state == domain.JobStateAccepted || state == domain.JobStateRunning } func isTerminalJobState(state domain.JobState) bool { return state == domain.JobStateSucceeded || state == domain.JobStateFailed || state == domain.JobStateCancelled }