admin343/run
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add persistent job claim and file artifact reads

Browse Source
This commit is contained in:
npc0-hue
2026-08-26 22:01:55 +08:00
parent bb051f6da1
commit 3b4e857ef7
8 changed files with 388 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/worker.go
+114 -28
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
@@ -46,7 +47,10 @@ type WorkerClient interface {
const (
jobActivePollInterval = 10 * time.Second
jobClaimWaitSeconds = 25
durableUploaderFlushTimeout = 5 * time.Second
fileArtifactChunkSize = 1024 * 1024
maxFileArtifactBytes = int64(512 * 1024 * 1024)
)
type Worker struct {
@@ -436,6 +440,7 @@ func (worker *Worker) ClaimAndRunOnce(ctx context.Context) (bool, error) {
SessionToken: state.SessionToken,
Capabilities: state.Capabilities,
Capacity: worker.capacityReportFor(state),
WaitSeconds: jobClaimWaitSeconds,
})
if err != nil {
if sessionInvalidError(err) {
@@ -728,7 +733,7 @@ func (worker *Worker) executeAssignment(ctx context.Context, assignment protocol
}
if supportedCapability(SupportedFileCapabilities(), assignment.Capability) {
log.Printf("RUN phase=job.dispatch status=selected job=%s executor=file", assignment.JobID)
return worker.executor.ExecuteContext(ctx, assignment)
return worker.executeFileJob(ctx, assignment)
}
if isSupportedDistributionCapability(assignment.Capability) {
log.Printf("RUN phase=job.dispatch status=selected job=%s executor=distribution", assignment.JobID)
@@ -742,6 +747,50 @@ func (worker *Worker) executeAssignment(ctx context.Context, assignment protocol
return lifecycleFailure("unsupported_run_capability", "unsupported run capability")
}
func (worker *Worker) executeFileJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
if assignment.Capability != protocol.RunCapabilityFilesRead {
return worker.executor.ExecuteContext(ctx, assignment)
}
fileExecutor := worker.executor.fileExecutor
if fileExecutor == nil {
return lifecycleExecutionFailure("file_executor_unavailable", "file executor is unavailable", false)
}
scope, targetPath, deploymentRoot, err := fileExecutor.existingReadTargetForAssignment(assignment)
if err != nil {
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
}
info, err := os.Stat(targetPath)
if err != nil {
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
}
limit := assignment.ExecutionInput.MaxReadBytes
if limit <= 0 || limit > maxExecutionContentBytes {
limit = maxExecutionContentBytes
}
if info.Size() <= int64(limit) {
return fileExecutor.read(ctx, scope, deploymentRoot, assignment)
}
if info.Size() > maxFileArtifactBytes {
return lifecycleExecutionFailure("file_read_too_large", "file exceeds artifact transfer limit", false)
}
checksum, err := checksumServerFileForArtifact(ctx, targetPath)
if err != nil {
if errors.Is(err, context.Canceled) {
return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false)
}
return lifecycleExecutionFailure("file_read_failed", err.Error(), false)
}
artifactID := "artifact-" + safeWorkspaceName(assignment.JobID) + "-file-read"
if err := worker.uploadFileArtifact(ctx, assignment, artifactID, targetPath, info.Size(), checksum); err != nil {
if errors.Is(err, context.Canceled) {
return lifecycleExecutionFailure("file_cancelled", "file read cancelled", false)
}
return lifecycleExecutionFailure("file_artifact_upload_failed", "file artifact upload failed", false)
}
metadata := fileExecutor.metadata(scope, assignment.TargetKey, checksum, info.Size())
return LifecycleExecutionResult{State: lifecycleResultStateSucceeded, Progress: protocol.RunJobProgressReport{Percent: 100, Message: "file read completed"}, ResultRef: "artifact://" + artifactID, Message: "file read completed", ExecutionResult: protocol.RunJobExecutionResult{Kind: "file.read", Version: metadata.Version, Checksum: checksum, SizeBytes: info.Size(), Summary: "large file transferred as artifact"}}
}
func (worker *Worker) executeProtectedRequestJob(ctx context.Context, assignment protocol.RunJobAssignment) LifecycleExecutionResult {
if err := protocol.ValidateRunJobAssignment(assignment); err != nil {
return protectedRequestFailure(assignment.Capability, ProtectedRequestStatusFailed, "protected_request_assignment_invalid")
@@ -988,42 +1037,66 @@ func (worker *Worker) reportRunUpdateHealth(ctx context.Context) error {
}
func (worker *Worker) runJobLoop(ctx context.Context, interval time.Duration) error {
log.Printf("RUN phase=job_loop status=starting pollMs=%d", interval.Milliseconds())
ticker := time.NewTicker(interval)
defer ticker.Stop()
log.Printf("RUN phase=job_loop status=starting fallbackPollMs=%d claimWaitSeconds=%d", interval.Milliseconds(), jobClaimWaitSeconds)
for {
select {
case <-ctx.Done():
log.Printf("RUN phase=job_loop status=context_done error=%s", RedactText(ctx.Err().Error()))
return ctx.Err()
case <-ticker.C:
if worker.journal.ActiveCount() > 0 {
log.Printf("RUN phase=job_loop status=active_jobs activeJobs=%d", worker.journal.ActiveCount())
if err := worker.ReconcileOnce(ctx); err != nil {
log.Printf("RUN phase=job_loop status=reconcile_failed error=%s", RedactText(err.Error()))
ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
continue
default:
}
if worker.journal.ActiveCount() > 0 {
log.Printf("RUN phase=job_loop status=active_jobs activeJobs=%d", worker.journal.ActiveCount())
if err := worker.ReconcileOnce(ctx); err != nil {
log.Printf("RUN phase=job_loop status=reconcile_failed error=%s", RedactText(err.Error()))
if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil {
return err
}
if err := worker.RecoverActiveJobs(ctx); err != nil {
log.Printf("RUN phase=job_loop status=recover_failed error=%s", RedactText(err.Error()))
ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
continue
}
}
if _, err := worker.ClaimAndRunOnce(ctx); err != nil {
log.Printf("RUN phase=job_loop status=claim_failed error=%s", RedactText(err.Error()))
ticker.Reset(boundedRetryBackoff(worker.cfg.RetryBackoff))
continue
}
worker.restartMu.Lock()
restartRequested := worker.restartRequested
worker.restartMu.Unlock()
if restartRequested {
log.Printf("RUN phase=job_loop status=restart_requested")
return ErrSelfUpdateRestartRequested
if err := worker.RecoverActiveJobs(ctx); err != nil {
log.Printf("RUN phase=job_loop status=recover_failed error=%s", RedactText(err.Error()))
if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil {
return err
}
continue
}
ticker.Reset(interval)
}
claimStartedAt := time.Now()
handled, err := worker.ClaimAndRunOnce(ctx)
if err != nil {
log.Printf("RUN phase=job_loop status=claim_failed error=%s", RedactText(err.Error()))
if err := waitWorkerLoop(ctx, boundedRetryBackoff(worker.cfg.RetryBackoff)); err != nil {
return err
}
continue
}
worker.restartMu.Lock()
restartRequested := worker.restartRequested
worker.restartMu.Unlock()
if restartRequested {
log.Printf("RUN phase=job_loop status=restart_requested")
return ErrSelfUpdateRestartRequested
}
if !handled && time.Since(claimStartedAt) < interval {
if err := waitWorkerLoop(ctx, interval-time.Since(claimStartedAt)); err != nil {
return err
}
}
}
}
func waitWorkerLoop(ctx context.Context, delay time.Duration) error {
if delay <= 0 {
return nil
}
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
@@ -1112,6 +1185,11 @@ func logBatchSessionMetadataMismatchError(err error) bool {
return errors.As(err, &mismatch) && mismatch.LogBatchSessionMetadataMismatch()
}
func artifactChunkMissingOnPlatform(err error) bool {
var httpErr interface{ HTTPStatus() int }
return errors.As(err, &httpErr) && httpErr.HTTPStatus() == http.StatusNotFound
}
type sessionArtifactChunkClient struct {
client durableArtifactClient
runEndpointID string
@@ -1121,7 +1199,15 @@ type sessionArtifactChunkClient struct {
func (client sessionArtifactChunkClient) UploadArtifactChunk(ctx context.Context, chunk protocol.ArtifactChunkUploadRequest) (protocol.ArtifactChunkUploadResponse, error) {
chunk.RunEndpointID = client.runEndpointID
chunk.SessionToken = client.sessionToken
return client.client.UploadArtifactChunk(ctx, chunk)
response, err := client.client.UploadArtifactChunk(ctx, chunk)
if err != nil {
if artifactChunkMissingOnPlatform(err) {
log.Printf("RUN phase=durable_uploaders.artifacts status=drop_stale transfer=%s artifact=%s chunk=%d reason=platform_transfer_missing", safeOptional(chunk.TransferID), safeOptional(chunk.ArtifactID), chunk.ChunkIndex)
return protocol.ArtifactChunkUploadResponse{Accepted: true, TransferID: chunk.TransferID, ArtifactID: chunk.ArtifactID, ChunkIndex: chunk.ChunkIndex}, nil
}
return response, err
}
return response, nil
}
func (worker *Worker) runDurableUploaders(ctx context.Context, done chan<- struct{}) {