init
This commit is contained in:
npc0-hue
@@ -0,0 +1,277 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/run/protocol"
|
||||
)
|
||||
|
||||
const dataTargetSnapshotManifestVersion = 1
|
||||
|
||||
type dataTargetSnapshotManifest struct {
|
||||
Version int `json:"version"`
|
||||
DataTargetKey string `json:"dataTargetKey"`
|
||||
Kind string `json:"kind"`
|
||||
WorkspaceKey string `json:"workspaceKey"`
|
||||
SourceRootKey string `json:"sourceRootKey"`
|
||||
SourcePathFingerprint string `json:"sourcePathFingerprint"`
|
||||
SourceSizeBytes int64 `json:"sourceSizeBytes"`
|
||||
SourceModUnixNano int64 `json:"sourceModUnixNano"`
|
||||
SnapshotSizeBytes int64 `json:"snapshotSizeBytes"`
|
||||
SnapshotChecksum string `json:"snapshotChecksum"`
|
||||
MaterializedAt time.Time `json:"materializedAt"`
|
||||
}
|
||||
|
||||
type dataTargetMaterializeError struct {
|
||||
code string
|
||||
retryable bool
|
||||
}
|
||||
|
||||
func (err dataTargetMaterializeError) Error() string { return err.code }
|
||||
|
||||
func newDataTargetMaterializeError(code string, retryable bool) error {
|
||||
return dataTargetMaterializeError{code: code, retryable: retryable}
|
||||
}
|
||||
|
||||
func sqliteProbeFailureForDataTarget(assignment protocol.RunJobAssignment, err error) LifecycleExecutionResult {
|
||||
var materializeErr dataTargetMaterializeError
|
||||
if errors.As(err, &materializeErr) {
|
||||
return sqliteSchemaProbeFailure(assignment, materializeErr.code, materializeErr.retryable)
|
||||
}
|
||||
return sqliteSchemaProbeFailure(assignment, "data_target_unavailable", true)
|
||||
}
|
||||
|
||||
func (worker *Worker) materializeSQLiteProbeDataTarget(ctx context.Context, assignment protocol.RunJobAssignment) (string, error) {
|
||||
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
|
||||
if err != nil {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_plan_invalid", false)
|
||||
}
|
||||
if !ok || plan == nil || len(plan.DataTargets) == 0 {
|
||||
return assignment.TargetKey, nil
|
||||
}
|
||||
if plan.ServerInstanceID != assignment.ServerInstanceID || plan.RunEndpointID != assignment.RunEndpointID || plan.PluginID != assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
||||
}
|
||||
if plan.ProfileKey != "" && assignment.ExecutionInput.WorkspaceScope != "" && plan.ProfileKey != assignment.ExecutionInput.WorkspaceScope {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
||||
}
|
||||
var matched *protocol.RunAutonomousDataTarget
|
||||
for i := range plan.DataTargets {
|
||||
target := &plan.DataTargets[i]
|
||||
if dataTargetMatchesSQLiteProbeAssignment(*target, assignment.TargetKey) {
|
||||
matched = target
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched == nil {
|
||||
return assignment.TargetKey, nil
|
||||
}
|
||||
if !dataTargetSupportsPlatform(matched.Platforms, runtime.GOOS) {
|
||||
return assignment.TargetKey, newDataTargetMaterializeError("data_target_platform_unsupported", false)
|
||||
}
|
||||
manifest, err := materializeSQLiteSnapshotDataTarget(ctx, worker.cfg.WorkspaceRoot, assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope, *matched, plan.RuntimeBindings)
|
||||
if err != nil {
|
||||
log.Printf("RUN phase=data_target.snapshot status=failed target=%s code=%s", safeOptional(matched.Key), safeOptional(dataTargetErrorCode(err)))
|
||||
return assignment.TargetKey, err
|
||||
}
|
||||
log.Printf("RUN phase=data_target.snapshot status=complete target=%s workspaceKey=%s bytes=%d checksum=%s", safeOptional(matched.Key), safeOptional(matched.WorkspaceKey), manifest.SnapshotSizeBytes, safeOptional(manifest.SnapshotChecksum))
|
||||
return matched.WorkspaceKey, nil
|
||||
}
|
||||
|
||||
func dataTargetMatchesSQLiteProbeAssignment(target protocol.RunAutonomousDataTarget, assignmentTargetKey string) bool {
|
||||
if target.Kind != "sqlite.snapshot" || assignmentTargetKey == "" {
|
||||
return false
|
||||
}
|
||||
return target.Key == assignmentTargetKey || target.TransportKey == assignmentTargetKey || target.WorkspaceKey == assignmentTargetKey
|
||||
}
|
||||
|
||||
func materializeSQLiteSnapshotDataTarget(ctx context.Context, workspaceRoot string, serverInstanceID string, profileKey string, target protocol.RunAutonomousDataTarget, bindings map[string]string) (dataTargetSnapshotManifest, error) {
|
||||
if target.Kind != "sqlite.snapshot" || target.RefreshPolicy != "on-demand-snapshot" || target.MaxBytes <= 0 {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_invalid", false)
|
||||
}
|
||||
sourceRoot := strings.TrimSpace(bindings[target.SourceRootKey])
|
||||
if sourceRoot == "" {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_source_unbound", false)
|
||||
}
|
||||
source, sourceInfo, err := resolveDataTargetSource(sourceRoot, target.SourcePath, target.MaxBytes)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
scope, err := NewWorkspaceResolver(workspaceRoot).Scope(serverInstanceID, profileKey)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_scope_invalid", false)
|
||||
}
|
||||
destination, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, target.WorkspaceKey)
|
||||
if err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_invalid", false)
|
||||
}
|
||||
if err := ensureDirectory(filepath.Dir(destination)); err != nil {
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
||||
}
|
||||
temporary := filepath.Join(filepath.Dir(destination), "."+safeWorkspaceName(filepath.Base(target.WorkspaceKey))+".snapshot.tmp")
|
||||
_ = os.Remove(temporary)
|
||||
if err := snapshotSQLiteDatabase(ctx, source, temporary, target.MaxBytes); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
snapshotInfo, err := os.Stat(temporary)
|
||||
if err != nil || !snapshotInfo.Mode().IsRegular() || snapshotInfo.Size() <= 0 || snapshotInfo.Size() > target.MaxBytes {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_snapshot_invalid", true)
|
||||
}
|
||||
checksum, err := fingerprintSQLiteSource(temporary)
|
||||
if err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_checksum_failed", true)
|
||||
}
|
||||
if err := replaceRegularFile(temporary, destination); err != nil {
|
||||
_ = os.Remove(temporary)
|
||||
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
||||
}
|
||||
manifest := dataTargetSnapshotManifest{
|
||||
Version: dataTargetSnapshotManifestVersion,
|
||||
DataTargetKey: target.Key,
|
||||
Kind: target.Kind,
|
||||
WorkspaceKey: target.WorkspaceKey,
|
||||
SourceRootKey: target.SourceRootKey,
|
||||
SourcePathFingerprint: digestValue(target.SourcePath),
|
||||
SourceSizeBytes: sourceInfo.Size(),
|
||||
SourceModUnixNano: sourceInfo.ModTime().UnixNano(),
|
||||
SnapshotSizeBytes: snapshotInfo.Size(),
|
||||
SnapshotChecksum: checksum,
|
||||
MaterializedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := writeDataTargetSnapshotManifest(workspaceRoot, scope, target.WorkspaceKey, manifest); err != nil {
|
||||
return dataTargetSnapshotManifest{}, err
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func resolveDataTargetSource(sourceRoot string, sourcePath string, maxBytes int64) (string, os.FileInfo, error) {
|
||||
if strings.TrimSpace(sourceRoot) == "" || strings.TrimSpace(sourcePath) == "" || filepath.IsAbs(sourcePath) || strings.Contains(sourcePath, `\`) || strings.Contains(sourcePath, "..") || !protocol.ValidLogicalFileKey(sourcePath) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
cleanRoot := filepath.Clean(sourceRoot)
|
||||
if !filepath.IsAbs(cleanRoot) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
source := filepath.Join(cleanRoot, filepath.FromSlash(sourcePath))
|
||||
rel, err := filepath.Rel(cleanRoot, source)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
info, err := os.Lstat(source)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_missing", true)
|
||||
}
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_unavailable", true)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
||||
}
|
||||
if info.Size() <= 0 || info.Size() > maxBytes {
|
||||
return "", nil, newDataTargetMaterializeError("data_target_source_size_invalid", false)
|
||||
}
|
||||
return source, info, nil
|
||||
}
|
||||
|
||||
func snapshotSQLiteDatabase(ctx context.Context, source string, destination string, maxBytes int64) error {
|
||||
snapshotCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+source+"?mode=ro")
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_sqlite_open_failed", true)
|
||||
}
|
||||
defer database.Close()
|
||||
database.SetMaxOpenConns(1)
|
||||
if _, err := database.ExecContext(snapshotCtx, "VACUUM INTO "+quoteSQLiteStringLiteral(destination)); err != nil {
|
||||
return dataTargetSQLiteError(snapshotCtx, err)
|
||||
}
|
||||
info, err := os.Stat(destination)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_snapshot_unavailable", true)
|
||||
}
|
||||
if info.Size() <= 0 || info.Size() > maxBytes {
|
||||
return newDataTargetMaterializeError("data_target_snapshot_limit_exceeded", false)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataTargetSQLiteError(ctx context.Context, err error) error {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return newDataTargetMaterializeError("data_target_cancelled", true)
|
||||
}
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return newDataTargetMaterializeError("data_target_timeout", true)
|
||||
}
|
||||
lower := strings.ToLower(fmt.Sprint(err))
|
||||
if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") {
|
||||
return newDataTargetMaterializeError("data_target_busy", true)
|
||||
}
|
||||
return newDataTargetMaterializeError("data_target_snapshot_failed", true)
|
||||
}
|
||||
|
||||
func quoteSQLiteStringLiteral(value string) string {
|
||||
return `'` + strings.ReplaceAll(value, `'`, `''`) + `'`
|
||||
}
|
||||
|
||||
func replaceRegularFile(source string, destination string) error {
|
||||
if info, err := os.Lstat(destination); err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("destination is not a regular file")
|
||||
}
|
||||
if err := os.Remove(destination); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return os.Rename(source, destination)
|
||||
}
|
||||
|
||||
func writeDataTargetSnapshotManifest(workspaceRoot string, scope string, workspaceKey string, manifest dataTargetSnapshotManifest) error {
|
||||
manifestKey := workspaceKey + ".snapshot.json"
|
||||
path, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, manifestKey)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
||||
}
|
||||
body, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
||||
}
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
return newDataTargetMaterializeError("data_target_manifest_failed", true)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataTargetSupportsPlatform(platforms []string, targetOS string) bool {
|
||||
if len(platforms) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, platform := range platforms {
|
||||
if platform == targetOS {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dataTargetErrorCode(err error) string {
|
||||
var materializeErr dataTargetMaterializeError
|
||||
if errors.As(err, &materializeErr) {
|
||||
return materializeErr.code
|
||||
}
|
||||
return "data_target_unavailable"
|
||||
}
|
||||
Reference in New Issue
Block a user