278 lines
11 KiB
Go
278 lines
11 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
const dataTargetSnapshotManifestVersion = 1
|
|
|
|
type dataTargetSnapshotManifest struct {
|
|
Version int `json:"version"`
|
|
DataTargetKey string `json:"dataTargetKey"`
|
|
Kind string `json:"kind"`
|
|
WorkspaceKey string `json:"workspaceKey"`
|
|
SourceRootKey string `json:"sourceRootKey"`
|
|
SourcePathFingerprint string `json:"sourcePathFingerprint"`
|
|
SourceSizeBytes int64 `json:"sourceSizeBytes"`
|
|
SourceModUnixNano int64 `json:"sourceModUnixNano"`
|
|
SnapshotSizeBytes int64 `json:"snapshotSizeBytes"`
|
|
SnapshotChecksum string `json:"snapshotChecksum"`
|
|
MaterializedAt time.Time `json:"materializedAt"`
|
|
}
|
|
|
|
type dataTargetMaterializeError struct {
|
|
code string
|
|
retryable bool
|
|
}
|
|
|
|
func (err dataTargetMaterializeError) Error() string { return err.code }
|
|
|
|
func newDataTargetMaterializeError(code string, retryable bool) error {
|
|
return dataTargetMaterializeError{code: code, retryable: retryable}
|
|
}
|
|
|
|
func sqliteProbeFailureForDataTarget(assignment protocol.RunJobAssignment, err error) LifecycleExecutionResult {
|
|
var materializeErr dataTargetMaterializeError
|
|
if errors.As(err, &materializeErr) {
|
|
return sqliteSchemaProbeFailure(assignment, materializeErr.code, materializeErr.retryable)
|
|
}
|
|
return sqliteSchemaProbeFailure(assignment, "data_target_unavailable", true)
|
|
}
|
|
|
|
func (worker *Worker) materializeSQLiteProbeDataTarget(ctx context.Context, assignment protocol.RunJobAssignment) (string, error) {
|
|
plan, _, ok, err := LoadAutonomousLifecyclePlan(worker.cfg)
|
|
if err != nil {
|
|
return assignment.TargetKey, newDataTargetMaterializeError("data_target_plan_invalid", false)
|
|
}
|
|
if !ok || plan == nil || len(plan.DataTargets) == 0 {
|
|
return assignment.TargetKey, nil
|
|
}
|
|
if plan.ServerInstanceID != assignment.ServerInstanceID || plan.RunEndpointID != assignment.RunEndpointID || plan.PluginID != assignment.ExecutionInput.SQLiteSchemaProbe.Binding.PluginID {
|
|
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
|
}
|
|
if plan.ProfileKey != "" && assignment.ExecutionInput.WorkspaceScope != "" && plan.ProfileKey != assignment.ExecutionInput.WorkspaceScope {
|
|
return assignment.TargetKey, newDataTargetMaterializeError("data_target_scope_mismatch", false)
|
|
}
|
|
var matched *protocol.RunAutonomousDataTarget
|
|
for i := range plan.DataTargets {
|
|
target := &plan.DataTargets[i]
|
|
if dataTargetMatchesSQLiteProbeAssignment(*target, assignment.TargetKey) {
|
|
matched = target
|
|
break
|
|
}
|
|
}
|
|
if matched == nil {
|
|
return assignment.TargetKey, nil
|
|
}
|
|
if !dataTargetSupportsPlatform(matched.Platforms, runtime.GOOS) {
|
|
return assignment.TargetKey, newDataTargetMaterializeError("data_target_platform_unsupported", false)
|
|
}
|
|
manifest, err := materializeSQLiteSnapshotDataTarget(ctx, worker.cfg.WorkspaceRoot, assignment.ServerInstanceID, assignment.ExecutionInput.WorkspaceScope, *matched, plan.RuntimeBindings)
|
|
if err != nil {
|
|
log.Printf("RUN phase=data_target.snapshot status=failed target=%s code=%s", safeOptional(matched.Key), safeOptional(dataTargetErrorCode(err)))
|
|
return assignment.TargetKey, err
|
|
}
|
|
log.Printf("RUN phase=data_target.snapshot status=complete target=%s workspaceKey=%s bytes=%d checksum=%s", safeOptional(matched.Key), safeOptional(matched.WorkspaceKey), manifest.SnapshotSizeBytes, safeOptional(manifest.SnapshotChecksum))
|
|
return matched.WorkspaceKey, nil
|
|
}
|
|
|
|
func dataTargetMatchesSQLiteProbeAssignment(target protocol.RunAutonomousDataTarget, assignmentTargetKey string) bool {
|
|
if target.Kind != "sqlite.snapshot" || assignmentTargetKey == "" {
|
|
return false
|
|
}
|
|
return target.Key == assignmentTargetKey || target.TransportKey == assignmentTargetKey || target.WorkspaceKey == assignmentTargetKey
|
|
}
|
|
|
|
func materializeSQLiteSnapshotDataTarget(ctx context.Context, workspaceRoot string, serverInstanceID string, profileKey string, target protocol.RunAutonomousDataTarget, bindings map[string]string) (dataTargetSnapshotManifest, error) {
|
|
if target.Kind != "sqlite.snapshot" || target.RefreshPolicy != "on-demand-snapshot" || target.MaxBytes <= 0 {
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_invalid", false)
|
|
}
|
|
sourceRoot := strings.TrimSpace(bindings[target.SourceRootKey])
|
|
if sourceRoot == "" {
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_source_unbound", false)
|
|
}
|
|
source, sourceInfo, err := resolveDataTargetSource(sourceRoot, target.SourcePath, target.MaxBytes)
|
|
if err != nil {
|
|
return dataTargetSnapshotManifest{}, err
|
|
}
|
|
scope, err := NewWorkspaceResolver(workspaceRoot).Scope(serverInstanceID, profileKey)
|
|
if err != nil {
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_scope_invalid", false)
|
|
}
|
|
destination, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, target.WorkspaceKey)
|
|
if err != nil {
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_invalid", false)
|
|
}
|
|
if err := ensureDirectory(filepath.Dir(destination)); err != nil {
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
|
}
|
|
temporary := filepath.Join(filepath.Dir(destination), "."+safeWorkspaceName(filepath.Base(target.WorkspaceKey))+".snapshot.tmp")
|
|
_ = os.Remove(temporary)
|
|
if err := snapshotSQLiteDatabase(ctx, source, temporary, target.MaxBytes); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return dataTargetSnapshotManifest{}, err
|
|
}
|
|
snapshotInfo, err := os.Stat(temporary)
|
|
if err != nil || !snapshotInfo.Mode().IsRegular() || snapshotInfo.Size() <= 0 || snapshotInfo.Size() > target.MaxBytes {
|
|
_ = os.Remove(temporary)
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_snapshot_invalid", true)
|
|
}
|
|
checksum, err := fingerprintSQLiteSource(temporary)
|
|
if err != nil {
|
|
_ = os.Remove(temporary)
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_checksum_failed", true)
|
|
}
|
|
if err := replaceRegularFile(temporary, destination); err != nil {
|
|
_ = os.Remove(temporary)
|
|
return dataTargetSnapshotManifest{}, newDataTargetMaterializeError("data_target_workspace_unavailable", true)
|
|
}
|
|
manifest := dataTargetSnapshotManifest{
|
|
Version: dataTargetSnapshotManifestVersion,
|
|
DataTargetKey: target.Key,
|
|
Kind: target.Kind,
|
|
WorkspaceKey: target.WorkspaceKey,
|
|
SourceRootKey: target.SourceRootKey,
|
|
SourcePathFingerprint: digestValue(target.SourcePath),
|
|
SourceSizeBytes: sourceInfo.Size(),
|
|
SourceModUnixNano: sourceInfo.ModTime().UnixNano(),
|
|
SnapshotSizeBytes: snapshotInfo.Size(),
|
|
SnapshotChecksum: checksum,
|
|
MaterializedAt: time.Now().UTC(),
|
|
}
|
|
if err := writeDataTargetSnapshotManifest(workspaceRoot, scope, target.WorkspaceKey, manifest); err != nil {
|
|
return dataTargetSnapshotManifest{}, err
|
|
}
|
|
return manifest, nil
|
|
}
|
|
|
|
func resolveDataTargetSource(sourceRoot string, sourcePath string, maxBytes int64) (string, os.FileInfo, error) {
|
|
if strings.TrimSpace(sourceRoot) == "" || strings.TrimSpace(sourcePath) == "" || filepath.IsAbs(sourcePath) || strings.Contains(sourcePath, `\`) || strings.Contains(sourcePath, "..") || !protocol.ValidLogicalFileKey(sourcePath) {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
|
}
|
|
cleanRoot := filepath.Clean(sourceRoot)
|
|
if !filepath.IsAbs(cleanRoot) {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
|
}
|
|
source := filepath.Join(cleanRoot, filepath.FromSlash(sourcePath))
|
|
rel, err := filepath.Rel(cleanRoot, source)
|
|
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
|
}
|
|
info, err := os.Lstat(source)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_missing", true)
|
|
}
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_unavailable", true)
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_invalid", false)
|
|
}
|
|
if info.Size() <= 0 || info.Size() > maxBytes {
|
|
return "", nil, newDataTargetMaterializeError("data_target_source_size_invalid", false)
|
|
}
|
|
return source, info, nil
|
|
}
|
|
|
|
func snapshotSQLiteDatabase(ctx context.Context, source string, destination string, maxBytes int64) error {
|
|
snapshotCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
database, err := sql.Open(sqliteSchemaProbeDriver, "file:"+source+"?mode=ro")
|
|
if err != nil {
|
|
return newDataTargetMaterializeError("data_target_sqlite_open_failed", true)
|
|
}
|
|
defer database.Close()
|
|
database.SetMaxOpenConns(1)
|
|
if _, err := database.ExecContext(snapshotCtx, "VACUUM INTO "+quoteSQLiteStringLiteral(destination)); err != nil {
|
|
return dataTargetSQLiteError(snapshotCtx, err)
|
|
}
|
|
info, err := os.Stat(destination)
|
|
if err != nil {
|
|
return newDataTargetMaterializeError("data_target_snapshot_unavailable", true)
|
|
}
|
|
if info.Size() <= 0 || info.Size() > maxBytes {
|
|
return newDataTargetMaterializeError("data_target_snapshot_limit_exceeded", false)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func dataTargetSQLiteError(ctx context.Context, err error) error {
|
|
if errors.Is(ctx.Err(), context.Canceled) {
|
|
return newDataTargetMaterializeError("data_target_cancelled", true)
|
|
}
|
|
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
|
return newDataTargetMaterializeError("data_target_timeout", true)
|
|
}
|
|
lower := strings.ToLower(fmt.Sprint(err))
|
|
if strings.Contains(lower, "locked") || strings.Contains(lower, "busy") {
|
|
return newDataTargetMaterializeError("data_target_busy", true)
|
|
}
|
|
return newDataTargetMaterializeError("data_target_snapshot_failed", true)
|
|
}
|
|
|
|
func quoteSQLiteStringLiteral(value string) string {
|
|
return `'` + strings.ReplaceAll(value, `'`, `''`) + `'`
|
|
}
|
|
|
|
func replaceRegularFile(source string, destination string) error {
|
|
if info, err := os.Lstat(destination); err == nil {
|
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
return fmt.Errorf("destination is not a regular file")
|
|
}
|
|
if err := os.Remove(destination); err != nil {
|
|
return err
|
|
}
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
return os.Rename(source, destination)
|
|
}
|
|
|
|
func writeDataTargetSnapshotManifest(workspaceRoot string, scope string, workspaceKey string, manifest dataTargetSnapshotManifest) error {
|
|
manifestKey := workspaceKey + ".snapshot.json"
|
|
path, _, err := NewWorkspaceResolver(workspaceRoot).WritableTarget(scope, manifestKey)
|
|
if err != nil {
|
|
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
|
}
|
|
body, err := json.Marshal(manifest)
|
|
if err != nil {
|
|
return newDataTargetMaterializeError("data_target_manifest_invalid", false)
|
|
}
|
|
if err := os.WriteFile(path, body, 0o600); err != nil {
|
|
return newDataTargetMaterializeError("data_target_manifest_failed", true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func dataTargetSupportsPlatform(platforms []string, targetOS string) bool {
|
|
if len(platforms) == 0 {
|
|
return true
|
|
}
|
|
for _, platform := range platforms {
|
|
if platform == targetOS {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func dataTargetErrorCode(err error) string {
|
|
var materializeErr dataTargetMaterializeError
|
|
if errors.As(err, &materializeErr) {
|
|
return materializeErr.code
|
|
}
|
|
return "data_target_unavailable"
|
|
}
|