feat: 完整游戏运维功能
This commit is contained in:
npc0-hue
@@ -0,0 +1,226 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
type ArtifactBodyStore interface {
|
||||
SaveTransfer(domain.ArtifactTransferSession) error
|
||||
LoadTransfers() ([]domain.ArtifactTransferSession, error)
|
||||
PutPayload(string, []byte) error
|
||||
GetPayload(string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemoryArtifactBodyStore struct {
|
||||
mu sync.Mutex
|
||||
transfers map[string]domain.ArtifactTransferSession
|
||||
payloads map[string][]byte
|
||||
}
|
||||
|
||||
func NewMemoryArtifactBodyStore() *MemoryArtifactBodyStore {
|
||||
return &MemoryArtifactBodyStore{transfers: map[string]domain.ArtifactTransferSession{}, payloads: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func (store *MemoryArtifactBodyStore) SaveTransfer(session domain.ArtifactTransferSession) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
store.transfers[session.TransferID] = domain.CopyArtifactTransferSession(session)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryArtifactBodyStore) LoadTransfers() ([]domain.ArtifactTransferSession, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
ids := make([]string, 0, len(store.transfers))
|
||||
for id := range store.transfers {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := make([]domain.ArtifactTransferSession, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, domain.CopyArtifactTransferSession(store.transfers[id]))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (store *MemoryArtifactBodyStore) PutPayload(artifactID string, payload []byte) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
store.payloads[artifactID] = domain.CopyBytes(payload)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryArtifactBodyStore) GetPayload(artifactID string) ([]byte, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
payload, exists := store.payloads[artifactID]
|
||||
if !exists {
|
||||
return nil, repo.ErrNotFound
|
||||
}
|
||||
return domain.CopyBytes(payload), nil
|
||||
}
|
||||
|
||||
type FileArtifactBodyStore struct {
|
||||
mu sync.Mutex
|
||||
rootDir string
|
||||
}
|
||||
|
||||
func NewFileArtifactBodyStore(rootDir string) (*FileArtifactBodyStore, error) {
|
||||
rootDir = strings.TrimSpace(rootDir)
|
||||
if rootDir == "" {
|
||||
return nil, fmt.Errorf("artifact directory is required")
|
||||
}
|
||||
for _, path := range []string{rootDir, filepath.Join(rootDir, "transfers"), filepath.Join(rootDir, "payloads")} {
|
||||
if err := os.MkdirAll(path, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create artifact body directory: %w", err)
|
||||
}
|
||||
}
|
||||
return &FileArtifactBodyStore{rootDir: rootDir}, nil
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) SaveTransfer(session domain.ArtifactTransferSession) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
dir := store.transferDir(session.TransferID)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return fmt.Errorf("create artifact transfer directory: %w", err)
|
||||
}
|
||||
manifest := domain.CopyArtifactTransferSession(session)
|
||||
for index, record := range manifest.ReceivedChunks {
|
||||
payload := domain.CopyBytes(record.Payload)
|
||||
if len(payload) != record.SizeBytes || validator.BytesChecksum(payload) != record.Checksum {
|
||||
return validationError("artifact chunk does not match durable manifest")
|
||||
}
|
||||
if err := writeAtomicFile(filepath.Join(dir, fmt.Sprintf("chunk-%08d.bin", index)), payload, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
record.Payload = nil
|
||||
manifest.ReceivedChunks[index] = record
|
||||
}
|
||||
body, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode artifact transfer manifest: %w", err)
|
||||
}
|
||||
return writeAtomicFile(filepath.Join(dir, "manifest.json"), body, 0o600)
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) LoadTransfers() ([]domain.ArtifactTransferSession, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
entries, err := os.ReadDir(filepath.Join(store.rootDir, "transfers"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact transfer directory: %w", err)
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
|
||||
out := make([]domain.ArtifactTransferSession, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Join(store.rootDir, "transfers", entry.Name())
|
||||
body, err := os.ReadFile(filepath.Join(dir, "manifest.json"))
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("read artifact transfer manifest: %w", err)
|
||||
}
|
||||
var session domain.ArtifactTransferSession
|
||||
if err := json.Unmarshal(body, &session); err != nil {
|
||||
return nil, fmt.Errorf("decode artifact transfer manifest: %w", err)
|
||||
}
|
||||
if session.TransferID == "" || store.transferDir(session.TransferID) != dir {
|
||||
return nil, fmt.Errorf("artifact transfer manifest identity mismatch")
|
||||
}
|
||||
for index, record := range session.ReceivedChunks {
|
||||
payload, err := os.ReadFile(filepath.Join(dir, fmt.Sprintf("chunk-%08d.bin", index)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact transfer chunk: %w", err)
|
||||
}
|
||||
if len(payload) != record.SizeBytes || validator.BytesChecksum(payload) != record.Checksum {
|
||||
return nil, validationError("durable artifact chunk checksum mismatch")
|
||||
}
|
||||
record.Payload = payload
|
||||
session.ReceivedChunks[index] = record
|
||||
}
|
||||
out = append(out, domain.CopyArtifactTransferSession(session))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) PutPayload(artifactID string, payload []byte) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
return writeAtomicFile(store.payloadPath(artifactID), payload, 0o600)
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) GetPayload(artifactID string) ([]byte, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
payload, err := os.ReadFile(store.payloadPath(artifactID))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, repo.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact payload: %w", err)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) transferDir(transferID string) string {
|
||||
return filepath.Join(store.rootDir, "transfers", stableStorageKey(transferID))
|
||||
}
|
||||
|
||||
func (store *FileArtifactBodyStore) payloadPath(artifactID string) string {
|
||||
return filepath.Join(store.rootDir, "payloads", stableStorageKey(artifactID)+".bin")
|
||||
}
|
||||
|
||||
func stableStorageKey(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func writeAtomicFile(path string, payload []byte, mode os.FileMode) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return fmt.Errorf("create durable body directory: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open durable body temporary file: %w", err)
|
||||
}
|
||||
if _, err := file.Write(payload); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("write durable body: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("sync durable body: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("close durable body: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("replace durable body: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
@@ -8,10 +9,11 @@ import (
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const artifactDownloadStorageBehavior = "platform-memory-transfer-session"
|
||||
const artifactDownloadStorageBehavior = "platform-durable-artifact-store"
|
||||
|
||||
func (svc *CoreService) GetArtifactForSession(sessionID string, artifactID string) (domain.Artifact, error) {
|
||||
artifact, err := svc.store.Artifacts().Get(strings.TrimSpace(artifactID))
|
||||
@@ -160,6 +162,12 @@ func (svc *CoreService) artifactPayload(artifactID string) ([]byte, error) {
|
||||
if payload, exists := svc.artifactPayloads[artifactID]; exists {
|
||||
return domain.CopyBytes(payload), nil
|
||||
}
|
||||
if payload, err := svc.artifactStore.GetPayload(artifactID); err == nil {
|
||||
svc.artifactPayloads[artifactID] = domain.CopyBytes(payload)
|
||||
return payload, nil
|
||||
} else if !errors.Is(err, repo.ErrNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sessions := make([]domain.ArtifactTransferSession, 0, len(svc.artifactTransfers))
|
||||
for _, session := range svc.artifactTransfers {
|
||||
@@ -183,6 +191,10 @@ func (svc *CoreService) artifactPayload(artifactID string) ([]byte, error) {
|
||||
if int64(len(payload)) != session.SizeBytes {
|
||||
return nil, validationError("artifact content size does not match transfer")
|
||||
}
|
||||
if err := svc.artifactStore.PutPayload(artifactID, payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
svc.artifactPayloads[artifactID] = domain.CopyBytes(payload)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -82,6 +82,9 @@ func (svc *CoreService) OpenArtifactTransfer(open domain.ArtifactTransferOpen) (
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
if err := svc.artifactStore.SaveTransfer(session); err != nil {
|
||||
return domain.ArtifactTransferOpenResult{}, err
|
||||
}
|
||||
svc.artifactTransfers[session.TransferID] = domain.CopyArtifactTransferSession(session)
|
||||
return artifactTransferOpenResult(session, artifact, false, stamp), nil
|
||||
}
|
||||
@@ -123,6 +126,9 @@ func (svc *CoreService) UploadArtifactChunk(chunk domain.ArtifactChunkUpload) (d
|
||||
ReceivedAt: stamp,
|
||||
}
|
||||
session.UpdatedAt = stamp
|
||||
if err := svc.artifactStore.SaveTransfer(session); err != nil {
|
||||
return domain.ArtifactChunkUploadResult{}, err
|
||||
}
|
||||
svc.artifactTransfers[session.TransferID] = domain.CopyArtifactTransferSession(session)
|
||||
return artifactChunkUploadResult(session, chunk.ChunkIndex, false, stamp), nil
|
||||
}
|
||||
@@ -201,11 +207,18 @@ func (svc *CoreService) CompleteArtifactTransfer(complete domain.ArtifactTransfe
|
||||
if err := validator.ValidateArtifact(artifact); err != nil {
|
||||
return domain.ArtifactTransferCompleteResult{}, err
|
||||
}
|
||||
if err := svc.artifactStore.PutPayload(artifact.ID, payload); err != nil {
|
||||
return domain.ArtifactTransferCompleteResult{}, err
|
||||
}
|
||||
if err := svc.store.Artifacts().Update(artifact); err != nil {
|
||||
return domain.ArtifactTransferCompleteResult{}, err
|
||||
}
|
||||
svc.artifactPayloads[artifact.ID] = domain.CopyBytes(payload)
|
||||
session.Completed = true
|
||||
session.UpdatedAt = stamp
|
||||
if err := svc.artifactStore.SaveTransfer(session); err != nil {
|
||||
return domain.ArtifactTransferCompleteResult{}, err
|
||||
}
|
||||
svc.artifactTransfers[session.TransferID] = domain.CopyArtifactTransferSession(session)
|
||||
return domain.ArtifactTransferCompleteResult{Accepted: true, TransferID: session.TransferID, Artifact: artifact, Completed: true, ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -76,6 +77,11 @@ func TestCoreServiceArtifactTransferWorkflow(t *testing.T) {
|
||||
if artifact.State != domain.ArtifactStateAvailable || artifact.Checksum != validator.BytesChecksum(payload) {
|
||||
t.Fatalf("expected available artifact, got %+v", artifact)
|
||||
}
|
||||
delete(svc.artifactTransfers, opened.TransferID)
|
||||
storedPayload, err := svc.artifactPayload("artifact-1")
|
||||
if err != nil || !bytes.Equal(storedPayload, payload) {
|
||||
t.Fatalf("expected completed upload payload to remain downloadable, payload=%q err=%v", storedPayload, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceRejectsInvalidArtifactTransferChunks(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const defaultAuthSessionTTL = 8 * time.Hour
|
||||
|
||||
func (svc *CoreService) issueAuthSession(user domain.User, message string) (domain.AuthSession, error) {
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
hash := tokenHash(token)
|
||||
stamp := svc.now()
|
||||
generation := 1
|
||||
existing, err := svc.store.AuthSessions().List(domain.AuthSessionFilter{UserID: user.ID})
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
for _, session := range existing {
|
||||
if session.Generation >= generation {
|
||||
generation = session.Generation + 1
|
||||
}
|
||||
}
|
||||
record := domain.AuthSessionRecord{
|
||||
ID: "auth-session-" + hash[:24],
|
||||
UserID: user.ID,
|
||||
TokenHash: hash,
|
||||
Status: domain.AuthSessionStatusActive,
|
||||
Generation: generation,
|
||||
IssuedAt: stamp,
|
||||
ExpiresAt: stamp.Add(defaultAuthSessionTTL),
|
||||
LastSeenAt: stamp,
|
||||
}
|
||||
if err := validator.ValidateAuthSessionRecord(record); err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
if err := svc.store.AuthSessions().Create(record); err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
svc.authSessions[token] = user.ID
|
||||
svc.authMu.Unlock()
|
||||
return domain.AuthSession{
|
||||
SessionID: token,
|
||||
User: domain.CopyUser(user),
|
||||
Status: "authenticated",
|
||||
Message: message,
|
||||
ExpiresAt: record.ExpiresAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) authenticatedSession(token string) (domain.AuthSessionRecord, error) {
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" {
|
||||
return domain.AuthSessionRecord{}, ErrUnauthorized
|
||||
}
|
||||
hash := tokenHash(token)
|
||||
sessions, err := svc.store.AuthSessions().List(domain.AuthSessionFilter{TokenHash: hash})
|
||||
if err != nil {
|
||||
return domain.AuthSessionRecord{}, err
|
||||
}
|
||||
if len(sessions) != 1 || subtle.ConstantTimeCompare([]byte(sessions[0].TokenHash), []byte(hash)) != 1 {
|
||||
return domain.AuthSessionRecord{}, ErrUnauthorized
|
||||
}
|
||||
session := sessions[0]
|
||||
stamp := svc.now()
|
||||
if session.Status != domain.AuthSessionStatusActive || !session.RevokedAt.IsZero() || !stamp.Before(session.ExpiresAt) {
|
||||
if session.Status == domain.AuthSessionStatusActive && !stamp.Before(session.ExpiresAt) {
|
||||
session.Status = domain.AuthSessionStatusRevoked
|
||||
session.RevokedAt = stamp
|
||||
_ = svc.store.AuthSessions().Update(session)
|
||||
}
|
||||
return domain.AuthSessionRecord{}, ErrUnauthorized
|
||||
}
|
||||
user, err := svc.store.Users().Get(session.UserID)
|
||||
if err != nil {
|
||||
if errors.Is(err, repo.ErrNotFound) {
|
||||
return domain.AuthSessionRecord{}, ErrUnauthorized
|
||||
}
|
||||
return domain.AuthSessionRecord{}, err
|
||||
}
|
||||
if user.Status != domain.UserStatusActive {
|
||||
return domain.AuthSessionRecord{}, ErrUnauthorized
|
||||
}
|
||||
if session.LastSeenAt.IsZero() || stamp.Sub(session.LastSeenAt) >= time.Minute {
|
||||
session.LastSeenAt = stamp
|
||||
if err := svc.store.AuthSessions().Update(session); err != nil {
|
||||
return domain.AuthSessionRecord{}, err
|
||||
}
|
||||
}
|
||||
return domain.CopyAuthSessionRecord(session), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) revokeAuthSession(token string) error {
|
||||
session, err := svc.authenticatedSession(token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stamp := svc.now()
|
||||
session.Status = domain.AuthSessionStatusRevoked
|
||||
session.RevokedAt = stamp
|
||||
session.LastSeenAt = stamp
|
||||
if err := validator.ValidateAuthSessionRecord(session); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.AuthSessions().Update(session); err != nil {
|
||||
return err
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
delete(svc.authSessions, token)
|
||||
svc.authMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RotateUserSession(token string) (domain.AuthSession, error) {
|
||||
session, err := svc.authenticatedSession(token)
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
user, err := svc.store.Users().Get(session.UserID)
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
if err := svc.revokeAuthSession(token); err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
return svc.issueAuthSession(user, "会话已安全轮换")
|
||||
}
|
||||
|
||||
func (svc *CoreService) revokeUserSessions(userID string) error {
|
||||
sessions, err := svc.store.AuthSessions().List(domain.AuthSessionFilter{UserID: userID, Status: domain.AuthSessionStatusActive})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stamp := svc.now()
|
||||
for _, session := range sessions {
|
||||
session.Status = domain.AuthSessionStatusRevoked
|
||||
session.RevokedAt = stamp
|
||||
session.LastSeenAt = stamp
|
||||
if err := validator.ValidateAuthSessionRecord(session); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.AuthSessions().Update(session); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tokenHash(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
func TestAuthSessionPersistsWithoutRawTokenAndRotates(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "metadata.json")
|
||||
store, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create file store: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 18, 8, 0, 0, 0, time.UTC)
|
||||
svc := newCoreService(store, func() time.Time { return now })
|
||||
user, err := svc.CreateUser(domain.User{
|
||||
ID: "user-owner", DisplayName: "Owner", Email: "owner@example.test",
|
||||
Status: domain.UserStatusActive, Roles: []string{"server-admin"}, PasswordHash: "secret-password",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
session, err := svc.LoginUser(domain.UserLogin{Account: user.Email, Password: "secret-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("login: %v", err)
|
||||
}
|
||||
if session.SessionID == "" || !session.ExpiresAt.Equal(now.Add(defaultAuthSessionTTL)) {
|
||||
t.Fatalf("unexpected bounded session: %+v", session)
|
||||
}
|
||||
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read snapshot: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), session.SessionID) || strings.Contains(string(payload), "secret-password") {
|
||||
t.Fatalf("snapshot contains raw session or password literal: %s", payload)
|
||||
}
|
||||
if !strings.Contains(string(payload), tokenHash(session.SessionID)) {
|
||||
t.Fatalf("snapshot does not contain the expected one-way session verifier")
|
||||
}
|
||||
|
||||
reloadedStore, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reload file store: %v", err)
|
||||
}
|
||||
reloaded := newCoreService(reloadedStore, func() time.Time { return now.Add(time.Minute) })
|
||||
if current, err := reloaded.GetCurrentUser(session.SessionID); err != nil || current.ID != user.ID {
|
||||
t.Fatalf("restored session was not accepted: current=%+v err=%v", current, err)
|
||||
}
|
||||
rotated, err := reloaded.RotateUserSession(session.SessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("rotate session: %v", err)
|
||||
}
|
||||
if rotated.SessionID == "" || rotated.SessionID == session.SessionID {
|
||||
t.Fatalf("rotation did not issue a distinct token")
|
||||
}
|
||||
if _, err := reloaded.GetCurrentUser(session.SessionID); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("revoked prior token should be unauthorized, got %v", err)
|
||||
}
|
||||
if current, err := reloaded.GetCurrentUser(rotated.SessionID); err != nil || current.ID != user.ID {
|
||||
t.Fatalf("rotated token was not accepted: current=%+v err=%v", current, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthSessionExpiryIsDurablyRevoked(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
now := time.Date(2026, 7, 18, 8, 0, 0, 0, time.UTC)
|
||||
svc := newCoreService(store, func() time.Time { return now })
|
||||
if _, err := svc.CreateUser(domain.User{ID: "user-expiry", DisplayName: "Expiry", Email: "expiry@example.test", Status: domain.UserStatusActive, Roles: []string{"server-admin"}, PasswordHash: "secret-password"}); err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
session, err := svc.LoginUser(domain.UserLogin{Account: "expiry@example.test", Password: "secret-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("login: %v", err)
|
||||
}
|
||||
now = now.Add(defaultAuthSessionTTL + time.Second)
|
||||
if _, err := svc.GetCurrentUser(session.SessionID); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("expired session should be unauthorized, got %v", err)
|
||||
}
|
||||
records, err := store.AuthSessions().List(domain.AuthSessionFilter{TokenHash: tokenHash(session.SessionID)})
|
||||
if err != nil || len(records) != 1 || records[0].Status != domain.AuthSessionStatusRevoked || records[0].RevokedAt.IsZero() {
|
||||
t.Fatalf("expired session was not durably revoked: records=%+v err=%v", records, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisablingUserRevokesActiveSessions(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
svc := NewCoreService(store)
|
||||
user, err := svc.CreateUser(domain.User{ID: "user-disabled", DisplayName: "Disabled", Email: "disabled@example.test", Status: domain.UserStatusActive, Roles: []string{"server-admin"}, PasswordHash: "secret-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
session, err := svc.LoginUser(domain.UserLogin{Account: user.Email, Password: "secret-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("login: %v", err)
|
||||
}
|
||||
user.Status = domain.UserStatusDisabled
|
||||
if _, err := svc.UpdateUser(user.ID, user); err != nil {
|
||||
t.Fatalf("disable user: %v", err)
|
||||
}
|
||||
if _, err := svc.GetCurrentUser(session.SessionID); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("disabled user session should be unauthorized, got %v", err)
|
||||
}
|
||||
records, err := store.AuthSessions().List(domain.AuthSessionFilter{UserID: user.ID})
|
||||
if err != nil || len(records) != 1 || records[0].Status != domain.AuthSessionStatusRevoked || records[0].RevokedAt.IsZero() {
|
||||
t.Fatalf("disabled user sessions were not revoked: records=%+v err=%v", records, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSessionPersistsAndSignedEnvelopeRejectsReplay(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "metadata.json")
|
||||
store, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create store: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 18, 8, 0, 0, 0, time.UTC)
|
||||
svc := newCoreService(store, func() time.Time { return now })
|
||||
hello, err := svc.RegisterRunHello(validRunControlHello())
|
||||
if err != nil {
|
||||
t.Fatalf("register run: %v", err)
|
||||
}
|
||||
record, err := store.RunControlSessions().Get("run-local")
|
||||
if err != nil {
|
||||
t.Fatalf("get run session: %v", err)
|
||||
}
|
||||
record.RequireSignedRequests = true
|
||||
if err := store.RunControlSessions().Update(record); err != nil {
|
||||
t.Fatalf("require signed requests: %v", err)
|
||||
}
|
||||
delete(svc.runSessions, "run-local")
|
||||
|
||||
request := domain.RunRequestSignature{
|
||||
RunEndpointID: "run-local",
|
||||
SessionToken: hello.SessionToken,
|
||||
Method: "POST",
|
||||
Path: "/api/v1/run/jobs/claim",
|
||||
Timestamp: strconv.FormatInt(now.Unix(), 10),
|
||||
Nonce: "nonce-1",
|
||||
BodyHash: strings.Repeat("a", 64),
|
||||
}
|
||||
request.Signature = signRunRequest(request)
|
||||
if err := svc.AuthorizeRunRequestSignature(request); err != nil {
|
||||
t.Fatalf("authorize signed request: %v", err)
|
||||
}
|
||||
if err := svc.AuthorizeRunRequestSignature(request); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("replayed nonce should be unauthorized, got %v", err)
|
||||
}
|
||||
stale := request
|
||||
stale.Nonce = "nonce-2"
|
||||
stale.Timestamp = strconv.FormatInt(now.Add(-maxRunRequestClockSkew-time.Second).Unix(), 10)
|
||||
stale.Signature = signRunRequest(stale)
|
||||
if err := svc.AuthorizeRunRequestSignature(stale); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("stale signature should be unauthorized, got %v", err)
|
||||
}
|
||||
|
||||
reloadedStore, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reload store: %v", err)
|
||||
}
|
||||
reloaded := newCoreService(reloadedStore, func() time.Time { return now.Add(time.Minute) })
|
||||
result, err := reloaded.AcceptRunHeartbeat(domain.RunControlHeartbeat{
|
||||
RunEndpointID: "run-local", SessionToken: hello.SessionToken, Version: "0.1.1",
|
||||
Status: domain.RunEndpointStatusOnline, CapabilityFingerprint: "cap-jobs",
|
||||
Capacity: domain.RunCapacity{MaxJobs: 4},
|
||||
})
|
||||
if err != nil || !result.Accepted {
|
||||
t.Fatalf("reloaded hashed Run session was not accepted: result=%+v err=%v", result, err)
|
||||
}
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read snapshot: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), hello.SessionToken) || strings.Contains(string(payload), "registration-token") {
|
||||
t.Fatalf("snapshot contains raw Run credential: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func signRunRequest(request domain.RunRequestSignature) string {
|
||||
canonical := strings.Join([]string{request.Method, request.Path, request.Timestamp, request.Nonce, request.BodyHash}, "\n")
|
||||
mac := hmac.New(sha256.New, []byte(request.SessionToken))
|
||||
_, _ = mac.Write([]byte(canonical))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,279 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func TestClientManagerLifecycleBuildDeployRegisterHealthUpdateRollbackAndUninstall(t *testing.T) {
|
||||
svc, ownerSession, instance := newDistributionTestFixture(t)
|
||||
baseTime := svc.now()
|
||||
svc.now = func() time.Time { return baseTime }
|
||||
distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-build-v1")
|
||||
view, err := svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleAvailable {
|
||||
t.Fatalf("expected available build projection, view=%+v err=%v", view, err)
|
||||
}
|
||||
|
||||
view, err = svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"})
|
||||
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleDeploying || view.Job.State != domain.JobStateQueued {
|
||||
t.Fatalf("queue deployment: view=%+v err=%v", view, err)
|
||||
}
|
||||
if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-deploy-v1"}); err != nil {
|
||||
t.Fatalf("idempotent deployment: %v", err)
|
||||
}
|
||||
runSession := registerClientManagerRun(t, svc)
|
||||
claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy)
|
||||
input, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
||||
if err != nil || input.ArtifactID != distribution.ArtifactID || input.KeyGeneration != distribution.KeyGeneration || input.DeploymentGeneration != view.Installation.DeploymentGeneration || strings.Contains(strings.Join(input.Arguments, " "), "/Users/") {
|
||||
t.Fatalf("get fenced deployment input: input=%+v err=%v", input, err)
|
||||
}
|
||||
chunk, err := svc.ReadClientManagerLifecycleChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Offset: 0, Length: 7})
|
||||
if err != nil || len(chunk.Payload) == 0 || chunk.ArtifactID != distribution.ArtifactID {
|
||||
t.Fatalf("read deployment chunk: chunk=%+v err=%v", chunk, err)
|
||||
}
|
||||
if _, err := svc.GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt + 1}); err == nil {
|
||||
t.Fatal("expected stale attempt to be rejected")
|
||||
}
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.Status != domain.ClientManagerLifecycleRegistering || view.Installation.ActiveArtifactID != distribution.ArtifactID {
|
||||
t.Fatalf("expected deployed registration state, got %+v", view.Installation)
|
||||
}
|
||||
|
||||
componentKey := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager")
|
||||
registerRequest := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, baseTime)
|
||||
registerRequest.Signature = clientManagerRegistrationSignature(componentKey, registerRequest)
|
||||
registration, err := svc.RegisterClientManager(registerRequest)
|
||||
if err != nil || !registration.Accepted || registration.SessionToken == "" {
|
||||
t.Fatalf("register client manager: result=%+v err=%v", registration, err)
|
||||
}
|
||||
if _, err := svc.RegisterClientManager(registerRequest); err == nil {
|
||||
t.Fatal("expected registration nonce replay rejection")
|
||||
}
|
||||
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: runSession, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil {
|
||||
t.Fatal("Run control session must not authenticate as a Client Manager session")
|
||||
}
|
||||
heartbeat, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, HealthReason: "ready", Capabilities: registerRequest.Capabilities, SentAt: baseTime})
|
||||
if err != nil || heartbeat.Status != domain.ClientManagerLifecycleOnline {
|
||||
t.Fatalf("accept heartbeat: result=%+v err=%v", heartbeat, err)
|
||||
}
|
||||
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: registerRequest.Capabilities, SentAt: baseTime}); err == nil {
|
||||
t.Fatal("expected replayed heartbeat sequence rejection")
|
||||
}
|
||||
|
||||
baseTime = baseTime.Add(50 * time.Second)
|
||||
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
||||
t.Fatalf("reconcile degraded health: %v", err)
|
||||
}
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.Status != domain.ClientManagerLifecycleDegraded {
|
||||
t.Fatalf("expected degraded heartbeat timeout, got %+v", view.Installation)
|
||||
}
|
||||
baseTime = baseTime.Add(80 * time.Second)
|
||||
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
||||
t.Fatalf("reconcile offline health: %v", err)
|
||||
}
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.Status != domain.ClientManagerLifecycleOffline {
|
||||
t.Fatalf("expected offline heartbeat timeout, got %+v", view.Installation)
|
||||
}
|
||||
|
||||
updatedDistribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.1.0", "lifecycle-build-v2")
|
||||
view, err = svc.UpdateClientManagerForSession(ownerSession, domain.ClientManagerUpdateRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: updatedDistribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Approved: true, IdempotencyKey: "lifecycle-update-v2"})
|
||||
if err != nil || view.Installation.Status != domain.ClientManagerLifecycleUpdating {
|
||||
t.Fatalf("queue staged update: view=%+v err=%v", view, err)
|
||||
}
|
||||
runSession = registerClientManagerRun(t, svc)
|
||||
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate)
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateFailed, "client-manager.rollback.restored", "running")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.ActiveArtifactID != distribution.ArtifactID || !strings.Contains(view.Installation.Phase, "previous deployment restored") || !view.Installation.Retryable {
|
||||
t.Fatalf("expected failed update to retain previous active slot, got %+v", view.Installation)
|
||||
}
|
||||
view, err = svc.RetryClientManagerLifecycleForSession(ownerSession, domain.ClientManagerRetryRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-update-v2-retry"})
|
||||
if err != nil {
|
||||
t.Fatalf("retry staged update: %v", err)
|
||||
}
|
||||
runSession = registerClientManagerRun(t, svc)
|
||||
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUpdate)
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.updated", "running")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.ActiveArtifactID != updatedDistribution.ArtifactID || view.Installation.PreviousArtifactID != distribution.ArtifactID || view.Installation.Status != domain.ClientManagerLifecycleRegistering {
|
||||
t.Fatalf("expected successful update slot commit, got %+v", view.Installation)
|
||||
}
|
||||
|
||||
view, err = svc.ControlClientManagerForSession(ownerSession, domain.ClientManagerControlRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", Operation: domain.ClientManagerOperationRollback, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, IdempotencyKey: "lifecycle-rollback-v1"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue explicit rollback: %v", err)
|
||||
}
|
||||
runSession = registerClientManagerRun(t, svc)
|
||||
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerRollback)
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.rolled-back", "running")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.ActiveArtifactID != distribution.ArtifactID || view.Installation.PreviousArtifactID != updatedDistribution.ArtifactID {
|
||||
t.Fatalf("expected rollback slot swap, got %+v", view.Installation)
|
||||
}
|
||||
|
||||
view, err = svc.UninstallClientManagerForSession(ownerSession, domain.ClientManagerUninstallRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration, Confirmed: true, IdempotencyKey: "lifecycle-uninstall"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue uninstall: %v", err)
|
||||
}
|
||||
runSession = registerClientManagerRun(t, svc)
|
||||
claim = claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerUninstall)
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.uninstalled", "stopped")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if view.Installation.Status != domain.ClientManagerLifecycleUninstalled || view.Installation.ActiveArtifactID != "" {
|
||||
t.Fatalf("expected durable uninstalled history, got %+v", view.Installation)
|
||||
}
|
||||
if _, err := svc.store.ClientManagerDistributions().Get(distribution.ID); err != nil {
|
||||
t.Fatalf("uninstall must retain distribution history: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientManagerLifecycleRejectsCrossScopeStaleAndRevokedIdentity(t *testing.T) {
|
||||
svc, ownerSession, instance := newDistributionTestFixture(t)
|
||||
now := svc.now()
|
||||
svc.now = func() time.Time { return now }
|
||||
distribution := buildLifecycleDistribution(t, svc, ownerSession, instance, "1.0.0", "lifecycle-scope-build")
|
||||
view, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, IdempotencyKey: "lifecycle-scope-deploy"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue deploy: %v", err)
|
||||
}
|
||||
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "other-owner", DisplayName: "Other", Email: "other-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
if _, err := svc.GetClientManagerLifecycleForSession(otherSession, instance.ID, "scum-client-manager"); err == nil {
|
||||
t.Fatal("expected cross-owner lifecycle read denial")
|
||||
}
|
||||
if _, err := svc.DeployClientManagerForSession(ownerSession, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", DistributionID: distribution.ID, ExpectedDeploymentGeneration: view.Installation.DeploymentGeneration + 1, IdempotencyKey: "lifecycle-stale-deploy"}); err == nil {
|
||||
t.Fatal("expected stale deployment generation denial")
|
||||
}
|
||||
runSession := registerClientManagerRun(t, svc)
|
||||
claim := claimClientManagerJob(t, svc, runSession, domain.JobCapabilityClientManagerDeploy)
|
||||
completeClientManagerJob(t, svc, runSession, claim, domain.JobStateSucceeded, "client-manager.deployed", "running")
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
plain := currentClientManagerPlainKey(t, svc, instance.ID, "scum-client-manager")
|
||||
request := lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
||||
request.ArtifactID = "cross-server-artifact"
|
||||
request.Signature = clientManagerRegistrationSignature(plain, request)
|
||||
if _, err := svc.RegisterClientManager(request); err == nil {
|
||||
t.Fatal("expected cross-artifact registration rejection")
|
||||
}
|
||||
request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
||||
request.KeyGeneration++
|
||||
request.Nonce = "nonce-stale-key-generation"
|
||||
request.Signature = clientManagerRegistrationSignature(plain, request)
|
||||
if _, err := svc.RegisterClientManager(request); err == nil {
|
||||
t.Fatal("expected stale key generation registration rejection")
|
||||
}
|
||||
request = lifecycleRegisterRequest(view.Installation, []string{"component.register", "component.heartbeat", "component.health"}, now)
|
||||
request.Nonce = "nonce-valid-component-identity"
|
||||
request.Signature = clientManagerRegistrationSignature(plain, request)
|
||||
registration, err := svc.RegisterClientManager(request)
|
||||
if err != nil {
|
||||
t.Fatalf("register valid component: %v", err)
|
||||
}
|
||||
now = now.Add(16 * time.Minute)
|
||||
if err := svc.ReconcileClientManagerLifecycle(); err != nil {
|
||||
t.Fatalf("expire component session: %v", err)
|
||||
}
|
||||
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil {
|
||||
t.Fatal("expected expired component session rejection")
|
||||
}
|
||||
request.Timestamp = now
|
||||
request.Nonce = "nonce-replacement-after-expiry"
|
||||
request.Signature = clientManagerRegistrationSignature(plain, request)
|
||||
registration, err = svc.RegisterClientManager(request)
|
||||
if err != nil {
|
||||
t.Fatalf("register replacement component session: %v", err)
|
||||
}
|
||||
if _, err := svc.ResetComponentKeyForSession(ownerSession, domain.ComponentKeyResetRequest{ServerInstanceID: instance.ID, ComponentKind: domain.DistributionComponentClientManager, ComponentKey: "scum-client-manager"}); err != nil {
|
||||
t.Fatalf("reset component key: %v", err)
|
||||
}
|
||||
if _, err := svc.AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat{InstallationID: view.Installation.ID, SessionToken: registration.SessionToken, Sequence: 1, Health: domain.ClientManagerHealthHealthy, Capabilities: request.Capabilities, SentAt: now}); err == nil {
|
||||
t.Fatal("expected reset to revoke component session")
|
||||
}
|
||||
view, _ = svc.GetClientManagerLifecycleForSession(ownerSession, instance.ID, "scum-client-manager")
|
||||
if !view.Installation.RequiresRedeploy || view.Installation.Status != domain.ClientManagerLifecycleFailed {
|
||||
t.Fatalf("expected key reset recovery projection, got %+v", view.Installation)
|
||||
}
|
||||
for _, forbidden := range []string{plain, registration.SessionToken, "secret://", "/Users/", "tcp://"} {
|
||||
payload := strings.Join([]string{view.Installation.Phase, view.Installation.HealthReason}, " ")
|
||||
if strings.Contains(payload, forbidden) {
|
||||
t.Fatalf("safe lifecycle view leaked %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildLifecycleDistribution(t *testing.T, svc *CoreService, session string, instance domain.ServerInstance, version, idempotency string) domain.ClientManagerDistribution {
|
||||
t.Helper()
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get lifecycle plugin: %v", err)
|
||||
}
|
||||
for i := range plugin.RuntimeProfiles.ClientManagers {
|
||||
if plugin.RuntimeProfiles.ClientManagers[i].Key == "scum-client-manager" {
|
||||
plugin.RuntimeProfiles.ClientManagers[i].Version = version
|
||||
}
|
||||
}
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("update lifecycle version: %v", err)
|
||||
}
|
||||
distribution, err := svc.GenerateClientManagerDistributionForSession(session, domain.ClientManagerBuildRequest{ServerInstanceID: instance.ID, ProfileKey: "scum-client-manager", TargetOS: "linux", TargetArch: "amd64", RepositoryURL: "https://github.com/F88888/scum_client.git", SourceRevision: "main", IdempotencyKey: idempotency})
|
||||
if err != nil {
|
||||
t.Fatalf("generate lifecycle distribution: %v", err)
|
||||
}
|
||||
return completeClientDistributionBuild(t, svc, distribution, []byte("client-manager-package-"+version))
|
||||
}
|
||||
|
||||
func registerClientManagerRun(t *testing.T, svc *CoreService) string {
|
||||
t.Helper()
|
||||
hello := validRunControlHello()
|
||||
hello.Platform = "linux"
|
||||
hello.Architecture = "amd64"
|
||||
hello.CapabilityReport.Capabilities = append(hello.CapabilityReport.Capabilities, domain.JobCapabilityClientManagerDeploy, domain.JobCapabilityClientManagerControl, domain.JobCapabilityClientManagerUpdate, domain.JobCapabilityClientManagerRollback, domain.JobCapabilityClientManagerUninstall)
|
||||
result, err := svc.RegisterRunHello(hello)
|
||||
if err != nil {
|
||||
t.Fatalf("register lifecycle Run: %v", err)
|
||||
}
|
||||
return result.SessionToken
|
||||
}
|
||||
|
||||
func claimClientManagerJob(t *testing.T, svc *CoreService, sessionToken, capability string) domain.RunJobClaimResult {
|
||||
t.Helper()
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: "run-local", SessionToken: sessionToken, Capabilities: []string{capability}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job.Capability != capability {
|
||||
t.Fatalf("claim %s job: claim=%+v err=%v", capability, claim, err)
|
||||
}
|
||||
if _, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Message: "typed lifecycle work started"}); err != nil {
|
||||
t.Fatalf("ack lifecycle job: %v", err)
|
||||
}
|
||||
return claim
|
||||
}
|
||||
|
||||
func completeClientManagerJob(t *testing.T, svc *CoreService, sessionToken string, claim domain.RunJobClaimResult, state domain.JobState, kind, processState string) {
|
||||
t.Helper()
|
||||
_, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: state, Progress: domain.RunJobProgressReport{Percent: 100, Message: "client-manager lifecycle terminal"}, Message: "client-manager lifecycle terminal", ExecutionResult: domain.JobExecutionResult{Kind: kind, ProcessState: processState, AuditSummary: "bounded lifecycle result"}})
|
||||
if err != nil {
|
||||
t.Fatalf("complete lifecycle job: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func currentClientManagerPlainKey(t *testing.T, svc *CoreService, serverID, profileKey string) string {
|
||||
t.Helper()
|
||||
key, err := svc.activeComponentKey(serverID, domain.DistributionComponentClientManager, profileKey)
|
||||
if err != nil {
|
||||
t.Fatalf("get active component key: %v", err)
|
||||
}
|
||||
plain, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
if err != nil {
|
||||
t.Fatalf("decrypt component key: %v", err)
|
||||
}
|
||||
return plain
|
||||
}
|
||||
|
||||
func lifecycleRegisterRequest(installation domain.ClientManagerInstallation, capabilities []string, stamp time.Time) domain.ClientManagerRegisterRequest {
|
||||
return domain.ClientManagerRegisterRequest{InstallationID: installation.ID, ServerInstanceID: installation.ServerInstanceID, ProfileKey: installation.ProfileKey, ArtifactID: installation.ActiveArtifactID, Version: installation.ActiveVersion, SourceRevision: installation.ActiveRevision, TargetOS: installation.TargetOS, TargetArch: installation.TargetArch, KeyGeneration: installation.KeyGeneration, DeploymentGeneration: installation.DeploymentGeneration, Capabilities: capabilities, Timestamp: stamp, Nonce: "nonce-client-manager-registration"}
|
||||
}
|
||||
+152
-10
@@ -1,8 +1,14 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
@@ -12,6 +18,9 @@ import (
|
||||
|
||||
const (
|
||||
defaultHeartbeatIntervalSeconds = 15
|
||||
defaultRunSessionTTL = 24 * time.Hour
|
||||
maxRunRequestClockSkew = 5 * time.Minute
|
||||
maxRunRequestNonces = 8192
|
||||
)
|
||||
|
||||
func (svc *CoreService) RegisterRunHello(hello domain.RunControlHello) (domain.RunControlHelloResult, error) {
|
||||
@@ -46,6 +55,8 @@ func (svc *CoreService) RegisterRunHello(hello domain.RunControlHello) (domain.R
|
||||
ID: hello.RunEndpointID,
|
||||
DisplayName: hello.DisplayName,
|
||||
Version: hello.Version,
|
||||
Platform: hello.Platform,
|
||||
Architecture: hello.Architecture,
|
||||
Status: domain.RunEndpointStatusOnline,
|
||||
Capabilities: domain.CopyStringSlice(hello.CapabilityReport.Capabilities),
|
||||
Capacity: hello.Capacity,
|
||||
@@ -61,23 +72,53 @@ func (svc *CoreService) RegisterRunHello(hello domain.RunControlHello) (domain.R
|
||||
if err := svc.upsertRunEndpoint(endpoint); err != nil {
|
||||
return domain.RunControlHelloResult{}, err
|
||||
}
|
||||
sessionToken := svc.nextSessionToken(hello.RunEndpointID, stamp)
|
||||
svc.runSessions[hello.RunEndpointID] = domain.RunControlSession{
|
||||
previous, previousErr := svc.store.RunControlSessions().Get(hello.RunEndpointID)
|
||||
generation := 1
|
||||
if previousErr == nil {
|
||||
generation = previous.Generation + 1
|
||||
} else if !errors.Is(previousErr, repo.ErrNotFound) {
|
||||
return domain.RunControlHelloResult{}, previousErr
|
||||
}
|
||||
sessionToken, err := svc.nextSessionToken()
|
||||
if err != nil {
|
||||
return domain.RunControlHelloResult{}, err
|
||||
}
|
||||
session := domain.RunControlSession{
|
||||
RunEndpointID: hello.RunEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
SessionTokenHash: tokenHash(sessionToken),
|
||||
Status: domain.AuthSessionStatusActive,
|
||||
Generation: generation,
|
||||
CapabilityFingerprint: hello.CapabilityReport.Fingerprint,
|
||||
HeartbeatIntervalSeconds: defaultHeartbeatIntervalSeconds,
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
ExpiresAt: stamp.Add(defaultRunSessionTTL),
|
||||
RequireSignedRequests: hasComponentAuthIdentity(hello),
|
||||
}
|
||||
if err := validator.ValidateRunControlSession(session); err != nil {
|
||||
return domain.RunControlHelloResult{}, err
|
||||
}
|
||||
if previousErr == nil {
|
||||
if err := svc.store.RunControlSessions().Update(session); err != nil {
|
||||
return domain.RunControlHelloResult{}, err
|
||||
}
|
||||
} else if err := svc.store.RunControlSessions().Create(session); err != nil {
|
||||
return domain.RunControlHelloResult{}, err
|
||||
}
|
||||
svc.runSessions[hello.RunEndpointID] = session
|
||||
featureFlags := []string{"control.hello", "control.heartbeat", "signed-envelope.v1.optional"}
|
||||
if session.RequireSignedRequests {
|
||||
featureFlags[2] = "signed-envelope.v1.required"
|
||||
}
|
||||
|
||||
return domain.CopyRunControlHelloResult(domain.RunControlHelloResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: hello.RunEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
ServerTime: stamp,
|
||||
HeartbeatIntervalSeconds: defaultHeartbeatIntervalSeconds,
|
||||
FeatureFlags: []string{"control.hello", "control.heartbeat"},
|
||||
SessionExpiresAt: session.ExpiresAt,
|
||||
FeatureFlags: featureFlags,
|
||||
}), nil
|
||||
}
|
||||
|
||||
@@ -96,9 +137,9 @@ func (svc *CoreService) AcceptRunHeartbeat(heartbeat domain.RunControlHeartbeat)
|
||||
svc.controlMu.Lock()
|
||||
defer svc.controlMu.Unlock()
|
||||
|
||||
session, exists := svc.runSessions[heartbeat.RunEndpointID]
|
||||
if !exists || session.SessionToken != heartbeat.SessionToken {
|
||||
return domain.RunControlHeartbeatResult{}, validationError("sessionToken is invalid")
|
||||
session, err := svc.currentRunSession(heartbeat.RunEndpointID, heartbeat.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunControlHeartbeatResult{}, err
|
||||
}
|
||||
|
||||
endpoint, err := svc.store.RunEndpoints().Get(heartbeat.RunEndpointID)
|
||||
@@ -119,6 +160,9 @@ func (svc *CoreService) AcceptRunHeartbeat(heartbeat domain.RunControlHeartbeat)
|
||||
refreshCapabilities := session.CapabilityFingerprint != heartbeat.CapabilityFingerprint
|
||||
session.CapabilityFingerprint = heartbeat.CapabilityFingerprint
|
||||
session.UpdatedAt = stamp
|
||||
if err := svc.store.RunControlSessions().Update(session); err != nil {
|
||||
return domain.RunControlHeartbeatResult{}, err
|
||||
}
|
||||
svc.runSessions[heartbeat.RunEndpointID] = session
|
||||
|
||||
return domain.CopyRunControlHeartbeatResult(domain.RunControlHeartbeatResult{
|
||||
@@ -140,7 +184,105 @@ func (svc *CoreService) upsertRunEndpoint(endpoint domain.RunEndpoint) error {
|
||||
return svc.store.RunEndpoints().Update(endpoint)
|
||||
}
|
||||
|
||||
func (svc *CoreService) nextSessionToken(runEndpointID string, stamp time.Time) string {
|
||||
svc.runSessionSeq++
|
||||
return fmt.Sprintf("session:%s:%d:%d", runEndpointID, stamp.UnixNano(), svc.runSessionSeq)
|
||||
func (svc *CoreService) nextSessionToken() (string, error) {
|
||||
return randomToken()
|
||||
}
|
||||
|
||||
func (svc *CoreService) currentRunSession(runEndpointID string, sessionToken string) (domain.RunControlSession, error) {
|
||||
session, exists := svc.runSessions[runEndpointID]
|
||||
if !exists {
|
||||
stored, err := svc.store.RunControlSessions().Get(runEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunControlSession{}, runAuthenticationError(true)
|
||||
}
|
||||
session = stored
|
||||
}
|
||||
presentedHash := tokenHash(strings.TrimSpace(sessionToken))
|
||||
if strings.TrimSpace(sessionToken) == "" || session.Status != domain.AuthSessionStatusActive || !session.RevokedAt.IsZero() || !svc.now().Before(session.ExpiresAt) || subtle.ConstantTimeCompare([]byte(session.SessionTokenHash), []byte(presentedHash)) != 1 {
|
||||
if session.Status == domain.AuthSessionStatusActive && !svc.now().Before(session.ExpiresAt) {
|
||||
session.Status = domain.AuthSessionStatusRevoked
|
||||
session.RevokedAt = svc.now()
|
||||
session.UpdatedAt = session.RevokedAt
|
||||
_ = svc.store.RunControlSessions().Update(session)
|
||||
}
|
||||
return domain.RunControlSession{}, runAuthenticationError(session.RequireSignedRequests)
|
||||
}
|
||||
return domain.CopyRunControlSession(session), nil
|
||||
}
|
||||
|
||||
func runAuthenticationError(requireSigned bool) error {
|
||||
if !requireSigned {
|
||||
return validationError("sessionToken is invalid")
|
||||
}
|
||||
return fmt.Errorf("sessionToken is invalid: %w", ErrUnauthorized)
|
||||
}
|
||||
|
||||
func (svc *CoreService) AuthorizeRunRequestSignature(request domain.RunRequestSignature) error {
|
||||
svc.controlMu.Lock()
|
||||
defer svc.controlMu.Unlock()
|
||||
|
||||
session, err := svc.currentRunSession(request.RunEndpointID, request.SessionToken)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(request.Signature) == "" && !session.RequireSignedRequests {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(request.Timestamp) == "" || strings.TrimSpace(request.Nonce) == "" || strings.TrimSpace(request.Signature) == "" {
|
||||
return runAuthenticationError(true)
|
||||
}
|
||||
unixSeconds, err := strconv.ParseInt(request.Timestamp, 10, 64)
|
||||
if err != nil {
|
||||
return runAuthenticationError(true)
|
||||
}
|
||||
stamp := time.Unix(unixSeconds, 0).UTC()
|
||||
delta := svc.now().Sub(stamp)
|
||||
if delta < -maxRunRequestClockSkew || delta > maxRunRequestClockSkew {
|
||||
return runAuthenticationError(true)
|
||||
}
|
||||
session.UsedNonces = activeRunNonces(session.UsedNonces, svc.now().Add(-maxRunRequestClockSkew))
|
||||
if len(request.Nonce) > 128 || runNonceSeen(session.UsedNonces, request.Nonce) || len(session.UsedNonces) >= maxRunRequestNonces {
|
||||
return runAuthenticationError(true)
|
||||
}
|
||||
canonical := strings.Join([]string{request.Method, request.Path, request.Timestamp, request.Nonce, request.BodyHash}, "\n")
|
||||
mac := hmac.New(sha256.New, []byte(request.SessionToken))
|
||||
_, _ = mac.Write([]byte(canonical))
|
||||
expected := hex.EncodeToString(mac.Sum(nil))
|
||||
provided, err := hex.DecodeString(request.Signature)
|
||||
if err != nil || subtle.ConstantTimeCompare([]byte(expected), []byte(hex.EncodeToString(provided))) != 1 {
|
||||
return runAuthenticationError(true)
|
||||
}
|
||||
session.UsedNonces = append(session.UsedNonces, request.Timestamp+":"+request.Nonce)
|
||||
session.UpdatedAt = svc.now()
|
||||
if err := svc.store.RunControlSessions().Update(session); err != nil {
|
||||
return err
|
||||
}
|
||||
svc.runSessions[request.RunEndpointID] = session
|
||||
return nil
|
||||
}
|
||||
|
||||
func activeRunNonces(entries []string, cutoff time.Time) []string {
|
||||
active := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
timestamp, _, ok := strings.Cut(entry, ":")
|
||||
if !ok {
|
||||
active = append(active, entry)
|
||||
continue
|
||||
}
|
||||
unixSeconds, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err == nil && !time.Unix(unixSeconds, 0).Before(cutoff) {
|
||||
active = append(active, entry)
|
||||
}
|
||||
}
|
||||
return active
|
||||
}
|
||||
|
||||
func runNonceSeen(entries []string, nonce string) bool {
|
||||
for _, entry := range entries {
|
||||
_, storedNonce, ok := strings.Cut(entry, ":")
|
||||
if (ok && storedNonce == nonce) || (!ok && entry == nonce) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const runUpdateChunkSize = 1024 * 1024
|
||||
|
||||
type dependencyResolution struct {
|
||||
instance domain.ServerInstance
|
||||
plugin domain.GamePlugin
|
||||
binding domain.RuntimeBinding
|
||||
endpoint domain.RunEndpoint
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetDependencyCatalogForSession(sessionID, serverInstanceID string) (domain.DependencyCatalog, error) {
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID); err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
resolution, err := svc.resolveDependencyContext(serverInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
statuses, err := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return domain.DependencyCatalog{}, err
|
||||
}
|
||||
statusByProbe := map[string]domain.DependencyStatus{}
|
||||
for _, status := range statuses {
|
||||
statusByProbe[status.ProbeKey] = status
|
||||
}
|
||||
|
||||
plans := make([]domain.DependencyPlanView, 0, len(resolution.plugin.RuntimeProfiles.InstallPlans))
|
||||
for _, plan := range resolution.plugin.RuntimeProfiles.InstallPlans {
|
||||
if !runtimePlatformsContain(plan.Platforms, resolution.endpoint.Platform) {
|
||||
continue
|
||||
}
|
||||
steps := make([]domain.DependencyPlanStepView, len(plan.Steps))
|
||||
for i, step := range plan.Steps {
|
||||
host := ""
|
||||
if parsed, parseErr := url.Parse(step.DownloadRef); parseErr == nil {
|
||||
host = parsed.Hostname()
|
||||
}
|
||||
steps[i] = domain.DependencyPlanStepView{Type: step.Type, TargetKey: step.TargetKey, PackageManager: step.PackageManager, PackageName: step.PackageName, Version: step.Version, DownloadHost: host}
|
||||
}
|
||||
var planProbe domain.RuntimeDependencyProbe
|
||||
for _, candidate := range resolution.plugin.RuntimeProfiles.DependencyProbes {
|
||||
if runtimePlatformsContain(candidate.Platforms, resolution.endpoint.Platform) && planTargetsProbe(plan, candidate) {
|
||||
planProbe = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
plans = append(plans, domain.DependencyPlanView{Key: plan.Key, Title: plan.Title, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, Digest: dependencyPlanDigest(resolution, planProbe, plan), Steps: steps})
|
||||
}
|
||||
sort.Slice(plans, func(i, j int) bool { return plans[i].Key < plans[j].Key })
|
||||
|
||||
probes := make([]domain.DependencyProbeView, 0, len(resolution.plugin.RuntimeProfiles.DependencyProbes))
|
||||
for _, probe := range resolution.plugin.RuntimeProfiles.DependencyProbes {
|
||||
if !runtimePlatformsContain(probe.Platforms, resolution.endpoint.Platform) {
|
||||
continue
|
||||
}
|
||||
status := statusByProbe[probe.Key]
|
||||
planKey := ""
|
||||
for _, plan := range resolution.plugin.RuntimeProfiles.InstallPlans {
|
||||
if runtimePlatformsContain(plan.Platforms, resolution.endpoint.Platform) && planTargetsProbe(plan, probe) {
|
||||
planKey = plan.Key
|
||||
break
|
||||
}
|
||||
}
|
||||
state := status.State
|
||||
if state == "" {
|
||||
state = domain.DependencyStateUnknown
|
||||
}
|
||||
probes = append(probes, domain.DependencyProbeView{Key: probe.Key, Kind: probe.Kind, Required: probe.Required, MinimumVersion: probe.MinimumVersion, State: state, Evidence: status.Evidence, InstallPlanKey: planKey})
|
||||
}
|
||||
sort.Slice(probes, func(i, j int) bool { return probes[i].Key < probes[j].Key })
|
||||
|
||||
return domain.CopyDependencyCatalog(domain.DependencyCatalog{ServerInstanceID: resolution.instance.ID, PluginID: resolution.plugin.ID, PluginVersion: resolution.plugin.Version, ProfileKey: resolution.binding.ProfileKey, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, Probes: probes, Plans: plans, UpdatedAt: svc.now()}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListRunUpdateJobsForSession(sessionID, serverInstanceID string) ([]domain.RunUpdateJob, error) {
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].UpdatedAt.After(items[j].UpdatedAt) })
|
||||
for i := range items {
|
||||
items[i] = domain.CopyRunUpdateJob(items[i])
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetDependencyExecutionInput(request domain.DependencyExecutionInputRequest) (domain.DependencyExecutionInput, error) {
|
||||
if err := validator.ValidateDependencyExecutionInputRequest(request); err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityDependenciesCheck && job.Capability != domain.JobCapabilityDependenciesInstall {
|
||||
return domain.DependencyExecutionInput{}, validationError("job is not a dependency operation")
|
||||
}
|
||||
resolution, err := svc.resolveDependencyContext(job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if resolution.endpoint.ID != job.RunEndpointID {
|
||||
return domain.DependencyExecutionInput{}, validationError("dependency endpoint no longer matches")
|
||||
}
|
||||
probeKey := strings.TrimPrefix(job.TargetKey, "dependencies/")
|
||||
if job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
probeKey = ""
|
||||
}
|
||||
var probe domain.RuntimeDependencyProbe
|
||||
if probeKey != "" {
|
||||
probe, err = declaredDependencyProbe(resolution.plugin, probeKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
}
|
||||
var plan domain.RuntimeInstallPlan
|
||||
if job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
planKey := strings.TrimPrefix(job.TargetKey, "dependencies/install/")
|
||||
plan, err = declaredInstallPlan(resolution.plugin, planKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
statuses, listErr := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if listErr != nil {
|
||||
return domain.DependencyExecutionInput{}, listErr
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.JobID == job.ID {
|
||||
probe, err = declaredDependencyProbe(resolution.plugin, status.ProbeKey, resolution.endpoint.Platform)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
digest := dependencyPlanDigest(resolution, probe, plan)
|
||||
status, err := svc.dependencyStatusForJob(job.ID, job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
if status.PlanDigest != digest {
|
||||
return domain.DependencyExecutionInput{}, validationError("dependency declaration changed after dispatch")
|
||||
}
|
||||
bindings, err := dependencyBindings(resolution.binding, probe, plan)
|
||||
if err != nil {
|
||||
return domain.DependencyExecutionInput{}, err
|
||||
}
|
||||
return domain.CopyDependencyExecutionInput(domain.DependencyExecutionInput{JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, PluginID: resolution.plugin.ID, PluginVersion: resolution.plugin.Version, ProfileKey: resolution.binding.ProfileKey, TargetOS: resolution.endpoint.Platform, TargetArch: resolution.endpoint.Architecture, PlanDigest: digest, Probe: probe, Plan: plan, Bindings: bindings}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetRunUpdateInput(request domain.RunUpdateInputRequest) (domain.RunUpdateInput, error) {
|
||||
if err := validator.ValidateRunUpdateInputRequest(request); err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return domain.RunUpdateInput{}, validationError("job is not a Run self-update")
|
||||
}
|
||||
update, distribution, artifact, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return domain.RunUpdateInput{}, err
|
||||
}
|
||||
return domain.RunUpdateInput{JobID: job.ID, ServerInstanceID: job.ServerInstanceID, RunEndpointID: job.RunEndpointID, ArtifactID: artifact.ID, Checksum: artifact.Checksum, SizeBytes: artifact.SizeBytes, TargetOS: distribution.TargetOS, TargetArch: distribution.TargetArch, PackageFormat: distribution.PackageFormat, ExecutableName: executableFilename("run", distribution.TargetOS), TargetRelease: update.TargetRelease, ChunkSizeBytes: runUpdateChunkSize}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ReadRunUpdateChunk(request domain.RunUpdateChunkRequest) (domain.RunUpdateChunk, error) {
|
||||
if err := validator.ValidateRunUpdateChunkRequest(request); err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
job, err := svc.activeFencedInputJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return domain.RunUpdateChunk{}, validationError("job is not a Run self-update")
|
||||
}
|
||||
_, _, artifact, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
payload, err := svc.artifactPayload(artifact.ID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateChunk{}, err
|
||||
}
|
||||
if int64(len(payload)) != artifact.SizeBytes || validator.BytesChecksum(payload) != artifact.Checksum {
|
||||
return domain.RunUpdateChunk{}, validationError("update artifact content does not match metadata")
|
||||
}
|
||||
if request.Offset >= artifact.SizeBytes {
|
||||
return domain.RunUpdateChunk{}, validationError("offset must be inside update artifact")
|
||||
}
|
||||
length := request.Length
|
||||
remaining := artifact.SizeBytes - request.Offset
|
||||
if int64(length) > remaining {
|
||||
length = int(remaining)
|
||||
}
|
||||
end := request.Offset + int64(length)
|
||||
return domain.CopyRunUpdateChunk(domain.RunUpdateChunk{JobID: job.ID, ArtifactID: artifact.ID, Offset: request.Offset, TotalBytes: artifact.SizeBytes, Checksum: artifact.Checksum, Payload: payload[int(request.Offset):int(end)], Complete: end == artifact.SizeBytes}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) activeFencedInputJob(endpointID, sessionToken, jobID, leaseToken string, attempt int) (domain.Job, error) {
|
||||
session, err := svc.validatedRunSession(endpointID, sessionToken)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
job, err := svc.fencedJob(session, jobID, leaseToken, attempt)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.Job{}, validationError("job input is not active")
|
||||
}
|
||||
if !job.CancelRequestedAt.IsZero() {
|
||||
return domain.Job{}, validationError("job input is cancelled")
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveDependencyContext(serverInstanceID string) (dependencyResolution, error) {
|
||||
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if plugin.Status != domain.GamePluginStatusInstalled || plugin.Version != instance.PluginVersion {
|
||||
return dependencyResolution{}, validationError("installed plugin version does not match server")
|
||||
}
|
||||
binding, err := svc.runtimeBindingForServer(instance.ID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
binding, err = normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if binding.Status != domain.RuntimeBindingStatusComplete || binding.PluginVersion != plugin.Version {
|
||||
return dependencyResolution{}, validationError("runtime binding is incomplete or stale")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return dependencyResolution{}, err
|
||||
}
|
||||
if endpoint.Platform == "" || endpoint.Architecture == "" {
|
||||
return dependencyResolution{}, validationError("Run endpoint target is not registered")
|
||||
}
|
||||
return dependencyResolution{instance: instance, plugin: plugin, binding: binding, endpoint: endpoint}, nil
|
||||
}
|
||||
|
||||
func declaredDependencyProbe(plugin domain.GamePlugin, key, targetOS string) (domain.RuntimeDependencyProbe, error) {
|
||||
for _, probe := range plugin.RuntimeProfiles.DependencyProbes {
|
||||
if probe.Key == key && runtimePlatformsContain(probe.Platforms, targetOS) {
|
||||
return probe, nil
|
||||
}
|
||||
}
|
||||
return domain.RuntimeDependencyProbe{}, validationError("dependency probe is not declared for endpoint target")
|
||||
}
|
||||
|
||||
func declaredInstallPlan(plugin domain.GamePlugin, key, targetOS string) (domain.RuntimeInstallPlan, error) {
|
||||
for _, plan := range plugin.RuntimeProfiles.InstallPlans {
|
||||
if plan.Key == key && runtimePlatformsContain(plan.Platforms, targetOS) {
|
||||
return plan, nil
|
||||
}
|
||||
}
|
||||
return domain.RuntimeInstallPlan{}, validationError("dependency install plan is not declared for endpoint target")
|
||||
}
|
||||
|
||||
func runtimePlatformsContain(platforms []string, target string) bool {
|
||||
if len(platforms) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, platform := range platforms {
|
||||
if platform == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func planTargetsProbe(plan domain.RuntimeInstallPlan, probe domain.RuntimeDependencyProbe) bool {
|
||||
for _, step := range plan.Steps {
|
||||
if step.TargetKey == probe.TargetKey {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dependencyPlanDigest(resolution dependencyResolution, probe domain.RuntimeDependencyProbe, plan domain.RuntimeInstallPlan) string {
|
||||
keys := make([]string, 0, len(resolution.binding.Bindings))
|
||||
for key := range resolution.binding.Bindings {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
bindingEvidence := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
bindingEvidence = append(bindingEvidence, key+"="+validator.BytesChecksum([]byte(resolution.binding.Bindings[key])))
|
||||
}
|
||||
payload := struct {
|
||||
PluginID string `json:"pluginId"`
|
||||
PluginVersion string `json:"pluginVersion"`
|
||||
ProfileKey string `json:"profileKey"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
Binding []string `json:"binding"`
|
||||
Probe domain.RuntimeDependencyProbe `json:"probe"`
|
||||
Plan domain.RuntimeInstallPlan `json:"plan"`
|
||||
}{resolution.plugin.ID, resolution.plugin.Version, resolution.binding.ProfileKey, resolution.endpoint.Platform, resolution.endpoint.Architecture, bindingEvidence, probe, plan}
|
||||
body, _ := json.Marshal(payload)
|
||||
return validator.BytesChecksum(body)
|
||||
}
|
||||
|
||||
func dependencyBindings(binding domain.RuntimeBinding, probe domain.RuntimeDependencyProbe, plan domain.RuntimeInstallPlan) (map[string]string, error) {
|
||||
keys := map[string]struct{}{}
|
||||
if probe.TargetKey != "" {
|
||||
keys[probe.TargetKey] = struct{}{}
|
||||
}
|
||||
for _, step := range plan.Steps {
|
||||
if step.TargetKey != "" {
|
||||
keys[step.TargetKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make(map[string]string, len(keys))
|
||||
for key := range keys {
|
||||
value := strings.TrimSpace(binding.Bindings[key])
|
||||
if value == "" {
|
||||
value = key
|
||||
}
|
||||
lower := strings.ToLower(value)
|
||||
if strings.HasPrefix(lower, "secret://") || strings.Contains(lower, "password=") || strings.Contains(lower, "token=") {
|
||||
return nil, validationError("dependency target binding cannot be a secret")
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) dependencyStatusForJob(jobID, serverInstanceID string) (domain.DependencyStatus, error) {
|
||||
statuses, err := svc.store.DependencyStatuses().List(domain.DependencyStatusFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return domain.DependencyStatus{}, err
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.JobID == jobID {
|
||||
return status, nil
|
||||
}
|
||||
}
|
||||
return domain.DependencyStatus{}, repo.ErrNotFound
|
||||
}
|
||||
|
||||
func (svc *CoreService) resolveRunUpdate(job domain.Job) (domain.RunUpdateJob, domain.RunDistribution, domain.Artifact, error) {
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
var update domain.RunUpdateJob
|
||||
for _, candidate := range updates {
|
||||
if candidate.JobID == job.ID {
|
||||
update = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if update.ID == "" || update.RunEndpointID != job.RunEndpointID {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update record does not match active job")
|
||||
}
|
||||
distributions, err := svc.store.RunDistributions().List(domain.RunDistributionFilter{ServerInstanceID: job.ServerInstanceID, Status: domain.DistributionStatusAvailable})
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
var distribution domain.RunDistribution
|
||||
for _, candidate := range distributions {
|
||||
if candidate.ArtifactID == update.ArtifactID {
|
||||
distribution = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if distribution.ID == "" || distribution.RunEndpointID != job.RunEndpointID || distribution.TargetOS != update.TargetOS || distribution.TargetArch != update.TargetArch || distribution.Checksum != update.Checksum {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run distribution no longer matches update")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(job.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
if endpoint.Platform != distribution.TargetOS || endpoint.Architecture != distribution.TargetArch {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update target no longer matches endpoint")
|
||||
}
|
||||
artifact, err := svc.store.Artifacts().Get(update.ArtifactID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, err
|
||||
}
|
||||
if artifact.State != domain.ArtifactStateAvailable || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != distribution.BuildJobID || artifact.Checksum != update.Checksum {
|
||||
return domain.RunUpdateJob{}, domain.RunDistribution{}, domain.Artifact{}, validationError("Run update artifact is unavailable or outside distribution scope")
|
||||
}
|
||||
return update, distribution, artifact, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectDependencyAndRunUpdateResult(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityDependenciesCheck || job.Capability == domain.JobCapabilityDependenciesInstall {
|
||||
status, err := svc.dependencyStatusForJob(job.ID, job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status.UpdatedAt = stamp
|
||||
status.CheckedAt = stamp
|
||||
status.JobID = job.ID
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
var evidence domain.DependencyExecutionEvidence
|
||||
if err := json.Unmarshal([]byte(job.ExecutionResult.Content), &evidence); err != nil {
|
||||
return validationError("dependency result evidence is invalid")
|
||||
}
|
||||
if evidence.ProbeKey != status.ProbeKey || evidence.PlanDigest != status.PlanDigest || job.ExecutionResult.Checksum != status.PlanDigest {
|
||||
return validationError("dependency result evidence does not match approved plan")
|
||||
}
|
||||
status.State = domain.DependencyState(evidence.State)
|
||||
status.Evidence = evidence.Evidence
|
||||
status.CompletedSteps = evidence.CompletedSteps
|
||||
status.Message = "dependency execution completed"
|
||||
} else if job.State == domain.JobStateCancelled {
|
||||
status.State = domain.DependencyStateFailed
|
||||
status.Message = "dependency execution cancelled"
|
||||
} else {
|
||||
status.State = domain.DependencyStateFailed
|
||||
status.Message = "dependency execution failed"
|
||||
}
|
||||
if err := validator.ValidateDependencyStatus(status); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.DependencyStatuses().Update(status); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "dependency.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), status.Message)
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityRunSelfUpdate {
|
||||
return nil
|
||||
}
|
||||
update, _, _, err := svc.resolveRunUpdate(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
update.UpdatedAt = stamp
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
var evidence domain.RunUpdateExecutionEvidence
|
||||
if err := json.Unmarshal([]byte(job.ExecutionResult.Content), &evidence); err != nil || evidence.TargetRelease != update.TargetRelease || evidence.Phase != "staged" || job.ExecutionResult.Checksum != update.Checksum {
|
||||
return validationError("Run update staged evidence is invalid")
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusRunning
|
||||
update.Phase = domain.RunUpdatePhaseRestartRequested
|
||||
update.Message = "verified update staged; restart requested"
|
||||
} else if job.State == domain.JobStateCancelled {
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseFailed
|
||||
update.Message = "Run update cancelled before activation"
|
||||
} else {
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseFailed
|
||||
update.Message = "Run update verification or staging failed"
|
||||
}
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run", "run.update.result", "server-instance", job.ServerInstanceID, auditResultForJob(job), update.Message)
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectDependencyAndRunUpdateProgress(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityRunSelfUpdate {
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, update := range updates {
|
||||
if update.JobID != job.ID || update.Status != domain.DistributionJobStatusQueued {
|
||||
continue
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusRunning
|
||||
update.Phase = domain.RunUpdatePhaseDownloading
|
||||
update.Message = "Run is downloading and verifying the update"
|
||||
update.UpdatedAt = stamp
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.store.RunUpdateJobs().Update(update)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ReportRunUpdateHealth(report domain.RunUpdateHealthReport) (domain.RunUpdateHealthResult, error) {
|
||||
if err := validator.ValidateRunUpdateHealthReport(report); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if _, err := svc.validatedRunSession(report.RunEndpointID, report.SessionToken); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
job, err := svc.store.Jobs().Get(report.JobID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if job.RunEndpointID != report.RunEndpointID || job.Capability != domain.JobCapabilityRunSelfUpdate || job.State != domain.JobStateSucceeded || job.Attempt != report.Attempt || !leaseTokenMatches(job.LeaseTokenHash, report.LeaseToken) {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health report does not match terminal attempt")
|
||||
}
|
||||
updates, err := svc.store.RunUpdateJobs().List(domain.RunUpdateJobFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
var update domain.RunUpdateJob
|
||||
for _, candidate := range updates {
|
||||
if candidate.JobID == job.ID && candidate.RunEndpointID == report.RunEndpointID {
|
||||
update = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if update.ID == "" || job.ExecutionResult.Checksum != update.Checksum {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health report does not match staged update")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(report.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if endpoint.Version != report.Version || endpoint.Status != domain.RunEndpointStatusOnline {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health version does not match online endpoint")
|
||||
}
|
||||
stamp := svc.now()
|
||||
if report.Outcome == "succeeded" {
|
||||
if report.Version != update.TargetRelease || update.Phase == domain.RunUpdatePhaseRolledBack || update.Phase == domain.RunUpdatePhaseFailed {
|
||||
return domain.RunUpdateHealthResult{}, validationError("Run update health version does not match target release")
|
||||
}
|
||||
if update.Phase == domain.RunUpdatePhaseSucceeded {
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusSucceeded
|
||||
update.Phase = domain.RunUpdatePhaseSucceeded
|
||||
update.Rollback = false
|
||||
update.Message = "updated Run registered, reconciled, and reported healthy"
|
||||
} else {
|
||||
if update.PreviousVersion != "" && report.Version != update.PreviousVersion {
|
||||
return domain.RunUpdateHealthResult{}, validationError("rolled-back Run version does not match previous release")
|
||||
}
|
||||
if update.Phase == domain.RunUpdatePhaseRolledBack {
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
update.Status = domain.DistributionJobStatusFailed
|
||||
update.Phase = domain.RunUpdatePhaseRolledBack
|
||||
update.Rollback = true
|
||||
update.Message = "Run update activation failed and previous executable was restored"
|
||||
}
|
||||
update.UpdatedAt = stamp
|
||||
if err := validator.ValidateRunUpdateJob(update); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
if err := svc.store.RunUpdateJobs().Update(update); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if report.Outcome == "rolled-back" {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
if err := svc.recordAuditEvent("run", "run.update.health", "server-instance", update.ServerInstanceID, auditResult, update.Message); err != nil {
|
||||
return domain.RunUpdateHealthResult{}, err
|
||||
}
|
||||
return domain.RunUpdateHealthResult{Accepted: true, JobID: job.ID, Phase: update.Phase, ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func auditResultForJob(job domain.Job) domain.AuditResult {
|
||||
if job.State == domain.JobStateSucceeded {
|
||||
return domain.AuditResultSuccess
|
||||
}
|
||||
if job.State == domain.JobStateCancelled {
|
||||
return domain.AuditResultDenied
|
||||
}
|
||||
return domain.AuditResultFailed
|
||||
}
|
||||
|
||||
func sameRunUpdateTarget(existing, expected domain.RunUpdateJob) bool {
|
||||
return existing.ServerInstanceID == expected.ServerInstanceID && existing.RunEndpointID == expected.RunEndpointID && existing.ArtifactID == expected.ArtifactID && existing.Checksum == expected.Checksum && existing.TargetOS == expected.TargetOS && existing.TargetArch == expected.TargetArch && existing.TargetRelease == expected.TargetRelease && existing.JobID == expected.JobID && existing.IdempotencyKey == expected.IdempotencyKey
|
||||
}
|
||||
|
||||
func findRunDistributionForArtifact(distributions []domain.RunDistribution, artifactID string) (domain.RunDistribution, error) {
|
||||
for _, distribution := range distributions {
|
||||
if distribution.ArtifactID == artifactID && distribution.Status == domain.DistributionStatusAvailable {
|
||||
return distribution, nil
|
||||
}
|
||||
}
|
||||
return domain.RunDistribution{}, errors.New("available Run distribution not found")
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func TestDependencyCatalogRequiresCurrentReviewedDigest(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "dependency-other-owner", DisplayName: "Other Owner", Email: "dependency-other@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
if _, err := svc.GetDependencyCatalogForSession(otherSession, instance.ID); !errors.Is(err, ErrForbidden) {
|
||||
t.Fatalf("expected cross-owner dependency catalog denial, got %v", err)
|
||||
}
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get dependency catalog: %v", err)
|
||||
}
|
||||
if catalog.TargetOS != "linux" || catalog.TargetArch != "amd64" || len(catalog.Probes) != 1 || len(catalog.Plans) != 1 || !strings.HasPrefix(catalog.Plans[0].Digest, "sha256:") {
|
||||
t.Fatalf("unexpected dependency catalog: %+v", catalog)
|
||||
}
|
||||
request := domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, Install: true, InstallPlanKey: catalog.Plans[0].Key, PlanDigest: "sha256:" + strings.Repeat("f", 64), IdempotencyKey: "dependency-stale-digest"}
|
||||
if _, err := svc.QueueDependencyJobForSession(session, request); err == nil || !strings.Contains(err.Error(), "planDigest") {
|
||||
t.Fatalf("expected stale digest rejection, got %v", err)
|
||||
}
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{ServerInstanceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list jobs: %v", err)
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if job.IdempotencyKey == request.IdempotencyKey {
|
||||
t.Fatalf("stale digest created a job: %+v", job)
|
||||
}
|
||||
}
|
||||
audits, err := svc.ListAuditEvents(domain.AuditEventFilter{ResourceID: instance.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("list audits: %v", err)
|
||||
}
|
||||
foundDenied := false
|
||||
for _, audit := range audits {
|
||||
foundDenied = foundDenied || audit.Action == "dependency.install.denied"
|
||||
}
|
||||
if !foundDenied {
|
||||
t.Fatalf("expected stale digest audit, got %+v", audits)
|
||||
}
|
||||
|
||||
request.PlanDigest = catalog.Plans[0].Digest
|
||||
request.IdempotencyKey = "dependency-current-digest"
|
||||
job, err := svc.QueueDependencyJobForSession(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("queue reviewed dependency plan: %v", err)
|
||||
}
|
||||
if job.Capability != domain.JobCapabilityDependenciesInstall || job.TargetKey != "dependencies/install/"+catalog.Plans[0].Key {
|
||||
t.Fatalf("unexpected dependency install job: %+v", job)
|
||||
}
|
||||
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get plugin: %v", err)
|
||||
}
|
||||
plugin.RuntimeProfiles.InstallPlans[0].Steps[0].PackageName = "openjdk-22-jre"
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("mutate plugin declaration: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesInstall}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob {
|
||||
t.Fatalf("claim dependency install: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
_, err = svc.GetDependencyExecutionInput(domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
||||
if err == nil || !strings.Contains(err.Error(), "changed after dispatch") {
|
||||
t.Fatalf("expected changed declaration rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDependencyInputFencingCancellationAndTerminalProjection(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("catalog: %v", err)
|
||||
}
|
||||
job, err := svc.QueueDependencyJobForSession(session, domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, IdempotencyKey: "dependency-check-fencing"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue dependency check: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesCheck}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job.JobID != job.ID {
|
||||
t.Fatalf("claim dependency check: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
base := domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}
|
||||
input, err := svc.GetDependencyExecutionInput(base)
|
||||
if err != nil || input.PlanDigest == "" || input.Bindings["java"] == "" {
|
||||
t.Fatalf("get fenced dependency input: input=%+v err=%v", input, err)
|
||||
}
|
||||
wrongSession := base
|
||||
wrongSession.SessionToken = "stale-session"
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongSession); err == nil {
|
||||
t.Fatal("expected wrong session rejection")
|
||||
}
|
||||
wrongAttempt := base
|
||||
wrongAttempt.Attempt++
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongAttempt); err == nil {
|
||||
t.Fatal("expected wrong attempt rejection")
|
||||
}
|
||||
wrongLease := base
|
||||
wrongLease.LeaseToken = "stale-lease"
|
||||
if _, err := svc.GetDependencyExecutionInput(wrongLease); err == nil {
|
||||
t.Fatal("expected wrong lease rejection")
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.DependencyExecutionEvidence{ProbeKey: input.Probe.Key, PlanDigest: input.PlanDigest, State: string(domain.DependencyStatePresent), Evidence: "OpenJDK 21"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "dependency probe completed"}, ResultRef: "artifact://jobs/dependency-check/result", Message: "dependency probe completed", ExecutionResult: domain.JobExecutionResult{Kind: "dependency.check", Checksum: input.PlanDigest, AuditSummary: "dependency probe completed", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete dependency result: %v", err)
|
||||
}
|
||||
projected, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil || projected.Probes[0].State != domain.DependencyStatePresent || projected.Probes[0].Evidence != "OpenJDK 21" {
|
||||
t.Fatalf("unexpected dependency projection: catalog=%+v err=%v", projected, err)
|
||||
}
|
||||
|
||||
cancelJob, err := svc.QueueDependencyJobForSession(session, domain.DependencyJobRequest{ServerInstanceID: instance.ID, ProbeKey: catalog.Probes[0].Key, IdempotencyKey: "dependency-check-cancel"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue cancellable dependency check: %v", err)
|
||||
}
|
||||
claim, err = svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityDependenciesCheck}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || claim.Job.JobID != cancelJob.ID {
|
||||
t.Fatalf("claim cancellable dependency check: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
if _, err := svc.RequestRunJobCancelForSession(session, domain.RunJobCancelRequest{JobID: cancelJob.ID, Reason: "operator cancelled"}); err != nil {
|
||||
t.Fatalf("request cancel: %v", err)
|
||||
}
|
||||
if _, err := svc.GetDependencyExecutionInput(domain.DependencyExecutionInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}); err == nil || !strings.Contains(err.Error(), "cancelled") {
|
||||
t.Fatalf("expected cancelled input rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginBridgeDependencyInstallUsesReviewedPlanDigest(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get plugin: %v", err)
|
||||
}
|
||||
plugin.Pages = append(plugin.Pages, domain.GamePluginPage{Key: "runtime", Title: "Runtime", Path: "/runtime", Permissions: []string{"server.dependencies.manage"}, BridgeActions: []string{string(domain.PluginBridgeActionDependenciesRequest)}})
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("add dependency bridge page: %v", err)
|
||||
}
|
||||
catalog, err := svc.GetDependencyCatalogForSession(session, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get dependency catalog: %v", err)
|
||||
}
|
||||
request := domain.PluginBridgeExecuteRequest{RequestID: "bridge-dependency-install", PluginID: plugin.ID, RouteKey: "runtime", ServerInstanceID: instance.ID, Action: domain.PluginBridgeActionDependenciesRequest, Payload: map[string]string{"operation": "install", "probeKey": catalog.Probes[0].Key, "planKey": catalog.Plans[0].Key, "idempotencyKey": "bridge-dependency-install"}}
|
||||
denied, err := svc.ExecutePluginBridgeAction(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("execute bridge without digest: %v", err)
|
||||
}
|
||||
if denied.Status == "queued" || denied.Error == nil {
|
||||
t.Fatalf("bridge install without reviewed digest must be denied: %+v", denied)
|
||||
}
|
||||
request.Payload["planDigest"] = catalog.Plans[0].Digest
|
||||
request.Payload["idempotencyKey"] = "bridge-dependency-install-approved"
|
||||
approved, err := svc.ExecutePluginBridgeAction(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("execute reviewed bridge install: %v", err)
|
||||
}
|
||||
if approved.Status != "queued" || approved.Result["capability"] != domain.JobCapabilityDependenciesInstall {
|
||||
t.Fatalf("expected reviewed bridge dependency job, got %+v", approved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUpdateTargetFencingChunksHealthAndRollbackProjection(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{ServerInstanceID: instance.ID, TargetOS: "linux", TargetArch: "amd64", IdempotencyKey: "run-update-build"})
|
||||
if err != nil {
|
||||
t.Fatalf("generate update distribution: %v", err)
|
||||
}
|
||||
payload := []byte("compiled target-matched run archive")
|
||||
distribution = completeDistributionBuild(t, svc, distribution, payload)
|
||||
otherInstance, err := svc.CreateServerInstanceForSession(session, domain.ServerInstance{ID: "server-update-other", PluginID: instance.PluginID, RunEndpointID: instance.RunEndpointID, Name: "Other Update Server", State: domain.ServerInstanceStateReady})
|
||||
if err != nil {
|
||||
t.Fatalf("create other update server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, otherInstance, "local")
|
||||
if _, err := svc.PushRunUpdateForSession(session, domain.RunUpdateRequest{ServerInstanceID: otherInstance.ID, ArtifactID: distribution.ArtifactID, Checksum: distribution.Checksum, IdempotencyKey: "run-update-cross-server"}); err == nil {
|
||||
t.Fatal("expected cross-server update artifact rejection")
|
||||
}
|
||||
|
||||
endpoint, _ := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
endpoint.Architecture = "arm64"
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("change endpoint target: %v", err)
|
||||
}
|
||||
request := domain.RunUpdateRequest{ServerInstanceID: instance.ID, ArtifactID: distribution.ArtifactID, Checksum: distribution.Checksum, IdempotencyKey: "run-update-target-check"}
|
||||
if _, err := svc.PushRunUpdateForSession(session, request); err == nil || !strings.Contains(err.Error(), "target-matched") {
|
||||
t.Fatalf("expected cross-target update rejection, got %v", err)
|
||||
}
|
||||
endpoint.Architecture = "amd64"
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("restore endpoint target: %v", err)
|
||||
}
|
||||
request.IdempotencyKey = "run-update-fenced"
|
||||
update, err := svc.PushRunUpdateForSession(session, request)
|
||||
if err != nil {
|
||||
t.Fatalf("push target-matched update: %v", err)
|
||||
}
|
||||
runSession := registerDependencyUpdateRun(t, svc, instance)
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, Capabilities: []string{domain.JobCapabilityRunSelfUpdate}, Capacity: domain.RunCapacity{MaxJobs: 1}})
|
||||
if err != nil || !claim.HasJob || claim.Job.JobID != update.JobID {
|
||||
t.Fatalf("claim Run update: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
inputRequest := domain.RunUpdateInputRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt}
|
||||
input, err := svc.GetRunUpdateInput(inputRequest)
|
||||
if err != nil || input.TargetRelease != update.TargetRelease || input.Checksum != distribution.Checksum {
|
||||
t.Fatalf("get Run update input: input=%+v err=%v", input, err)
|
||||
}
|
||||
chunk, err := svc.ReadRunUpdateChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Offset: 0, Length: 8})
|
||||
if err != nil || string(chunk.Payload) != string(payload[:8]) || chunk.Offset != 0 || chunk.TotalBytes != int64(len(payload)) {
|
||||
t.Fatalf("read bounded update chunk: chunk=%+v err=%v", chunk, err)
|
||||
}
|
||||
if _, err := svc.ReadRunUpdateChunk(domain.RunUpdateChunkRequest{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt + 1, Offset: 0, Length: 8}); err == nil {
|
||||
t.Fatal("expected stale update chunk attempt rejection")
|
||||
}
|
||||
|
||||
evidence, _ := json.Marshal(domain.RunUpdateExecutionEvidence{TargetRelease: update.TargetRelease, Phase: "staged"})
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: instance.RunEndpointID, SessionToken: runSession, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "Run update verified and staged"}, ResultRef: "artifact://jobs/run-update/staged", Message: "Run update verified and staged", ExecutionResult: domain.JobExecutionResult{Kind: "run.update.staged", Checksum: update.Checksum, SizeBytes: int64(len(payload)), AuditSummary: "verified update staged", Content: string(evidence)}}); err != nil {
|
||||
t.Fatalf("complete staged Run update: %v", err)
|
||||
}
|
||||
updates, err := svc.ListRunUpdateJobsForSession(session, instance.ID)
|
||||
if err != nil || len(updates) != 1 || updates[0].Phase != domain.RunUpdatePhaseRestartRequested {
|
||||
t.Fatalf("expected restart-requested projection, updates=%+v err=%v", updates, err)
|
||||
}
|
||||
|
||||
newHello := dependencyUpdateHello(instance)
|
||||
newHello.Version = update.TargetRelease
|
||||
newRegistration, err := svc.RegisterRunHello(newHello)
|
||||
if err != nil {
|
||||
t.Fatalf("register updated Run: %v", err)
|
||||
}
|
||||
health := domain.RunUpdateHealthReport{RunEndpointID: instance.RunEndpointID, SessionToken: newRegistration.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Outcome: "succeeded", Version: update.TargetRelease}
|
||||
if _, err := svc.ReportRunUpdateHealth(domain.RunUpdateHealthReport{RunEndpointID: health.RunEndpointID, SessionToken: health.SessionToken, JobID: health.JobID, LeaseToken: "stale-lease", Attempt: health.Attempt, Outcome: health.Outcome, Version: health.Version}); err == nil {
|
||||
t.Fatal("expected stale health lease rejection")
|
||||
}
|
||||
result, err := svc.ReportRunUpdateHealth(health)
|
||||
if err != nil || !result.Accepted || result.Phase != domain.RunUpdatePhaseSucceeded {
|
||||
t.Fatalf("report updated Run health: result=%+v err=%v", result, err)
|
||||
}
|
||||
|
||||
rollbackHello := dependencyUpdateHello(instance)
|
||||
rollbackHello.Version = update.PreviousVersion
|
||||
rollbackRegistration, err := svc.RegisterRunHello(rollbackHello)
|
||||
if err != nil {
|
||||
t.Fatalf("register rolled-back Run: %v", err)
|
||||
}
|
||||
health.SessionToken = rollbackRegistration.SessionToken
|
||||
health.Outcome = "rolled-back"
|
||||
health.Version = update.PreviousVersion
|
||||
result, err = svc.ReportRunUpdateHealth(health)
|
||||
if err != nil || result.Phase != domain.RunUpdatePhaseRolledBack {
|
||||
t.Fatalf("report rollback: result=%+v err=%v", result, err)
|
||||
}
|
||||
updates, _ = svc.ListRunUpdateJobsForSession(session, instance.ID)
|
||||
if !updates[0].Rollback || updates[0].Status != domain.DistributionJobStatusFailed || updates[0].Phase != domain.RunUpdatePhaseRolledBack {
|
||||
t.Fatalf("unexpected rollback projection: %+v", updates[0])
|
||||
}
|
||||
}
|
||||
|
||||
func registerDependencyUpdateRun(t *testing.T, svc *CoreService, instance domain.ServerInstance) string {
|
||||
t.Helper()
|
||||
result, err := svc.RegisterRunHello(dependencyUpdateHello(instance))
|
||||
if err != nil || !result.Accepted {
|
||||
t.Fatalf("register dependency/update Run: result=%+v err=%v", result, err)
|
||||
}
|
||||
return result.SessionToken
|
||||
}
|
||||
|
||||
func dependencyUpdateHello(instance domain.ServerInstance) domain.RunControlHello {
|
||||
return domain.RunControlHello{
|
||||
RegistrationToken: "registration-token",
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
DisplayName: "Dependency Update Run",
|
||||
Version: "0.1.0",
|
||||
Status: domain.RunEndpointStatusOnline,
|
||||
Platform: "linux",
|
||||
Architecture: "amd64",
|
||||
CapabilityReport: domain.RunCapabilityReport{Capabilities: []string{domain.JobCapabilityDependenciesCheck, domain.JobCapabilityDependenciesInstall, domain.JobCapabilityRunSelfUpdate}, Fingerprint: "dependency-update-v1"},
|
||||
Capacity: domain.RunCapacity{MaxJobs: 2},
|
||||
}
|
||||
}
|
||||
@@ -14,12 +14,13 @@ func (svc *CoreService) GetDistributionBuildInput(request domain.DistributionBui
|
||||
if err := validator.ValidateDistributionBuildInputRequest(request); err != nil {
|
||||
return domain.DistributionBuildInput{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(request.RunEndpointID, request.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(request.RunEndpointID, request.SessionToken)
|
||||
if err != nil {
|
||||
return domain.DistributionBuildInput{}, err
|
||||
}
|
||||
|
||||
svc.jobMu.Lock()
|
||||
job, _, err := svc.activeLeasedJob(request.RunEndpointID, request.SessionToken, request.JobID, request.LeaseToken, request.Attempt)
|
||||
job, err := svc.fencedJob(session, request.JobID, request.LeaseToken, request.Attempt)
|
||||
svc.jobMu.Unlock()
|
||||
if err != nil {
|
||||
return domain.DistributionBuildInput{}, err
|
||||
@@ -46,7 +47,7 @@ func (svc *CoreService) GetDistributionBuildInput(request domain.DistributionBui
|
||||
if key.Generation != distribution.KeyGeneration {
|
||||
return domain.DistributionBuildInput{}, validationError("run build key generation is no longer current")
|
||||
}
|
||||
plainKey, err := decryptRuntimeKey(key.EncryptedKey)
|
||||
plainKey, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
if err != nil {
|
||||
return domain.DistributionBuildInput{}, err
|
||||
}
|
||||
@@ -58,6 +59,7 @@ func (svc *CoreService) GetDistributionBuildInput(request domain.DistributionBui
|
||||
RunEndpointID: distribution.RunEndpointID,
|
||||
TargetOS: distribution.TargetOS,
|
||||
TargetArch: distribution.TargetArch,
|
||||
TargetRelease: distribution.ID,
|
||||
PackageFormat: distribution.PackageFormat,
|
||||
ArtifactID: distribution.ArtifactID,
|
||||
OutputFilename: executableFilename("run", distribution.TargetOS),
|
||||
@@ -82,7 +84,7 @@ func (svc *CoreService) GetDistributionBuildInput(request domain.DistributionBui
|
||||
if key.Generation != distribution.KeyGeneration {
|
||||
return domain.DistributionBuildInput{}, validationError("client-manager build key generation is no longer current")
|
||||
}
|
||||
plainKey, err := decryptRuntimeKey(key.EncryptedKey)
|
||||
plainKey, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
if err != nil {
|
||||
return domain.DistributionBuildInput{}, err
|
||||
}
|
||||
@@ -134,6 +136,9 @@ func (svc *CoreService) projectDistributionBuildResult(job domain.Job, stamp tim
|
||||
if job.Capability != domain.JobCapabilityDistributionBuild {
|
||||
return nil
|
||||
}
|
||||
if err := svc.validateDistributionBuildResult(job); err != nil {
|
||||
return err
|
||||
}
|
||||
status := domain.DistributionStatusFailed
|
||||
buildStatus := domain.DistributionJobStatusFailed
|
||||
var artifact domain.Artifact
|
||||
@@ -198,6 +203,9 @@ func (svc *CoreService) projectDistributionBuildResult(job domain.Job, stamp tim
|
||||
if err := svc.store.ClientManagerDistributions().Update(distribution); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.ProjectClientManagerDistribution(distribution); err != nil {
|
||||
return err
|
||||
}
|
||||
build, err := svc.store.ClientManagerBuildJobs().Get(job.ID)
|
||||
if err != nil && !errors.Is(err, repo.ErrNotFound) {
|
||||
return err
|
||||
@@ -221,6 +229,55 @@ func (svc *CoreService) projectDistributionBuildResult(job domain.Job, stamp tim
|
||||
return repo.ErrNotFound
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateDistributionBuildResult(job domain.Job) error {
|
||||
if job.Capability != domain.JobCapabilityDistributionBuild {
|
||||
return nil
|
||||
}
|
||||
|
||||
expectedArtifactID := ""
|
||||
runDistributions, err := svc.store.RunDistributions().List(domain.RunDistributionFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, distribution := range runDistributions {
|
||||
if distribution.BuildJobID == job.ID {
|
||||
expectedArtifactID = distribution.ArtifactID
|
||||
break
|
||||
}
|
||||
}
|
||||
if expectedArtifactID == "" {
|
||||
clientDistributions, err := svc.store.ClientManagerDistributions().List(domain.ClientManagerDistributionFilter{ServerInstanceID: job.ServerInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, distribution := range clientDistributions {
|
||||
if distribution.BuildJobID == job.ID {
|
||||
expectedArtifactID = distribution.ArtifactID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if expectedArtifactID == "" {
|
||||
return repo.ErrNotFound
|
||||
}
|
||||
if job.State != domain.JobStateSucceeded {
|
||||
return nil
|
||||
}
|
||||
|
||||
artifactID := strings.TrimPrefix(job.ResultRef, "artifact://")
|
||||
if artifactID == "" || artifactID == job.ResultRef || artifactID != expectedArtifactID {
|
||||
return validationError("distribution build result must reference the expected artifact")
|
||||
}
|
||||
artifact, err := svc.store.Artifacts().Get(artifactID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if artifact.State != domain.ArtifactStateAvailable || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != job.ID {
|
||||
return validationError("distribution build artifact is unavailable or outside the job scope")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func executableFilename(base string, targetOS string) string {
|
||||
if targetOS == "windows" {
|
||||
return base + ".exe"
|
||||
|
||||
+150
-194
@@ -1,12 +1,8 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -17,36 +13,6 @@ import (
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
type generatedPackageConfig struct {
|
||||
Kind string `json:"kind"`
|
||||
ServerInstanceID string `json:"serverInstanceId"`
|
||||
PluginID string `json:"pluginId"`
|
||||
RunEndpointID string `json:"runEndpointId,omitempty"`
|
||||
ProfileKey string `json:"profileKey,omitempty"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
SecretRef string `json:"secretRef"`
|
||||
KeyGeneration int `json:"keyGeneration"`
|
||||
AuthKey string `json:"authKey"`
|
||||
}
|
||||
|
||||
type generatedClientManagerPackage struct {
|
||||
Kind string `json:"kind"`
|
||||
Checkout clientManagerCheckoutPlan `json:"checkout"`
|
||||
Config generatedPackageConfig `json:"config"`
|
||||
OutputArtifacts []string `json:"outputArtifacts"`
|
||||
BuildLogRef string `json:"buildLogRef"`
|
||||
KeyFingerprint string `json:"keyFingerprint"`
|
||||
}
|
||||
|
||||
type clientManagerCheckoutPlan struct {
|
||||
RepositoryURL string `json:"repositoryUrl"`
|
||||
SourceRevision string `json:"sourceRevision"`
|
||||
CheckoutRef string `json:"checkoutRef"`
|
||||
TargetOS string `json:"targetOs"`
|
||||
TargetArch string `json:"targetArch"`
|
||||
}
|
||||
|
||||
func (svc *CoreService) GenerateRunDistributionForSession(sessionID string, request domain.RunDistributionGenerateRequest) (domain.RunDistribution, error) {
|
||||
request = domain.CopyRunDistributionGenerateRequest(request)
|
||||
if strings.TrimSpace(request.IdempotencyKey) == "" {
|
||||
@@ -159,9 +125,6 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
if strings.TrimSpace(request.IdempotencyKey) == "" {
|
||||
request.IdempotencyKey = "client-manager-" + request.ServerInstanceID + "-" + request.ProfileKey + "-" + request.TargetOS + "-" + request.TargetArch
|
||||
}
|
||||
if strings.TrimSpace(request.SourceRevision) == "" {
|
||||
request.SourceRevision = "main"
|
||||
}
|
||||
if err := validator.ValidateClientManagerBuildRequest(request); err != nil {
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
@@ -184,6 +147,18 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: unsupported target")
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
profile, err := findRuntimeClientManagerProfile(plugin, request.ProfileKey)
|
||||
if err != nil {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: profile is not declared")
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if strings.TrimSpace(request.SourceRevision) == "" {
|
||||
request.SourceRevision = clientManagerProfileRevision(profile)
|
||||
}
|
||||
if !clientManagerProfileSupportsTarget(profile, request.TargetOS, request.TargetArch) || request.RepositoryURL != profile.RepositoryURL || !clientManagerProfileAllowsRevision(profile, request.SourceRevision) {
|
||||
_ = svc.recordAuditEvent(user.ID, "client-manager.build.denied", "server-instance", instance.ID, domain.AuditResultDenied, "client-manager build denied: repository, revision, or target is not declared")
|
||||
return domain.ClientManagerDistribution{}, validationError("client-manager build must match the declared profile repository, revision, and target")
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "client-manager.build.denied"); err != nil {
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
@@ -215,6 +190,7 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
ServerInstanceID: instance.ID,
|
||||
PluginID: plugin.ID,
|
||||
ProfileKey: request.ProfileKey,
|
||||
Version: profile.Version,
|
||||
TargetOS: request.TargetOS,
|
||||
TargetArch: request.TargetArch,
|
||||
RepositoryURL: request.RepositoryURL,
|
||||
@@ -246,6 +222,7 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
ServerInstanceID: instance.ID,
|
||||
PluginID: plugin.ID,
|
||||
ProfileKey: request.ProfileKey,
|
||||
Version: profile.Version,
|
||||
TargetOS: request.TargetOS,
|
||||
TargetArch: request.TargetArch,
|
||||
RepositoryURL: request.RepositoryURL,
|
||||
@@ -271,6 +248,9 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
}
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if err := svc.ProjectClientManagerDistribution(distribution); err != nil {
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: buildJobID,
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -288,6 +268,7 @@ func (svc *CoreService) GenerateClientManagerDistributionForSession(sessionID st
|
||||
distribution.UpdatedAt = buildJob.UpdatedAt
|
||||
_ = svc.store.ClientManagerBuildJobs().Update(buildJob)
|
||||
_ = svc.store.ClientManagerDistributions().Update(distribution)
|
||||
_ = svc.ProjectClientManagerDistribution(distribution)
|
||||
return domain.ClientManagerDistribution{}, err
|
||||
}
|
||||
if job.ID != buildJobID || job.Capability != domain.JobCapabilityDistributionBuild {
|
||||
@@ -388,6 +369,11 @@ func (svc *CoreService) ResetComponentKeyForSession(sessionID string, request do
|
||||
if err := svc.revokeComponentDistributions(instance.ID, request.ComponentKind, normalizedComponentKey(request.ComponentKind, request.ComponentKey), nextGeneration); err != nil {
|
||||
return domain.EncryptedComponentKey{}, err
|
||||
}
|
||||
if request.ComponentKind == domain.DistributionComponentClientManager {
|
||||
if err := svc.fenceClientManagerAfterKeyReset(instance.ID, normalizedComponentKey(request.ComponentKind, request.ComponentKey), nextGeneration); err != nil {
|
||||
return domain.EncryptedComponentKey{}, err
|
||||
}
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "runtime-key.reset", "server-instance", instance.ID, domain.AuditResultSuccess, "reset "+string(request.ComponentKind)+" key; previous packages revoked"); err != nil {
|
||||
return domain.EncryptedComponentKey{}, err
|
||||
}
|
||||
@@ -419,7 +405,7 @@ func (svc *CoreService) AuthenticateComponent(request domain.ComponentAuthentica
|
||||
_ = svc.recordAuditEvent("runtime", "runtime-key.auth", "server-instance", request.ServerInstanceID, domain.AuditResultDenied, "component authentication denied: stale generation")
|
||||
return domain.CopyComponentAuthenticationResult(result), nil
|
||||
}
|
||||
plainKey, err := decryptRuntimeKey(key.EncryptedKey)
|
||||
plainKey, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
if err != nil {
|
||||
return domain.ComponentAuthenticationResult{}, err
|
||||
}
|
||||
@@ -468,8 +454,7 @@ func (svc *CoreService) GetServerRuntimeActionsForSession(sessionID string, serv
|
||||
break
|
||||
}
|
||||
}
|
||||
bindingsComplete := svc.runtimeBindingsComplete(instance.ID)
|
||||
bindingReason := "runtime binding is incomplete"
|
||||
bindingsComplete, bindingReason := svc.runtimeBindingReadiness(instance.ID)
|
||||
actions := domain.ServerRuntimeActions{
|
||||
ServerInstanceID: instance.ID,
|
||||
PluginID: plugin.ID,
|
||||
@@ -489,6 +474,7 @@ func (svc *CoreService) GetServerRuntimeActionsForSession(sessionID string, serv
|
||||
runtimeAction("historical-logs", "Historical logs", endpointSupports(endpoint, domain.JobCapabilityLogsBackfill) && bindingsComplete, fallbackReason(!endpointSupports(endpoint, domain.JobCapabilityLogsBackfill), "run endpoint cannot backfill logs", bindingReason)),
|
||||
},
|
||||
}
|
||||
actions.Actions = append(actions.Actions, svc.clientManagerRuntimeActionProjection(instance, plugin, endpoint, bindingsComplete, bindingReason)...)
|
||||
return domain.CopyServerRuntimeActions(actions), nil
|
||||
}
|
||||
|
||||
@@ -500,11 +486,7 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
if err := validateRunUpdateRequest(request); err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID)
|
||||
user, instance, err := svc.requireServerOwner(sessionID, request.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
@@ -533,6 +515,22 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: checksum mismatch")
|
||||
return domain.RunUpdateJob{}, validationError("checksum must match artifact")
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
if endpoint.Platform == "" || endpoint.Architecture == "" {
|
||||
return domain.RunUpdateJob{}, validationError("Run endpoint target is not registered")
|
||||
}
|
||||
distributions, err := svc.store.RunDistributions().List(domain.RunDistributionFilter{ServerInstanceID: instance.ID, Status: domain.DistributionStatusAvailable})
|
||||
if err != nil {
|
||||
return domain.RunUpdateJob{}, err
|
||||
}
|
||||
distribution, err := findRunDistributionForArtifact(distributions, artifact.ID)
|
||||
if err != nil || distribution.RunEndpointID != endpoint.ID || distribution.TargetOS != endpoint.Platform || distribution.TargetArch != endpoint.Architecture || distribution.Checksum != artifact.Checksum || artifact.OwnerKind != domain.ArtifactOwnerKindJob || artifact.OwnerID != distribution.BuildJobID {
|
||||
_ = svc.recordAuditEvent(user.ID, "run.update.denied", "server-instance", instance.ID, domain.AuditResultDenied, "run update denied: artifact is not an approved target-matched Run distribution")
|
||||
return domain.RunUpdateJob{}, validationError("artifact must be an approved target-matched Run distribution")
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: jobIDFromParts("job-run-update", request.ServerInstanceID, request.IdempotencyKey),
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -554,9 +552,15 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
ArtifactID: artifact.ID,
|
||||
Checksum: artifact.Checksum,
|
||||
TargetOS: distribution.TargetOS,
|
||||
TargetArch: distribution.TargetArch,
|
||||
TargetRelease: distribution.ID,
|
||||
PreviousVersion: endpoint.Version,
|
||||
JobID: job.ID,
|
||||
IdempotencyKey: request.IdempotencyKey,
|
||||
Status: domain.DistributionJobStatusQueued,
|
||||
Phase: domain.RunUpdatePhaseQueued,
|
||||
Message: "Run update queued",
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
@@ -569,7 +573,7 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
if getErr != nil {
|
||||
return domain.RunUpdateJob{}, getErr
|
||||
}
|
||||
if !sameRunUpdateJob(existing, updateJob) {
|
||||
if !sameRunUpdateTarget(existing, updateJob) {
|
||||
return domain.RunUpdateJob{}, validationError("run update job already exists with different target")
|
||||
}
|
||||
return domain.CopyRunUpdateJob(existing), nil
|
||||
@@ -585,16 +589,16 @@ func (svc *CoreService) PushRunUpdateForSession(sessionID string, request domain
|
||||
func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request domain.DependencyJobRequest) (domain.Job, error) {
|
||||
request = domain.CopyDependencyJobRequest(request)
|
||||
if strings.TrimSpace(request.IdempotencyKey) == "" {
|
||||
request.IdempotencyKey = "dependencies-" + request.ServerInstanceID + "-" + request.ProbeKey
|
||||
operation := "check"
|
||||
if request.Install {
|
||||
operation = "install-" + request.InstallPlanKey
|
||||
}
|
||||
request.IdempotencyKey = "dependencies-" + operation + "-" + request.ServerInstanceID + "-" + request.ProbeKey
|
||||
}
|
||||
if err := validateDependencyJobRequest(request); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID)
|
||||
user, instance, err := svc.requireServerOwner(sessionID, request.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
@@ -609,6 +613,35 @@ func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request d
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "dependency.install.denied"); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
resolution, err := svc.resolveDependencyContext(instance.ID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if request.TargetOS != "" && request.TargetOS != resolution.endpoint.Platform || request.TargetArch != "" && request.TargetArch != resolution.endpoint.Architecture {
|
||||
return domain.Job{}, validationError("dependency request target does not match Run endpoint")
|
||||
}
|
||||
request.TargetOS = resolution.endpoint.Platform
|
||||
request.TargetArch = resolution.endpoint.Architecture
|
||||
probe, err := declaredDependencyProbe(plugin, request.ProbeKey, request.TargetOS)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
var plan domain.RuntimeInstallPlan
|
||||
if request.Install {
|
||||
plan, err = declaredInstallPlan(plugin, request.InstallPlanKey, request.TargetOS)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if !planTargetsProbe(plan, probe) {
|
||||
return domain.Job{}, validationError("install plan does not target requested dependency probe")
|
||||
}
|
||||
}
|
||||
expectedDigest := dependencyPlanDigest(resolution, probe, plan)
|
||||
if request.Install && request.PlanDigest != expectedDigest {
|
||||
_ = svc.recordAuditEvent(user.ID, "dependency.install.denied", "server-instance", instance.ID, domain.AuditResultDenied, "dependency install denied: reviewed plan digest is stale or missing")
|
||||
return domain.Job{}, validationError("planDigest must match the current reviewed install plan")
|
||||
}
|
||||
request.PlanDigest = expectedDigest
|
||||
capability := domain.JobCapabilityDependenciesCheck
|
||||
targetKey := "dependencies/" + request.ProbeKey
|
||||
message := "dependency check queued"
|
||||
@@ -634,7 +667,7 @@ func (svc *CoreService) QueueDependencyJobForSession(sessionID string, request d
|
||||
_ = svc.recordAuditEvent(user.ID, auditAction+".denied", "server-instance", instance.ID, domain.AuditResultDenied, "dependency operation denied: endpoint unsupported or offline")
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.upsertDependencyStatus(instance, request, state, "queued through platform job"); err != nil {
|
||||
if err := svc.upsertDependencyStatus(instance, request, job.ID, probe.Required, state, "queued through platform job"); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, auditAction, "server-instance", instance.ID, domain.AuditResultQueued, message); err != nil {
|
||||
@@ -706,7 +739,7 @@ func (svc *CoreService) ensureActiveComponentKey(serverInstanceID string, kind d
|
||||
normalized := normalizedComponentKey(kind, componentKey)
|
||||
key, err := svc.activeComponentKey(serverInstanceID, kind, normalized)
|
||||
if err == nil {
|
||||
plainKey, err := decryptRuntimeKey(key.EncryptedKey)
|
||||
plainKey, err := svc.decryptRuntimeKey(key.EncryptedKey)
|
||||
return key, plainKey, err
|
||||
}
|
||||
if !errors.Is(err, repo.ErrNotFound) {
|
||||
@@ -742,7 +775,7 @@ func (svc *CoreService) createEncryptedComponentKey(serverInstanceID string, kin
|
||||
if err != nil {
|
||||
return domain.EncryptedComponentKey{}, "", err
|
||||
}
|
||||
encryptedKey, err := encryptRuntimeKey(plainKey)
|
||||
encryptedKey, err := svc.encryptRuntimeKey(plainKey)
|
||||
if err != nil {
|
||||
return domain.EncryptedComponentKey{}, "", err
|
||||
}
|
||||
@@ -894,9 +927,21 @@ func (svc *CoreService) ensureArtifactPayload(artifactID string, payload []byte,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if existingPayload, err := svc.artifactStore.GetPayload(artifactID); err == nil {
|
||||
if int64(len(existingPayload)) != artifact.SizeBytes || validator.BytesChecksum(existingPayload) != artifact.Checksum {
|
||||
return validationError("artifact payload does not match metadata")
|
||||
}
|
||||
svc.artifactPayloads[artifactID] = domain.CopyBytes(existingPayload)
|
||||
return nil
|
||||
} else if !errors.Is(err, repo.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
if int64(len(payload)) != artifact.SizeBytes || validator.BytesChecksum(payload) != artifact.Checksum {
|
||||
return validationError("artifact payload does not match metadata")
|
||||
}
|
||||
if err := svc.artifactStore.PutPayload(artifactID, payload); err != nil {
|
||||
return err
|
||||
}
|
||||
svc.artifactPayloads[artifactID] = domain.CopyBytes(payload)
|
||||
return nil
|
||||
}
|
||||
@@ -905,6 +950,7 @@ func sameClientManagerBuildJobArtifacts(existing domain.ClientManagerBuildJob, e
|
||||
return existing.ServerInstanceID == expected.ServerInstanceID &&
|
||||
existing.PluginID == expected.PluginID &&
|
||||
existing.ProfileKey == expected.ProfileKey &&
|
||||
existing.Version == expected.Version &&
|
||||
existing.TargetOS == expected.TargetOS &&
|
||||
existing.TargetArch == expected.TargetArch &&
|
||||
existing.RepositoryURL == expected.RepositoryURL &&
|
||||
@@ -916,17 +962,7 @@ func sameClientManagerBuildJobArtifacts(existing domain.ClientManagerBuildJob, e
|
||||
existing.Status == expected.Status
|
||||
}
|
||||
|
||||
func sameRunUpdateJob(existing domain.RunUpdateJob, expected domain.RunUpdateJob) bool {
|
||||
return existing.ServerInstanceID == expected.ServerInstanceID &&
|
||||
existing.RunEndpointID == expected.RunEndpointID &&
|
||||
existing.ArtifactID == expected.ArtifactID &&
|
||||
existing.Checksum == expected.Checksum &&
|
||||
existing.JobID == expected.JobID &&
|
||||
existing.IdempotencyKey == expected.IdempotencyKey &&
|
||||
existing.Status == expected.Status
|
||||
}
|
||||
|
||||
func (svc *CoreService) upsertDependencyStatus(instance domain.ServerInstance, request domain.DependencyJobRequest, state domain.DependencyState, message string) error {
|
||||
func (svc *CoreService) upsertDependencyStatus(instance domain.ServerInstance, request domain.DependencyJobRequest, jobID string, required bool, state domain.DependencyState, message string) error {
|
||||
statusID := distributionID("dependency-status", instance.ID, request.ProbeKey)
|
||||
stamp := svc.now()
|
||||
status := domain.DependencyStatus{
|
||||
@@ -937,8 +973,10 @@ func (svc *CoreService) upsertDependencyStatus(instance domain.ServerInstance, r
|
||||
TargetOS: request.TargetOS,
|
||||
TargetArch: request.TargetArch,
|
||||
State: state,
|
||||
Required: true,
|
||||
Required: required,
|
||||
InstallPlanKey: request.InstallPlanKey,
|
||||
PlanDigest: request.PlanDigest,
|
||||
JobID: jobID,
|
||||
Message: message,
|
||||
CheckedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
@@ -955,25 +993,34 @@ func (svc *CoreService) upsertDependencyStatus(instance domain.ServerInstance, r
|
||||
}
|
||||
|
||||
func (svc *CoreService) recordAuditEvent(actorID string, action string, resourceKind string, resourceID string, result domain.AuditResult, summary string) error {
|
||||
_, err := svc.recordAuditEventWithID(actorID, action, resourceKind, resourceID, result, summary)
|
||||
return err
|
||||
}
|
||||
|
||||
func (svc *CoreService) recordAuditEventWithID(actorID string, action string, resourceKind string, resourceID string, result domain.AuditResult, summary string) (string, error) {
|
||||
svc.auditMu.Lock()
|
||||
svc.auditSeq++
|
||||
seq := svc.auditSeq
|
||||
svc.auditMu.Unlock()
|
||||
|
||||
stamp := svc.now()
|
||||
event := domain.AuditEvent{
|
||||
ID: fmt.Sprintf("audit-%s-%d", strings.ReplaceAll(action, ".", "-"), seq),
|
||||
ID: fmt.Sprintf("audit-%s-%d-%d", strings.ReplaceAll(action, ".", "-"), stamp.UnixNano(), seq),
|
||||
ActorID: actorID,
|
||||
Action: action,
|
||||
ResourceKind: resourceKind,
|
||||
ResourceID: resourceID,
|
||||
Result: result,
|
||||
Summary: safeBridgeReason(summary),
|
||||
CreatedAt: svc.now(),
|
||||
CreatedAt: stamp,
|
||||
}
|
||||
if err := validator.ValidateAuditEvent(event); err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
return svc.store.AuditEvents().Create(event)
|
||||
if err := svc.store.AuditEvents().Create(event); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return event.ID, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) auditArtifactDownload(sessionID string, artifact domain.Artifact) error {
|
||||
@@ -1085,17 +1132,6 @@ func fingerprintForString(value string) string {
|
||||
return hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
|
||||
func clientManagerCheckoutRef(repositoryURL string, sourceRevision string) string {
|
||||
sourceRevision = strings.TrimSpace(sourceRevision)
|
||||
if sourceRevision == "" {
|
||||
sourceRevision = "main"
|
||||
}
|
||||
if looksLikeCommitRevision(sourceRevision) {
|
||||
return "commit/" + sourceRevision
|
||||
}
|
||||
return "branch/" + sanitizeIDPart(sourceRevision)
|
||||
}
|
||||
|
||||
func clientManagerOutputName(profileKey string, targetOS string) string {
|
||||
name := sanitizeIDPart(profileKey)
|
||||
if targetOS == "windows" {
|
||||
@@ -1104,57 +1140,6 @@ func clientManagerOutputName(profileKey string, targetOS string) string {
|
||||
return name
|
||||
}
|
||||
|
||||
func clientManagerBuildLog(checkout clientManagerCheckoutPlan, config generatedPackageConfig, outputs []string) string {
|
||||
lines := []string{
|
||||
"client-manager checkout prepared",
|
||||
"repository=" + checkout.RepositoryURL,
|
||||
"sourceRevision=" + checkout.SourceRevision,
|
||||
"checkoutRef=" + checkout.CheckoutRef,
|
||||
"target=" + checkout.TargetOS + "/" + checkout.TargetArch,
|
||||
"dependencyCheck=typed build profile accepted",
|
||||
"configInjection=secret ref " + config.SecretRef + " generation " + fmt.Sprintf("%d", config.KeyGeneration),
|
||||
"keyFingerprint=" + fingerprintForString(config.AuthKey),
|
||||
"outputs=" + strings.Join(outputs, ","),
|
||||
}
|
||||
return redactDistributionLog(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
func looksLikeCommitRevision(value string) bool {
|
||||
if len(value) < 7 || len(value) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F') || (char >= '0' && char <= '9') {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func redactDistributionLog(value string) string {
|
||||
replacements := []string{
|
||||
"/Users/", "[host]/",
|
||||
"password=", "password=[redacted]",
|
||||
"api_key=", "api_key=[redacted]",
|
||||
"secret=", "secret=[redacted]",
|
||||
"Bearer ", "Bearer [redacted] ",
|
||||
"sk-", "sk-[redacted]",
|
||||
"unix://", "socket://",
|
||||
"tcp://", "endpoint://",
|
||||
"mysql://", "db://",
|
||||
"sqlite://", "db://",
|
||||
}
|
||||
redacted := value
|
||||
for i := 0; i+1 < len(replacements); i += 2 {
|
||||
redacted = strings.ReplaceAll(redacted, replacements[i], replacements[i+1])
|
||||
}
|
||||
if len(redacted) > 4096 {
|
||||
return redacted[:4096]
|
||||
}
|
||||
return redacted
|
||||
}
|
||||
|
||||
func minInt(a int, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
@@ -1178,24 +1163,43 @@ func fallbackReason(primary bool, primaryReason string, fallback string) string
|
||||
}
|
||||
|
||||
func (svc *CoreService) runtimeBindingsComplete(serverInstanceID string) bool {
|
||||
bindings, err := svc.store.RuntimeBindings().List(domain.RuntimeBindingFilter{ServerInstanceID: serverInstanceID})
|
||||
complete, _ := svc.runtimeBindingReadiness(serverInstanceID)
|
||||
return complete
|
||||
}
|
||||
|
||||
func (svc *CoreService) runtimeBindingReadiness(serverInstanceID string) (bool, string) {
|
||||
binding, err := svc.runtimeBindingForServer(serverInstanceID)
|
||||
if errors.Is(err, repo.ErrNotFound) {
|
||||
return false, "runtime profile is not configured"
|
||||
}
|
||||
if err != nil {
|
||||
return false
|
||||
return false, "runtime binding cannot be verified"
|
||||
}
|
||||
for _, binding := range bindings {
|
||||
if binding.Status == domain.RuntimeBindingStatusIncomplete {
|
||||
return false
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(serverInstanceID)
|
||||
if err != nil || binding.PluginID != instance.PluginID || binding.PluginVersion != instance.PluginVersion {
|
||||
return false, "runtime binding does not match the server plugin"
|
||||
}
|
||||
return true
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return false, "runtime profile cannot be verified"
|
||||
}
|
||||
binding, err = normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return false, "runtime binding cannot be verified"
|
||||
}
|
||||
if binding.Status != domain.RuntimeBindingStatusComplete {
|
||||
return false, "missing logical bindings: " + strings.Join(binding.MissingKeys, ", ")
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func (svc *CoreService) requireCompleteRuntimeBindings(actorID string, serverInstanceID string, deniedAction string) error {
|
||||
if svc.runtimeBindingsComplete(serverInstanceID) {
|
||||
complete, reason := svc.runtimeBindingReadiness(serverInstanceID)
|
||||
if complete {
|
||||
return nil
|
||||
}
|
||||
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "operation denied: runtime binding is incomplete")
|
||||
return ErrForbidden
|
||||
_ = svc.recordAuditEvent(actorID, deniedAction, "server-instance", serverInstanceID, domain.AuditResultDenied, "operation denied: "+reason)
|
||||
return validationError(reason)
|
||||
}
|
||||
|
||||
func pluginDeclares(plugin domain.GamePlugin, permission string) bool {
|
||||
@@ -1239,6 +1243,9 @@ func validateDependencyJobRequest(request domain.DependencyJobRequest) error {
|
||||
if request.Install && !safeDistributionKey(request.InstallPlanKey) {
|
||||
return validationError("installPlanKey is invalid")
|
||||
}
|
||||
if request.Install && (request.PlanDigest == "" || !strings.HasPrefix(request.PlanDigest, "sha256:") || len(request.PlanDigest) != len("sha256:")+64) {
|
||||
return validationError("planDigest must be a sha256 digest")
|
||||
}
|
||||
if containsUnsafeRequestText(request.IdempotencyKey) || containsUnsafeRequestText(request.TargetOS) || containsUnsafeRequestText(request.TargetArch) {
|
||||
return validationError("dependency request contains unsafe content")
|
||||
}
|
||||
@@ -1291,54 +1298,3 @@ func containsUnsafeRequestText(value string) bool {
|
||||
strings.Contains(lowered, "tcp://") ||
|
||||
strings.Contains(lowered, "/users/")
|
||||
}
|
||||
|
||||
func encryptRuntimeKey(plain string) (string, error) {
|
||||
key := runtimeEncryptionKey()
|
||||
block, err := aes.NewCipher(key[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ciphertext := gcm.Seal(nil, nonce, []byte(plain), nil)
|
||||
return "enc:v1:" + base64.RawURLEncoding.EncodeToString(nonce) + ":" + base64.RawURLEncoding.EncodeToString(ciphertext), nil
|
||||
}
|
||||
|
||||
func decryptRuntimeKey(encrypted string) (string, error) {
|
||||
parts := strings.Split(encrypted, ":")
|
||||
if len(parts) != 4 || parts[0] != "enc" || parts[1] != "v1" {
|
||||
return "", validationError("encrypted key format is invalid")
|
||||
}
|
||||
nonce, err := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ciphertext, err := base64.RawURLEncoding.DecodeString(parts[3])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := runtimeEncryptionKey()
|
||||
block, err := aes.NewCipher(key[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plain, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(plain), nil
|
||||
}
|
||||
|
||||
func runtimeEncryptionKey() [32]byte {
|
||||
return sha256.Sum256([]byte("browser.local/platform/runtime-component-key/v1"))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,19 @@ import (
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
type generatedPackageConfig struct {
|
||||
Kind string
|
||||
ServerInstanceID string
|
||||
PluginID string
|
||||
RunEndpointID string
|
||||
ProfileKey string
|
||||
TargetOS string
|
||||
TargetArch string
|
||||
SecretRef string
|
||||
KeyGeneration int
|
||||
AuthKey string
|
||||
}
|
||||
|
||||
func TestCoreServiceGeneratesRunDistributionWithEncryptedSingletonKey(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
|
||||
@@ -83,6 +96,64 @@ func TestCoreServiceGeneratesRunDistributionWithEncryptedSingletonKey(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceDistributionBuildRejectsPrematureSuccessAndCanRetryAfterUpload(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
TargetOS: "linux",
|
||||
TargetArch: "amd64",
|
||||
IdempotencyKey: "idem-premature-result",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("generate run distribution: %v", err)
|
||||
}
|
||||
|
||||
helloRequest := validRunControlHello()
|
||||
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, domain.JobCapabilityDistributionBuild)
|
||||
hello, err := svc.RegisterRunHello(helloRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("register build worker: %v", err)
|
||||
}
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
SessionToken: hello.SessionToken,
|
||||
Capabilities: []string{domain.JobCapabilityDistributionBuild},
|
||||
Capacity: domain.RunCapacity{MaxJobs: 1},
|
||||
})
|
||||
if err != nil || !claim.HasJob || claim.Job.JobID != distribution.BuildJobID {
|
||||
t.Fatalf("claim distribution build job: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
result := domain.RunJobResult{
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
SessionToken: hello.SessionToken,
|
||||
JobID: claim.Job.JobID,
|
||||
LeaseToken: claim.Job.LeaseToken,
|
||||
Attempt: claim.Job.Attempt,
|
||||
State: domain.JobStateSucceeded,
|
||||
Progress: domain.RunJobProgressReport{Percent: 100, Message: "package_finalize: done"},
|
||||
ResultRef: "artifact://" + distribution.ArtifactID,
|
||||
Message: "done",
|
||||
}
|
||||
if _, err := svc.CompleteRunJob(result); err == nil {
|
||||
t.Fatal("expected premature success without uploaded artifact to be rejected")
|
||||
}
|
||||
stored, err := svc.GetJob(distribution.BuildJobID)
|
||||
if err != nil || stored.State != domain.JobStateAccepted {
|
||||
t.Fatalf("premature success must not make the job terminal, job=%+v err=%v", stored, err)
|
||||
}
|
||||
|
||||
if _, err := svc.createPlatformArtifactPayload(distribution.ArtifactID, domain.ArtifactOwnerKindJob, distribution.BuildJobID, []byte("actual compiled archive")); err != nil {
|
||||
t.Fatalf("publish uploaded build output: %v", err)
|
||||
}
|
||||
if _, err := svc.CompleteRunJob(result); err != nil {
|
||||
t.Fatalf("retry success after artifact upload: %v", err)
|
||||
}
|
||||
stored, err = svc.GetJob(distribution.BuildJobID)
|
||||
if err != nil || stored.State != domain.JobStateSucceeded {
|
||||
t.Fatalf("expected terminal success after upload, job=%+v err=%v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceRunDistributionRetryReusesPartialArtifact(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
key, plainKey, err := svc.ensureActiveComponentKey(instance.ID, domain.DistributionComponentRun, "")
|
||||
@@ -124,7 +195,7 @@ func TestCoreServicePushRunUpdateReusesExistingUpdateJob(t *testing.T) {
|
||||
svc, session, instance := newDistributionTestFixture(t)
|
||||
distribution, err := svc.GenerateRunDistributionForSession(session, domain.RunDistributionGenerateRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
TargetOS: "windows",
|
||||
TargetOS: "linux",
|
||||
TargetArch: "amd64",
|
||||
IdempotencyKey: "idem-run-before-update",
|
||||
})
|
||||
@@ -347,6 +418,11 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
|
||||
domain.JobCapabilityDependenciesCheck,
|
||||
domain.JobCapabilityDependenciesInstall,
|
||||
domain.JobCapabilityLogsBackfill,
|
||||
domain.JobCapabilityClientManagerDeploy,
|
||||
domain.JobCapabilityClientManagerControl,
|
||||
domain.JobCapabilityClientManagerUpdate,
|
||||
domain.JobCapabilityClientManagerRollback,
|
||||
domain.JobCapabilityClientManagerUninstall,
|
||||
)
|
||||
plugin.BridgeActions = append(plugin.BridgeActions,
|
||||
string(domain.PluginBridgeActionRunDistribution),
|
||||
@@ -354,6 +430,9 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
|
||||
string(domain.PluginBridgeActionDependenciesRequest),
|
||||
string(domain.PluginBridgeActionLogsBackfillRequest),
|
||||
)
|
||||
plugin.RuntimeProfiles.DependencyProbes = []domain.RuntimeDependencyProbe{{Key: "java-runtime", Kind: "command.version", TargetKey: "java", Platforms: []string{"linux"}}}
|
||||
plugin.RuntimeProfiles.InstallPlans = []domain.RuntimeInstallPlan{{Key: "java-install", Title: "Install Java", Platforms: []string{"linux"}, Steps: []domain.RuntimeInstallStep{{Type: "package", TargetKey: "java", PackageManager: "apt", PackageName: "openjdk-21-jre"}}}}
|
||||
plugin.RuntimeProfiles.ClientManagers = []domain.RuntimeClientManagerProfile{{Key: "scum-client-manager", DisplayName: "SCUM Client Manager", Version: "1.0.0", RepositoryURL: "https://github.com/F88888/scum_client.git", RevisionPolicy: "branch", Branch: "main", SupportedTargets: []domain.RuntimeTarget{{OS: "windows", Arch: "amd64"}, {OS: "linux", Arch: "amd64"}}, BuildSystem: "go", EntryRef: "main.go", OutputArtifacts: []string{"scum_client.exe"}, Deployment: domain.RuntimeClientManagerDeployment{Mode: "run-supervised", ExecutableRef: "scum_client.exe", RequiredRunCapabilities: []string{domain.JobCapabilityClientManagerDeploy, domain.JobCapabilityClientManagerControl, domain.JobCapabilityClientManagerUpdate, domain.JobCapabilityClientManagerRollback, domain.JobCapabilityClientManagerUninstall}}, Lifecycle: domain.RuntimeClientManagerLifecycle{Actions: []string{"start", "stop", "restart", "status", "update", "rollback", "uninstall"}, StartupTimeoutSeconds: 60, StopTimeoutSeconds: 30}, Health: domain.RuntimeClientManagerHealth{Mode: "component-heartbeat", IntervalSeconds: 15, DegradedAfterSeconds: 45, OfflineAfterSeconds: 120, RequiredCapabilities: []string{"component.register", "component.heartbeat", "component.health"}}, Compatibility: domain.RuntimeClientManagerCompatibility{MinimumVersion: "1.0.0"}, UpdatePolicy: domain.RuntimeClientManagerUpdatePolicy{Strategy: "manual-staged", RequireApproval: true, HealthConfirmationSeconds: 60, RetainPrevious: true}}}
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("update plugin fixture: %v", err)
|
||||
}
|
||||
@@ -363,7 +442,14 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
|
||||
domain.JobCapabilityDependenciesCheck,
|
||||
domain.JobCapabilityDependenciesInstall,
|
||||
domain.JobCapabilityLogsBackfill,
|
||||
domain.JobCapabilityClientManagerDeploy,
|
||||
domain.JobCapabilityClientManagerControl,
|
||||
domain.JobCapabilityClientManagerUpdate,
|
||||
domain.JobCapabilityClientManagerRollback,
|
||||
domain.JobCapabilityClientManagerUninstall,
|
||||
)
|
||||
endpoint.Platform = "linux"
|
||||
endpoint.Architecture = "amd64"
|
||||
if err := svc.store.RunEndpoints().Update(endpoint); err != nil {
|
||||
t.Fatalf("update endpoint fixture: %v", err)
|
||||
}
|
||||
@@ -384,6 +470,7 @@ func newDistributionTestFixture(t *testing.T) (*CoreService, string, domain.Serv
|
||||
if err != nil {
|
||||
t.Fatalf("create distribution server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, instance, "local")
|
||||
return svc, session, instance
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
func TestFileArtifactBodyStoreResumesTransferAfterServiceRestart(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
metadata := filepath.Join(root, "metadata.json")
|
||||
store, err := repo.NewFileStore(metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("new file store: %v", err)
|
||||
}
|
||||
logStore, err := NewFileLogBodyStore(filepath.Join(root, "logs"))
|
||||
if err != nil {
|
||||
t.Fatalf("new log store: %v", err)
|
||||
}
|
||||
artifactStore, err := NewFileArtifactBodyStore(filepath.Join(root, "artifacts"))
|
||||
if err != nil {
|
||||
t.Fatalf("new artifact store: %v", err)
|
||||
}
|
||||
svc, err := NewCoreServiceWithDurableStores(store, logStore, artifactStore)
|
||||
if err != nil {
|
||||
t.Fatalf("new durable service: %v", err)
|
||||
}
|
||||
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
|
||||
if _, err := svc.CreateServerInstance(domain.ServerInstance{ID: "durable-server", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Durable"}); err != nil {
|
||||
t.Fatalf("create instance: %v", err)
|
||||
}
|
||||
if _, err := svc.CreateJob(domain.Job{ID: "durable-job", ServerInstanceID: "durable-server", RunEndpointID: endpoint.ID, Capability: "process.start", IdempotencyKey: "durable-job"}); err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
hello, err := svc.RegisterRunHello(validRunControlHello())
|
||||
if err != nil {
|
||||
t.Fatalf("register run: %v", err)
|
||||
}
|
||||
payload := []byte("durable transfer payload")
|
||||
open, err := svc.OpenArtifactTransfer(domain.ArtifactTransferOpen{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, ArtifactID: "durable-artifact", Direction: domain.ArtifactTransferDirectionUpload, OwnerKind: domain.ArtifactOwnerKindJob, OwnerID: "durable-job", SizeBytes: int64(len(payload)), ChunkSizeBytes: 8, Checksum: validator.BytesChecksum(payload), IdempotencyKey: "durable-transfer"})
|
||||
if err != nil {
|
||||
t.Fatalf("open transfer: %v", err)
|
||||
}
|
||||
first := validArtifactChunk(hello.SessionToken, open.TransferID, payload, 0, 8)
|
||||
first.ArtifactID = "durable-artifact"
|
||||
if _, err := svc.UploadArtifactChunk(first); err != nil {
|
||||
t.Fatalf("upload first chunk: %v", err)
|
||||
}
|
||||
|
||||
reloadedStore, err := repo.NewFileStore(metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("reload metadata store: %v", err)
|
||||
}
|
||||
reloadedLogStore, err := NewFileLogBodyStore(filepath.Join(root, "logs"))
|
||||
if err != nil {
|
||||
t.Fatalf("reload log store: %v", err)
|
||||
}
|
||||
reloadedArtifacts, err := NewFileArtifactBodyStore(filepath.Join(root, "artifacts"))
|
||||
if err != nil {
|
||||
t.Fatalf("reload artifact store: %v", err)
|
||||
}
|
||||
restarted, err := NewCoreServiceWithDurableStores(reloadedStore, reloadedLogStore, reloadedArtifacts)
|
||||
if err != nil {
|
||||
t.Fatalf("restart service: %v", err)
|
||||
}
|
||||
status, err := restarted.QueryArtifactTransferStatus(domain.ArtifactTransferStatusQuery{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, TransferID: open.TransferID, ArtifactID: "durable-artifact"})
|
||||
if err != nil {
|
||||
t.Fatalf("query resumed status: %v", err)
|
||||
}
|
||||
if status.NextMissingChunkIndex != 1 || len(status.ReceivedChunkIndexes) != 1 {
|
||||
t.Fatalf("unexpected resumed transfer status: %+v", status)
|
||||
}
|
||||
for index := 1; index < open.TotalChunks; index++ {
|
||||
chunk := validArtifactChunk(hello.SessionToken, open.TransferID, payload, index, 8)
|
||||
chunk.ArtifactID = "durable-artifact"
|
||||
if _, err := restarted.UploadArtifactChunk(chunk); err != nil {
|
||||
t.Fatalf("upload resumed chunk %d: %v", index, err)
|
||||
}
|
||||
}
|
||||
if _, err := restarted.CompleteArtifactTransfer(domain.ArtifactTransferComplete{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, TransferID: open.TransferID, ArtifactID: "durable-artifact", Checksum: validator.BytesChecksum(payload), SizeBytes: int64(len(payload))}); err != nil {
|
||||
t.Fatalf("complete resumed transfer: %v", err)
|
||||
}
|
||||
finalStore, _ := repo.NewFileStore(metadata)
|
||||
finalArtifacts, _ := NewFileArtifactBodyStore(filepath.Join(root, "artifacts"))
|
||||
finalService, err := NewCoreServiceWithDurableStores(finalStore, reloadedLogStore, finalArtifacts)
|
||||
if err != nil {
|
||||
t.Fatalf("final restart service: %v", err)
|
||||
}
|
||||
stored, err := finalService.artifactPayload("durable-artifact")
|
||||
if err != nil || !bytes.Equal(stored, payload) {
|
||||
t.Fatalf("expected durable payload after restart, payload=%q err=%v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAndBackupsPersistWithRetentionRecovery(t *testing.T) {
|
||||
svc := newTestCoreService()
|
||||
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
|
||||
ownerSession := createServiceUserAndLogin(t, svc, domain.User{ID: "observability-owner", DisplayName: "Owner", Email: "observability-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
instance, err := svc.CreateServerInstanceForSession(ownerSession, domain.ServerInstance{ID: "observability-server", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Observability"})
|
||||
if err != nil {
|
||||
t.Fatalf("create instance: %v", err)
|
||||
}
|
||||
runHello := validRunControlHello()
|
||||
runHello.RunEndpointID = endpoint.ID
|
||||
registered, err := svc.RegisterRunHello(runHello)
|
||||
if err != nil {
|
||||
t.Fatalf("register run: %v", err)
|
||||
}
|
||||
collectedAt := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC)
|
||||
cpu := 42.0
|
||||
if _, err := svc.IngestMetricBatch(domain.MetricBatchIngest{RunEndpointID: endpoint.ID, SessionToken: registered.SessionToken, Samples: []domain.MetricSample{{ServerInstanceID: instance.ID, CPUPercent: &cpu, Source: "run", CollectedAt: collectedAt}}}); err != nil {
|
||||
t.Fatalf("ingest metrics: %v", err)
|
||||
}
|
||||
metrics, err := svc.ListMetricSamplesForSession(ownerSession, domain.MetricSampleFilter{ServerInstanceID: instance.ID, Limit: 10})
|
||||
if err != nil || len(metrics) != 1 || metrics[0].CPUPercent == nil || *metrics[0].CPUPercent != cpu {
|
||||
t.Fatalf("unexpected persisted metrics: %+v err=%v", metrics, err)
|
||||
}
|
||||
artifact, err := svc.CreateArtifact(domain.Artifact{ID: "backup-artifact", OwnerKind: domain.ArtifactOwnerKindServerInstance, OwnerID: instance.ID, SizeBytes: 12, Checksum: validator.BytesChecksum([]byte("backup bytes")), State: domain.ArtifactStateAvailable})
|
||||
if err != nil {
|
||||
t.Fatalf("create backup artifact: %v", err)
|
||||
}
|
||||
backup, err := svc.CreateBackupForSession(ownerSession, domain.BackupRecord{ID: "backup-1", ServerInstanceID: instance.ID, ArtifactID: artifact.ID})
|
||||
if err != nil || backup.State != domain.BackupStatePending {
|
||||
t.Fatalf("create backup record: %+v err=%v", backup, err)
|
||||
}
|
||||
if err := svc.RecoverIncompleteBackups(); err != nil {
|
||||
t.Fatalf("recover backups: %v", err)
|
||||
}
|
||||
recovered, err := svc.GetBackupForSession(ownerSession, backup.ID)
|
||||
if err != nil || recovered.State != domain.BackupStateFailed || recovered.RecoveryStatus == "" {
|
||||
t.Fatalf("expected recoverable failed backup, record=%+v err=%v", recovered, err)
|
||||
}
|
||||
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "observability-other", DisplayName: "Other", Email: "observability-other@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
if _, err := svc.ListMetricSamplesForSession(otherSession, domain.MetricSampleFilter{ServerInstanceID: instance.ID, Limit: 10}); err != ErrForbidden {
|
||||
t.Fatalf("expected cross-owner metric denial, got %v", err)
|
||||
}
|
||||
if _, err := svc.GetBackupForSession(otherSession, backup.ID); err != ErrForbidden {
|
||||
t.Fatalf("expected cross-owner backup denial, got %v", err)
|
||||
}
|
||||
}
|
||||
+435
-167
@@ -1,6 +1,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -10,14 +11,22 @@ import (
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const defaultJobPollSeconds = 2
|
||||
const (
|
||||
defaultJobPollSeconds = 2
|
||||
defaultJobMaxAttempts = 3
|
||||
defaultJobInitialBackoffSeconds = 2
|
||||
defaultJobMaxBackoffSeconds = 60
|
||||
defaultJobAckTimeout = 15 * time.Second
|
||||
defaultJobLeaseDuration = 60 * time.Second
|
||||
)
|
||||
|
||||
func (svc *CoreService) ClaimRunJob(claim domain.RunJobClaim) (domain.RunJobClaimResult, error) {
|
||||
claim = domain.CopyRunJobClaim(claim)
|
||||
if err := validator.ValidateRunJobClaim(claim); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(claim.RunEndpointID, claim.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(claim.RunEndpointID, claim.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
|
||||
@@ -25,31 +34,40 @@ func (svc *CoreService) ClaimRunJob(claim domain.RunJobClaim) (domain.RunJobClai
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: claim.RunEndpointID, State: domain.JobStateQueued})
|
||||
if err := svc.sweepExpiredJobs(claim.RunEndpointID, stamp); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
if claim.Capacity.MaxJobs > 0 && claim.Capacity.RunningJobs >= claim.Capacity.MaxJobs {
|
||||
return emptyJobClaim(claim.RunEndpointID, stamp), nil
|
||||
}
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: claim.RunEndpointID})
|
||||
if err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
job, ok := firstSupportedJob(jobs, claim.Capabilities)
|
||||
job, ok := firstEligibleSupportedJob(jobs, claim.Capabilities, stamp)
|
||||
if !ok {
|
||||
return domain.RunJobClaimResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: claim.RunEndpointID,
|
||||
NextPollSeconds: defaultJobPollSeconds,
|
||||
ServerTime: stamp,
|
||||
}, nil
|
||||
return emptyJobClaim(claim.RunEndpointID, stamp), nil
|
||||
}
|
||||
|
||||
lease := svc.newJobLease(job.ID, claim.RunEndpointID, claim.SessionToken, stamp)
|
||||
svc.jobLeases[job.ID] = lease
|
||||
leaseToken, err := randomToken()
|
||||
if err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
job.Attempt++
|
||||
job.State = domain.JobStateAccepted
|
||||
job.Progress = domain.JobProgress{Percent: 0, Message: "claimed; awaiting Run acknowledgement"}
|
||||
job.NextAttemptAt = time.Time{}
|
||||
job.LeaseTokenHash = tokenHash(leaseToken)
|
||||
job.LeaseSessionGen = session.Generation
|
||||
job.AckDeadlineAt = stamp.Add(defaultJobAckTimeout)
|
||||
job.LeaseExpiresAt = stamp.Add(defaultJobLeaseDuration)
|
||||
job.LastProgressSeq = 0
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobClaimResult{}, err
|
||||
}
|
||||
assignment := assignmentFromJob(job, lease)
|
||||
assignment := assignmentFromJob(job, leaseToken)
|
||||
return domain.CopyRunJobClaimResult(domain.RunJobClaimResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: claim.RunEndpointID,
|
||||
@@ -64,20 +82,26 @@ func (svc *CoreService) AckRunJob(ack domain.RunJobAck) (domain.RunJobAckResult,
|
||||
if err := validator.ValidateRunJobAck(ack); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(ack.RunEndpointID, ack.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(ack.RunEndpointID, ack.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(ack.RunEndpointID, ack.SessionToken, ack.JobID, ack.LeaseToken, ack.Attempt)
|
||||
job, err := svc.fencedJob(session, ack.JobID, ack.LeaseToken, ack.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if isTerminalJobState(job.State) {
|
||||
return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
return domain.RunJobAckResult{}, validationError("late ack rejected for terminal job")
|
||||
}
|
||||
if job.State == domain.JobStateAccepted && deadlineExpired(job.AckDeadlineAt, stamp) {
|
||||
if err := svc.expireJobAttempt(&job, stamp, "Run acknowledgement deadline expired"); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
return domain.RunJobAckResult{}, validationError("ack deadline expired")
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.RunJobAckResult{}, validationError("job is not claimable for ack")
|
||||
@@ -86,92 +110,134 @@ func (svc *CoreService) AckRunJob(ack domain.RunJobAck) (domain.RunJobAckResult,
|
||||
if strings.TrimSpace(ack.Message) != "" {
|
||||
job.Progress.Message = ack.Message
|
||||
}
|
||||
job.AckDeadlineAt = time.Time{}
|
||||
job.LeaseExpiresAt = stamp.Add(defaultJobLeaseDuration)
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
return domain.RunJobAckResult{}, err
|
||||
}
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
return domain.RunJobAckResult{Accepted: true, Job: assignmentFromJob(job, ack.LeaseToken), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) UpdateRunJobProgress(progress domain.RunJobProgress) (domain.RunJobProgressResult, error) {
|
||||
if err := validator.ValidateRunJobProgress(progress); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(progress.RunEndpointID, progress.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(progress.RunEndpointID, progress.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(progress.RunEndpointID, progress.SessionToken, progress.JobID, progress.LeaseToken, progress.Attempt)
|
||||
job, err := svc.fencedJob(session, progress.JobID, progress.LeaseToken, progress.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.RunJobProgressResult{}, validationError("job is not active")
|
||||
if job.State != domain.JobStateRunning {
|
||||
return domain.RunJobProgressResult{}, validationError("job is not running")
|
||||
}
|
||||
if deadlineExpired(job.LeaseExpiresAt, stamp) {
|
||||
if err := svc.expireJobAttempt(&job, stamp, "Run execution lease expired"); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
return domain.RunJobProgressResult{}, validationError("job lease expired")
|
||||
}
|
||||
if progress.Sequence > 0 && progress.Sequence <= job.LastProgressSeq {
|
||||
return domain.RunJobProgressResult{}, validationError("progress sequence is stale")
|
||||
}
|
||||
job.State = domain.JobStateRunning
|
||||
job.Progress = domain.JobProgress{Percent: progress.Progress.Percent, Message: progress.Progress.Message}
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
if progress.Sequence > 0 {
|
||||
job.LastProgressSeq = progress.Sequence
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
job.LeaseExpiresAt = stamp.Add(defaultJobLeaseDuration)
|
||||
job.UpdatedAt = stamp
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if err := svc.projectDistributionBuildProgress(job, stamp); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobProgressResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
if err := svc.projectDependencyAndRunUpdateProgress(job, stamp); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
if err := svc.projectClientManagerLifecycleProgress(job, stamp); err != nil {
|
||||
return domain.RunJobProgressResult{}, err
|
||||
}
|
||||
return domain.RunJobProgressResult{Accepted: true, Job: assignmentFromJob(job, progress.LeaseToken), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CompleteRunJob(result domain.RunJobResult) (domain.RunJobResultResult, error) {
|
||||
if err := validator.ValidateRunJobResult(result); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(result.RunEndpointID, result.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(result.RunEndpointID, result.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
job, lease, err := svc.activeLeasedJob(result.RunEndpointID, result.SessionToken, result.JobID, result.LeaseToken, result.Attempt)
|
||||
job, err := svc.fencedJob(session, result.JobID, result.LeaseToken, result.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
fingerprint := terminalFingerprint(result)
|
||||
if isTerminalJobState(job.State) {
|
||||
if lease.TerminalFingerprint != "" && lease.TerminalFingerprint == fingerprint {
|
||||
if err := svc.projectLifecycleJobResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectDistributionBuildResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
if job.TerminalFingerprint == fingerprint {
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, result.LeaseToken), ServerTime: stamp}, nil
|
||||
}
|
||||
return domain.RunJobResultResult{}, validationError("terminal result conflicts with existing job result")
|
||||
}
|
||||
if job.State != domain.JobStateAccepted && job.State != domain.JobStateRunning {
|
||||
return domain.RunJobResultResult{}, validationError("job attempt is no longer active")
|
||||
}
|
||||
if deadlineExpired(job.LeaseExpiresAt, stamp) {
|
||||
if err := svc.expireJobAttempt(&job, stamp, "Run execution lease expired"); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
return domain.RunJobResultResult{}, validationError("job lease expired")
|
||||
}
|
||||
if !job.CancelRequestedAt.IsZero() && result.State != domain.JobStateCancelled {
|
||||
return domain.RunJobResultResult{}, validationError("cancel intent requires a cancelled terminal result")
|
||||
}
|
||||
if err := validateExecutionResultForJob(job, result); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
|
||||
if result.State == domain.JobStateFailed && result.Retryable && job.Attempt < job.RetryPolicy.MaxAttempts && job.CancelRequestedAt.IsZero() {
|
||||
job.Progress = domain.JobProgress{Percent: result.Progress.Percent, Message: terminalMessage(result)}
|
||||
if err := svc.scheduleJobRetry(&job, stamp, "retryable Run failure"); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, ""), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
job.State = result.State
|
||||
job.Progress = domain.JobProgress{Percent: result.Progress.Percent, Message: terminalMessage(result)}
|
||||
job.ResultRef = result.ResultRef
|
||||
job.ExecutionResult = result.ExecutionResult
|
||||
job.TerminalAt = stamp
|
||||
job.TerminalFingerprint = fingerprint
|
||||
job.AckDeadlineAt = time.Time{}
|
||||
job.LeaseExpiresAt = time.Time{}
|
||||
if result.State == domain.JobStateCancelled {
|
||||
job.CancelCompletedAt = stamp
|
||||
if job.CancelRequestedAt.IsZero() {
|
||||
job.CancelRequestedAt = stamp
|
||||
job.CancelReason = terminalMessage(result)
|
||||
}
|
||||
}
|
||||
job.UpdatedAt = stamp
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.store.Jobs().Update(job); err != nil {
|
||||
if err := svc.validateDistributionBuildResult(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectLifecycleJobResult(job, stamp); err != nil {
|
||||
@@ -180,17 +246,84 @@ func (svc *CoreService) CompleteRunJob(result domain.RunJobResult) (domain.RunJo
|
||||
if err := svc.projectDistributionBuildResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
lease.TerminalFingerprint = fingerprint
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, lease), ServerTime: stamp}, nil
|
||||
if err := svc.projectRemoteAdapterJobResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectDependencyAndRunUpdateResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
if err := svc.projectClientManagerLifecycleResult(job, stamp); err != nil {
|
||||
return domain.RunJobResultResult{}, err
|
||||
}
|
||||
return domain.RunJobResultResult{Accepted: true, Job: assignmentFromJob(job, result.LeaseToken), ServerTime: stamp}, nil
|
||||
}
|
||||
|
||||
func validateExecutionResultForJob(job domain.Job, result domain.RunJobResult) error {
|
||||
if result.ExecutionResult.Kind == "" {
|
||||
return nil
|
||||
}
|
||||
switch job.Capability {
|
||||
case domain.JobCapabilityConfigWrite:
|
||||
if result.ExecutionResult.Kind != "file.write" {
|
||||
return validationError("config write result type is invalid")
|
||||
}
|
||||
if result.State != domain.JobStateSucceeded {
|
||||
return nil
|
||||
}
|
||||
if result.ExecutionResult.Version != job.ExecutionInput.ExpectedVersion+1 {
|
||||
return validationError("config write result version is invalid")
|
||||
}
|
||||
if result.ExecutionResult.Checksum == "" || result.ExecutionResult.Checksum != validator.BytesChecksum([]byte(job.ExecutionInput.Content)) {
|
||||
return validationError("config write result checksum is invalid")
|
||||
}
|
||||
case domain.JobCapabilityFilesRead:
|
||||
if result.ExecutionResult.Kind != "file.read" {
|
||||
return validationError("file read result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityFilesWrite:
|
||||
if result.ExecutionResult.Kind != "file.write" {
|
||||
return validationError("file write result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityDependenciesCheck:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "dependency.check" {
|
||||
return validationError("dependency check result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityDependenciesInstall:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "dependency.install" {
|
||||
return validationError("dependency install result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityRunSelfUpdate:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "run.update.staged" {
|
||||
return validationError("Run self-update result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerDeploy:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.deployed" {
|
||||
return validationError("client-manager deploy result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerControl:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.controlled" {
|
||||
return validationError("client-manager control result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerUpdate:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.updated" || result.State == domain.JobStateFailed && result.ExecutionResult.Kind != "" && result.ExecutionResult.Kind != "client-manager.rollback.restored" {
|
||||
return validationError("client-manager update result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerRollback:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.rolled-back" {
|
||||
return validationError("client-manager rollback result type is invalid")
|
||||
}
|
||||
case domain.JobCapabilityClientManagerUninstall:
|
||||
if result.State == domain.JobStateSucceeded && result.ExecutionResult.Kind != "client-manager.uninstalled" {
|
||||
return validationError("client-manager uninstall result type is invalid")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RequestRunJobCancel(request domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error) {
|
||||
if err := validator.ValidateRunJobCancelRequest(request); err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
@@ -199,43 +332,59 @@ func (svc *CoreService) RequestRunJobCancel(request domain.RunJobCancelRequest)
|
||||
if err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
if !isActiveJobState(job.State) {
|
||||
return domain.RunJobCancelRequestResult{}, validationError("job is not active")
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
if job.State == domain.JobStateCancelled {
|
||||
return cancelRequestResult(job), nil
|
||||
}
|
||||
lease, exists := svc.jobLeases[job.ID]
|
||||
if !exists {
|
||||
return domain.RunJobCancelRequestResult{}, validationError("job lease is missing")
|
||||
if isTerminalJobState(job.State) {
|
||||
return domain.RunJobCancelRequestResult{}, validationError("job is already terminal")
|
||||
}
|
||||
lease.CancelReason = request.Reason
|
||||
lease.CancelRequestedAt = stamp
|
||||
lease.UpdatedAt = stamp
|
||||
svc.jobLeases[job.ID] = lease
|
||||
return domain.RunJobCancelRequestResult{Accepted: true, JobID: job.ID, Reason: request.Reason, RequestedAt: stamp}, nil
|
||||
if job.CancelRequestedAt.IsZero() {
|
||||
job.CancelReason = request.Reason
|
||||
job.CancelRequestedAt = stamp
|
||||
}
|
||||
if job.State == domain.JobStateQueued || job.State == domain.JobStateRetrying {
|
||||
terminalizeCancelled(&job, stamp, job.CancelReason)
|
||||
}
|
||||
job.UpdatedAt = stamp
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
return cancelRequestResult(job), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) PollRunJobCancel(poll domain.RunJobCancelPoll) (domain.RunJobCancelPollResult, error) {
|
||||
if err := validator.ValidateRunJobCancelPoll(poll); err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(poll.RunEndpointID, poll.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(poll.RunEndpointID, poll.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
lease, ok := svc.findCancelLease(poll)
|
||||
if !ok {
|
||||
job, err := svc.fencedJob(session, poll.JobID, poll.LeaseToken, poll.Attempt)
|
||||
if err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
if jobAttemptExpired(job, stamp) {
|
||||
if err := svc.expireJobAttempt(&job, stamp, "Run job deadline expired before cancel poll"); err != nil {
|
||||
return domain.RunJobCancelPollResult{}, err
|
||||
}
|
||||
return domain.RunJobCancelPollResult{}, validationError("job lease expired")
|
||||
}
|
||||
if job.CancelRequestedAt.IsZero() || !isActiveJobState(job.State) {
|
||||
return domain.RunJobCancelPollResult{Accepted: true, RunEndpointID: poll.RunEndpointID, ServerTime: stamp}, nil
|
||||
}
|
||||
return domain.RunJobCancelPollResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: poll.RunEndpointID,
|
||||
HasCancel: true,
|
||||
JobID: lease.JobID,
|
||||
Reason: lease.CancelReason,
|
||||
RequestedAt: lease.CancelRequestedAt,
|
||||
JobID: job.ID,
|
||||
Reason: job.CancelReason,
|
||||
RequestedAt: job.CancelRequestedAt,
|
||||
ServerTime: stamp,
|
||||
}, nil
|
||||
}
|
||||
@@ -245,140 +394,198 @@ func (svc *CoreService) ReconcileRunJobs(reconcile domain.RunJobReconcile) (doma
|
||||
if err := validator.ValidateRunJobReconcile(reconcile); err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
if err := svc.validateRunSession(reconcile.RunEndpointID, reconcile.SessionToken); err != nil {
|
||||
session, err := svc.validatedRunSession(reconcile.RunEndpointID, reconcile.SessionToken)
|
||||
if err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
|
||||
stamp := svc.now()
|
||||
svc.jobMu.Lock()
|
||||
defer svc.jobMu.Unlock()
|
||||
|
||||
confirmed := make([]domain.RunJobAssignment, 0, len(reconcile.ActiveJobs))
|
||||
discard := make([]string, 0)
|
||||
confirmedIDs := map[string]struct{}{}
|
||||
for _, entry := range reconcile.ActiveJobs {
|
||||
job, getErr := svc.store.Jobs().Get(entry.JobID)
|
||||
if getErr != nil || job.RunEndpointID != reconcile.RunEndpointID || !isActiveJobState(job.State) || job.Attempt != entry.Attempt || !leaseTokenMatches(job.LeaseTokenHash, entry.LeaseToken) || jobAttemptExpired(job, stamp) {
|
||||
discard = append(discard, entry.JobID)
|
||||
continue
|
||||
}
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
job.LeaseSessionGen = session.Generation
|
||||
job.LeaseExpiresAt = stamp.Add(defaultJobLeaseDuration)
|
||||
job.LastReconciledAt = stamp
|
||||
job.ReconcileCount++
|
||||
job.ReconcileOutcome = "confirmed active attempt"
|
||||
job.UpdatedAt = stamp
|
||||
if err := svc.updateScheduledJob(job); err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
confirmedIDs[job.ID] = struct{}{}
|
||||
confirmed = append(confirmed, assignmentFromJob(job, entry.LeaseToken))
|
||||
}
|
||||
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: reconcile.RunEndpointID})
|
||||
if err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
activeByID := map[string]domain.Job{}
|
||||
for _, job := range jobs {
|
||||
if isActiveJobState(job.State) {
|
||||
activeByID[job.ID] = job
|
||||
if !isActiveJobState(job.State) {
|
||||
continue
|
||||
}
|
||||
if _, ok := confirmedIDs[job.ID]; ok {
|
||||
continue
|
||||
}
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
job.LastReconciledAt = stamp
|
||||
job.ReconcileCount++
|
||||
job.ReconcileOutcome = "missing from Run journal"
|
||||
if err := svc.expireJobAttempt(&job, stamp, "active attempt missing during Run reconciliation"); err != nil {
|
||||
return domain.RunJobReconcileResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
activeJobs := make([]domain.RunJobAssignment, 0, len(activeByID))
|
||||
ids := make([]string, 0, len(activeByID))
|
||||
for id := range activeByID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
job := activeByID[id]
|
||||
lease := svc.jobLeases[job.ID]
|
||||
if lease.JobID == "" || lease.SessionToken != reconcile.SessionToken {
|
||||
lease = svc.newJobLease(job.ID, reconcile.RunEndpointID, reconcile.SessionToken, stamp)
|
||||
} else {
|
||||
lease.UpdatedAt = stamp
|
||||
}
|
||||
svc.jobLeases[job.ID] = lease
|
||||
activeJobs = append(activeJobs, assignmentFromJob(job, lease))
|
||||
}
|
||||
|
||||
unknown := make([]string, 0)
|
||||
for _, reportedID := range reconcile.ActiveJobIDs {
|
||||
if _, exists := activeByID[reportedID]; !exists {
|
||||
unknown = append(unknown, reportedID)
|
||||
}
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
sort.Strings(discard)
|
||||
sort.Slice(confirmed, func(i, j int) bool { return confirmed[i].JobID < confirmed[j].JobID })
|
||||
return domain.CopyRunJobReconcileResult(domain.RunJobReconcileResult{
|
||||
Accepted: true,
|
||||
RunEndpointID: reconcile.RunEndpointID,
|
||||
ActiveJobs: activeJobs,
|
||||
UnknownJobIDs: unknown,
|
||||
ConfirmedJobs: confirmed,
|
||||
DiscardJobIDs: discard,
|
||||
ServerTime: stamp,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateRunSession(runEndpointID string, sessionToken string) error {
|
||||
func (svc *CoreService) validatedRunSession(runEndpointID string, sessionToken string) (domain.RunControlSession, error) {
|
||||
svc.controlMu.Lock()
|
||||
defer svc.controlMu.Unlock()
|
||||
session, exists := svc.runSessions[runEndpointID]
|
||||
if !exists || session.SessionToken != sessionToken {
|
||||
return validationError("sessionToken is invalid")
|
||||
return svc.currentRunSession(runEndpointID, sessionToken)
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateRunSession(runEndpointID string, sessionToken string) error {
|
||||
_, err := svc.validatedRunSession(runEndpointID, sessionToken)
|
||||
return err
|
||||
}
|
||||
|
||||
func (svc *CoreService) fencedJob(session domain.RunControlSession, jobID string, leaseToken string, attempt int) (domain.Job, error) {
|
||||
job, err := svc.store.Jobs().Get(jobID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
job = normalizeJobScheduling(job, svc.now())
|
||||
if job.RunEndpointID != session.RunEndpointID {
|
||||
return domain.Job{}, validationError("job runEndpointId does not match request")
|
||||
}
|
||||
if job.Attempt != attempt || job.LeaseSessionGen != session.Generation || !leaseTokenMatches(job.LeaseTokenHash, leaseToken) {
|
||||
return domain.Job{}, validationError("attempt or leaseToken is invalid")
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) sweepExpiredJobs(runEndpointID string, stamp time.Time) error {
|
||||
jobs, err := svc.store.Jobs().List(domain.JobFilter{RunEndpointID: runEndpointID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
expired := job.State == domain.JobStateAccepted && deadlineExpired(job.AckDeadlineAt, stamp)
|
||||
expired = expired || job.State == domain.JobStateRunning && deadlineExpired(job.LeaseExpiresAt, stamp)
|
||||
if expired {
|
||||
if err := svc.expireJobAttempt(&job, stamp, "Run job deadline expired"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) newJobLease(jobID string, runEndpointID string, sessionToken string, stamp time.Time) domain.RunJobLease {
|
||||
svc.jobLeaseSeq++
|
||||
return domain.RunJobLease{
|
||||
JobID: jobID,
|
||||
RunEndpointID: runEndpointID,
|
||||
SessionToken: sessionToken,
|
||||
LeaseToken: fmt.Sprintf("job-lease:%s:%d:%d", jobID, stamp.UnixNano(), svc.jobLeaseSeq),
|
||||
Attempt: int(svc.jobLeaseSeq),
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
func (svc *CoreService) expireJobAttempt(job *domain.Job, stamp time.Time, reason string) error {
|
||||
if !job.CancelRequestedAt.IsZero() {
|
||||
terminalizeCancelled(job, stamp, job.CancelReason)
|
||||
return svc.updateScheduledJob(*job)
|
||||
}
|
||||
if job.Attempt < job.RetryPolicy.MaxAttempts {
|
||||
return svc.scheduleJobRetry(job, stamp, reason)
|
||||
}
|
||||
job.State = domain.JobStateFailed
|
||||
job.Progress = domain.JobProgress{Percent: job.Progress.Percent, Message: reason + "; retry budget exhausted"}
|
||||
job.TerminalAt = stamp
|
||||
job.TerminalFingerprint = fmt.Sprintf("scheduler-failed|%d|%s", job.Attempt, reason)
|
||||
job.AckDeadlineAt = time.Time{}
|
||||
job.LeaseExpiresAt = time.Time{}
|
||||
job.UpdatedAt = stamp
|
||||
return svc.updateScheduledJob(*job)
|
||||
}
|
||||
|
||||
func (svc *CoreService) activeLeasedJob(runEndpointID string, sessionToken string, jobID string, leaseToken string, attempt int) (domain.Job, domain.RunJobLease, error) {
|
||||
job, err := svc.store.Jobs().Get(jobID)
|
||||
if err != nil {
|
||||
return domain.Job{}, domain.RunJobLease{}, err
|
||||
}
|
||||
if job.RunEndpointID != runEndpointID {
|
||||
return domain.Job{}, domain.RunJobLease{}, validationError("job runEndpointId does not match request")
|
||||
}
|
||||
lease, exists := svc.jobLeases[jobID]
|
||||
if !exists || lease.SessionToken != sessionToken || lease.LeaseToken != leaseToken || lease.Attempt != attempt {
|
||||
return domain.Job{}, domain.RunJobLease{}, validationError("leaseToken is invalid")
|
||||
}
|
||||
return job, lease, nil
|
||||
func (svc *CoreService) scheduleJobRetry(job *domain.Job, stamp time.Time, reason string) error {
|
||||
job.State = domain.JobStateRetrying
|
||||
job.Progress.Message = reason
|
||||
job.NextAttemptAt = stamp.Add(jobRetryBackoff(job.RetryPolicy, job.Attempt))
|
||||
job.LeaseTokenHash = ""
|
||||
job.LeaseSessionGen = 0
|
||||
job.AckDeadlineAt = time.Time{}
|
||||
job.LeaseExpiresAt = time.Time{}
|
||||
job.LastProgressSeq = 0
|
||||
job.UpdatedAt = stamp
|
||||
return svc.updateScheduledJob(*job)
|
||||
}
|
||||
|
||||
func (svc *CoreService) findCancelLease(poll domain.RunJobCancelPoll) (domain.RunJobLease, bool) {
|
||||
if poll.JobID != "" {
|
||||
lease, exists := svc.jobLeases[poll.JobID]
|
||||
if !exists || lease.RunEndpointID != poll.RunEndpointID || lease.SessionToken != poll.SessionToken {
|
||||
return domain.RunJobLease{}, false
|
||||
}
|
||||
if poll.LeaseToken != "" && lease.LeaseToken != poll.LeaseToken {
|
||||
return domain.RunJobLease{}, false
|
||||
}
|
||||
return lease, lease.CancelReason != ""
|
||||
func (svc *CoreService) updateScheduledJob(job domain.Job) error {
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(svc.jobLeases))
|
||||
for id := range svc.jobLeases {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
lease := svc.jobLeases[id]
|
||||
if lease.RunEndpointID == poll.RunEndpointID && lease.SessionToken == poll.SessionToken && lease.CancelReason != "" {
|
||||
return lease, true
|
||||
}
|
||||
}
|
||||
return domain.RunJobLease{}, false
|
||||
return svc.store.Jobs().Update(job)
|
||||
}
|
||||
|
||||
func firstSupportedJob(jobs []domain.Job, capabilities []string) (domain.Job, bool) {
|
||||
func normalizeJobScheduling(job domain.Job, stamp time.Time) domain.Job {
|
||||
if job.RetryPolicy.MaxAttempts <= 0 {
|
||||
job.RetryPolicy.MaxAttempts = defaultJobMaxAttempts
|
||||
}
|
||||
if job.RetryPolicy.InitialBackoffSeconds <= 0 {
|
||||
job.RetryPolicy.InitialBackoffSeconds = defaultJobInitialBackoffSeconds
|
||||
}
|
||||
if job.RetryPolicy.MaxBackoffSeconds < job.RetryPolicy.InitialBackoffSeconds {
|
||||
job.RetryPolicy.MaxBackoffSeconds = defaultJobMaxBackoffSeconds
|
||||
}
|
||||
if job.QueueEligibleAt.IsZero() {
|
||||
if !job.CreatedAt.IsZero() {
|
||||
job.QueueEligibleAt = job.CreatedAt
|
||||
} else {
|
||||
job.QueueEligibleAt = stamp
|
||||
}
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func firstEligibleSupportedJob(jobs []domain.Job, capabilities []string, stamp time.Time) (domain.Job, bool) {
|
||||
capabilitySet := map[string]struct{}{}
|
||||
for _, capability := range capabilities {
|
||||
capabilitySet[capability] = struct{}{}
|
||||
}
|
||||
sort.SliceStable(jobs, func(i, j int) bool {
|
||||
if jobs[i].CreatedAt.Equal(jobs[j].CreatedAt) {
|
||||
return jobs[i].ID < jobs[j].ID
|
||||
}
|
||||
return jobs[i].CreatedAt.Before(jobs[j].CreatedAt)
|
||||
})
|
||||
for _, job := range jobs {
|
||||
if len(capabilitySet) == 0 {
|
||||
return job, true
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
eligible := job.State == domain.JobStateQueued && !stamp.Before(job.QueueEligibleAt)
|
||||
eligible = eligible || job.State == domain.JobStateRetrying && !stamp.Before(job.NextAttemptAt)
|
||||
if !eligible || !job.CancelRequestedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if _, supported := capabilitySet[job.Capability]; supported {
|
||||
return job, true
|
||||
if len(capabilitySet) > 0 {
|
||||
if _, supported := capabilitySet[job.Capability]; !supported {
|
||||
continue
|
||||
}
|
||||
}
|
||||
return job, true
|
||||
}
|
||||
return domain.Job{}, false
|
||||
}
|
||||
|
||||
func assignmentFromJob(job domain.Job, lease domain.RunJobLease) domain.RunJobAssignment {
|
||||
func assignmentFromJob(job domain.Job, leaseToken string) domain.RunJobAssignment {
|
||||
return domain.RunJobAssignment{
|
||||
JobID: job.ID,
|
||||
ServerInstanceID: job.ServerInstanceID,
|
||||
@@ -390,15 +597,76 @@ func assignmentFromJob(job domain.Job, lease domain.RunJobLease) domain.RunJobAs
|
||||
State: job.State,
|
||||
Progress: domain.RunJobProgressReport{Percent: job.Progress.Percent, Message: job.Progress.Message},
|
||||
ResultRef: job.ResultRef,
|
||||
LeaseToken: lease.LeaseToken,
|
||||
Attempt: lease.Attempt,
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: job.ExecutionInput.WorkspaceScope, Content: job.ExecutionInput.Content, ExpectedVersion: job.ExecutionInput.ExpectedVersion, ExpectedChecksum: job.ExecutionInput.ExpectedChecksum, MaxReadBytes: job.ExecutionInput.MaxReadBytes, RemoteAdapterKey: job.ExecutionInput.RemoteAdapterKey, RemoteAdapterKind: job.ExecutionInput.RemoteAdapterKind, TimeoutSeconds: job.ExecutionInput.TimeoutSeconds},
|
||||
LeaseToken: leaseToken,
|
||||
Attempt: job.Attempt,
|
||||
MaxAttempts: job.RetryPolicy.MaxAttempts,
|
||||
AckDeadlineAt: job.AckDeadlineAt,
|
||||
LeaseExpiresAt: job.LeaseExpiresAt,
|
||||
NextAttemptAt: job.NextAttemptAt,
|
||||
ProgressSequence: job.LastProgressSeq,
|
||||
CreatedAt: job.CreatedAt,
|
||||
UpdatedAt: job.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func emptyJobClaim(runEndpointID string, stamp time.Time) domain.RunJobClaimResult {
|
||||
return domain.RunJobClaimResult{Accepted: true, RunEndpointID: runEndpointID, NextPollSeconds: defaultJobPollSeconds, ServerTime: stamp}
|
||||
}
|
||||
|
||||
func cancelRequestResult(job domain.Job) domain.RunJobCancelRequestResult {
|
||||
return domain.RunJobCancelRequestResult{
|
||||
Accepted: true, JobID: job.ID, Reason: job.CancelReason, RequestedAt: job.CancelRequestedAt,
|
||||
CompletedAt: job.CancelCompletedAt, State: job.State,
|
||||
}
|
||||
}
|
||||
|
||||
func terminalizeCancelled(job *domain.Job, stamp time.Time, reason string) {
|
||||
job.State = domain.JobStateCancelled
|
||||
job.Progress = domain.JobProgress{Percent: job.Progress.Percent, Message: reason}
|
||||
job.CancelCompletedAt = stamp
|
||||
job.TerminalAt = stamp
|
||||
job.TerminalFingerprint = fmt.Sprintf("scheduler-cancelled|%d|%s", job.Attempt, reason)
|
||||
job.NextAttemptAt = time.Time{}
|
||||
job.AckDeadlineAt = time.Time{}
|
||||
job.LeaseExpiresAt = time.Time{}
|
||||
}
|
||||
|
||||
func leaseTokenMatches(expectedHash string, token string) bool {
|
||||
if expectedHash == "" || strings.TrimSpace(token) == "" {
|
||||
return false
|
||||
}
|
||||
actual := tokenHash(token)
|
||||
return subtle.ConstantTimeCompare([]byte(expectedHash), []byte(actual)) == 1
|
||||
}
|
||||
|
||||
func deadlineExpired(deadline time.Time, stamp time.Time) bool {
|
||||
return deadline.IsZero() || !stamp.Before(deadline)
|
||||
}
|
||||
|
||||
func jobAttemptExpired(job domain.Job, stamp time.Time) bool {
|
||||
if job.State == domain.JobStateAccepted {
|
||||
return deadlineExpired(job.AckDeadlineAt, stamp)
|
||||
}
|
||||
if job.State == domain.JobStateRunning {
|
||||
return deadlineExpired(job.LeaseExpiresAt, stamp)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func jobRetryBackoff(policy domain.JobRetryPolicy, attempt int) time.Duration {
|
||||
delay := int64(policy.InitialBackoffSeconds)
|
||||
for current := 1; current < attempt && delay < int64(policy.MaxBackoffSeconds); current++ {
|
||||
delay *= 2
|
||||
if delay > int64(policy.MaxBackoffSeconds) {
|
||||
delay = int64(policy.MaxBackoffSeconds)
|
||||
}
|
||||
}
|
||||
return time.Duration(delay) * time.Second
|
||||
}
|
||||
|
||||
func terminalFingerprint(result domain.RunJobResult) string {
|
||||
return fmt.Sprintf("%s|%d|%s|%s|%s|%s", result.State, result.Progress.Percent, result.ResultRef, result.Message, result.ErrorCode, result.Progress.Message)
|
||||
return fmt.Sprintf("%s|%d|%s|%s|%s|%s|%t", result.State, result.Progress.Percent, result.ResultRef, result.Message, result.ErrorCode, result.Progress.Message, result.Retryable)
|
||||
}
|
||||
|
||||
func terminalMessage(result domain.RunJobResult) string {
|
||||
|
||||
@@ -164,7 +164,7 @@ func TestCoreServiceRunJobCancelPoll(t *testing.T) {
|
||||
t.Fatalf("unexpected cancel request: %+v", cancel)
|
||||
}
|
||||
|
||||
poll, err := svc.PollRunJobCancel(domain.RunJobCancelPoll{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: "job-1", LeaseToken: claim.Job.LeaseToken})
|
||||
poll, err := svc.PollRunJobCancel(domain.RunJobCancelPoll{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: "job-1", LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt})
|
||||
if err != nil {
|
||||
t.Fatalf("poll cancel: %v", err)
|
||||
}
|
||||
@@ -223,15 +223,18 @@ func TestCoreServiceRunJobReconcile(t *testing.T) {
|
||||
t.Fatalf("ack job: %v", err)
|
||||
}
|
||||
|
||||
reconcile, err := svc.ReconcileRunJobs(domain.RunJobReconcile{RunEndpointID: "run-local", SessionToken: sessionToken, ActiveJobIDs: []string{"job-1", "local-only"}})
|
||||
reconcile, err := svc.ReconcileRunJobs(domain.RunJobReconcile{RunEndpointID: "run-local", SessionToken: sessionToken, ActiveJobs: []domain.RunJobReconcileEntry{
|
||||
{JobID: "job-1", LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt},
|
||||
{JobID: "local-only", LeaseToken: "local-lease", Attempt: 1},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("reconcile jobs: %v", err)
|
||||
}
|
||||
if len(reconcile.ActiveJobs) != 1 || reconcile.ActiveJobs[0].JobID != "job-1" {
|
||||
if len(reconcile.ConfirmedJobs) != 1 || reconcile.ConfirmedJobs[0].JobID != "job-1" {
|
||||
t.Fatalf("expected platform active job, got %+v", reconcile)
|
||||
}
|
||||
if len(reconcile.UnknownJobIDs) != 1 || reconcile.UnknownJobIDs[0] != "local-only" {
|
||||
t.Fatalf("expected unknown local job, got %+v", reconcile.UnknownJobIDs)
|
||||
if len(reconcile.DiscardJobIDs) != 1 || reconcile.DiscardJobIDs[0] != "local-only" {
|
||||
t.Fatalf("expected unknown local job, got %+v", reconcile.DiscardJobIDs)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
func TestDurableJobPlatformRestartPreservesLeaseFencing(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
stamp := fixedTime
|
||||
now := func() time.Time { return stamp }
|
||||
svc, sessionToken := newMutableRunJobService(t, store, now)
|
||||
createQueuedRunJob(t, svc, "job-restart", "idem-restart")
|
||||
claim := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
|
||||
restarted := newCoreService(store, now)
|
||||
ack, err := restarted.AckRunJob(domain.RunJobAck{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID,
|
||||
LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, Message: "recovered after restart",
|
||||
})
|
||||
if err != nil || ack.Job.State != domain.JobStateRunning {
|
||||
t.Fatalf("restart ack failed: ack=%+v err=%v", ack, err)
|
||||
}
|
||||
stored, err := store.Jobs().Get("job-restart")
|
||||
if err != nil {
|
||||
t.Fatalf("get stored job: %v", err)
|
||||
}
|
||||
if stored.LeaseTokenHash == "" || stored.LeaseTokenHash == claim.Job.LeaseToken || stored.Attempt != 1 || stored.LeaseSessionGen != 1 {
|
||||
t.Fatalf("expected hashed durable lease metadata, got %+v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableJobAckTimeoutBackoffAndAttemptFencing(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
stamp := fixedTime
|
||||
now := func() time.Time { return stamp }
|
||||
svc, sessionToken := newMutableRunJobService(t, store, now)
|
||||
createQueuedRunJob(t, svc, "job-timeout", "idem-timeout")
|
||||
first := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
|
||||
stamp = stamp.Add(defaultJobAckTimeout)
|
||||
_, err := svc.AckRunJob(domain.RunJobAck{RunEndpointID: "run-local", SessionToken: sessionToken, JobID: first.Job.JobID, LeaseToken: first.Job.LeaseToken, Attempt: first.Job.Attempt})
|
||||
if err == nil || !strings.Contains(err.Error(), "ack deadline") {
|
||||
t.Fatalf("expected late ack rejection, got %v", err)
|
||||
}
|
||||
retrying, _ := svc.GetJob(first.Job.JobID)
|
||||
if retrying.State != domain.JobStateRetrying || !retrying.NextAttemptAt.Equal(stamp.Add(2*time.Second)) {
|
||||
t.Fatalf("expected persisted retry wait, got %+v", retrying)
|
||||
}
|
||||
empty, err := svc.ClaimRunJob(runJobClaim(sessionToken, "run-local"))
|
||||
if err != nil || empty.HasJob {
|
||||
t.Fatalf("job claimed before backoff elapsed: %+v err=%v", empty, err)
|
||||
}
|
||||
|
||||
stamp = retrying.NextAttemptAt
|
||||
second := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
if second.Job.Attempt != first.Job.Attempt+1 || second.Job.LeaseToken == first.Job.LeaseToken {
|
||||
t.Fatalf("expected fenced second attempt, first=%+v second=%+v", first.Job, second.Job)
|
||||
}
|
||||
_, err = svc.CompleteRunJob(domain.RunJobResult{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: first.Job.JobID,
|
||||
LeaseToken: first.Job.LeaseToken, Attempt: first.Job.Attempt, State: domain.JobStateSucceeded,
|
||||
Progress: domain.RunJobProgressReport{Percent: 100}, Message: "late result",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "attempt or leaseToken") {
|
||||
t.Fatalf("expected old attempt result rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableJobLeaseExpiryAndRetryBudget(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
stamp := fixedTime
|
||||
now := func() time.Time { return stamp }
|
||||
svc, sessionToken := newMutableRunJobService(t, store, now)
|
||||
_, err := svc.CreateJob(domain.Job{
|
||||
ID: "job-retry", RunEndpointID: "run-local", Capability: "process.start", IdempotencyKey: "idem-retry",
|
||||
RetryPolicy: domain.JobRetryPolicy{MaxAttempts: 2, InitialBackoffSeconds: 3, MaxBackoffSeconds: 3},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create retry job: %v", err)
|
||||
}
|
||||
first := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
mustAckJob(t, svc, sessionToken, first.Job)
|
||||
stamp = stamp.Add(defaultJobLeaseDuration)
|
||||
_, err = svc.UpdateRunJobProgress(domain.RunJobProgress{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: first.Job.JobID,
|
||||
LeaseToken: first.Job.LeaseToken, Attempt: first.Job.Attempt, Sequence: 1,
|
||||
Progress: domain.RunJobProgressReport{Percent: 20, Message: "late progress"},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "lease expired") {
|
||||
t.Fatalf("expected expired lease rejection, got %v", err)
|
||||
}
|
||||
retrying, _ := svc.GetJob(first.Job.JobID)
|
||||
if retrying.State != domain.JobStateRetrying {
|
||||
t.Fatalf("expected retrying after lease expiry, got %+v", retrying)
|
||||
}
|
||||
|
||||
stamp = retrying.NextAttemptAt
|
||||
second := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
mustAckJob(t, svc, sessionToken, second.Job)
|
||||
result, err := svc.CompleteRunJob(domain.RunJobResult{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: second.Job.JobID,
|
||||
LeaseToken: second.Job.LeaseToken, Attempt: second.Job.Attempt, State: domain.JobStateFailed,
|
||||
Progress: domain.RunJobProgressReport{Percent: 100, Message: "still failing"}, Message: "still failing", Retryable: true,
|
||||
})
|
||||
if err != nil || result.Job.State != domain.JobStateFailed {
|
||||
t.Fatalf("expected terminal failure after retry budget, result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableJobCancellationBeforeAndAfterClaimIsIdempotent(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
stamp := fixedTime
|
||||
now := func() time.Time { return stamp }
|
||||
svc, sessionToken := newMutableRunJobService(t, store, now)
|
||||
createQueuedRunJob(t, svc, "job-cancel-queued", "idem-cancel-queued")
|
||||
firstCancel, err := svc.RequestRunJobCancel(domain.RunJobCancelRequest{JobID: "job-cancel-queued", Reason: "operator cancelled queue"})
|
||||
if err != nil || firstCancel.State != domain.JobStateCancelled || firstCancel.CompletedAt.IsZero() {
|
||||
t.Fatalf("cancel queued job: result=%+v err=%v", firstCancel, err)
|
||||
}
|
||||
repeated, err := svc.RequestRunJobCancel(domain.RunJobCancelRequest{JobID: "job-cancel-queued", Reason: "operator cancelled queue"})
|
||||
if err != nil || !repeated.CompletedAt.Equal(firstCancel.CompletedAt) {
|
||||
t.Fatalf("repeat cancel was not idempotent: result=%+v err=%v", repeated, err)
|
||||
}
|
||||
|
||||
createQueuedRunJob(t, svc, "job-cancel-running", "idem-cancel-running")
|
||||
claim := mustClaimJob(t, svc, sessionToken, "run-local")
|
||||
mustAckJob(t, svc, sessionToken, claim.Job)
|
||||
intent, err := svc.RequestRunJobCancel(domain.RunJobCancelRequest{JobID: claim.Job.JobID, Reason: "operator stop"})
|
||||
if err != nil || intent.State != domain.JobStateRunning || !intent.CompletedAt.IsZero() {
|
||||
t.Fatalf("cancel active intent: result=%+v err=%v", intent, err)
|
||||
}
|
||||
poll, err := svc.PollRunJobCancel(domain.RunJobCancelPoll{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID,
|
||||
LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt,
|
||||
})
|
||||
if err != nil || !poll.HasCancel || poll.Reason != "operator stop" {
|
||||
t.Fatalf("poll cancel: result=%+v err=%v", poll, err)
|
||||
}
|
||||
terminalRequest := domain.RunJobResult{
|
||||
RunEndpointID: "run-local", SessionToken: sessionToken, JobID: claim.Job.JobID,
|
||||
LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateCancelled,
|
||||
Progress: domain.RunJobProgressReport{Percent: 100, Message: "cancelled"}, Message: "cancelled",
|
||||
}
|
||||
terminal, err := svc.CompleteRunJob(terminalRequest)
|
||||
if err != nil || terminal.Job.State != domain.JobStateCancelled {
|
||||
t.Fatalf("complete cancellation: result=%+v err=%v", terminal, err)
|
||||
}
|
||||
if _, err := svc.CompleteRunJob(terminalRequest); err != nil {
|
||||
t.Fatalf("duplicate cancelled result should be idempotent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableJobReconcileRotatedSessionAndMissingAttempt(t *testing.T) {
|
||||
store := repo.NewMemoryStore()
|
||||
stamp := fixedTime
|
||||
now := func() time.Time { return stamp }
|
||||
svc, oldSession := newMutableRunJobService(t, store, now)
|
||||
createQueuedRunJob(t, svc, "job-confirmed", "idem-confirmed")
|
||||
confirmedClaim := mustClaimJob(t, svc, oldSession, "run-local")
|
||||
mustAckJob(t, svc, oldSession, confirmedClaim.Job)
|
||||
createQueuedRunJob(t, svc, "job-missing", "idem-missing")
|
||||
missingClaim := mustClaimJob(t, svc, oldSession, "run-local")
|
||||
mustAckJob(t, svc, oldSession, missingClaim.Job)
|
||||
|
||||
hello := validRunControlHello()
|
||||
hello.CapabilityReport.Capabilities = append(hello.CapabilityReport.Capabilities, "process.start")
|
||||
hello.CapabilityReport.Fingerprint = "cap-jobs-rotated"
|
||||
rotated, err := svc.RegisterRunHello(hello)
|
||||
if err != nil {
|
||||
t.Fatalf("rotate Run session: %v", err)
|
||||
}
|
||||
_, err = svc.UpdateRunJobProgress(domain.RunJobProgress{
|
||||
RunEndpointID: "run-local", SessionToken: oldSession, JobID: confirmedClaim.Job.JobID,
|
||||
LeaseToken: confirmedClaim.Job.LeaseToken, Attempt: confirmedClaim.Job.Attempt,
|
||||
Progress: domain.RunJobProgressReport{Percent: 20}, Sequence: 1,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected rotated Run session to reject progress")
|
||||
}
|
||||
|
||||
reconciled, err := svc.ReconcileRunJobs(domain.RunJobReconcile{
|
||||
RunEndpointID: "run-local", SessionToken: rotated.SessionToken,
|
||||
ActiveJobs: []domain.RunJobReconcileEntry{{JobID: confirmedClaim.Job.JobID, LeaseToken: confirmedClaim.Job.LeaseToken, Attempt: confirmedClaim.Job.Attempt}},
|
||||
})
|
||||
if err != nil || len(reconciled.ConfirmedJobs) != 1 || len(reconciled.DiscardJobIDs) != 0 {
|
||||
t.Fatalf("reconcile rotated session: result=%+v err=%v", reconciled, err)
|
||||
}
|
||||
progress, err := svc.UpdateRunJobProgress(domain.RunJobProgress{
|
||||
RunEndpointID: "run-local", SessionToken: rotated.SessionToken, JobID: confirmedClaim.Job.JobID,
|
||||
LeaseToken: confirmedClaim.Job.LeaseToken, Attempt: confirmedClaim.Job.Attempt,
|
||||
Progress: domain.RunJobProgressReport{Percent: 30, Message: "reconciled"}, Sequence: 1,
|
||||
})
|
||||
if err != nil || progress.Job.Progress.Percent != 30 {
|
||||
t.Fatalf("progress after reconcile: result=%+v err=%v", progress, err)
|
||||
}
|
||||
missing, _ := svc.GetJob(missingClaim.Job.JobID)
|
||||
if missing.State != domain.JobStateRetrying || missing.ReconcileOutcome != "missing from Run journal" {
|
||||
t.Fatalf("expected missing active attempt to retry, got %+v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func newMutableRunJobService(t *testing.T, store repo.Store, now func() time.Time) (*CoreService, string) {
|
||||
t.Helper()
|
||||
svc := newCoreService(store, now)
|
||||
hello := validRunControlHello()
|
||||
hello.CapabilityReport.Capabilities = append(hello.CapabilityReport.Capabilities, "process.start")
|
||||
hello.CapabilityReport.Fingerprint = "cap-jobs"
|
||||
result, err := svc.RegisterRunHello(hello)
|
||||
if err != nil {
|
||||
t.Fatalf("register Run: %v", err)
|
||||
}
|
||||
return svc, result.SessionToken
|
||||
}
|
||||
|
||||
func runJobClaim(sessionToken string, endpointID string) domain.RunJobClaim {
|
||||
return domain.RunJobClaim{RunEndpointID: endpointID, SessionToken: sessionToken, Capabilities: []string{"process.start"}, Capacity: domain.RunCapacity{MaxJobs: 4}}
|
||||
}
|
||||
|
||||
func mustClaimJob(t *testing.T, svc *CoreService, sessionToken string, endpointID string) domain.RunJobClaimResult {
|
||||
t.Helper()
|
||||
claim, err := svc.ClaimRunJob(runJobClaim(sessionToken, endpointID))
|
||||
if err != nil || !claim.HasJob || claim.Job == nil {
|
||||
t.Fatalf("claim job: result=%+v err=%v", claim, err)
|
||||
}
|
||||
return claim
|
||||
}
|
||||
|
||||
func mustAckJob(t *testing.T, svc *CoreService, sessionToken string, job *domain.RunJobAssignment) {
|
||||
t.Helper()
|
||||
if _, err := svc.AckRunJob(domain.RunJobAck{
|
||||
RunEndpointID: job.RunEndpointID, SessionToken: sessionToken, JobID: job.JobID,
|
||||
LeaseToken: job.LeaseToken, Attempt: job.Attempt,
|
||||
}); err != nil {
|
||||
t.Fatalf("ack job %s: %v", job.JobID, err)
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,18 @@ func (store *MemoryLogBodyStore) Query(streamID string, afterSeq uint64, limit i
|
||||
return selected, nextSeq, nil
|
||||
}
|
||||
|
||||
func (store *MemoryLogBodyStore) LatestSeq(streamID string) (uint64, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
var latest uint64
|
||||
for _, entry := range store.entries[streamID] {
|
||||
if entry.Seq > latest {
|
||||
latest = entry.Seq
|
||||
}
|
||||
}
|
||||
return latest, nil
|
||||
}
|
||||
|
||||
type FileLogBodyStore struct {
|
||||
mu sync.Mutex
|
||||
rootDir string
|
||||
@@ -101,7 +113,7 @@ func NewFileLogBodyStore(rootDir string) (*FileLogBodyStore, error) {
|
||||
rootDir: rootDir,
|
||||
memory: NewMemoryLogBodyStore(),
|
||||
}
|
||||
if err := os.MkdirAll(rootDir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(rootDir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create log directory: %w", err)
|
||||
}
|
||||
if err := store.load(); err != nil {
|
||||
@@ -124,7 +136,7 @@ func (store *FileLogBodyStore) AppendBatch(streamID string, record domain.LogBat
|
||||
return store.memory.AppendBatch(streamID, record)
|
||||
}
|
||||
streamDir := store.streamDir(streamID)
|
||||
if err := os.MkdirAll(streamDir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(streamDir, 0o700); err != nil {
|
||||
return fmt.Errorf("create log stream directory: %w", err)
|
||||
}
|
||||
segmentPath := store.segmentPath(streamID, record.FirstSeq)
|
||||
@@ -133,23 +145,15 @@ func (store *FileLogBodyStore) AppendBatch(streamID string, record domain.LogBat
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("stat log segment: %w", err)
|
||||
}
|
||||
tmpPath := segmentPath + ".tmp"
|
||||
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open log segment: %w", err)
|
||||
}
|
||||
encoder := json.NewEncoder(file)
|
||||
var body strings.Builder
|
||||
encoder := json.NewEncoder(&body)
|
||||
for _, entry := range record.Entries {
|
||||
if err := encoder.Encode(domain.CopyLogEntry(entry)); err != nil {
|
||||
_ = file.Close()
|
||||
return fmt.Errorf("write log segment: %w", err)
|
||||
}
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("close log segment: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, segmentPath); err != nil {
|
||||
return fmt.Errorf("replace log segment: %w", err)
|
||||
if err := writeAtomicFile(segmentPath, []byte(body.String()), 0o600); err != nil {
|
||||
return fmt.Errorf("persist log segment: %w", err)
|
||||
}
|
||||
return store.memory.AppendBatch(streamID, record)
|
||||
}
|
||||
@@ -162,6 +166,10 @@ func (store *FileLogBodyStore) Query(streamID string, afterSeq uint64, limit int
|
||||
return store.memory.Query(streamID, afterSeq, limit)
|
||||
}
|
||||
|
||||
func (store *FileLogBodyStore) LatestSeq(streamID string) (uint64, error) {
|
||||
return store.memory.LatestSeq(streamID)
|
||||
}
|
||||
|
||||
func (store *FileLogBodyStore) load() error {
|
||||
entries, err := os.ReadDir(store.rootDir)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
const (
|
||||
maxMetricSamplesPerServer = 1000
|
||||
maxBackupsPerServer = 100
|
||||
maxBackupBytesPerServer = int64(4 * 1024 * 1024 * 1024)
|
||||
)
|
||||
|
||||
func (svc *CoreService) IngestMetricBatch(batch domain.MetricBatchIngest) (domain.MetricBatchIngestResult, error) {
|
||||
batch = domain.CopyMetricBatchIngest(batch)
|
||||
if len(batch.Samples) == 0 || len(batch.Samples) > 256 {
|
||||
return domain.MetricBatchIngestResult{}, validationError("metric batch must contain between 1 and 256 samples")
|
||||
}
|
||||
if err := svc.validateRunSession(batch.RunEndpointID, batch.SessionToken); err != nil {
|
||||
return domain.MetricBatchIngestResult{}, err
|
||||
}
|
||||
latest := svc.now()
|
||||
for index, sample := range batch.Samples {
|
||||
instance, err := svc.store.ServerInstances().Get(sample.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.MetricBatchIngestResult{}, err
|
||||
}
|
||||
if instance.RunEndpointID != batch.RunEndpointID {
|
||||
return domain.MetricBatchIngestResult{}, validationError("metric sample server must belong to runEndpointId")
|
||||
}
|
||||
sample.RunEndpointID = batch.RunEndpointID
|
||||
if sample.ID == "" {
|
||||
sample.ID = fmt.Sprintf("metric:%s:%d:%d", sample.ServerInstanceID, sample.CollectedAt.UnixNano(), index)
|
||||
}
|
||||
if sample.CollectedAt.IsZero() {
|
||||
sample.CollectedAt = svc.now()
|
||||
}
|
||||
if err := validator.ValidateMetricSample(sample); err != nil {
|
||||
return domain.MetricBatchIngestResult{}, err
|
||||
}
|
||||
if err := svc.store.MetricSamples().Create(sample); err != nil {
|
||||
if !errors.Is(err, repo.ErrDuplicate) {
|
||||
return domain.MetricBatchIngestResult{}, err
|
||||
}
|
||||
existing, getErr := svc.store.MetricSamples().Get(sample.ID)
|
||||
if getErr != nil || existing.ServerInstanceID != sample.ServerInstanceID || existing.CollectedAt != sample.CollectedAt {
|
||||
return domain.MetricBatchIngestResult{}, validationError("metric sample id conflicts with persisted sample")
|
||||
}
|
||||
}
|
||||
if sample.CollectedAt.After(latest) {
|
||||
latest = sample.CollectedAt
|
||||
}
|
||||
if err := svc.pruneMetricSamples(sample.ServerInstanceID); err != nil {
|
||||
return domain.MetricBatchIngestResult{}, err
|
||||
}
|
||||
}
|
||||
return domain.MetricBatchIngestResult{Accepted: true, AcceptedCount: len(batch.Samples), LatestAt: latest, ServerTime: svc.now()}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListMetricSamplesForSession(sessionID string, filter domain.MetricSampleFilter) ([]domain.MetricSample, error) {
|
||||
if err := validator.ValidateMetricSampleFilter(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(filter.ServerInstanceID) == "" {
|
||||
return nil, validationError("serverInstanceId is required")
|
||||
}
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, filter.ServerInstanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := svc.store.MetricSamples().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].CollectedAt.Before(items[j].CollectedAt) })
|
||||
limit := filter.Limit
|
||||
if limit == 0 {
|
||||
limit = 100
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[len(items)-limit:]
|
||||
}
|
||||
for _, sample := range items {
|
||||
if sample.ServerInstanceID != instance.ID || sample.RunEndpointID != instance.RunEndpointID {
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
}
|
||||
return domain.CopyMetricSamples(items), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CreateBackupForSession(sessionID string, record domain.BackupRecord) (domain.BackupRecord, error) {
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, record.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
artifact, err := svc.store.Artifacts().Get(record.ArtifactID)
|
||||
if err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.validateBackupArtifactOwner(instance, artifact); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
stamp := svc.now()
|
||||
if record.ID == "" {
|
||||
record.ID = fmt.Sprintf("backup:%s:%d", instance.ID, stamp.UnixNano())
|
||||
}
|
||||
if record.State == "" {
|
||||
record.State = domain.BackupStatePending
|
||||
}
|
||||
if record.Checksum == "" {
|
||||
record.Checksum = artifact.Checksum
|
||||
}
|
||||
if record.SizeBytes == 0 {
|
||||
record.SizeBytes = artifact.SizeBytes
|
||||
}
|
||||
if record.CreatedAt.IsZero() {
|
||||
record.CreatedAt = stamp
|
||||
}
|
||||
record.UpdatedAt = stamp
|
||||
if record.State == domain.BackupStateAvailable && artifact.State != domain.ArtifactStateAvailable {
|
||||
return domain.BackupRecord{}, validationError("backup artifact must be available")
|
||||
}
|
||||
if err := validator.ValidateBackupRecord(record); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.store.Backups().Create(record); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.recordAuditEvent(user.ID, "backup.create", "server-instance", instance.ID, domain.AuditResultQueued, "created bounded backup record with artifact checksum"); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if err := svc.pruneBackups(instance.ID); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
return domain.CopyBackupRecord(record), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetBackupForSession(sessionID string, backupID string) (domain.BackupRecord, error) {
|
||||
record, err := svc.store.Backups().Get(strings.TrimSpace(backupID))
|
||||
if err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, record.ServerInstanceID); err != nil {
|
||||
return domain.BackupRecord{}, err
|
||||
}
|
||||
return domain.CopyBackupRecord(record), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListBackupsForSession(sessionID string, filter domain.BackupFilter) ([]domain.BackupRecord, error) {
|
||||
if err := validator.ValidateBackupFilter(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(filter.ServerInstanceID) == "" {
|
||||
return nil, validationError("serverInstanceId is required")
|
||||
}
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, filter.ServerInstanceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := svc.store.Backups().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.After(items[j].CreatedAt) })
|
||||
if len(items) > validator.MaxBackupRecordsPerQuery {
|
||||
items = items[:validator.MaxBackupRecordsPerQuery]
|
||||
}
|
||||
return domain.CopyBackupRecords(items), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RecoverIncompleteBackups() error {
|
||||
items, err := svc.store.Backups().List(domain.BackupFilter{State: domain.BackupStatePending})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range items {
|
||||
record.State = domain.BackupStateFailed
|
||||
record.RecoveryStatus = "recoverable-after-interrupted-transfer"
|
||||
record.UpdatedAt = svc.now()
|
||||
if err := svc.store.Backups().Update(record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.recordAuditEvent("platform-recovery", "backup.recover", "server-instance", record.ServerInstanceID, domain.AuditResultFailed, "marked interrupted backup recoverable without exposing storage details"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) pruneMetricSamples(serverInstanceID string) error {
|
||||
items, err := svc.store.MetricSamples().List(domain.MetricSampleFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil || len(items) <= maxMetricSamplesPerServer {
|
||||
return err
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].CollectedAt.Before(items[j].CollectedAt) })
|
||||
for _, sample := range items[:len(items)-maxMetricSamplesPerServer] {
|
||||
if err := svc.store.MetricSamples().Delete(sample.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return svc.recordAuditEvent("platform-retention", "metrics.retention", "server-instance", serverInstanceID, domain.AuditResultSuccess, "pruned oldest metric samples to bounded retention")
|
||||
}
|
||||
|
||||
func (svc *CoreService) pruneBackups(serverInstanceID string) error {
|
||||
items, err := svc.store.Backups().List(domain.BackupFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.Before(items[j].CreatedAt) })
|
||||
total := int64(0)
|
||||
for _, record := range items {
|
||||
if record.State != domain.BackupStateExpired {
|
||||
total += record.SizeBytes
|
||||
}
|
||||
}
|
||||
pruned := false
|
||||
for len(items) > maxBackupsPerServer || total > maxBackupBytesPerServer {
|
||||
record := items[0]
|
||||
items = items[1:]
|
||||
if record.State != domain.BackupStateExpired {
|
||||
total -= record.SizeBytes
|
||||
}
|
||||
record.State = domain.BackupStateExpired
|
||||
record.RecoveryStatus = "retention-expired"
|
||||
record.UpdatedAt = svc.now()
|
||||
if err := svc.store.Backups().Update(record); err != nil {
|
||||
return err
|
||||
}
|
||||
pruned = true
|
||||
}
|
||||
if pruned {
|
||||
return svc.recordAuditEvent("platform-retention", "backup.retention", "server-instance", serverInstanceID, domain.AuditResultSuccess, "expired oldest backup records to bounded retention")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateBackupArtifactOwner(instance domain.ServerInstance, artifact domain.Artifact) error {
|
||||
switch artifact.OwnerKind {
|
||||
case domain.ArtifactOwnerKindServerInstance:
|
||||
if artifact.OwnerID != instance.ID {
|
||||
return ErrForbidden
|
||||
}
|
||||
case domain.ArtifactOwnerKindJob:
|
||||
job, err := svc.store.Jobs().Get(artifact.OwnerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if job.ServerInstanceID != instance.ID || job.RunEndpointID != instance.RunEndpointID {
|
||||
return ErrForbidden
|
||||
}
|
||||
default:
|
||||
return ErrForbidden
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
func (svc *CoreService) ListRemoteAdapterDeclarationsForSession(sessionID string, serverInstanceID string) ([]domain.RemoteAdapterDeclaration, error) {
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpoint, err := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !plugin.Permissions.RemoteAccess {
|
||||
return []domain.RemoteAdapterDeclaration{}, nil
|
||||
}
|
||||
declarations := make([]domain.RemoteAdapterDeclaration, 0, len(plugin.RuntimeProfiles.TransportProfiles))
|
||||
for _, profile := range plugin.RuntimeProfiles.TransportProfiles {
|
||||
capabilities := intersectRemoteCapabilities(profile.Capabilities, plugin.RemoteAccess.RunCapabilities, endpoint.Capabilities)
|
||||
if len(capabilities) == 0 || strings.TrimSpace(profile.TargetKey) == "" {
|
||||
continue
|
||||
}
|
||||
declaration := domain.RemoteAdapterDeclaration{Key: profile.Key, Kind: remoteAdapterKind(profile.Kind), TargetKeys: []string{profile.TargetKey}, Capabilities: capabilities, TimeoutSeconds: 30, MaxAttempts: 3}
|
||||
if err := validator.ValidateRemoteAdapterDeclaration(declaration); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
declarations = append(declarations, declaration)
|
||||
}
|
||||
return domain.CopyRemoteAdapterDeclarations(declarations), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RequestRemoteAdapterForSession(sessionID string, request domain.RemoteAdapterRequest) (domain.RemoteAdapterResult, error) {
|
||||
request = domain.CopyRemoteAdapterRequest(request)
|
||||
if err := validator.ValidateRemoteAdapterRequest(request); err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID)
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
declarations, err := svc.ListRemoteAdapterDeclarationsForSession(sessionID, instance.ID)
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
var selected domain.RemoteAdapterDeclaration
|
||||
for _, declaration := range declarations {
|
||||
if declaration.Key == request.DeclarationKey && containsString(declaration.TargetKeys, request.TargetKey) && containsString(declaration.Capabilities, request.Capability) {
|
||||
selected = declaration
|
||||
break
|
||||
}
|
||||
}
|
||||
if selected.Key == "" {
|
||||
plugin, pluginErr := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
endpoint, endpointErr := svc.store.RunEndpoints().Get(instance.RunEndpointID)
|
||||
if pluginErr == nil && endpointErr == nil && plugin.Permissions.RemoteAccess && containsString(plugin.RemoteAccess.RunCapabilities, request.Capability) && containsString(endpoint.Capabilities, request.Capability) {
|
||||
selected = domain.RemoteAdapterDeclaration{Key: "legacy-" + string(remoteAdapterKindForCapability(request.Capability)), Kind: remoteAdapterKindForCapability(request.Capability), TargetKeys: []string{request.TargetKey}, Capabilities: []string{request.Capability}, TimeoutSeconds: 30, MaxAttempts: 3}
|
||||
}
|
||||
if selected.Key == "" {
|
||||
user, _ := svc.GetCurrentUser(sessionID)
|
||||
_ = svc.recordAuditEvent(user.ID, "remote-adapter.authorize", "server-instance", instance.ID, domain.AuditResultDenied, "remote adapter declaration, target, or capability was not approved")
|
||||
return domain.RemoteAdapterResult{}, ErrForbidden
|
||||
}
|
||||
}
|
||||
timeout := request.TimeoutSeconds
|
||||
if timeout == 0 {
|
||||
timeout = selected.TimeoutSeconds
|
||||
}
|
||||
attempts := request.MaxAttempts
|
||||
if attempts == 0 {
|
||||
attempts = selected.MaxAttempts
|
||||
}
|
||||
if timeout > selected.TimeoutSeconds || attempts > selected.MaxAttempts {
|
||||
return domain.RemoteAdapterResult{}, validationError("remote adapter timeout or retry exceeds declaration")
|
||||
}
|
||||
job := domain.Job{
|
||||
ID: jobIDFromParts("job-remote-adapter", instance.ID, request.IdempotencyKey),
|
||||
ServerInstanceID: instance.ID,
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
Capability: request.Capability,
|
||||
TargetKey: request.TargetKey,
|
||||
InputRef: fmt.Sprintf("input://remote-adapters/%s/%s", instance.ID, request.DeclarationKey),
|
||||
IdempotencyKey: request.IdempotencyKey,
|
||||
Progress: domain.JobProgress{Percent: 0, Message: "scoped remote adapter queued"},
|
||||
RetryPolicy: domain.JobRetryPolicy{MaxAttempts: attempts, InitialBackoffSeconds: 2, MaxBackoffSeconds: 30},
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: svc.runtimeProfileScope(instance.ID), RemoteAdapterKey: selected.Key, RemoteAdapterKind: string(selected.Kind), TimeoutSeconds: timeout},
|
||||
}
|
||||
created, err := svc.CreateJob(job)
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
auditID, err := svc.recordAuditEventWithID(user.ID, "remote-adapter.authorize", "server-instance", instance.ID, domain.AuditResultQueued, "authorized declared remote adapter target with bounded timeout and retry")
|
||||
if err != nil {
|
||||
return domain.RemoteAdapterResult{}, err
|
||||
}
|
||||
return domain.RemoteAdapterResult{RequestID: created.ID, ServerInstanceID: instance.ID, DeclarationKey: selected.Key, TargetKey: request.TargetKey, Kind: selected.Kind, Status: string(created.State), Retryable: attempts > 1, Message: "scoped remote adapter queued", ResultRef: "job://" + created.ID, AuditEventID: auditID}, nil
|
||||
}
|
||||
|
||||
func intersectRemoteCapabilities(profile []string, declared []string, endpoint []string) []string {
|
||||
result := make([]string, 0, len(profile))
|
||||
for _, capability := range profile {
|
||||
if isRemoteAdapterCapability(capability) && containsString(declared, capability) && containsString(endpoint, capability) {
|
||||
result = append(result, capability)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func isRemoteAdapterCapability(capability string) bool {
|
||||
switch capability {
|
||||
case domain.JobCapabilityRemoteFTPRead, domain.JobCapabilityRemoteFTPWrite,
|
||||
domain.JobCapabilityRemoteRsyncRead, domain.JobCapabilityRemoteRsyncWrite,
|
||||
domain.JobCapabilityRemoteRunFilesRead, domain.JobCapabilityRemoteRunFilesWrite,
|
||||
domain.JobCapabilityRemoteRunProcessStart, domain.JobCapabilityRemoteRunProcessStop,
|
||||
domain.JobCapabilityRemoteRunDBMySQLQuery, domain.JobCapabilityRemoteRunDBSQLiteQuery,
|
||||
domain.JobCapabilityRemoteRunLogsTransfer, domain.JobCapabilityRemoteRunRCONCommand:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func remoteAdapterKind(kind string) domain.RemoteAdapterKind {
|
||||
switch strings.ToLower(strings.TrimSpace(kind)) {
|
||||
case "ftp":
|
||||
return domain.RemoteAdapterFTP
|
||||
case "rsync":
|
||||
return domain.RemoteAdapterRsync
|
||||
case "file":
|
||||
return domain.RemoteAdapterRunFile
|
||||
case "process":
|
||||
return domain.RemoteAdapterRunProcess
|
||||
case "sqlite", "mysql", "database":
|
||||
return domain.RemoteAdapterDatabase
|
||||
case "rcon":
|
||||
return domain.RemoteAdapterRCON
|
||||
default:
|
||||
return domain.RemoteAdapterKind(strings.ToLower(strings.TrimSpace(kind)))
|
||||
}
|
||||
}
|
||||
|
||||
func remoteAdapterKindForCapability(capability string) domain.RemoteAdapterKind {
|
||||
switch capability {
|
||||
case domain.JobCapabilityRemoteFTPRead, domain.JobCapabilityRemoteFTPWrite:
|
||||
return domain.RemoteAdapterFTP
|
||||
case domain.JobCapabilityRemoteRsyncRead, domain.JobCapabilityRemoteRsyncWrite:
|
||||
return domain.RemoteAdapterRsync
|
||||
case domain.JobCapabilityRemoteRunFilesRead, domain.JobCapabilityRemoteRunFilesWrite:
|
||||
return domain.RemoteAdapterRunFile
|
||||
case domain.JobCapabilityRemoteRunProcessStart, domain.JobCapabilityRemoteRunProcessStop:
|
||||
return domain.RemoteAdapterRunProcess
|
||||
case domain.JobCapabilityRemoteRunDBMySQLQuery, domain.JobCapabilityRemoteRunDBSQLiteQuery:
|
||||
return domain.RemoteAdapterDatabase
|
||||
case domain.JobCapabilityRemoteRunRCONCommand:
|
||||
return domain.RemoteAdapterRCON
|
||||
case domain.JobCapabilityRemoteRunLogsTransfer:
|
||||
return domain.RemoteAdapterKind("log-transfer")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"browser.local/platform/domain"
|
||||
)
|
||||
|
||||
func (svc *CoreService) AuthorizePluginBridgeActionForSession(sessionID string, request domain.PluginBridgeAuthorizeRequest) (domain.PluginBridgeAuthorization, error) {
|
||||
if _, err := svc.GetCurrentUser(sessionID); err != nil {
|
||||
return domain.PluginBridgeAuthorization{}, err
|
||||
}
|
||||
if request.ServerInstanceID != "" {
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID); err != nil {
|
||||
return domain.PluginBridgeAuthorization{}, err
|
||||
}
|
||||
}
|
||||
return svc.AuthorizePluginBridgeAction(request)
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetJobForSession(sessionID string, id string) (domain.Job, error) {
|
||||
job, err := svc.store.Jobs().Get(id)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
if err := svc.authorizeJobAccess(sessionID, job); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
return domain.CopyJob(job), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListJobsForSession(sessionID string, filter domain.JobFilter) ([]domain.Job, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter.ServerInstanceID != "" {
|
||||
instance, err := svc.store.ServerInstances().Get(filter.ServerInstanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !canAccessServer(user, instance) {
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
}
|
||||
jobs, err := svc.store.Jobs().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if isPlatformAdmin(user) {
|
||||
return jobs, nil
|
||||
}
|
||||
visible := make([]domain.Job, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
if job.ServerInstanceID == "" {
|
||||
continue
|
||||
}
|
||||
instance, getErr := svc.store.ServerInstances().Get(job.ServerInstanceID)
|
||||
if getErr == nil && canAccessServer(user, instance) {
|
||||
visible = append(visible, domain.CopyJob(job))
|
||||
}
|
||||
}
|
||||
return visible, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) RequestRunJobCancelForSession(sessionID string, request domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error) {
|
||||
job, err := svc.GetJobForSession(sessionID, request.JobID)
|
||||
if err != nil {
|
||||
return domain.RunJobCancelRequestResult{}, err
|
||||
}
|
||||
request.JobID = job.ID
|
||||
return svc.RequestRunJobCancel(request)
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListArtifactsForSession(sessionID string, filter domain.ArtifactFilter) ([]domain.Artifact, error) {
|
||||
if _, err := svc.GetCurrentUser(sessionID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts, err := svc.store.Artifacts().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := make([]domain.Artifact, 0, len(artifacts))
|
||||
for _, artifact := range artifacts {
|
||||
if err := svc.authorizeArtifactAccess(sessionID, artifact); err == nil {
|
||||
visible = append(visible, domain.CopyArtifact(artifact))
|
||||
}
|
||||
}
|
||||
if filter.OwnerID != "" && len(artifacts) > 0 && len(visible) == 0 {
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
return visible, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetLogStreamForSession(sessionID string, id string) (domain.LogStream, error) {
|
||||
stream, err := svc.store.LogStreams().Get(id)
|
||||
if err != nil {
|
||||
return domain.LogStream{}, err
|
||||
}
|
||||
if _, err := svc.GetServerInstanceForSession(sessionID, stream.ServerInstanceID); err != nil {
|
||||
return domain.LogStream{}, err
|
||||
}
|
||||
return domain.CopyLogStream(stream), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListLogStreamsForSession(sessionID string, filter domain.LogStreamFilter) ([]domain.LogStream, error) {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter.ServerInstanceID != "" {
|
||||
instance, err := svc.store.ServerInstances().Get(filter.ServerInstanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !canAccessServer(user, instance) {
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
}
|
||||
streams, err := svc.store.LogStreams().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if isPlatformAdmin(user) {
|
||||
return streams, nil
|
||||
}
|
||||
visible := make([]domain.LogStream, 0, len(streams))
|
||||
for _, stream := range streams {
|
||||
instance, getErr := svc.store.ServerInstances().Get(stream.ServerInstanceID)
|
||||
if getErr == nil && canAccessServer(user, instance) {
|
||||
visible = append(visible, domain.CopyLogStream(stream))
|
||||
}
|
||||
}
|
||||
return visible, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) QueryLogStreamForSession(sessionID string, query domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error) {
|
||||
if _, err := svc.GetLogStreamForSession(sessionID, query.LogStreamID); err != nil {
|
||||
return domain.LogStreamCursorResult{}, err
|
||||
}
|
||||
return svc.QueryLogStream(query)
|
||||
}
|
||||
|
||||
func (svc *CoreService) authorizeJobAccess(sessionID string, job domain.Job) error {
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if job.ServerInstanceID == "" {
|
||||
if !isPlatformAdmin(user) {
|
||||
return ErrForbidden
|
||||
}
|
||||
return nil
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !canAccessServer(user, instance) {
|
||||
return ErrForbidden
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+306
-100
@@ -31,6 +31,7 @@ type Core interface {
|
||||
RegisterUser(domain.UserRegistration) (domain.AuthSession, error)
|
||||
LoginUser(domain.UserLogin) (domain.AuthSession, error)
|
||||
LogoutUser(string) error
|
||||
RotateUserSession(string) (domain.AuthSession, error)
|
||||
GetCurrentUser(string) (domain.User, error)
|
||||
UpdateCurrentUserProfile(string, domain.UserProfile) (domain.User, error)
|
||||
UpdateCurrentUserTheme(string, domain.UserThemePreference) (domain.UserThemePreference, error)
|
||||
@@ -50,12 +51,14 @@ type Core interface {
|
||||
GetMarketplacePlugin(string) (domain.PluginMarketplacePlugin, error)
|
||||
SetMarketplacePluginState(string, domain.PluginMarketplaceStateAction) (domain.PluginMarketplacePlugin, error)
|
||||
AuthorizePluginBridgeAction(domain.PluginBridgeAuthorizeRequest) (domain.PluginBridgeAuthorization, error)
|
||||
AuthorizePluginBridgeActionForSession(string, domain.PluginBridgeAuthorizeRequest) (domain.PluginBridgeAuthorization, error)
|
||||
ExecutePluginBridgeAction(string, domain.PluginBridgeExecuteRequest) (domain.PluginBridgeExecuteResponse, error)
|
||||
CreateRunEndpoint(domain.RunEndpoint) (domain.RunEndpoint, error)
|
||||
GetRunEndpoint(string) (domain.RunEndpoint, error)
|
||||
ListRunEndpoints(domain.RunEndpointFilter) ([]domain.RunEndpoint, error)
|
||||
RegisterRunHello(domain.RunControlHello) (domain.RunControlHelloResult, error)
|
||||
AcceptRunHeartbeat(domain.RunControlHeartbeat) (domain.RunControlHeartbeatResult, error)
|
||||
AuthorizeRunRequestSignature(domain.RunRequestSignature) error
|
||||
CreateServerInstance(domain.ServerInstance) (domain.ServerInstance, error)
|
||||
CreateServerInstanceForSession(string, domain.ServerInstance) (domain.ServerInstance, error)
|
||||
CreateServerInstanceWorkflow(domain.ServerLifecycleCreate) (domain.ServerLifecycleResult, error)
|
||||
@@ -64,6 +67,7 @@ type Core interface {
|
||||
StartServerInstanceForSession(string, domain.ServerLifecycleCommand) (domain.ServerLifecycleResult, error)
|
||||
StopServerInstance(domain.ServerLifecycleCommand) (domain.ServerLifecycleResult, error)
|
||||
StopServerInstanceForSession(string, domain.ServerLifecycleCommand) (domain.ServerLifecycleResult, error)
|
||||
QueryServerInstanceProcessForSession(string, domain.ServerLifecycleCommand) (domain.ServerLifecycleResult, error)
|
||||
GetServerInstance(string) (domain.ServerInstance, error)
|
||||
GetServerInstanceForSession(string, string) (domain.ServerInstance, error)
|
||||
UpdateServerInstanceForSession(string, string, domain.ServerInstanceUpdate) (domain.ServerInstance, error)
|
||||
@@ -75,6 +79,13 @@ type Core interface {
|
||||
ArchiveServerInstanceForSession(string, string) (domain.ServerInstance, error)
|
||||
GetPlatformResourceUsage() (domain.PlatformResourceUsage, error)
|
||||
ListServerMetricsForSession(string) ([]domain.ServerMetrics, error)
|
||||
IngestMetricBatch(domain.MetricBatchIngest) (domain.MetricBatchIngestResult, error)
|
||||
ListMetricSamplesForSession(string, domain.MetricSampleFilter) ([]domain.MetricSample, error)
|
||||
CreateBackupForSession(string, domain.BackupRecord) (domain.BackupRecord, error)
|
||||
GetBackupForSession(string, string) (domain.BackupRecord, error)
|
||||
ListBackupsForSession(string, domain.BackupFilter) ([]domain.BackupRecord, error)
|
||||
ListRemoteAdapterDeclarationsForSession(string, string) ([]domain.RemoteAdapterDeclaration, error)
|
||||
RequestRemoteAdapterForSession(string, domain.RemoteAdapterRequest) (domain.RemoteAdapterResult, error)
|
||||
GetServerConfigForSession(string, string) (domain.ServerConfig, error)
|
||||
PreviewServerConfigWriteForSession(string, domain.ServerConfigDiffRequest) (domain.ServerConfigDiffPreview, error)
|
||||
ApproveServerConfigWriteForSession(string, domain.ServerConfigWriteApproval) (domain.ServerConfigWriteDispatch, error)
|
||||
@@ -82,28 +93,53 @@ type Core interface {
|
||||
CreateJob(domain.Job) (domain.Job, error)
|
||||
GetJob(string) (domain.Job, error)
|
||||
ListJobs(domain.JobFilter) ([]domain.Job, error)
|
||||
GetJobForSession(string, string) (domain.Job, error)
|
||||
ListJobsForSession(string, domain.JobFilter) ([]domain.Job, error)
|
||||
RequestRunJobCancelForSession(string, domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error)
|
||||
ClaimRunJob(domain.RunJobClaim) (domain.RunJobClaimResult, error)
|
||||
AckRunJob(domain.RunJobAck) (domain.RunJobAckResult, error)
|
||||
UpdateRunJobProgress(domain.RunJobProgress) (domain.RunJobProgressResult, error)
|
||||
CompleteRunJob(domain.RunJobResult) (domain.RunJobResultResult, error)
|
||||
GetDistributionBuildInput(domain.DistributionBuildInputRequest) (domain.DistributionBuildInput, error)
|
||||
GetDependencyExecutionInput(domain.DependencyExecutionInputRequest) (domain.DependencyExecutionInput, error)
|
||||
GetRunUpdateInput(domain.RunUpdateInputRequest) (domain.RunUpdateInput, error)
|
||||
ReadRunUpdateChunk(domain.RunUpdateChunkRequest) (domain.RunUpdateChunk, error)
|
||||
ReportRunUpdateHealth(domain.RunUpdateHealthReport) (domain.RunUpdateHealthResult, error)
|
||||
RequestRunJobCancel(domain.RunJobCancelRequest) (domain.RunJobCancelRequestResult, error)
|
||||
PollRunJobCancel(domain.RunJobCancelPoll) (domain.RunJobCancelPollResult, error)
|
||||
ReconcileRunJobs(domain.RunJobReconcile) (domain.RunJobReconcileResult, error)
|
||||
CreateArtifact(domain.Artifact) (domain.Artifact, error)
|
||||
GetArtifact(string) (domain.Artifact, error)
|
||||
ListArtifacts(domain.ArtifactFilter) ([]domain.Artifact, error)
|
||||
ListArtifactsForSession(string, domain.ArtifactFilter) ([]domain.Artifact, error)
|
||||
GetArtifactForSession(string, string) (domain.Artifact, error)
|
||||
OpenArtifactDownloadForSession(string, domain.ArtifactDownloadReferenceRequest) (domain.ArtifactDownloadReference, error)
|
||||
ReadArtifactContentForSession(string, domain.ArtifactContentRequest) (domain.ArtifactContent, error)
|
||||
GetServerRuntimeActionsForSession(string, string) (domain.ServerRuntimeActions, error)
|
||||
GetServerRuntimeBindingForSession(string, string) (domain.RuntimeBindingView, error)
|
||||
UpdateServerRuntimeBindingForSession(string, string, domain.RuntimeBindingUpdate) (domain.RuntimeBindingView, error)
|
||||
GenerateRunDistributionForSession(string, domain.RunDistributionGenerateRequest) (domain.RunDistribution, error)
|
||||
GenerateClientManagerDistributionForSession(string, domain.ClientManagerBuildRequest) (domain.ClientManagerDistribution, error)
|
||||
OpenLatestRunDistributionDownloadForSession(string, string) (domain.ArtifactDownloadReference, error)
|
||||
OpenLatestClientManagerDistributionDownloadForSession(string, string, string) (domain.ArtifactDownloadReference, error)
|
||||
ResetComponentKeyForSession(string, domain.ComponentKeyResetRequest) (domain.EncryptedComponentKey, error)
|
||||
AuthenticateComponent(domain.ComponentAuthenticationRequest) (domain.ComponentAuthenticationResult, error)
|
||||
DeployClientManagerForSession(string, domain.ClientManagerDeployRequest) (domain.ClientManagerLifecycleView, error)
|
||||
ControlClientManagerForSession(string, domain.ClientManagerControlRequest) (domain.ClientManagerLifecycleView, error)
|
||||
UpdateClientManagerForSession(string, domain.ClientManagerUpdateRequest) (domain.ClientManagerLifecycleView, error)
|
||||
UninstallClientManagerForSession(string, domain.ClientManagerUninstallRequest) (domain.ClientManagerLifecycleView, error)
|
||||
RetryClientManagerLifecycleForSession(string, domain.ClientManagerRetryRequest) (domain.ClientManagerLifecycleView, error)
|
||||
RevokeClientManagerSessionForSession(string, domain.ClientManagerRevokeSessionRequest) (domain.ClientManagerLifecycleView, error)
|
||||
GetClientManagerLifecycleForSession(string, string, string) (domain.ClientManagerLifecycleView, error)
|
||||
ListClientManagerLifecyclesForSession(string, string) ([]domain.ClientManagerLifecycleView, error)
|
||||
GetClientManagerLifecycleInput(domain.ClientManagerLifecycleInputRequest) (domain.ClientManagerLifecycleInput, error)
|
||||
ReadClientManagerLifecycleChunk(domain.RunUpdateChunkRequest) (domain.RunUpdateChunk, error)
|
||||
RegisterClientManager(domain.ClientManagerRegisterRequest) (domain.ClientManagerRegisterResult, error)
|
||||
AcceptClientManagerHeartbeat(domain.ClientManagerHeartbeat) (domain.ClientManagerHeartbeatResult, error)
|
||||
ReconcileClientManagerLifecycle() error
|
||||
PushRunUpdateForSession(string, domain.RunUpdateRequest) (domain.RunUpdateJob, error)
|
||||
ListRunUpdateJobsForSession(string, string) ([]domain.RunUpdateJob, error)
|
||||
GetDependencyCatalogForSession(string, string) (domain.DependencyCatalog, error)
|
||||
QueueDependencyJobForSession(string, domain.DependencyJobRequest) (domain.Job, error)
|
||||
QueueLogBackfillForSession(string, domain.LogBackfillRequest) (domain.Job, error)
|
||||
OpenArtifactTransfer(domain.ArtifactTransferOpen) (domain.ArtifactTransferOpenResult, error)
|
||||
@@ -113,11 +149,15 @@ type Core interface {
|
||||
CreateLogStream(domain.LogStream) (domain.LogStream, error)
|
||||
GetLogStream(string) (domain.LogStream, error)
|
||||
ListLogStreams(domain.LogStreamFilter) ([]domain.LogStream, error)
|
||||
GetLogStreamForSession(string, string) (domain.LogStream, error)
|
||||
ListLogStreamsForSession(string, domain.LogStreamFilter) ([]domain.LogStream, error)
|
||||
QueryLogStreamForSession(string, domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
|
||||
IngestLogBatch(domain.LogBatchIngest) (domain.LogBatchIngestResult, error)
|
||||
QueryLogStream(domain.LogStreamCursorQuery) (domain.LogStreamCursorResult, error)
|
||||
CreateAuditEvent(domain.AuditEvent) (domain.AuditEvent, error)
|
||||
GetAuditEvent(string) (domain.AuditEvent, error)
|
||||
ListAuditEvents(domain.AuditEventFilter) ([]domain.AuditEvent, error)
|
||||
SeedPlatformAdmin(string, string) error
|
||||
}
|
||||
|
||||
type CoreService struct {
|
||||
@@ -129,9 +169,8 @@ type CoreService struct {
|
||||
runSessions map[string]domain.RunControlSession
|
||||
runSessionSeq uint64
|
||||
jobMu sync.Mutex
|
||||
jobLeases map[string]domain.RunJobLease
|
||||
jobLeaseSeq uint64
|
||||
logStore LogBodyStore
|
||||
artifactStore ArtifactBodyStore
|
||||
artifactMu sync.Mutex
|
||||
artifactTransfers map[string]domain.ArtifactTransferSession
|
||||
artifactPayloads map[string][]byte
|
||||
@@ -139,6 +178,7 @@ type CoreService struct {
|
||||
auditMu sync.Mutex
|
||||
auditSeq uint64
|
||||
aiProviderClient AIProviderClient
|
||||
secretEnvelope SecretEnvelope
|
||||
}
|
||||
|
||||
var _ Core = (*CoreService)(nil)
|
||||
@@ -159,17 +199,71 @@ func newCoreServiceWithLogStore(store repo.Store, logStore LogBodyStore, now fun
|
||||
if logStore == nil {
|
||||
logStore = NewMemoryLogBodyStore()
|
||||
}
|
||||
return &CoreService{
|
||||
artifactStore := NewMemoryArtifactBodyStore()
|
||||
service := &CoreService{
|
||||
store: store,
|
||||
now: now,
|
||||
authSessions: map[string]string{},
|
||||
runSessions: map[string]domain.RunControlSession{},
|
||||
jobLeases: map[string]domain.RunJobLease{},
|
||||
logStore: logStore,
|
||||
artifactStore: artifactStore,
|
||||
artifactTransfers: map[string]domain.ArtifactTransferSession{},
|
||||
artifactPayloads: map[string][]byte{},
|
||||
aiProviderClient: MockAIProviderClient{},
|
||||
secretEnvelope: newSecretEnvelope(developmentSecretEnvelopeKey),
|
||||
}
|
||||
return service
|
||||
}
|
||||
|
||||
func NewCoreServiceWithDurableStores(store repo.Store, logStore LogBodyStore, artifactStore ArtifactBodyStore) (*CoreService, error) {
|
||||
if artifactStore == nil {
|
||||
artifactStore = NewMemoryArtifactBodyStore()
|
||||
}
|
||||
service := newCoreServiceWithLogStore(store, logStore, func() time.Time { return time.Now().UTC() })
|
||||
service.artifactStore = artifactStore
|
||||
sessions, err := artifactStore.LoadTransfers()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, session := range sessions {
|
||||
service.artifactTransfers[session.TransferID] = domain.CopyArtifactTransferSession(session)
|
||||
}
|
||||
if err := service.recoverLogCursors(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := service.RecoverIncompleteBackups(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := service.ReconcileClientManagerLifecycle(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return service, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) recoverLogCursors() error {
|
||||
store, ok := svc.logStore.(interface{ LatestSeq(string) (uint64, error) })
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
streams, err := svc.store.LogStreams().List(domain.LogStreamFilter{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stream := range streams {
|
||||
latest, latestErr := store.LatestSeq(stream.ID)
|
||||
if latestErr != nil || latest <= stream.LatestSeq {
|
||||
if latestErr != nil {
|
||||
return latestErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
stream.LatestSeq = latest
|
||||
stream.UpdatedAt = svc.now()
|
||||
if err := svc.store.LogStreams().Update(stream); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CreateUser(user domain.User) (domain.User, error) {
|
||||
@@ -240,6 +334,11 @@ func (svc *CoreService) UpdateUser(id string, user domain.User) (domain.User, er
|
||||
if err := svc.store.Users().Update(user); err != nil {
|
||||
return domain.User{}, err
|
||||
}
|
||||
if user.Status != domain.UserStatusActive {
|
||||
if err := svc.revokeUserSessions(user.ID); err != nil {
|
||||
return domain.User{}, err
|
||||
}
|
||||
}
|
||||
return domain.CopyUser(user), nil
|
||||
}
|
||||
|
||||
@@ -282,19 +381,7 @@ func (svc *CoreService) RegisterUser(registration domain.UserRegistration) (doma
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
if firstUser {
|
||||
sessionID, err := randomToken()
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
svc.authSessions[sessionID] = created.ID
|
||||
svc.authMu.Unlock()
|
||||
return domain.AuthSession{
|
||||
SessionID: sessionID,
|
||||
User: created,
|
||||
Status: "authenticated",
|
||||
Message: "首个账号已创建为平台管理员。",
|
||||
}, nil
|
||||
return svc.issueAuthSession(created, "首个账号已创建为平台管理员。")
|
||||
}
|
||||
return domain.AuthSession{
|
||||
User: created,
|
||||
@@ -325,27 +412,11 @@ func (svc *CoreService) LoginUser(login domain.UserLogin) (domain.AuthSession, e
|
||||
if matched.Status == domain.UserStatusDisabled {
|
||||
return domain.AuthSession{}, ErrForbidden
|
||||
}
|
||||
sessionID, err := randomToken()
|
||||
if err != nil {
|
||||
return domain.AuthSession{}, err
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
svc.authSessions[sessionID] = matched.ID
|
||||
svc.authMu.Unlock()
|
||||
return domain.AuthSession{SessionID: sessionID, User: matched, Status: "authenticated", Message: "登录成功"}, nil
|
||||
return svc.issueAuthSession(matched, "登录成功")
|
||||
}
|
||||
|
||||
func (svc *CoreService) LogoutUser(sessionID string) error {
|
||||
if strings.TrimSpace(sessionID) == "" {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
defer svc.authMu.Unlock()
|
||||
if _, exists := svc.authSessions[sessionID]; !exists {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
delete(svc.authSessions, sessionID)
|
||||
return nil
|
||||
return svc.revokeAuthSession(sessionID)
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetCurrentUser(sessionID string) (domain.User, error) {
|
||||
@@ -381,7 +452,17 @@ func (svc *CoreService) UpdateCurrentUserTheme(sessionID string, preference doma
|
||||
}
|
||||
|
||||
func (svc *CoreService) SeedLocalPlatformAdmin() error {
|
||||
const adminEmail = "operator.local@example.test"
|
||||
return svc.SeedPlatformAdmin("operator.local@example.test", "operator-local")
|
||||
}
|
||||
|
||||
func (svc *CoreService) SeedPlatformAdmin(email string, password string) error {
|
||||
email = strings.TrimSpace(email)
|
||||
if email == "" {
|
||||
email = "operator.local@example.test"
|
||||
}
|
||||
if len([]rune(password)) < 12 {
|
||||
return validationError("bootstrap admin password must be at least 12 characters")
|
||||
}
|
||||
_, err := svc.store.Users().Get("user-admin")
|
||||
if err == nil {
|
||||
return nil
|
||||
@@ -392,11 +473,11 @@ func (svc *CoreService) SeedLocalPlatformAdmin() error {
|
||||
return svc.store.Users().Create(domain.User{
|
||||
ID: "user-admin",
|
||||
DisplayName: "Operator",
|
||||
Email: adminEmail,
|
||||
Email: email,
|
||||
Status: domain.UserStatusActive,
|
||||
Roles: []string{"platform-admin"},
|
||||
PasswordHash: mustHashPassword("operator-local"),
|
||||
Profile: domain.UserProfile{ContactNote: "local development admin"},
|
||||
PasswordHash: mustHashPassword(password),
|
||||
Profile: domain.UserProfile{ContactNote: "bootstrap platform admin"},
|
||||
CreatedAt: svc.now(),
|
||||
UpdatedAt: svc.now(),
|
||||
})
|
||||
@@ -541,6 +622,7 @@ func gamePluginFromManifestRegistration(registration domain.GamePluginManifestRe
|
||||
Tags: manifest.Tags,
|
||||
AIPurposes: manifest.AI.Purposes,
|
||||
RemoteAccess: manifest.RemoteAccess,
|
||||
RuntimeProfiles: manifest.RuntimeProfiles,
|
||||
Status: domain.GamePluginStatusInstalled,
|
||||
}
|
||||
}
|
||||
@@ -623,11 +705,11 @@ func (svc *CoreService) ExecutePluginBridgeAction(sessionID string, request doma
|
||||
case domain.PluginBridgeActionFilesRequest:
|
||||
base = svc.executeBridgeFileRequest(sessionID, base, request)
|
||||
case domain.PluginBridgeActionRemoteAccessRequest:
|
||||
base = svc.executeBridgeRemoteAccessRequest(base, plugin, instance, request.Payload)
|
||||
base = svc.executeBridgeRemoteAccessRequest(sessionID, base, plugin, instance, request.Payload)
|
||||
case domain.PluginBridgeActionRunDistribution:
|
||||
base = svc.executeBridgeRunDistribution(sessionID, base, request)
|
||||
case domain.PluginBridgeActionDependenciesRequest:
|
||||
base = svc.executeBridgeDependenciesRequest(base, plugin, instance, request.Payload)
|
||||
base = svc.executeBridgeDependenciesRequest(sessionID, base, plugin, instance, request.Payload)
|
||||
case domain.PluginBridgeActionLogsBackfillRequest:
|
||||
base = svc.executeBridgeLogsBackfillRequest(base, plugin, instance, request.Payload)
|
||||
case domain.PluginBridgeActionClientManager:
|
||||
@@ -843,39 +925,44 @@ func (svc *CoreService) executeBridgeFileRequest(sessionID string, base domain.P
|
||||
return base
|
||||
}
|
||||
|
||||
func (svc *CoreService) executeBridgeRemoteAccessRequest(base domain.PluginBridgeExecuteResponse, plugin domain.GamePlugin, instance domain.ServerInstance, payload map[string]string) domain.PluginBridgeExecuteResponse {
|
||||
func (svc *CoreService) executeBridgeRemoteAccessRequest(sessionID string, base domain.PluginBridgeExecuteResponse, plugin domain.GamePlugin, instance domain.ServerInstance, payload map[string]string) domain.PluginBridgeExecuteResponse {
|
||||
capability := strings.TrimSpace(payload["capability"])
|
||||
if capability == "" {
|
||||
base.Status = "error"
|
||||
base.Error = &domain.PluginBridgeSafeError{Code: "validation", Message: "capability is required"}
|
||||
return base
|
||||
}
|
||||
if !containsString(plugin.RequiredRunCapabilities, capability) {
|
||||
if !containsString(plugin.RequiredRunCapabilities, capability) || !containsString(plugin.RemoteAccess.RunCapabilities, capability) {
|
||||
base.Status = "denied"
|
||||
base.Error = &domain.PluginBridgeSafeError{Code: "capability_denied", Message: "requested remote capability is not declared by plugin"}
|
||||
return base
|
||||
}
|
||||
job := domain.Job{
|
||||
ID: jobIDFromParts("job-remote", base.RequestID, capability),
|
||||
ServerInstanceID: instance.ID,
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
Capability: capability,
|
||||
TargetKey: payload["targetKey"],
|
||||
InputRef: payload["inputRef"],
|
||||
IdempotencyKey: defaultBridgeValue(payload["idempotencyKey"], base.RequestID),
|
||||
Progress: domain.JobProgress{Percent: 0, Message: "remote access job queued"},
|
||||
declarationKey := strings.TrimSpace(payload["declarationKey"])
|
||||
if declarationKey == "" {
|
||||
for _, profile := range plugin.RuntimeProfiles.TransportProfiles {
|
||||
if profile.TargetKey == payload["targetKey"] && containsString(profile.Capabilities, capability) {
|
||||
declarationKey = profile.Key
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
created, err := svc.CreateJob(job)
|
||||
if declarationKey == "" {
|
||||
declarationKey = "legacy-" + string(remoteAdapterKindForCapability(capability))
|
||||
}
|
||||
timeoutSeconds, _ := strconv.Atoi(payload["timeoutSeconds"])
|
||||
maxAttempts, _ := strconv.Atoi(payload["maxAttempts"])
|
||||
result, err := svc.RequestRemoteAdapterForSession(sessionID, domain.RemoteAdapterRequest{ServerInstanceID: instance.ID, DeclarationKey: declarationKey, TargetKey: payload["targetKey"], Capability: capability, TimeoutSeconds: timeoutSeconds, MaxAttempts: maxAttempts, IdempotencyKey: defaultBridgeValue(payload["idempotencyKey"], base.RequestID)})
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = "queued"
|
||||
base.Result = map[string]string{
|
||||
"jobId": created.ID,
|
||||
"state": string(created.State),
|
||||
"capability": created.Capability,
|
||||
"targetKey": created.TargetKey,
|
||||
"serverInstanceId": created.ServerInstanceID,
|
||||
"jobId": result.RequestID,
|
||||
"state": result.Status,
|
||||
"capability": capability,
|
||||
"targetKey": result.TargetKey,
|
||||
"serverInstanceId": result.ServerInstanceID,
|
||||
"adapterKind": string(result.Kind),
|
||||
}
|
||||
return base
|
||||
}
|
||||
@@ -896,60 +983,116 @@ func (svc *CoreService) executeBridgeRunDistribution(sessionID string, base doma
|
||||
"artifactId": distribution.ArtifactID,
|
||||
"checksum": distribution.Checksum,
|
||||
"keyGeneration": strconv.Itoa(distribution.KeyGeneration),
|
||||
"secretRef": distribution.SecretRef,
|
||||
"status": string(distribution.Status),
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func (svc *CoreService) executeBridgeClientManager(sessionID string, base domain.PluginBridgeExecuteResponse, request domain.PluginBridgeExecuteRequest) domain.PluginBridgeExecuteResponse {
|
||||
distribution, err := svc.GenerateClientManagerDistributionForSession(sessionID, domain.ClientManagerBuildRequest{
|
||||
ServerInstanceID: request.ServerInstanceID,
|
||||
ProfileKey: request.Payload["profileKey"],
|
||||
TargetOS: defaultBridgeValue(request.Payload["targetOs"], "windows"),
|
||||
TargetArch: defaultBridgeValue(request.Payload["targetArch"], "amd64"),
|
||||
RepositoryURL: request.Payload["repositoryUrl"],
|
||||
SourceRevision: request.Payload["sourceRevision"],
|
||||
IdempotencyKey: defaultBridgeValue(request.Payload["idempotencyKey"], request.RequestID),
|
||||
})
|
||||
profileKey := request.Payload["profileKey"]
|
||||
operation := defaultBridgeValue(request.Payload["operation"], "status")
|
||||
idempotencyKey := defaultBridgeValue(request.Payload["idempotencyKey"], request.RequestID)
|
||||
generation, _ := strconv.Atoi(request.Payload["expectedDeploymentGeneration"])
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, request.ServerInstanceID)
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
profile, err := findRuntimeClientManagerProfile(plugin, profileKey)
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, ErrForbidden)
|
||||
}
|
||||
var view domain.ClientManagerLifecycleView
|
||||
switch operation {
|
||||
case "generate":
|
||||
distribution, err := svc.GenerateClientManagerDistributionForSession(sessionID, domain.ClientManagerBuildRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, TargetOS: defaultBridgeValue(request.Payload["targetOs"], "windows"), TargetArch: defaultBridgeValue(request.Payload["targetArch"], "amd64"), RepositoryURL: profile.RepositoryURL, SourceRevision: clientManagerProfileRevision(profile), IdempotencyKey: idempotencyKey})
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = "queued"
|
||||
base.Result = map[string]string{"distributionId": distribution.ID, "buildJobId": distribution.BuildJobID, "artifactId": distribution.ArtifactID, "checksum": distribution.Checksum, "keyGeneration": strconv.Itoa(distribution.KeyGeneration), "version": distribution.Version, "status": string(distribution.Status)}
|
||||
return base
|
||||
case "download":
|
||||
reference, err := svc.OpenLatestClientManagerDistributionDownloadForSession(sessionID, instance.ID, profileKey)
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = "ok"
|
||||
base.Result = map[string]string{"artifactId": reference.ArtifactID, "downloadUrl": reference.DownloadURL, "checksum": reference.Checksum, "sizeBytes": strconv.FormatInt(reference.SizeBytes, 10), "expiresAt": reference.ExpiresAt.Format(time.RFC3339), "rangeSupported": strconv.FormatBool(reference.RangeSupported), "chunkSizeBytes": strconv.Itoa(reference.ChunkSizeBytes)}
|
||||
return base
|
||||
case "reset-key":
|
||||
key, err := svc.ResetComponentKeyForSession(sessionID, domain.ComponentKeyResetRequest{ServerInstanceID: instance.ID, ComponentKind: domain.DistributionComponentClientManager, ComponentKey: profileKey})
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = "ok"
|
||||
base.Result = map[string]string{"profileKey": profileKey, "keyGeneration": strconv.Itoa(key.Generation), "status": string(key.Status), "requiresRedeploy": "true"}
|
||||
return base
|
||||
case "status":
|
||||
view, err = svc.GetClientManagerLifecycleForSession(sessionID, instance.ID, profileKey)
|
||||
case "deploy":
|
||||
distributionID, resolveErr := svc.resolveClientManagerDistributionID(instance.ID, profileKey, request.Payload["artifactId"])
|
||||
if resolveErr != nil {
|
||||
return bridgeExecutionError(base, resolveErr)
|
||||
}
|
||||
view, err = svc.DeployClientManagerForSession(sessionID, domain.ClientManagerDeployRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, DistributionID: distributionID, ExpectedDeploymentGeneration: generation, IdempotencyKey: idempotencyKey})
|
||||
case "start", "stop", "restart", "rollback":
|
||||
view, err = svc.ControlClientManagerForSession(sessionID, domain.ClientManagerControlRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, Operation: domain.ClientManagerLifecycleOperation(operation), ExpectedDeploymentGeneration: generation, IdempotencyKey: idempotencyKey})
|
||||
case "update":
|
||||
distributionID, resolveErr := svc.resolveClientManagerDistributionID(instance.ID, profileKey, request.Payload["artifactId"])
|
||||
if resolveErr != nil {
|
||||
return bridgeExecutionError(base, resolveErr)
|
||||
}
|
||||
view, err = svc.UpdateClientManagerForSession(sessionID, domain.ClientManagerUpdateRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, DistributionID: distributionID, ExpectedDeploymentGeneration: generation, Approved: true, IdempotencyKey: idempotencyKey})
|
||||
case "retry":
|
||||
view, err = svc.RetryClientManagerLifecycleForSession(sessionID, domain.ClientManagerRetryRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, ExpectedDeploymentGeneration: generation, IdempotencyKey: idempotencyKey})
|
||||
case "revoke-session":
|
||||
view, err = svc.RevokeClientManagerSessionForSession(sessionID, domain.ClientManagerRevokeSessionRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, Reason: "plugin bridge operator request"})
|
||||
case "uninstall":
|
||||
view, err = svc.UninstallClientManagerForSession(sessionID, domain.ClientManagerUninstallRequest{ServerInstanceID: instance.ID, ProfileKey: profileKey, ExpectedDeploymentGeneration: generation, Confirmed: true, IdempotencyKey: idempotencyKey})
|
||||
default:
|
||||
base.Status = "denied"
|
||||
base.Error = &domain.PluginBridgeSafeError{Code: "invalid_client_manager_operation", Message: "client-manager operation is not supported"}
|
||||
return base
|
||||
}
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
}
|
||||
base.Status = "ok"
|
||||
base.Result = map[string]string{
|
||||
"distributionId": distribution.ID,
|
||||
"buildJobId": distribution.BuildJobID,
|
||||
"artifactId": distribution.ArtifactID,
|
||||
"checksum": distribution.Checksum,
|
||||
"keyGeneration": strconv.Itoa(distribution.KeyGeneration),
|
||||
"secretRef": distribution.SecretRef,
|
||||
"status": string(distribution.Status),
|
||||
if view.Job.ID != "" && !isTerminalJobState(view.Job.State) {
|
||||
base.Status = "queued"
|
||||
}
|
||||
base.Result = safeClientManagerBridgeResult(view)
|
||||
return base
|
||||
}
|
||||
|
||||
func (svc *CoreService) executeBridgeDependenciesRequest(base domain.PluginBridgeExecuteResponse, plugin domain.GamePlugin, instance domain.ServerInstance, payload map[string]string) domain.PluginBridgeExecuteResponse {
|
||||
action := defaultBridgeValue(payload["action"], "check")
|
||||
func (svc *CoreService) executeBridgeDependenciesRequest(sessionID string, base domain.PluginBridgeExecuteResponse, plugin domain.GamePlugin, instance domain.ServerInstance, payload map[string]string) domain.PluginBridgeExecuteResponse {
|
||||
action := defaultBridgeValue(payload["operation"], "check")
|
||||
capability := domain.JobCapabilityDependenciesCheck
|
||||
message := "dependency check queued"
|
||||
if action == "install" {
|
||||
capability = domain.JobCapabilityDependenciesInstall
|
||||
message = "dependency install queued"
|
||||
} else if action != "check" {
|
||||
base.Status = "denied"
|
||||
base.Error = &domain.PluginBridgeSafeError{Code: "invalid_dependency_operation", Message: "dependency operation must be check or install"}
|
||||
return base
|
||||
}
|
||||
if !containsString(plugin.RequiredRunCapabilities, capability) {
|
||||
base.Status = "denied"
|
||||
base.Error = &domain.PluginBridgeSafeError{Code: "capability_denied", Message: "dependency capability is not declared by plugin"}
|
||||
return base
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: jobIDFromParts("job-dependencies", base.RequestID, capability),
|
||||
job, err := svc.QueueDependencyJobForSession(sessionID, domain.DependencyJobRequest{
|
||||
ServerInstanceID: instance.ID,
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
Capability: capability,
|
||||
TargetKey: defaultBridgeValue(payload["probeKey"], "dependencies/default"),
|
||||
InputRef: payload["inputRef"],
|
||||
ProbeKey: payload["probeKey"],
|
||||
InstallPlanKey: payload["planKey"],
|
||||
PlanDigest: payload["planDigest"],
|
||||
TargetOS: payload["targetOS"],
|
||||
TargetArch: payload["targetArch"],
|
||||
IdempotencyKey: defaultBridgeValue(payload["idempotencyKey"], base.RequestID),
|
||||
Progress: domain.JobProgress{Percent: 0, Message: message},
|
||||
Install: action == "install",
|
||||
})
|
||||
if err != nil {
|
||||
return bridgeExecutionError(base, err)
|
||||
@@ -1136,6 +1279,7 @@ func marketplacePluginFromGamePlugin(plugin domain.GamePlugin) domain.PluginMark
|
||||
Tags: plugin.Tags,
|
||||
AIPurposes: plugin.AIPurposes,
|
||||
RemoteAccess: plugin.RemoteAccess,
|
||||
RuntimeProfiles: plugin.RuntimeProfiles,
|
||||
ValidationViolations: plugin.ValidationViolations,
|
||||
Status: plugin.Status,
|
||||
Source: "platform-registry",
|
||||
@@ -1390,11 +1534,22 @@ func (svc *CoreService) GetServerConfigForSession(sessionID string, serverInstan
|
||||
Key: "server.properties",
|
||||
Source: "platform-derived",
|
||||
UpdatedAt: instance.UpdatedAt,
|
||||
Checksum: instance.ConfigChecksum,
|
||||
}
|
||||
if config.UpdatedAt.IsZero() {
|
||||
config.UpdatedAt = svc.now()
|
||||
}
|
||||
config.Content = buildLogicalServerConfig(instance)
|
||||
config.Key = instance.ConfigKey
|
||||
if config.Key == "" {
|
||||
config.Key = "server.properties"
|
||||
}
|
||||
config.Content = instance.ConfigContent
|
||||
if config.Content == "" {
|
||||
config.Content = buildLogicalServerConfig(instance)
|
||||
}
|
||||
if config.Checksum == "" {
|
||||
config.Checksum = validator.BytesChecksum([]byte(config.Content))
|
||||
}
|
||||
if err := validator.ValidateServerConfig(config); err != nil {
|
||||
return domain.ServerConfig{}, err
|
||||
}
|
||||
@@ -1415,12 +1570,16 @@ func (svc *CoreService) PreviewServerConfigWriteForSession(sessionID string, req
|
||||
if config.ConfigVersion != request.ExpectedConfigVersion {
|
||||
return domain.ServerConfigDiffPreview{}, validationError("expectedConfigVersion must match server instance")
|
||||
}
|
||||
if request.ExpectedChecksum != "" && config.Checksum != request.ExpectedChecksum {
|
||||
return domain.ServerConfigDiffPreview{}, validationError("expectedChecksum must match server config")
|
||||
}
|
||||
if config.Key != request.Key {
|
||||
return domain.ServerConfigDiffPreview{}, validationError("key must match server config")
|
||||
}
|
||||
preview := domain.ServerConfigDiffPreview{
|
||||
ServerInstanceID: request.ServerInstanceID,
|
||||
ConfigVersion: config.ConfigVersion,
|
||||
Checksum: config.Checksum,
|
||||
Key: request.Key,
|
||||
CurrentContent: config.Content,
|
||||
ProposedContent: request.ProposedContent,
|
||||
@@ -1446,6 +1605,7 @@ func (svc *CoreService) ApproveServerConfigWriteForSession(sessionID string, app
|
||||
preview, err := svc.PreviewServerConfigWriteForSession(sessionID, domain.ServerConfigDiffRequest{
|
||||
ServerInstanceID: approval.ServerInstanceID,
|
||||
ExpectedConfigVersion: approval.ExpectedConfigVersion,
|
||||
ExpectedChecksum: approval.ExpectedChecksum,
|
||||
Key: approval.Key,
|
||||
ProposedContent: approval.ProposedContent,
|
||||
ProposedContentInputRef: approval.ProposedContentInputRef,
|
||||
@@ -1460,6 +1620,13 @@ func (svc *CoreService) ApproveServerConfigWriteForSession(sessionID string, app
|
||||
if err != nil {
|
||||
return domain.ServerConfigWriteDispatch{}, err
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.ServerConfigWriteDispatch{}, err
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "config.write.denied"); err != nil {
|
||||
return domain.ServerConfigWriteDispatch{}, err
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: jobIDFromParts("job-config-write", approval.ServerInstanceID, approval.IdempotencyKey),
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -1467,6 +1634,7 @@ func (svc *CoreService) ApproveServerConfigWriteForSession(sessionID string, app
|
||||
Capability: domain.JobCapabilityConfigWrite,
|
||||
TargetKey: approval.Key,
|
||||
InputRef: approval.ProposedContentInputRef,
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: svc.runtimeProfileScope(instance.ID), Content: approval.ProposedContent, ExpectedVersion: approval.ExpectedConfigVersion, ExpectedChecksum: preview.Checksum, MaxReadBytes: 64 * 1024},
|
||||
IdempotencyKey: approval.IdempotencyKey,
|
||||
Progress: domain.JobProgress{Percent: 0, Message: "config write queued"},
|
||||
})
|
||||
@@ -1487,6 +1655,29 @@ func (svc *CoreService) DispatchFileOperationForSession(sessionID string, reques
|
||||
if request.ExpectedConfigVersion > 0 && request.ExpectedConfigVersion != instance.ConfigVersion {
|
||||
return domain.FileOperationDispatchResult{}, validationError("expectedConfigVersion must match server instance")
|
||||
}
|
||||
user, err := svc.GetCurrentUser(sessionID)
|
||||
if err != nil {
|
||||
return domain.FileOperationDispatchResult{}, err
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(user.ID, instance.ID, "file.operation.denied"); err != nil {
|
||||
return domain.FileOperationDispatchResult{}, err
|
||||
}
|
||||
content := request.Content
|
||||
if request.Operation == domain.FileOperationWrite && content == "" && strings.HasPrefix(request.InputRef, "artifact://") {
|
||||
artifactID := strings.TrimPrefix(request.InputRef, "artifact://")
|
||||
artifact, artifactErr := svc.store.Artifacts().Get(artifactID)
|
||||
if artifactErr != nil {
|
||||
return domain.FileOperationDispatchResult{}, artifactErr
|
||||
}
|
||||
if artifact.OwnerKind != domain.ArtifactOwnerKindServerInstance || artifact.OwnerID != instance.ID || artifact.State != domain.ArtifactStateAvailable {
|
||||
return domain.FileOperationDispatchResult{}, ErrForbidden
|
||||
}
|
||||
payload, payloadErr := svc.artifactPayload(artifactID)
|
||||
if payloadErr != nil {
|
||||
return domain.FileOperationDispatchResult{}, payloadErr
|
||||
}
|
||||
content = string(payload)
|
||||
}
|
||||
if request.PluginID != "" {
|
||||
plugin, err := svc.store.GamePlugins().Get(request.PluginID)
|
||||
if err != nil {
|
||||
@@ -1518,6 +1709,7 @@ func (svc *CoreService) DispatchFileOperationForSession(sessionID string, reques
|
||||
Capability: capability,
|
||||
TargetKey: request.Key,
|
||||
InputRef: request.InputRef,
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: svc.runtimeProfileScope(instance.ID), Content: content, ExpectedVersion: request.ExpectedConfigVersion, ExpectedChecksum: request.ExpectedChecksum, MaxReadBytes: 64 * 1024},
|
||||
IdempotencyKey: request.IdempotencyKey,
|
||||
Progress: domain.JobProgress{Percent: 0, Message: message},
|
||||
})
|
||||
@@ -1535,6 +1727,14 @@ func (svc *CoreService) DispatchFileOperationForSession(sessionID string, reques
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) runtimeProfileScope(serverInstanceID string) string {
|
||||
binding, err := svc.runtimeBindingForServer(serverInstanceID)
|
||||
if err != nil {
|
||||
return "default"
|
||||
}
|
||||
return binding.ProfileKey
|
||||
}
|
||||
|
||||
func (svc *CoreService) metricsForServer(instance domain.ServerInstance) domain.ServerMetrics {
|
||||
metrics := domain.ServerMetrics{
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -1732,6 +1932,7 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
|
||||
if job.UpdatedAt.IsZero() {
|
||||
job.UpdatedAt = stamp
|
||||
}
|
||||
job = normalizeJobScheduling(job, stamp)
|
||||
if err := validator.ValidateJob(job); err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
@@ -1772,11 +1973,22 @@ func (svc *CoreService) CreateJob(job domain.Job) (domain.Job, error) {
|
||||
}
|
||||
|
||||
func (svc *CoreService) GetJob(id string) (domain.Job, error) {
|
||||
return svc.store.Jobs().Get(id)
|
||||
job, err := svc.store.Jobs().Get(id)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
return normalizeJobScheduling(job, svc.now()), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ListJobs(filter domain.JobFilter) ([]domain.Job, error) {
|
||||
return svc.store.Jobs().List(filter)
|
||||
jobs, err := svc.store.Jobs().List(filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range jobs {
|
||||
jobs[i] = normalizeJobScheduling(jobs[i], svc.now())
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) CreateArtifact(artifact domain.Artifact) (domain.Artifact, error) {
|
||||
@@ -1891,17 +2103,11 @@ func validationError(violation string) error {
|
||||
}
|
||||
|
||||
func (svc *CoreService) userIDForSession(sessionID string) (string, error) {
|
||||
sessionID = strings.TrimSpace(sessionID)
|
||||
if sessionID == "" {
|
||||
return "", ErrUnauthorized
|
||||
session, err := svc.authenticatedSession(sessionID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
svc.authMu.Lock()
|
||||
defer svc.authMu.Unlock()
|
||||
userID, exists := svc.authSessions[sessionID]
|
||||
if !exists {
|
||||
return "", ErrUnauthorized
|
||||
}
|
||||
return userID, nil
|
||||
return session.UserID, nil
|
||||
}
|
||||
|
||||
func userIDFromEmail(email string) string {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
var fixedTime = time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC)
|
||||
@@ -351,10 +352,10 @@ func TestCoreServiceScopesServerAccessAndMembership(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create owned server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, instance, "local")
|
||||
if instance.OwnerUserID != "user-owner" {
|
||||
t.Fatalf("expected owner to be recorded, got %+v", instance)
|
||||
}
|
||||
|
||||
ownerServers, err := svc.ListServerInstancesForSession(ownerSession, domain.ServerInstanceFilter{})
|
||||
if err != nil || len(ownerServers) != 1 {
|
||||
t.Fatalf("expected owner server visibility, len=%d err=%v", len(ownerServers), err)
|
||||
@@ -513,6 +514,7 @@ func TestCoreServiceConfigWriteAndFileDispatchAreScoped(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, instance, "local")
|
||||
current, err := svc.GetServerConfigForSession(ownerSession, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get config: %v", err)
|
||||
@@ -603,6 +605,51 @@ func TestCoreServiceConfigWriteAndFileDispatchAreScoped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigWriteTerminalResultAppliesDurableTypedProjection(t *testing.T) {
|
||||
svc := newTestCoreService()
|
||||
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
|
||||
ownerSession := createServiceUserAndLogin(t, svc, domain.User{ID: "typed-config-owner", DisplayName: "Typed Config Owner", Email: "typed-config-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
instance, err := svc.CreateServerInstanceForSession(ownerSession, domain.ServerInstance{ID: "typed-config-server", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Typed Config", State: domain.ServerInstanceStateRunning})
|
||||
if err != nil {
|
||||
t.Fatalf("create typed config server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, instance, "local")
|
||||
current, err := svc.GetServerConfigForSession(ownerSession, instance.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("read typed config: %v", err)
|
||||
}
|
||||
proposed := current.Content + "motd=typed\n"
|
||||
dispatch, err := svc.ApproveServerConfigWriteForSession(ownerSession, domain.ServerConfigWriteApproval{ServerInstanceID: instance.ID, ExpectedConfigVersion: current.ConfigVersion, ExpectedChecksum: current.Checksum, Key: current.Key, ProposedContent: proposed, IdempotencyKey: "typed-config-write"})
|
||||
if err != nil {
|
||||
t.Fatalf("queue typed config write: %v", err)
|
||||
}
|
||||
helloRequest := validRunControlHello()
|
||||
helloRequest.CapabilityReport.Capabilities = append(helloRequest.CapabilityReport.Capabilities, domain.JobCapabilityConfigWrite)
|
||||
hello, err := svc.RegisterRunHello(helloRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("register typed config Run: %v", err)
|
||||
}
|
||||
claim, err := svc.ClaimRunJob(domain.RunJobClaim{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, Capabilities: []string{domain.JobCapabilityConfigWrite}, Capacity: domain.RunCapacity{MaxJobs: 2}})
|
||||
if err != nil || !claim.HasJob {
|
||||
t.Fatalf("claim typed config job: claim=%+v err=%v", claim, err)
|
||||
}
|
||||
checksum := validator.BytesChecksum([]byte(proposed))
|
||||
if _, err := svc.CompleteRunJob(domain.RunJobResult{RunEndpointID: endpoint.ID, SessionToken: hello.SessionToken, JobID: claim.Job.JobID, LeaseToken: claim.Job.LeaseToken, Attempt: claim.Job.Attempt, State: domain.JobStateSucceeded, Progress: domain.RunJobProgressReport{Percent: 100, Message: "config write completed"}, Message: "config write completed", ExecutionResult: domain.JobExecutionResult{Kind: "file.write", Version: dispatch.Job.ExecutionInput.ExpectedVersion + 1, Checksum: checksum, SizeBytes: int64(len(proposed)), AuditSummary: "atomic compare-and-swap file write"}}); err != nil {
|
||||
t.Fatalf("complete typed config job: %v", err)
|
||||
}
|
||||
updated, err := svc.GetServerConfigForSession(ownerSession, instance.ID)
|
||||
if err != nil || updated.Content != proposed || updated.ConfigVersion != current.ConfigVersion+1 || updated.Checksum != checksum {
|
||||
t.Fatalf("expected durable typed config projection, config=%+v err=%v", updated, err)
|
||||
}
|
||||
stored, err := svc.GetJobForSession(ownerSession, dispatch.Job.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("read typed config job: %v", err)
|
||||
}
|
||||
if stored.ExecutionResult.Content != "" || stored.ExecutionResult.Checksum != checksum || stored.ExecutionResult.Version != current.ConfigVersion+1 {
|
||||
t.Fatalf("unexpected safe/private job result projection: %+v", stored.ExecutionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreServiceUpdatesUsersProfileAndTheme(t *testing.T) {
|
||||
svc := newTestCoreService()
|
||||
if _, err := svc.CreateUser(domain.User{
|
||||
@@ -1145,6 +1192,7 @@ func createPluginAndRunEndpoint(t *testing.T, svc *CoreService) (domain.GamePlug
|
||||
Files: true,
|
||||
Jobs: true,
|
||||
},
|
||||
RuntimeProfiles: domain.GamePluginRuntimeProfiles{LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{"process.install", "process.start", "process.stop"}}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create plugin fixture: %v", err)
|
||||
@@ -1164,6 +1212,22 @@ func createPluginAndRunEndpoint(t *testing.T, svc *CoreService) (domain.GamePlug
|
||||
return plugin, endpoint
|
||||
}
|
||||
|
||||
func createCompleteRuntimeBinding(t *testing.T, svc *CoreService, instance domain.ServerInstance, profileKey string) domain.RuntimeBinding {
|
||||
t.Helper()
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
t.Fatalf("get plugin for runtime binding: %v", err)
|
||||
}
|
||||
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: profileKey, Bindings: map[string]string{}}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("build runtime binding: %v", err)
|
||||
}
|
||||
if err := svc.store.RuntimeBindings().Create(binding); err != nil {
|
||||
t.Fatalf("create runtime binding: %v", err)
|
||||
}
|
||||
return binding
|
||||
}
|
||||
|
||||
func validPluginManifestRegistration() domain.GamePluginManifestRegistration {
|
||||
return domain.GamePluginManifestRegistration{
|
||||
ManifestRef: "artifact://manifests/game.example/0.1.0",
|
||||
@@ -1205,7 +1269,8 @@ func validPluginManifestRegistration() domain.GamePluginManifestRegistration {
|
||||
BridgeActions: []string{string(domain.PluginBridgeActionLogsQuery), string(domain.PluginBridgeActionFilesRequest), string(domain.PluginBridgeActionAIInvoke)},
|
||||
},
|
||||
},
|
||||
AI: domain.GamePluginManifestAI{Purposes: []string{"logs.diagnose"}},
|
||||
AI: domain.GamePluginManifestAI{Purposes: []string{"logs.diagnose"}},
|
||||
RuntimeProfiles: domain.GamePluginRuntimeProfiles{LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{"process.install", "process.start", "process.stop"}}}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
func (svc *CoreService) GetServerRuntimeBindingForSession(sessionID, serverInstanceID string) (domain.RuntimeBindingView, error) {
|
||||
instance, err := svc.GetServerInstanceForSession(sessionID, serverInstanceID)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
binding, err := svc.runtimeBindingForServer(instance.ID)
|
||||
if errors.Is(err, repo.ErrNotFound) {
|
||||
return domain.RuntimeBindingView{ServerInstanceID: instance.ID, PluginID: instance.PluginID, Status: domain.RuntimeBindingStatusIncomplete, Reason: "runtime profile is not configured"}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
return runtimeBindingView(plugin, binding)
|
||||
}
|
||||
|
||||
func (svc *CoreService) UpdateServerRuntimeBindingForSession(sessionID, serverInstanceID string, update domain.RuntimeBindingUpdate) (domain.RuntimeBindingView, error) {
|
||||
_, instance, err := svc.requireServerOwner(sessionID, serverInstanceID)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
existing, existingErr := svc.runtimeBindingForServer(instance.ID)
|
||||
if existingErr != nil && !errors.Is(existingErr, repo.ErrNotFound) {
|
||||
return domain.RuntimeBindingView{}, existingErr
|
||||
}
|
||||
if (instance.State == domain.ServerInstanceStateInstalling || instance.State == domain.ServerInstanceStateRunning) && existingErr == nil || instance.State == domain.ServerInstanceStateDeleted {
|
||||
return domain.RuntimeBindingView{}, validationError("runtime binding cannot be changed while the server is active")
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
binding, err := svc.buildRuntimeBinding(instance, plugin, update, false)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
if existingErr == nil {
|
||||
binding.CreatedAt = existing.CreatedAt
|
||||
if existing.ProfileKey == binding.ProfileKey {
|
||||
merged := domain.CopyStringMap(existing.Bindings)
|
||||
for key, value := range update.Bindings {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
delete(merged, key)
|
||||
} else {
|
||||
merged[key] = value
|
||||
}
|
||||
}
|
||||
binding, err = svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: update.ProfileKey, Bindings: merged}, false)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
binding.CreatedAt = existing.CreatedAt
|
||||
}
|
||||
if err := svc.store.RuntimeBindings().Update(binding); err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
} else if errors.Is(existingErr, repo.ErrNotFound) {
|
||||
if err := svc.store.RuntimeBindings().Create(binding); err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
}
|
||||
return runtimeBindingView(plugin, binding)
|
||||
}
|
||||
|
||||
func (svc *CoreService) buildRuntimeBinding(instance domain.ServerInstance, plugin domain.GamePlugin, update domain.RuntimeBindingUpdate, requireComplete bool) (domain.RuntimeBinding, error) {
|
||||
update = domain.CopyRuntimeBindingUpdate(update)
|
||||
profile, ok := runtimeLifecycleProfile(plugin.RuntimeProfiles, update.ProfileKey)
|
||||
if !ok {
|
||||
return domain.RuntimeBinding{}, validationError("profileKey must reference a declared lifecycle profile")
|
||||
}
|
||||
stamp := svc.now()
|
||||
binding := domain.RuntimeBinding{ID: "runtime-binding-" + instance.ID, ServerInstanceID: instance.ID, PluginID: plugin.ID, PluginVersion: plugin.Version, ProfileKey: profile.Key, Mode: profile.Mode, Bindings: update.Bindings, CreatedAt: stamp, UpdatedAt: stamp}
|
||||
binding, err := normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return domain.RuntimeBinding{}, err
|
||||
}
|
||||
if requireComplete && binding.Status != domain.RuntimeBindingStatusComplete {
|
||||
return domain.RuntimeBinding{}, validationError("missing runtime bindings: " + strings.Join(binding.MissingKeys, ", "))
|
||||
}
|
||||
return binding, nil
|
||||
}
|
||||
|
||||
func runtimeLifecycleProfile(profiles domain.GamePluginRuntimeProfiles, key string) (domain.RuntimeLifecycleProfile, bool) {
|
||||
for _, profile := range profiles.LifecycleProfiles {
|
||||
if profile.Key == key {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
return domain.RuntimeLifecycleProfile{}, false
|
||||
}
|
||||
|
||||
func runtimeBindingKeys(profiles domain.GamePluginRuntimeProfiles, profile domain.RuntimeLifecycleProfile) ([]string, map[string]struct{}) {
|
||||
requiredSet := map[string]struct{}{}
|
||||
allowed := map[string]struct{}{}
|
||||
add := func(key string, required bool) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
allowed[key] = struct{}{}
|
||||
if required {
|
||||
requiredSet[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, probe := range profiles.Discovery {
|
||||
add(probe.TargetKey, probe.Required)
|
||||
}
|
||||
for _, probe := range profiles.DependencyProbes {
|
||||
add(probe.TargetKey, probe.Required)
|
||||
}
|
||||
for _, source := range profiles.LogSources {
|
||||
add(source.TargetKey, source.TargetKey != "")
|
||||
}
|
||||
for _, plan := range profiles.InstallPlans {
|
||||
for _, step := range plan.Steps {
|
||||
add(step.TargetKey, false)
|
||||
}
|
||||
}
|
||||
for _, transport := range profiles.TransportProfiles {
|
||||
if containsString(profile.TransportKeys, transport.Key) {
|
||||
if transport.TargetKey != "" {
|
||||
add(transport.TargetKey, true)
|
||||
} else {
|
||||
add(transport.Key, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
if profile.ClientManagerRef != "" {
|
||||
add(profile.ClientManagerRef, true)
|
||||
}
|
||||
required := make([]string, 0, len(requiredSet))
|
||||
for key := range requiredSet {
|
||||
required = append(required, key)
|
||||
}
|
||||
sort.Strings(required)
|
||||
return required, allowed
|
||||
}
|
||||
|
||||
func normalizeRuntimeBinding(plugin domain.GamePlugin, binding domain.RuntimeBinding) (domain.RuntimeBinding, error) {
|
||||
profile, _ := runtimeLifecycleProfile(plugin.RuntimeProfiles, binding.ProfileKey)
|
||||
if profile.Key == "" {
|
||||
return domain.RuntimeBinding{}, validationError("runtime profile is no longer declared")
|
||||
}
|
||||
required, allowed := runtimeBindingKeys(plugin.RuntimeProfiles, profile)
|
||||
for key := range binding.Bindings {
|
||||
if _, ok := allowed[key]; !ok {
|
||||
return domain.RuntimeBinding{}, validationError(fmt.Sprintf("bindings.%s is not declared by runtime profile", key))
|
||||
}
|
||||
}
|
||||
missing := make([]string, 0)
|
||||
for _, key := range required {
|
||||
if strings.TrimSpace(binding.Bindings[key]) == "" {
|
||||
missing = append(missing, key)
|
||||
}
|
||||
}
|
||||
binding.Mode = profile.Mode
|
||||
binding.MissingKeys = missing
|
||||
binding.Status = domain.RuntimeBindingStatusComplete
|
||||
if len(missing) > 0 {
|
||||
binding.Status = domain.RuntimeBindingStatusIncomplete
|
||||
}
|
||||
if err := validator.ValidateRuntimeBinding(binding); err != nil {
|
||||
return domain.RuntimeBinding{}, err
|
||||
}
|
||||
return binding, nil
|
||||
}
|
||||
|
||||
func runtimeBindingView(plugin domain.GamePlugin, binding domain.RuntimeBinding) (domain.RuntimeBindingView, error) {
|
||||
binding, err := normalizeRuntimeBinding(plugin, binding)
|
||||
if err != nil {
|
||||
return domain.RuntimeBindingView{}, err
|
||||
}
|
||||
profile, _ := runtimeLifecycleProfile(plugin.RuntimeProfiles, binding.ProfileKey)
|
||||
required, allowed := runtimeBindingKeys(plugin.RuntimeProfiles, profile)
|
||||
requiredSet := map[string]struct{}{}
|
||||
for _, key := range required {
|
||||
requiredSet[key] = struct{}{}
|
||||
}
|
||||
keys := make([]string, 0, len(allowed))
|
||||
for key := range allowed {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
items := make([]domain.RuntimeBindingKeyView, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
value := strings.TrimSpace(binding.Bindings[key])
|
||||
_, isRequired := requiredSet[key]
|
||||
items = append(items, domain.RuntimeBindingKeyView{Key: key, Required: isRequired, Configured: value != "", Secret: strings.HasPrefix(value, "secret://")})
|
||||
}
|
||||
reason := ""
|
||||
if binding.Status != domain.RuntimeBindingStatusComplete {
|
||||
reason = "required logical bindings are missing"
|
||||
}
|
||||
return domain.RuntimeBindingView{ServerInstanceID: binding.ServerInstanceID, PluginID: binding.PluginID, ProfileKey: binding.ProfileKey, Mode: binding.Mode, Configured: true, Keys: items, MissingKeys: domain.CopyStringSlice(binding.MissingKeys), Status: binding.Status, Reason: reason, CreatedAt: binding.CreatedAt, UpdatedAt: binding.UpdatedAt}, nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) runtimeBindingForServer(serverInstanceID string) (domain.RuntimeBinding, error) {
|
||||
bindings, err := svc.store.RuntimeBindings().List(domain.RuntimeBindingFilter{ServerInstanceID: serverInstanceID})
|
||||
if err != nil {
|
||||
return domain.RuntimeBinding{}, err
|
||||
}
|
||||
if len(bindings) == 0 {
|
||||
return domain.RuntimeBinding{}, repo.ErrNotFound
|
||||
}
|
||||
if len(bindings) > 1 {
|
||||
return domain.RuntimeBinding{}, validationError("server has multiple runtime bindings")
|
||||
}
|
||||
return bindings[0], nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
func TestRegisteredRuntimeProfilesSurviveFileStoreReload(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "metadata.json")
|
||||
store, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create file store: %v", err)
|
||||
}
|
||||
svc := newCoreService(store, func() time.Time { return fixedTime })
|
||||
registration := validPluginManifestRegistration()
|
||||
registration.Manifest.RuntimeProfiles = requiredRuntimeProfilesFixture()
|
||||
registration.Manifest.Capabilities = append(registration.Manifest.Capabilities, "remote.run.rcon.command")
|
||||
registered, err := svc.RegisterGamePluginManifest(registration)
|
||||
if err != nil {
|
||||
t.Fatalf("register manifest: %v", err)
|
||||
}
|
||||
if len(registered.RuntimeProfiles.LifecycleProfiles) != 1 {
|
||||
t.Fatalf("expected registered profiles, got %+v", registered.RuntimeProfiles)
|
||||
}
|
||||
|
||||
reloaded, err := repo.NewFileStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reload file store: %v", err)
|
||||
}
|
||||
plugin, err := reloaded.GamePlugins().Get(registration.Manifest.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get reloaded plugin: %v", err)
|
||||
}
|
||||
if plugin.RuntimeProfiles.LifecycleProfiles[0].TransportKeys[0] != "rcon" || plugin.RuntimeProfiles.TransportProfiles[0].TargetKey != "rcon.password" {
|
||||
t.Fatalf("runtime profiles were not preserved: %+v", plugin.RuntimeProfiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeBindingValidationAndLifecycleGating(t *testing.T) {
|
||||
svc := newTestCoreService()
|
||||
plugin, endpoint := createPluginAndRunEndpoint(t, svc)
|
||||
plugin.RuntimeProfiles = requiredRuntimeProfilesFixture()
|
||||
if err := svc.store.GamePlugins().Update(plugin); err != nil {
|
||||
t.Fatalf("update plugin profiles: %v", err)
|
||||
}
|
||||
ownerSession := createServiceUserAndLogin(t, svc, domain.User{ID: "runtime-owner", DisplayName: "Runtime Owner", Email: "runtime-owner@example.test", Roles: []string{"server-owner"}, PasswordHash: "secret-password"})
|
||||
otherSession := createServiceUserAndLogin(t, svc, domain.User{ID: "runtime-other", DisplayName: "Runtime Other", Email: "runtime-other@example.test", Roles: []string{"server-admin"}, PasswordHash: "secret-password"})
|
||||
instance, err := svc.CreateServerInstanceForSession(ownerSession, domain.ServerInstance{ID: "runtime-server", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Runtime Server", State: domain.ServerInstanceStateReady})
|
||||
if err != nil {
|
||||
t.Fatalf("create server: %v", err)
|
||||
}
|
||||
forged, err := svc.CreateServerInstanceForSession(ownerSession, domain.ServerInstance{ID: "runtime-forged-complete", PluginID: plugin.ID, RunEndpointID: endpoint.ID, Name: "Forged Complete", State: domain.ServerInstanceStateReady})
|
||||
if err != nil {
|
||||
t.Fatalf("create forged-status server: %v", err)
|
||||
}
|
||||
if err := svc.store.RuntimeBindings().Create(domain.RuntimeBinding{ID: "runtime-binding-" + forged.ID, ServerInstanceID: forged.ID, PluginID: plugin.ID, PluginVersion: plugin.Version, ProfileKey: "local", Mode: "local-process", Bindings: map[string]string{}, Status: domain.RuntimeBindingStatusComplete, CreatedAt: fixedTime, UpdatedAt: fixedTime}); err != nil {
|
||||
t.Fatalf("store forged complete binding: %v", err)
|
||||
}
|
||||
if _, err := svc.StartServerInstanceForSession(ownerSession, domain.ServerLifecycleCommand{ServerInstanceID: forged.ID, ExpectedConfigVersion: forged.ConfigVersion, IdempotencyKey: "start-forged-complete"}); err == nil || !strings.Contains(err.Error(), "rcon.password") {
|
||||
t.Fatalf("expected derived missing keys to override stored complete status, got %v", err)
|
||||
}
|
||||
|
||||
view, err := svc.GetServerRuntimeBindingForSession(ownerSession, instance.ID)
|
||||
if err != nil || view.Configured || view.Reason != "runtime profile is not configured" {
|
||||
t.Fatalf("unexpected unconfigured view: view=%+v err=%v", view, err)
|
||||
}
|
||||
if _, err := svc.StartServerInstanceForSession(ownerSession, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "start-without-binding"}); err == nil || !strings.Contains(err.Error(), "runtime profile is not configured") {
|
||||
t.Fatalf("expected missing binding to block start, got %v", err)
|
||||
}
|
||||
if _, err := svc.UpdateServerRuntimeBindingForSession(otherSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local"}); err != ErrForbidden {
|
||||
t.Fatalf("expected non-owner update forbidden, got %v", err)
|
||||
}
|
||||
if _, err := svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "unknown"}); err == nil {
|
||||
t.Fatal("expected undeclared profile rejection")
|
||||
}
|
||||
if _, err := svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"server-root": "/srv/game"}}); err == nil {
|
||||
t.Fatal("expected raw host path rejection")
|
||||
}
|
||||
|
||||
view, err = svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"server-root": "runtime.server-root"}})
|
||||
if err != nil || view.Status != domain.RuntimeBindingStatusIncomplete || len(view.MissingKeys) != 1 || view.MissingKeys[0] != "rcon.password" {
|
||||
t.Fatalf("unexpected incomplete binding: view=%+v err=%v", view, err)
|
||||
}
|
||||
if _, err := svc.StartServerInstanceForSession(ownerSession, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "start-incomplete-binding"}); err == nil || !strings.Contains(err.Error(), "rcon.password") {
|
||||
t.Fatalf("expected missing logical key to block start, got %v", err)
|
||||
}
|
||||
|
||||
view, err = svc.UpdateServerRuntimeBindingForSession(ownerSession, instance.ID, domain.RuntimeBindingUpdate{ProfileKey: "local", Bindings: map[string]string{"rcon.password": "secret://runtime-server/rcon"}})
|
||||
if err != nil || view.Status != domain.RuntimeBindingStatusComplete || len(view.MissingKeys) != 0 {
|
||||
t.Fatalf("unexpected complete binding: view=%+v err=%v", view, err)
|
||||
}
|
||||
result, err := svc.StartServerInstanceForSession(ownerSession, domain.ServerLifecycleCommand{ServerInstanceID: instance.ID, ExpectedConfigVersion: instance.ConfigVersion, IdempotencyKey: "start-complete-binding"})
|
||||
if err != nil || result.Job.TargetKey != "local" {
|
||||
t.Fatalf("expected complete binding to permit start, result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func requiredRuntimeProfilesFixture() domain.GamePluginRuntimeProfiles {
|
||||
return domain.GamePluginRuntimeProfiles{
|
||||
Discovery: []domain.RuntimeDiscoveryProbe{{Key: "server-root-check", Kind: "file.exists", TargetKey: "server-root", Required: true}},
|
||||
LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{"process.install", "process.start", "process.stop"}, TransportKeys: []string{"rcon"}}},
|
||||
TransportProfiles: []domain.RuntimeTransportProfile{{Key: "rcon", Kind: "rcon", TargetKey: "rcon.password", Capabilities: []string{"remote.run.rcon.command"}}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const developmentSecretEnvelopeKey = "browser.local/platform/development-secret-envelope/v1"
|
||||
|
||||
type SecretEnvelope interface {
|
||||
Seal(string) (string, error)
|
||||
Open(string) (string, error)
|
||||
}
|
||||
|
||||
type aesGCMSecretEnvelope struct {
|
||||
key [32]byte
|
||||
}
|
||||
|
||||
func newSecretEnvelope(secret string) *aesGCMSecretEnvelope {
|
||||
return &aesGCMSecretEnvelope{key: sha256.Sum256([]byte(secret))}
|
||||
}
|
||||
|
||||
func (envelope *aesGCMSecretEnvelope) Seal(plain string) (string, error) {
|
||||
block, err := aes.NewCipher(envelope.key[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ciphertext := gcm.Seal(nil, nonce, []byte(plain), nil)
|
||||
return "enc:v1:" + base64.RawURLEncoding.EncodeToString(nonce) + ":" + base64.RawURLEncoding.EncodeToString(ciphertext), nil
|
||||
}
|
||||
|
||||
func (envelope *aesGCMSecretEnvelope) Open(encrypted string) (string, error) {
|
||||
parts := strings.Split(encrypted, ":")
|
||||
if len(parts) != 4 || parts[0] != "enc" || parts[1] != "v1" {
|
||||
return "", validationError("encrypted key format is invalid")
|
||||
}
|
||||
nonce, err := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ciphertext, err := base64.RawURLEncoding.DecodeString(parts[3])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
block, err := aes.NewCipher(envelope.key[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plain, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(plain), nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) ConfigureSecretEnvelopeKey(secret string) error {
|
||||
secret = strings.TrimSpace(secret)
|
||||
if secret == "" {
|
||||
return nil
|
||||
}
|
||||
if len([]rune(secret)) < 32 {
|
||||
return validationError("PLATFORM_SECRET_ENVELOPE_KEY must be at least 32 characters")
|
||||
}
|
||||
svc.secretEnvelope = newSecretEnvelope(secret)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svc *CoreService) encryptRuntimeKey(plain string) (string, error) {
|
||||
return svc.secretEnvelope.Seal(plain)
|
||||
}
|
||||
|
||||
func (svc *CoreService) decryptRuntimeKey(encrypted string) (string, error) {
|
||||
return svc.secretEnvelope.Open(encrypted)
|
||||
}
|
||||
|
||||
func encryptRuntimeKey(plain string) (string, error) {
|
||||
return newSecretEnvelope(developmentSecretEnvelopeKey).Seal(plain)
|
||||
}
|
||||
|
||||
func decryptRuntimeKey(encrypted string) (string, error) {
|
||||
return newSecretEnvelope(developmentSecretEnvelopeKey).Open(encrypted)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"browser.local/platform/repo"
|
||||
)
|
||||
|
||||
func TestConfiguredSecretEnvelopeIsOpaqueAndRestartStable(t *testing.T) {
|
||||
const key = "test-secret-envelope-key-at-least-32-characters"
|
||||
const plain = "raw-component-secret"
|
||||
svc := NewCoreService(repo.NewMemoryStore())
|
||||
if err := svc.ConfigureSecretEnvelopeKey(key); err != nil {
|
||||
t.Fatalf("configure envelope: %v", err)
|
||||
}
|
||||
encrypted, err := svc.encryptRuntimeKey(plain)
|
||||
if err != nil {
|
||||
t.Fatalf("seal secret: %v", err)
|
||||
}
|
||||
if strings.Contains(encrypted, plain) || !strings.HasPrefix(encrypted, "enc:v1:") {
|
||||
t.Fatalf("unexpected envelope ciphertext %q", encrypted)
|
||||
}
|
||||
restarted := NewCoreService(repo.NewMemoryStore())
|
||||
if err := restarted.ConfigureSecretEnvelopeKey(key); err != nil {
|
||||
t.Fatalf("configure restarted envelope: %v", err)
|
||||
}
|
||||
decrypted, err := restarted.decryptRuntimeKey(encrypted)
|
||||
if err != nil || decrypted != plain {
|
||||
t.Fatalf("open restarted envelope: plain=%q err=%v", decrypted, err)
|
||||
}
|
||||
if _, err := NewCoreService(repo.NewMemoryStore()).decryptRuntimeKey(encrypted); err == nil {
|
||||
t.Fatal("development fallback must not decrypt a custom-key envelope")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretEnvelopeRejectsShortConfiguredKey(t *testing.T) {
|
||||
svc := NewCoreService(repo.NewMemoryStore())
|
||||
if err := svc.ConfigureSecretEnvelopeKey("too-short"); err == nil {
|
||||
t.Fatal("expected short secret envelope key rejection")
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ func (svc *CoreService) requireServerOwner(sessionID string, serverInstanceID st
|
||||
if err != nil {
|
||||
return domain.User{}, domain.ServerInstance{}, err
|
||||
}
|
||||
if instance.OwnerUserID != user.ID {
|
||||
if !isPlatformAdmin(user) && instance.OwnerUserID != user.ID {
|
||||
return domain.User{}, domain.ServerInstance{}, ErrForbidden
|
||||
}
|
||||
return user, instance, nil
|
||||
|
||||
@@ -36,9 +36,13 @@ func (svc *CoreService) CreateServerInstanceWorkflow(create domain.ServerLifecyc
|
||||
OwnerUserID: create.OwnerUserID,
|
||||
State: domain.ServerInstanceStateInstalling,
|
||||
ConfigVersion: 1,
|
||||
ConfigKey: "server.properties",
|
||||
CreatedAt: stamp,
|
||||
UpdatedAt: stamp,
|
||||
}
|
||||
instance.ConfigContent = buildLogicalServerConfig(instance)
|
||||
instance.ConfigChecksum = validator.BytesChecksum([]byte(instance.ConfigContent))
|
||||
instance.ConfigUpdatedAt = stamp
|
||||
if err := validator.ValidateServerInstance(instance); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
@@ -51,9 +55,16 @@ func (svc *CoreService) CreateServerInstanceWorkflow(create domain.ServerLifecyc
|
||||
if err := svc.validateLifecycleIdempotency(instance.RunEndpointID, create.IdempotencyKey, instance.ID, domain.LifecycleCapabilityForAction(domain.ServerLifecycleActionCreate)); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
binding, err := svc.buildRuntimeBinding(instance, plugin, domain.RuntimeBindingUpdate{ProfileKey: create.ProfileKey, Bindings: create.Bindings}, true)
|
||||
if err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
if err := svc.store.ServerInstances().Create(instance); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
if err := svc.store.RuntimeBindings().Create(binding); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
|
||||
job, err := svc.dispatchLifecycleJob(instance, domain.ServerLifecycleActionCreate, create.IdempotencyKey)
|
||||
if err != nil {
|
||||
@@ -108,6 +119,18 @@ func (svc *CoreService) StopServerInstanceForSession(sessionID string, command d
|
||||
return svc.StopServerInstance(command)
|
||||
}
|
||||
|
||||
func (svc *CoreService) QueryServerInstanceProcessForSession(sessionID string, command domain.ServerLifecycleCommand) (domain.ServerLifecycleResult, error) {
|
||||
if err := svc.authorizeServerLifecycle(sessionID, command.ServerInstanceID); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
return svc.dispatchExistingServerLifecycle(command, domain.ServerLifecycleActionStatus, []domain.ServerInstanceState{
|
||||
domain.ServerInstanceStateReady,
|
||||
domain.ServerInstanceStateStopped,
|
||||
domain.ServerInstanceStateRunning,
|
||||
domain.ServerInstanceStateFailed,
|
||||
})
|
||||
}
|
||||
|
||||
func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLifecycleCommand, action domain.ServerLifecycleAction, allowedStates []domain.ServerInstanceState) (domain.ServerLifecycleResult, error) {
|
||||
command = domain.CopyServerLifecycleCommand(command)
|
||||
if err := validator.ValidateServerLifecycleCommand(command); err != nil {
|
||||
@@ -138,6 +161,9 @@ func (svc *CoreService) dispatchExistingServerLifecycle(command domain.ServerLif
|
||||
if err := validator.ValidateServerInstanceDependencies(instance, plugin, endpoint); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
if err := svc.requireCompleteRuntimeBindings(instance.OwnerUserID, instance.ID, "server.lifecycle."+string(action)+".denied"); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
if err := validateRunnableEndpoint(endpoint, domain.LifecycleCapabilityForAction(action)); err != nil {
|
||||
return domain.ServerLifecycleResult{}, err
|
||||
}
|
||||
@@ -168,12 +194,33 @@ func (svc *CoreService) lifecycleDependencies(pluginID string, runEndpointID str
|
||||
|
||||
func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, action domain.ServerLifecycleAction, idempotencyKey string) (domain.Job, error) {
|
||||
capability := domain.LifecycleCapabilityForAction(action)
|
||||
binding, err := svc.runtimeBindingForServer(instance.ID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
plugin, err := svc.store.GamePlugins().Get(instance.PluginID)
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
}
|
||||
actionRef := binding.ProfileKey
|
||||
if profile, ok := runtimeLifecycleProfileForKey(plugin.RuntimeProfiles, binding.ProfileKey); ok {
|
||||
if ref := runtimeProfileActionRef(profile.ActionRefs, action); ref != "" {
|
||||
actionRef = ref
|
||||
}
|
||||
} else if ref := lifecycleActionRef(plugin, action); ref != "" {
|
||||
actionRef = ref
|
||||
}
|
||||
if strings.TrimSpace(actionRef) == "" {
|
||||
return domain.Job{}, validationError(fmt.Sprintf("plugin %s lifecycle action is required", action))
|
||||
}
|
||||
job, err := svc.CreateJob(domain.Job{
|
||||
ID: lifecycleJobID(instance.ID, action, idempotencyKey),
|
||||
ServerInstanceID: instance.ID,
|
||||
RunEndpointID: instance.RunEndpointID,
|
||||
Capability: capability,
|
||||
TargetKey: actionRef,
|
||||
IdempotencyKey: idempotencyKey,
|
||||
ExecutionInput: domain.JobExecutionInput{WorkspaceScope: binding.ProfileKey},
|
||||
})
|
||||
if err != nil {
|
||||
return domain.Job{}, err
|
||||
@@ -184,6 +231,30 @@ func (svc *CoreService) dispatchLifecycleJob(instance domain.ServerInstance, act
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func runtimeProfileActionRef(actions domain.PluginLifecycleActions, action domain.ServerLifecycleAction) string {
|
||||
switch action {
|
||||
case domain.ServerLifecycleActionCreate:
|
||||
return actions.Install
|
||||
case domain.ServerLifecycleActionStart:
|
||||
return actions.Start
|
||||
case domain.ServerLifecycleActionStop:
|
||||
return actions.Stop
|
||||
case domain.ServerLifecycleActionStatus:
|
||||
return actions.Status
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func runtimeLifecycleProfileForKey(profiles domain.GamePluginRuntimeProfiles, key string) (domain.RuntimeLifecycleProfile, bool) {
|
||||
for _, profile := range profiles.LifecycleProfiles {
|
||||
if profile.Key == key {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
return domain.RuntimeLifecycleProfile{}, false
|
||||
}
|
||||
|
||||
func (svc *CoreService) validateLifecycleIdempotency(runEndpointID string, idempotencyKey string, serverInstanceID string, capability string) error {
|
||||
existing, err := svc.store.Jobs().GetByIdempotency(runEndpointID, idempotencyKey)
|
||||
if errors.Is(err, repo.ErrNotFound) {
|
||||
@@ -213,6 +284,8 @@ func lifecycleActionRef(plugin domain.GamePlugin, action domain.ServerLifecycleA
|
||||
return plugin.LifecycleActions.Start
|
||||
case domain.ServerLifecycleActionStop:
|
||||
return plugin.LifecycleActions.Stop
|
||||
case domain.ServerLifecycleActionStatus:
|
||||
return plugin.LifecycleActions.Status
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -1,13 +1,55 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"time"
|
||||
|
||||
"browser.local/platform/domain"
|
||||
"browser.local/platform/validator"
|
||||
)
|
||||
|
||||
func (svc *CoreService) projectRemoteAdapterJobResult(job domain.Job, stamp time.Time) error {
|
||||
if !strings.HasPrefix(job.Capability, "remote.") || job.ServerInstanceID == "" || !isTerminalJobState(job.State) {
|
||||
return nil
|
||||
}
|
||||
result := domain.AuditResultSuccess
|
||||
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
|
||||
result = domain.AuditResultFailed
|
||||
}
|
||||
summary := "remote adapter " + job.Capability + " completed with bounded result reference"
|
||||
if job.State == domain.JobStateFailed {
|
||||
summary = "remote adapter " + job.Capability + " failed or timed out; retry/fencing remained platform-owned"
|
||||
}
|
||||
if job.State == domain.JobStateCancelled {
|
||||
summary = "remote adapter " + job.Capability + " was cancelled before terminal projection"
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "remote-adapter.result", "server-instance", job.ServerInstanceID, result, summary)
|
||||
}
|
||||
|
||||
func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Time) error {
|
||||
if job.Capability == domain.JobCapabilityConfigWrite {
|
||||
if job.State != domain.JobStateSucceeded {
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "config.write.result", "server-instance", job.ServerInstanceID, domain.AuditResultFailed, job.ExecutionResult.AuditSummary)
|
||||
}
|
||||
instance, err := svc.store.ServerInstances().Get(job.ServerInstanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instance.ConfigKey = job.TargetKey
|
||||
instance.ConfigContent = job.ExecutionInput.Content
|
||||
instance.ConfigChecksum = job.ExecutionResult.Checksum
|
||||
instance.ConfigVersion = job.ExecutionResult.Version
|
||||
instance.ConfigUpdatedAt = stamp
|
||||
instance.UpdatedAt = stamp
|
||||
if err := validator.ValidateServerInstance(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "config.write.result", "server-instance", instance.ID, domain.AuditResultSuccess, job.ExecutionResult.AuditSummary)
|
||||
}
|
||||
nextState, ok := lifecycleProjectedState(job.Capability, job.State)
|
||||
if !ok || job.ServerInstanceID == "" {
|
||||
return nil
|
||||
@@ -21,7 +63,14 @@ func (svc *CoreService) projectLifecycleJobResult(job domain.Job, stamp time.Tim
|
||||
if err := validator.ValidateServerInstance(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.store.ServerInstances().Update(instance)
|
||||
if err := svc.store.ServerInstances().Update(instance); err != nil {
|
||||
return err
|
||||
}
|
||||
auditResult := domain.AuditResultSuccess
|
||||
if job.State == domain.JobStateFailed || job.State == domain.JobStateCancelled {
|
||||
auditResult = domain.AuditResultFailed
|
||||
}
|
||||
return svc.recordAuditEvent("run:"+job.RunEndpointID, "lifecycle.result", "server-instance", instance.ID, auditResult, job.Progress.Message)
|
||||
}
|
||||
|
||||
func lifecycleProjectedState(capability string, jobState domain.JobState) (domain.ServerInstanceState, bool) {
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestCoreServiceServerLifecycleWorkflows(t *testing.T) {
|
||||
RunEndpointID: "run-local",
|
||||
Name: "SCUM #1",
|
||||
IdempotencyKey: "idem-create",
|
||||
ProfileKey: "local",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create lifecycle workflow: %v", err)
|
||||
@@ -88,6 +89,7 @@ func TestCoreServiceServerLifecycleRejectsInvalidCommands(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create ready server: %v", err)
|
||||
}
|
||||
createCompleteRuntimeBinding(t, svc, instance, "local")
|
||||
|
||||
_, err = svc.StartServerInstance(domain.ServerLifecycleCommand{
|
||||
ServerInstanceID: instance.ID,
|
||||
@@ -121,6 +123,7 @@ func TestCoreServiceServerLifecycleRejectsInvalidCommands(t *testing.T) {
|
||||
State: domain.ServerInstanceStateRunning,
|
||||
})
|
||||
if err == nil {
|
||||
createCompleteRuntimeBinding(t, svc, running, "local")
|
||||
_, err = svc.StopServerInstance(domain.ServerLifecycleCommand{
|
||||
ServerInstanceID: running.ID,
|
||||
ExpectedConfigVersion: running.ConfigVersion,
|
||||
@@ -141,6 +144,7 @@ func TestCoreServiceLifecycleFailureProjectsFailedState(t *testing.T) {
|
||||
RunEndpointID: "run-local",
|
||||
Name: "SCUM #1",
|
||||
IdempotencyKey: "idem-create",
|
||||
ProfileKey: "local",
|
||||
}); err != nil {
|
||||
t.Fatalf("create lifecycle workflow: %v", err)
|
||||
}
|
||||
@@ -166,6 +170,7 @@ func TestCoreServicePluginLifecycleManagesMultipleInstancesIndependently(t *test
|
||||
RunEndpointID: "run-local",
|
||||
Name: id,
|
||||
IdempotencyKey: "idem-create-" + id,
|
||||
ProfileKey: "local",
|
||||
}); err != nil {
|
||||
t.Fatalf("create %s: %v", id, err)
|
||||
}
|
||||
@@ -249,7 +254,8 @@ func createLifecyclePlugin(t *testing.T, svc *CoreService) domain.GamePlugin {
|
||||
Start: "actions/start.json",
|
||||
Stop: "actions/stop.json",
|
||||
},
|
||||
Permissions: domain.PluginPermissions{Jobs: true, Logs: true},
|
||||
Permissions: domain.PluginPermissions{Jobs: true, Logs: true},
|
||||
RuntimeProfiles: domain.GamePluginRuntimeProfiles{LifecycleProfiles: []domain.RuntimeLifecycleProfile{{Key: "local", Mode: "local-process", Capabilities: []string{domain.LifecycleCapabilityInstall, domain.LifecycleCapabilityStart, domain.LifecycleCapabilityStop}}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create lifecycle plugin: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user