#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # shellcheck source=scripts/local-debug/env.sh source "$ROOT_DIR/scripts/local-debug/env.sh" PLATFORM_URL="$(local_debug_platform_url)" API_URL="$PLATFORM_URL/api/v1" WORK_DIR="$LOCAL_DEBUG_ROOT/smoke" SMOKE_INVOCATION_ID="${SMOKE_INVOCATION_ID:-$(date -u +%Y%m%d%H%M%S)-$$}" SERVER_LOCAL_ID="server-local-debug-$SMOKE_INVOCATION_ID" LOG_SESSION_SERVER_ID="server-log-session-$SMOKE_INVOCATION_ID" SCUM_ALPHA_ID="scum-alpha-$SMOKE_INVOCATION_ID" SCUM_BETA_ID="scum-beta-$SMOKE_INVOCATION_ID" SCUM_DYNAMIC_ID="scum-dynamic-$SMOKE_INVOCATION_ID" GENERATED_RUN_ENDPOINT_ID="server-run-$SERVER_LOCAL_ID" GENERATED_RUN_BIN="$WORK_DIR/generated-$SERVER_LOCAL_ID-run" GENERATED_RUN_LOG="$LOCAL_DEBUG_LOG_DIR/generated-$SERVER_LOCAL_ID-run.log" GENERATED_RUN_PID_FILE="$LOCAL_DEBUG_PID_DIR/generated-$SERVER_LOCAL_ID-run.pid" LOG_SESSION_SCOPE="$RUN_WORKSPACE_ROOT/instances/$LOG_SESSION_SERVER_ID/run-local" LOG_SESSION_MARKER_PREFIX="SMOKE-CURRENT-SESSION-$SMOKE_INVOCATION_ID" LOG_SESSION_SSE_FILE="$WORK_DIR/current-log-session.events.sse" LOG_SESSION_SSE_ERROR_FILE="$WORK_DIR/current-log-session.events.stderr.log" LOG_SESSION_SSE_PID="" LOG_SESSION_LEGACY_STREAM_ID="legacy.$LOG_SESSION_SERVER_ID.stdout" LOG_SESSION_JOB_STREAM_ID="job.smoke-$SMOKE_INVOCATION_ID.stdout" LOG_SESSION_FILE_STREAM_ID="file.$LOG_SESSION_SERVER_ID.backfill" LOG_SESSION_REAL_JOB_STREAM_IDS="" mkdir -p "$WORK_DIR" cat >"$WORK_DIR/run-build-config.env" <"$WORK_DIR/current-log-session-fixture.sh" <&2 printf '%s\n' "\$\$" >smoke-process.pid last_command="" while :; do command="\$(cat smoke-command 2>/dev/null || true)" if [ -n "\$command" ] && [ "\$command" != "\$last_command" ]; then case "\$command" in resume) printf '%s-%s-RUN-RESUME-STDOUT\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation" printf '%s-%s-RUN-RESUME-STDERR\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation" >&2 ;; esac last_command="\$command" printf '%s\n' "\$command" >smoke-last-command fi sleep 0.2 done EOF cat >"$WORK_DIR/current-log-session-install-fixture.sh" <&2 EOF cp "$WORK_DIR/current-log-session-fixture.sh" "$LOG_SESSION_SCOPE/bin/game-server" cp "$WORK_DIR/current-log-session-install-fixture.sh" "$LOG_SESSION_SCOPE/bin/install-server" chmod 700 "$LOG_SESSION_SCOPE/bin/game-server" chmod 700 "$LOG_SESSION_SCOPE/bin/install-server" printf 'A\n' >"$LOG_SESSION_SCOPE/smoke-generation" rm -f "$LOG_SESSION_SCOPE/smoke-command" "$LOG_SESSION_SCOPE/smoke-last-command" "$LOG_SESSION_SCOPE/smoke-process.pid" SELF_STARTED_PIDS=() cleanup_self_started() { local pid for pid in "${SELF_STARTED_PIDS[@]:-}"; do if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then kill "$pid" 2>/dev/null || true fi done } cleanup_log_session_fixture() { if [[ -n "${LOG_SESSION_SSE_PID:-}" ]] && kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then kill "$LOG_SESSION_SSE_PID" 2>/dev/null || true wait "$LOG_SESSION_SSE_PID" 2>/dev/null || true fi local fixture_pid_file="$LOG_SESSION_SCOPE/smoke-process.pid" if [[ -f "$fixture_pid_file" ]]; then local fixture_pid fixture_pid="$(<"$fixture_pid_file")" if [[ "$fixture_pid" =~ ^[0-9]+$ ]] && kill -0 "$fixture_pid" 2>/dev/null; then local fixture_command fixture_command="$(ps -p "$fixture_pid" -o command= 2>/dev/null || true)" if [[ "$fixture_command" == *"$LOG_SESSION_SCOPE/bin/game-server"* ]]; then kill "$fixture_pid" 2>/dev/null || true fi fi fi } cleanup_smoke() { cleanup_log_session_fixture cleanup_self_started } trap cleanup_smoke EXIT launch_bootstrap_run() { local log_mode="${1:-overwrite}" mkdir -p "$LOCAL_DEBUG_LOG_DIR" "$LOCAL_DEBUG_PID_DIR" if [[ "$log_mode" == "append" ]]; then ( cd "$(dirname "$RUN_BOOTSTRAP_BIN")" exec env \ GOCACHE="$GOCACHE" \ RUN_MODE="$RUN_MODE" \ RUN_PLATFORM_URL="$RUN_PLATFORM_URL" \ RUN_ENDPOINT_ID="$RUN_ENDPOINT_ID" \ RUN_DISPLAY_NAME="$RUN_DISPLAY_NAME" \ RUN_VERSION="$RUN_VERSION" \ RUN_REGISTRATION_TOKEN="$RUN_REGISTRATION_TOKEN" \ RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \ RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \ RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT" \ RUN_MAX_JOBS="$RUN_MAX_JOBS" \ RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \ RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \ RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \ "$RUN_BOOTSTRAP_BIN" ) >>"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 & else ( cd "$(dirname "$RUN_BOOTSTRAP_BIN")" exec env \ GOCACHE="$GOCACHE" \ RUN_MODE="$RUN_MODE" \ RUN_PLATFORM_URL="$RUN_PLATFORM_URL" \ RUN_ENDPOINT_ID="$RUN_ENDPOINT_ID" \ RUN_DISPLAY_NAME="$RUN_DISPLAY_NAME" \ RUN_VERSION="$RUN_VERSION" \ RUN_REGISTRATION_TOKEN="$RUN_REGISTRATION_TOKEN" \ RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \ RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \ RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT" \ RUN_MAX_JOBS="$RUN_MAX_JOBS" \ RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \ RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \ RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \ "$RUN_BOOTSTRAP_BIN" ) >"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 & fi local run_pid="$!" printf '%s' "$run_pid" >"$LOCAL_DEBUG_PID_DIR/run.pid" if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then SELF_STARTED_PIDS+=("$run_pid") fi } wait_for_url() { local name="$1" local url="$2" local attempts="${3:-30}" printf 'waiting for %s at %s\n' "$name" "$url" for _ in $(seq 1 "$attempts"); do if curl -fsS "$url" >/dev/null 2>&1; then printf '%s is ready\n' "$name" return 0 fi sleep 1 done printf '%s did not become ready at %s\n' "$name" "$url" >&2 return 1 } assert_no_active_run_session() { local endpoint_file="$WORK_DIR/bootstrap-run-endpoint.response.json" local endpoint_url="${RUN_PLATFORM_URL%/}/api/v1/run/endpoints/$RUN_ENDPOINT_ID" local status status="$(curl -sS -o "$endpoint_file" -w '%{http_code}' "$endpoint_url" || true)" case "$status" in 404) return 0 ;; 200) if node - "$endpoint_file" <<'NODE' const fs = require("fs"); const endpoint = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const heartbeat = endpoint.lastHeartbeatAt || endpoint.LastHeartbeatAt; const ageMilliseconds = heartbeat ? Date.now() - Date.parse(heartbeat) : Number.POSITIVE_INFINITY; process.exit(endpoint.status === "online" && Number.isFinite(ageMilliseconds) && ageMilliseconds >= 0 && ageMilliseconds < 30_000 ? 0 : 1); NODE then printf 'refusing to self-start Run endpoint %s: an active session already exists at %s\n' "$RUN_ENDPOINT_ID" "$RUN_PLATFORM_URL" >&2 printf 'run the smoke against that existing stack, or stop the existing Run before using LOCAL_DEBUG_SELF_START=true\n' >&2 return 1 fi return 0 ;; *) printf 'refusing to self-start Run endpoint %s: could not verify its session state at %s (HTTP %s)\n' "$RUN_ENDPOINT_ID" "$RUN_PLATFORM_URL" "$status" >&2 return 1 ;; esac } assert_no_managed_run_process() { local run_pid_file="$LOCAL_DEBUG_PID_DIR/run.pid" if [[ ! -f "$run_pid_file" ]]; then return 0 fi local pid pid="$(<"$run_pid_file")" if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then printf 'refusing to self-start a second Run while managed Run pid %s is active\n' "$pid" >&2 printf 'run the smoke against that existing stack, or stop it before using LOCAL_DEBUG_SELF_START=true\n' >&2 return 1 fi rm -f "$run_pid_file" } start_self_hosted_stack() { mkdir -p "$LOCAL_DEBUG_LOG_DIR" "$LOCAL_DEBUG_PID_DIR" "$PLATFORM_DATA_DIR" "$PLATFORM_LOG_DIR" "$PLATFORM_ARTIFACT_DIR" "$RUN_WORKSPACE_ROOT" "$RUN_SPOOL_ROOT" "$RUN_BUILD_BUCKET_ROOT" "$GOCACHE" if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" != "true" ]]; then local_debug_prepare_distribution_builder fi printf 'self-starting platform for local debug smoke\n' ( cd "$ROOT_DIR/platform" exec env \ GOCACHE="$GOCACHE" \ PLATFORM_ADDR="$PLATFORM_ADDR" \ PLATFORM_STORAGE_BACKEND="$PLATFORM_STORAGE_BACKEND" \ PLATFORM_DATA_DIR="$PLATFORM_DATA_DIR" \ PLATFORM_METADATA_PATH="$PLATFORM_METADATA_PATH" \ PLATFORM_LOG_BODY_BACKEND="$PLATFORM_LOG_BODY_BACKEND" \ PLATFORM_LOG_DIR="$PLATFORM_LOG_DIR" \ PLATFORM_ARTIFACT_DIR="$PLATFORM_ARTIFACT_DIR" \ PLATFORM_BOOTSTRAP_ADMIN_EMAIL="$PLATFORM_BOOTSTRAP_ADMIN_EMAIL" \ PLATFORM_BOOTSTRAP_ADMIN_PASSWORD="$PLATFORM_BOOTSTRAP_ADMIN_PASSWORD" \ PLATFORM_SECRET_ENVELOPE_KEY="$PLATFORM_SECRET_ENVELOPE_KEY" \ PLATFORM_RUN_RELEASE_URL="$PLATFORM_RUN_RELEASE_URL" \ PLATFORM_BUILDER_DOCKER_BINARY="$PLATFORM_BUILDER_DOCKER_BINARY" \ PLATFORM_BUILDER_IMAGE="$PLATFORM_BUILDER_IMAGE" \ PLATFORM_BUILDER_SOURCE_DIR="$PLATFORM_BUILDER_SOURCE_DIR" \ PLATFORM_BUILDER_SOURCE_REPOSITORY="$PLATFORM_BUILDER_SOURCE_REPOSITORY" \ PLATFORM_BUILDER_SOURCE_REVISION="$PLATFORM_BUILDER_SOURCE_REVISION" \ PLATFORM_BUILDER_WORKSPACE_DIR="$PLATFORM_BUILDER_WORKSPACE_DIR" \ PLATFORM_BUILDER_CACHE_DIR="$PLATFORM_BUILDER_CACHE_DIR" \ PLATFORM_BUILDER_TIMEOUT_SECONDS="$PLATFORM_BUILDER_TIMEOUT_SECONDS" \ go run ./cmd/platform ) >"$LOCAL_DEBUG_LOG_DIR/platform.log" 2>&1 & SELF_STARTED_PIDS+=("$!") printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform.pid" wait_for_url platform "$PLATFORM_URL/healthz" 45 printf 'self-starting platform_web for local debug smoke\n' ( cd "$ROOT_DIR" exec env \ PLATFORM_API_PROXY="$PLATFORM_API_PROXY" \ VITE_PLATFORM_API_BASE_URL="$VITE_PLATFORM_API_BASE_URL" \ VITE_ENABLE_LOCAL_AUTH_FALLBACK="$VITE_ENABLE_LOCAL_AUTH_FALLBACK" \ npm --prefix platform_web run dev -- --port "$LOCAL_DEBUG_WEB_PORT" ) >"$LOCAL_DEBUG_LOG_DIR/platform_web.log" 2>&1 & SELF_STARTED_PIDS+=("$!") printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/platform_web.pid" wait_for_url platform_web "$(local_debug_web_url)" 45 printf 'self-starting run worker for local debug smoke\n' local_debug_build_bootstrap_run launch_bootstrap_run overwrite } if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then start_self_hosted_stack fi json_post() { local url="$1" local body_file="$2" local output_file="$3" shift 3 local status status="$(curl -sS -H 'Content-Type: application/json' "$@" --data-binary "@$body_file" "$url" -o "$output_file" -w '%{http_code}')" case "$status" in 2*) return 0 ;; *) if [[ ! -s "$output_file" ]] || ! response_code_is_duplicate "$output_file"; then printf 'POST %s failed with HTTP %s\n' "$url" "$status" >&2 fi return 1 ;; esac } json_get() { local url="$1" local output_file="$2" shift 2 curl -fsS "$@" "$url" >"$output_file" } response_code_is_duplicate() { local file="$1" node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.exit(data.code === "duplicate" || data.code === "duplicate_resource" ? 0 : 1);' "$file" 2>/dev/null } register_plugin_manifest() { local label="$1" local plugin_id="$2" local request_file="$3" local response_file="$4" if json_post "$API_URL/game-plugins/register-manifest" "$request_file" "$response_file" "${AUTH_HEADER[@]}"; then return 0 fi if [[ -s "$response_file" ]] && response_code_is_duplicate "$response_file"; then json_get "$API_URL/game-plugins/$plugin_id" "$response_file" "${AUTH_HEADER[@]}" return 0 fi printf '%s manifest registration failed; platform response:\n' "$label" >&2 sed -n '1,160p' "$response_file" >&2 exit 1 } create_server_workflow() { local label="$1" local server_id="$2" local request_file="$3" local response_file="$4" if json_post "$API_URL/server-instances/workflows/create" "$request_file" "$response_file" "${AUTH_HEADER[@]}"; then return 0 fi if [[ -s "$response_file" ]] && response_code_is_duplicate "$response_file"; then json_get "$API_URL/server-instances/$server_id" "$response_file" "${AUTH_HEADER[@]}" return 0 fi printf '%s server workflow creation failed; platform response:\n' "$label" >&2 sed -n '1,160p' "$response_file" >&2 exit 1 } create_server_instance() { local label="$1" local server_id="$2" local request_file="$3" local response_file="$4" if json_post "$API_URL/server-instances" "$request_file" "$response_file" "${AUTH_HEADER[@]}"; then return 0 fi if [[ -s "$response_file" ]] && response_code_is_duplicate "$response_file"; then json_get "$API_URL/server-instances/$server_id" "$response_file" "${AUTH_HEADER[@]}" return 0 fi printf '%s server creation failed; platform response:\n' "$label" >&2 sed -n '1,160p' "$response_file" >&2 exit 1 } require_file_contains() { local file="$1" local pattern="$2" if ! grep -Eq "$pattern" "$file"; then printf 'expected %s to contain pattern: %s\n' "$file" "$pattern" >&2 sed -n '1,120p' "$file" >&2 exit 1 fi } reject_forbidden_fragments() { local file="$1" local pattern pattern="$(local_debug_forbidden_pattern)" if grep -Eq "$pattern" "$file"; then printf 'forbidden local-debug fragment found in %s\n' "$file" >&2 grep -En "$pattern" "$file" >&2 exit 1 fi } session_token_from_login() { node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!data.sessionId) process.exit(2); process.stdout.write(data.sessionId);' "$1" } json_id() { node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const id=data.instance?.id || data.serverInstance?.id || data.id || ((data.items||[])[0]||{}).id; if (!id) process.exit(2); process.stdout.write(id);' "$1" } require_scum_manifest_contract() { local file="$1" local expected_version="$2" node - "$file" "$expected_version" <<'NODE' const fs = require("fs"); const file = process.argv[2]; const expectedVersion = process.argv[3]; const plugin = JSON.parse(fs.readFileSync(file, "utf8")); const missing = []; const declaredPermissions = Array.isArray(plugin.declaredPermissions) ? plugin.declaredPermissions : []; const bridgeActions = Array.isArray(plugin.bridgeActions) ? plugin.bridgeActions : []; const lifecycleProfiles = plugin.runtimeProfiles?.lifecycleProfiles ?? []; const logSources = plugin.runtimeProfiles?.logSources ?? []; const pageKeys = (plugin.pages ?? []).map((page) => page.key); if (plugin.version !== expectedVersion) { missing.push(`version ${expectedVersion}`); } if (!declaredPermissions.includes("server.run.distribution")) { missing.push("server.run.distribution permission"); } if (!bridgeActions.includes("run.distribution.request")) { missing.push("run.distribution.request bridge action"); } if (!lifecycleProfiles.some((profile) => profile.key === "run-local")) { missing.push("run-local runtime profile"); } if (!logSources.some((source) => source.kind === "process.stdout" && source.streamKey === "scum.console.stdout")) { missing.push("SCUM process stdout log source"); } if (!logSources.some((source) => source.kind === "process.stderr" && source.streamKey === "scum.console.stderr")) { missing.push("SCUM process stderr log source"); } if (!plugin.gameClientBridge?.commands?.length) { missing.push("game client bridge declarations"); } for (const pageKey of ["players", "squads", "live-map", "gifts"]) { if (!pageKeys.includes(pageKey)) { missing.push(`SCUM page ${pageKey}`); } } if (missing.length > 0) { console.error(`SCUM plugin registration is stale or incomplete: missing ${missing.join(", ")}`); console.error("Run scripts/dev-reset.sh, restart the local debug stack, then rerun scripts/dev-smoke.sh."); process.exit(1); } NODE } wait_for_distribution_build() { local distribution_file="$1" local job_file="$2" local artifact_file="$3" local job_id local artifact_id job_id="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!data.buildJobId) process.exit(2); process.stdout.write(data.buildJobId);' "$distribution_file")" artifact_id="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!data.artifactId) process.exit(2); process.stdout.write(data.artifactId);' "$distribution_file")" rm -f "$job_file" "$artifact_file" for _ in $(seq 1 60); do json_get "$API_URL/jobs/$job_id" "$job_file" "${AUTH_HEADER[@]}" || true if [[ ! -s "$job_file" ]]; then sleep 1 continue fi if node - "$job_file" <<'NODE'; then const fs = require("fs"); const job = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); process.exit(job.state === "succeeded" && job.resultRef ? 0 : 1); NODE break fi if node - "$job_file" <<'NODE'; then const fs = require("fs"); const job = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); process.exit(job.state === "failed" || job.state === "cancelled" ? 0 : 1); NODE printf 'distribution build job reached terminal failure\n' >&2 sed -n '1,120p' "$job_file" >&2 exit 1 fi sleep 1 done json_get "$API_URL/jobs/$job_id" "$job_file" "${AUTH_HEADER[@]}" json_get "$API_URL/artifacts/$artifact_id" "$artifact_file" "${AUTH_HEADER[@]}" node - "$job_file" "$artifact_file" "$artifact_id" <<'NODE' const fs = require("fs"); const job = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const artifact = JSON.parse(fs.readFileSync(process.argv[3], "utf8")); const artifactId = process.argv[4]; if (job.state !== "succeeded" || job.resultRef !== `artifact://${artifactId}`) { console.error("expected distribution build job to succeed with artifact result"); console.error(JSON.stringify(job, null, 2)); process.exit(1); } if (artifact.id !== artifactId || artifact.state !== "available" || !/^sha256:/.test(artifact.checksum || "")) { console.error("expected available distribution artifact with sha256 checksum"); console.error(JSON.stringify(artifact, null, 2)); process.exit(1); } NODE } read_latest_run_download_content() { local server_id="$1" local download_file="$2" local payload_file="$3" local chunk_dir="$4" local request_file="$download_file.request.json" mkdir -p "$chunk_dir" printf '{}\n' >"$request_file" json_post "$API_URL/server-instances/$server_id/run/download" "$request_file" "$download_file" "${AUTH_HEADER[@]}" reject_forbidden_fragments "$download_file" local artifact_id local download_url local total_size local checksum local chunk_size artifact_id="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!data.artifactId) process.exit(2); process.stdout.write(data.artifactId);' "$download_file")" download_url="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!data.downloadUrl) process.exit(2); process.stdout.write(data.downloadUrl);' "$download_file")" total_size="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!Number.isSafeInteger(data.sizeBytes) || data.sizeBytes <= 0) process.exit(2); process.stdout.write(String(data.sizeBytes));' "$download_file")" checksum="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!/^sha256:/.test(data.checksum || "")) process.exit(2); process.stdout.write(data.checksum);' "$download_file")" chunk_size="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const size=Number(data.chunkSizeBytes || 1048576); if (!Number.isSafeInteger(size) || size <= 0) process.exit(2); process.stdout.write(String(size));' "$download_file")" : >"$payload_file" local offset=0 local index=0 while (( offset < total_size )); do local limit="$chunk_size" local remaining=$((total_size - offset)) if (( remaining < limit )); then limit="$remaining" fi local headers_file="$chunk_dir/chunk-$index.headers" local chunk_file="$chunk_dir/chunk-$index.bin" curl -fsS -D "$headers_file" -H "Authorization: Bearer $SESSION_ID" "$PLATFORM_URL$download_url?offset=$offset&limit=$limit" -o "$chunk_file" reject_forbidden_fragments "$headers_file" cat "$chunk_file" >>"$payload_file" node - "$download_file" "$headers_file" "$chunk_file" "$offset" "$limit" <<'NODE' const crypto = require("crypto"); const fs = require("fs"); const reference = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const headerLines = fs.readFileSync(process.argv[3], "utf8").split(/\r?\n/); const body = fs.readFileSync(process.argv[4]); const offset = Number(process.argv[5]); const limit = Number(process.argv[6]); const headers = new Map(); for (const line of headerLines) { const index = line.indexOf(":"); if (index > 0) { headers.set(line.slice(0, index).trim().toLowerCase(), line.slice(index + 1).trim()); } } const bodyChecksum = `sha256:${crypto.createHash("sha256").update(body).digest("hex")}`; const fail = (message) => { console.error(message); console.error({ reference, offset, limit, headers: Object.fromEntries(headers), bodyBytes: body.length }); process.exit(1); }; if (headers.get("x-artifact-id") !== reference.artifactId) { fail("artifact content route returned the wrong artifact id"); } if (headers.get("x-artifact-checksum") !== reference.checksum) { fail("artifact content route returned the wrong full checksum"); } if (headers.get("x-artifact-content-checksum") !== bodyChecksum) { fail("artifact content route returned the wrong chunk checksum"); } if (Number(headers.get("content-length")) !== body.length || body.length !== limit) { fail("artifact content chunk size did not match request"); } if (offset > 0 || body.length !== reference.sizeBytes) { const range = headers.get("content-range") || ""; if (!range.includes(`/${reference.sizeBytes}`)) { fail("partial artifact response did not include the expected total size"); } } NODE offset=$((offset + limit)) index=$((index + 1)) done node - "$download_file" "$payload_file" "$index" "$artifact_id" "$checksum" <<'NODE' const crypto = require("crypto"); const fs = require("fs"); const reference = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const payload = fs.readFileSync(process.argv[3]); const chunkCount = Number(process.argv[4]); const artifactId = process.argv[5]; const checksum = process.argv[6]; const actualChecksum = `sha256:${crypto.createHash("sha256").update(payload).digest("hex")}`; if (reference.artifactId !== artifactId || reference.checksum !== checksum || payload.length !== reference.sizeBytes || actualChecksum !== reference.checksum || chunkCount < 1) { console.error("downloaded artifact content did not match reference metadata"); console.error({ reference, payloadBytes: payload.length, actualChecksum, chunkCount }); process.exit(1); } NODE } local_debug_host_target() { local os_name local arch_name case "$(uname -s)" in Darwin) os_name="darwin" ;; Linux) os_name="linux" ;; *) printf 'unsupported local-debug generated Run host OS: %s\n' "$(uname -s)" >&2; return 1 ;; esac case "$(uname -m)" in arm64 | aarch64) arch_name="arm64" ;; x86_64 | amd64) arch_name="amd64" ;; *) printf 'unsupported local-debug generated Run host architecture: %s\n' "$(uname -m)" >&2; return 1 ;; esac printf '%s/%s' "$os_name" "$arch_name" } launch_generated_run() { local payload_file="$1" mkdir -p "$LOCAL_DEBUG_LOG_DIR" "$LOCAL_DEBUG_PID_DIR" "$(dirname "$GENERATED_RUN_BIN")" "$RUN_SPOOL_ROOT/$GENERATED_RUN_ENDPOINT_ID" cp "$payload_file" "$GENERATED_RUN_BIN" chmod 700 "$GENERATED_RUN_BIN" ( unset RUN_PLATFORM_URL RUN_ENDPOINT_ID RUN_DISPLAY_NAME RUN_VERSION RUN_REGISTRATION_TOKEN unset RUN_SERVER_INSTANCE_ID RUN_PLUGIN_ID RUN_COMPONENT_KIND RUN_COMPONENT_KEY RUN_KEY_GENERATION RUN_PACKAGE_CONFIG exec env \ GOCACHE="$GOCACHE" \ RUN_MODE=worker \ RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \ RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \ RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT/$GENERATED_RUN_ENDPOINT_ID" \ RUN_MAX_JOBS="$RUN_MAX_JOBS" \ RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \ RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \ RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \ "$GENERATED_RUN_BIN" ) >"$GENERATED_RUN_LOG" 2>&1 & local generated_pid="$!" printf '%s' "$generated_pid" >"$GENERATED_RUN_PID_FILE" if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then SELF_STARTED_PIDS+=("$generated_pid") fi } wait_for_generated_run_heartbeat() { local registration_file="$WORK_DIR/generated-run-registration.response.json" local heartbeat_file="$WORK_DIR/generated-run-heartbeat.response.json" local first_heartbeat="" rm -f "$registration_file" "$heartbeat_file" printf 'waiting for generated Run heartbeat %s\n' "$GENERATED_RUN_ENDPOINT_ID" for _ in $(seq 1 45); do if [[ -f "$GENERATED_RUN_PID_FILE" ]] && ! kill -0 "$(<"$GENERATED_RUN_PID_FILE")" 2>/dev/null; then printf 'generated Run exited before heartbeat; see %s\n' "$GENERATED_RUN_LOG" >&2 return 1 fi if json_get "$API_URL/run/endpoints/$GENERATED_RUN_ENDPOINT_ID" "$registration_file" "${AUTH_HEADER[@]}" 2>/dev/null; then if first_heartbeat="$(node - "$registration_file" "$GENERATED_RUN_ENDPOINT_ID" <<'NODE' const fs = require("fs"); const endpoint = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const expectedID = process.argv[3]; const heartbeat = endpoint.lastHeartbeatAt || endpoint.LastHeartbeatAt || ""; const capabilities = endpoint.capabilities || endpoint.Capabilities || []; if (endpoint.id !== expectedID || endpoint.status !== "online" || !heartbeat || capabilities.includes("distribution.build")) { process.exit(1); } process.stdout.write(heartbeat); NODE )"; then break fi fi sleep 1 done if [[ -z "$first_heartbeat" ]]; then printf 'generated Run endpoint %s did not report a safe heartbeat\n' "$GENERATED_RUN_ENDPOINT_ID" >&2 [[ -f "$GENERATED_RUN_LOG" ]] && sed -n '1,160p' "$GENERATED_RUN_LOG" >&2 return 1 fi reject_forbidden_fragments "$registration_file" printf 'waiting for a subsequent generated Run heartbeat %s\n' "$GENERATED_RUN_ENDPOINT_ID" for _ in $(seq 1 45); do if json_get "$API_URL/run/endpoints/$GENERATED_RUN_ENDPOINT_ID" "$heartbeat_file" "${AUTH_HEADER[@]}" 2>/dev/null && node - "$heartbeat_file" "$GENERATED_RUN_ENDPOINT_ID" "$first_heartbeat" <<'NODE' const fs = require("fs"); const endpoint = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const expectedID = process.argv[3]; const firstHeartbeat = Date.parse(process.argv[4]); const heartbeat = Date.parse(endpoint.lastHeartbeatAt || endpoint.LastHeartbeatAt || ""); const capabilities = endpoint.capabilities || endpoint.Capabilities || []; process.exit(endpoint.id === expectedID && endpoint.status === "online" && Number.isFinite(heartbeat) && heartbeat > firstHeartbeat && !capabilities.includes("distribution.build") ? 0 : 1); NODE then reject_forbidden_fragments "$heartbeat_file" return 0 fi sleep 1 done printf 'generated Run %s did not report a subsequent heartbeat\n' "$GENERATED_RUN_ENDPOINT_ID" >&2 [[ -f "$GENERATED_RUN_LOG" ]] && sed -n '1,160p' "$GENERATED_RUN_LOG" >&2 return 1 } wait_for_lifecycle_install_success() { local server_id="$1" local output_file="$2" local state rm -f "$output_file" for _ in $(seq 1 30); do json_get "$API_URL/jobs?serverInstanceId=$server_id" "$output_file" "${AUTH_HEADER[@]}" || true if [[ ! -s "$output_file" ]]; then sleep 1 continue fi state="$(node - "$output_file" <<'NODE' const fs = require("fs"); const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const job = (response.items || []).find((candidate) => candidate.capability === "process.install"); if (!job) { process.stdout.write("missing"); } else { process.stdout.write(job.state || "unknown"); } NODE )" case "$state" in succeeded) return 0 ;; failed | cancelled) printf 'lifecycle install job for %s reached terminal failure\n' "$server_id" >&2 sed -n '1,160p' "$output_file" >&2 exit 1 ;; esac sleep 1 done printf 'lifecycle install job for %s did not succeed before timeout\n' "$server_id" >&2 sed -n '1,160p' "$output_file" >&2 exit 1 } wait_for_job_success() { local response_file="$1" local output_file="$2" local label="$3" local job_id job_id="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const id=data.job?.id || data.id; if (!id) process.exit(2); process.stdout.write(id);' "$response_file")" rm -f "$output_file" for _ in $(seq 1 60); do json_get "$API_URL/jobs/$job_id" "$output_file" "${AUTH_HEADER[@]}" || true if [[ -s "$output_file" ]]; then local state state="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(data.state || "unknown");' "$output_file")" case "$state" in succeeded) return 0 ;; failed | cancelled) printf '%s job %s reached %s\n' "$label" "$job_id" "$state" >&2 sed -n '1,200p' "$output_file" >&2 return 1 ;; esac fi sleep 1 done printf '%s job %s did not succeed before timeout\n' "$label" "$job_id" >&2 [[ -s "$output_file" ]] && sed -n '1,200p' "$output_file" >&2 return 1 } dump_log_session_diagnostics() { local reason="$1" printf 'current log session smoke diagnostics: %s\n' "$reason" >&2 for file in \ "$WORK_DIR/log-session-streams.response.json" \ "$LOG_SESSION_SSE_FILE" \ "$LOG_SESSION_SSE_ERROR_FILE" \ "$LOCAL_DEBUG_LOG_DIR/run.log"; do if [[ -s "$file" ]]; then printf '%s (last 160 lines):\n' "$file" >&2 tail -n 160 "$file" >&2 fi done } wait_for_file_literal() { local label="$1" local file="$2" local literal="$3" local attempts="${4:-60}" for _ in $(seq 1 "$attempts"); do if [[ -f "$file" ]] && grep -Fq -- "$literal" "$file"; then return 0 fi if [[ -n "${LOG_SESSION_SSE_PID:-}" ]] && ! kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then dump_log_session_diagnostics "$label: SSE client exited" return 1 fi sleep 1 done dump_log_session_diagnostics "$label: timed out waiting for $literal" return 1 } wait_for_persisted_log_marker() { local marker="$1" local evidence_file="$2" local stream_scope="${3:-session}" local streams_file="$WORK_DIR/log-session-streams.response.json" for attempt in $(seq 1 60); do json_get "$API_URL/log-streams?serverInstanceId=$LOG_SESSION_SERVER_ID" "$streams_file" "${AUTH_HEADER[@]}" || true if [[ -s "$streams_file" ]]; then local stream_ids stream_ids="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const scope=process.argv[2]; for (const item of data.items || []) if (scope === "history" || item.logSessionId) console.log(item.id);' "$streams_file" "$stream_scope" 2>/dev/null || true)" local stream_id for stream_id in $stream_ids; do local request_file="$WORK_DIR/log-session-query-$attempt.request.json" printf '{"logStreamId":"%s","afterSeq":0,"limit":200}\n' "$stream_id" >"$request_file" if json_post "$API_URL/log-streams/query" "$request_file" "$evidence_file" "${AUTH_HEADER[@]}" && node -e 'const fs=require("fs"); const marker=process.argv[2]; const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.exit((data.entries || []).some((entry) => entry.line === marker) ? 0 : 1);' "$evidence_file" "$marker"; then return 0 fi done fi sleep 1 done dump_log_session_diagnostics "persisted marker timeout: $marker" return 1 } current_log_session_id() { json_get "$API_URL/log-streams?serverInstanceId=$LOG_SESSION_SERVER_ID" "$WORK_DIR/log-session-streams.response.json" "${AUTH_HEADER[@]}" node - "$WORK_DIR/log-session-streams.response.json" <<'NODE' const fs = require("fs"); const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const current = (response.items || []) .filter((item) => item.source === "process" && item.logSessionId && item.sessionStartedAt) .sort((left, right) => Date.parse(right.sessionStartedAt) - Date.parse(left.sessionStartedAt) || right.logSessionId.localeCompare(left.logSessionId))[0]; if (!current) process.exit(2); process.stdout.write(current.logSessionId); NODE } wait_for_run_heartbeat_after() { local previous_heartbeat="$1" local endpoint_file="$WORK_DIR/log-session-run-endpoint.response.json" for _ in $(seq 1 45); do if json_get "$API_URL/run/endpoints/$RUN_ENDPOINT_ID" "$endpoint_file" "${AUTH_HEADER[@]}" 2>/dev/null && node - "$endpoint_file" "$RUN_ENDPOINT_ID" "$previous_heartbeat" <<'NODE' const fs = require("fs"); const endpoint = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const current = Date.parse(endpoint.lastHeartbeatAt || endpoint.LastHeartbeatAt || ""); const previous = Date.parse(process.argv[4] || ""); process.exit(endpoint.id === process.argv[3] && endpoint.status === "online" && Number.isFinite(current) && current > previous ? 0 : 1); NODE then return 0 fi sleep 1 done dump_log_session_diagnostics "Run did not heartbeat after restart" return 1 } restart_bootstrap_run_preserving_process() { local endpoint_file="$WORK_DIR/log-session-run-endpoint-before-restart.response.json" json_get "$API_URL/run/endpoints/$RUN_ENDPOINT_ID" "$endpoint_file" "${AUTH_HEADER[@]}" local previous_heartbeat previous_heartbeat="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(data.lastHeartbeatAt || data.LastHeartbeatAt || "");' "$endpoint_file")" local run_pid run_pid="$(<"$LOCAL_DEBUG_PID_DIR/run.pid")" if [[ ! "$run_pid" =~ ^[0-9]+$ ]] || ! kill -0 "$run_pid" 2>/dev/null; then dump_log_session_diagnostics "managed Run pid is unavailable before restart" return 1 fi printf 'stopping Run pid %s while leaving its supervised process alive\n' "$run_pid" kill "$run_pid" for _ in $(seq 1 40); do if ! kill -0 "$run_pid" 2>/dev/null; then break fi sleep 0.25 done if kill -0 "$run_pid" 2>/dev/null; then dump_log_session_diagnostics "Run pid $run_pid did not stop" return 1 fi printf 'resume\n' >"$LOG_SESSION_SCOPE/smoke-command.next" mv "$LOG_SESSION_SCOPE/smoke-command.next" "$LOG_SESSION_SCOPE/smoke-command" wait_for_file_literal "supervised process output while Run is down" "$LOG_SESSION_SCOPE/smoke-last-command" "resume" 40 launch_bootstrap_run append wait_for_run_heartbeat_after "$previous_heartbeat" } write_server_lifecycle_request() { local action="$1" local output_file="$2" json_get "$API_URL/server-instances/$LOG_SESSION_SERVER_ID" "$WORK_DIR/log-session-server-current.response.json" "${AUTH_HEADER[@]}" local config_version config_version="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!Number.isInteger(data.configVersion)) process.exit(2); process.stdout.write(String(data.configVersion));' "$WORK_DIR/log-session-server-current.response.json")" cat >"$output_file" <"$request_file" <"$WORK_DIR/login.request.json" <<'JSON' {"account":"operator.local@example.test","password":"operator-local"} JSON json_post "$API_URL/auth/login" "$WORK_DIR/login.request.json" "$WORK_DIR/login.response.json" -H 'X-Auth-Token-Response: bearer' SESSION_ID="$(session_token_from_login "$WORK_DIR/login.response.json")" AUTH_HEADER=(-H "Authorization: Bearer $SESSION_ID") require_file_contains "$WORK_DIR/login.response.json" '"status"[[:space:]]*:[[:space:]]*"active"' printf 'validating plugin manifests\n' (cd "$ROOT_DIR/plugins" && npm run validate:manifest) node - "$ROOT_DIR/plugins/examples/dev-game-plugin/manifest.json" "$WORK_DIR/register-plugin.request.json" <<'NODE' const fs = require("fs"); const path = require("path"); const manifestPath = process.argv[2]; const outputPath = process.argv[3]; const source = JSON.parse(fs.readFileSync(manifestPath, "utf8")); const manifestDir = path.dirname(manifestPath); function readAssetFiles(manifest) { return (manifest.assetFiles ?? []).map((file) => ({ path: file.path, mode: file.mode, ...readAssetFileContent(path.join(manifestDir, file.path), file.path) })); } function readAssetFileContent(assetPath, logicalPath) { const body = fs.readFileSync(assetPath); if (/\.(?:json|cmd|sh|sql|txt|ya?ml)$/i.test(logicalPath)) return { content: body.toString("utf8") }; return { content: body.toString("base64"), encoding: "base64" }; } const manifest = { id: source.id, name: source.name, description: "Development plugin", version: source.version, kind: "game-plugin", tags: ["example", "development"], server: { type: source.server.type, displayName: source.server.displayName, supportedOs: ["linux", "darwin"], createFormSchema: source.server.createFormSchema }, capabilities: ["process.install", "process.start", "process.stop", "config.write"], permissions: ["server.read", "server.lifecycle", "server.run.distribution", "server.logs.read", "server.artifacts.read", "ai.invoke"], actions: { install: source.actions.install, start: source.actions.start, stop: source.actions.stop, restart: source.actions.restart }, assetFiles: source.assetFiles, pages: [ { key: "logs", title: "Logs", path: "/logs", permissions: ["server.read", "server.lifecycle", "server.run.distribution", "server.logs.read", "server.artifacts.read", "ai.invoke"], bridgeActions: ["server.instances.read", "jobs.dispatch", "logs.query", "artifacts.open", "run.distribution.request", "plugin-lifecycle.request", "ai.invoke"] } ], bridge: { actions: ["server.instances.read", "jobs.dispatch", "logs.query", "artifacts.open", "run.distribution.request", "plugin-lifecycle.request", "ai.invoke"] }, ai: source.ai, productionLifecycle: source.productionLifecycle, runtimeProfiles: { lifecycleProfiles: [{ key: "run-local", mode: "local-process", capabilities: ["process.install", "process.start", "process.stop", "config.write"], actionRefs: { install: "actions/install.json", start: "actions/start.json", stop: "actions/stop.json" } }], logSources: [ { key: "smoke-stdout", kind: "process.stdout", streamKey: "smoke.console.stdout", cursorKind: "sequence", retentionDays: 1 }, { key: "smoke-stderr", kind: "process.stderr", streamKey: "smoke.console.stderr", cursorKind: "sequence", retentionDays: 1 } ] } }; fs.writeFileSync(outputPath, JSON.stringify({ manifestRef: "artifact://manifests/game.example/0.1.0", manifest, assetFiles: readAssetFiles(source) }, null, 2)); NODE printf 'registering dev plugin through platform API\n' register_plugin_manifest "dev plugin" "game.example" "$WORK_DIR/register-plugin.request.json" "$WORK_DIR/register-plugin.response.json" reject_forbidden_fragments "$WORK_DIR/register-plugin.response.json" node - "$ROOT_DIR/plugins/examples/scum-server-plugin/manifest.json" "$WORK_DIR/register-scum-plugin.request.json" <<'NODE' const fs = require("fs"); const path = require("path"); const manifestPath = process.argv[2]; const outputPath = process.argv[3]; const source = JSON.parse(fs.readFileSync(manifestPath, "utf8")); const manifestDir = path.dirname(manifestPath); function readAssetFiles(manifest) { return (manifest.assetFiles ?? []).map((file) => ({ path: file.path, mode: file.mode, ...readAssetFileContent(path.join(manifestDir, file.path), file.path) })); } function readAssetFileContent(assetPath, logicalPath) { const body = fs.readFileSync(assetPath); if (/\.(?:json|cmd|sh|sql|txt|ya?ml)$/i.test(logicalPath)) return { content: body.toString("utf8") }; return { content: body.toString("base64"), encoding: "base64" }; } const localRunCapabilities = [ "process.install", "process.start", "process.stop", "logs.read", "run.self-update", "dependencies.check", "dependencies.install", "logs.backfill", ...source.capabilities.filter((capability) => capability.startsWith("remote.")) ]; const localRuntimeProfiles = { lifecycleProfiles: [{ key: "run-local", mode: "local-process", capabilities: ["process.install", "process.start", "process.stop"], actionRefs: { install: "actions/install.json", start: "actions/start.json", stop: "actions/stop.json" } }], logSources: source.runtimeProfiles?.logSources ?? [], transportProfiles: source.runtimeProfiles?.transportProfiles ?? [], clientManagers: [] }; const localGameClientBridge = JSON.parse(JSON.stringify(source.gameClientBridge ?? {})); delete localGameClientBridge.companion; const manifest = { id: source.id, name: source.name, description: source.description, version: source.version, kind: source.kind, tags: source.tags, server: { type: source.server.type, displayName: source.server.displayName, supportedOs: source.server.supportedOS || source.server.supportedOs || [], createFormSchema: source.server.createFormSchema }, capabilities: localRunCapabilities, permissions: source.permissions, actions: source.actions, assetFiles: source.assetFiles, pages: source.pages, bridge: source.bridge, ai: source.ai, productionLifecycle: source.productionLifecycle, remoteAccess: source.remoteAccess, runtimeProfiles: localRuntimeProfiles, gameClientBridge: localGameClientBridge }; fs.writeFileSync(outputPath, JSON.stringify({ manifestRef: `artifact://manifests/game.scum/${source.version}`, manifest, assetFiles: readAssetFiles(source) }, null, 2)); NODE printf 'registering SCUM plugin through platform API\n' register_plugin_manifest "SCUM plugin" "game.scum" "$WORK_DIR/register-scum-plugin.request.json" "$WORK_DIR/register-scum-plugin.response.json" reject_forbidden_fragments "$WORK_DIR/register-scum-plugin.response.json" SCUM_PLUGIN_VERSION="$(node -e 'const fs=require("fs"); process.stdout.write(JSON.parse(fs.readFileSync(process.argv[1],"utf8")).version);' "$ROOT_DIR/plugins/examples/scum-server-plugin/manifest.json")" require_scum_manifest_contract "$WORK_DIR/register-scum-plugin.response.json" "$SCUM_PLUGIN_VERSION" printf 'waiting for run endpoint heartbeat\n' for _ in $(seq 1 20); do if json_get "$API_URL/run/endpoints?status=online" "$WORK_DIR/run-endpoints.response.json" "${AUTH_HEADER[@]}" && grep -q "$RUN_ENDPOINT_ID" "$WORK_DIR/run-endpoints.response.json"; then break fi sleep 1 done require_file_contains "$WORK_DIR/run-endpoints.response.json" "$RUN_ENDPOINT_ID" reject_forbidden_fragments "$WORK_DIR/run-endpoints.response.json" GENERATED_RUN_TARGET="$(local_debug_host_target)" IFS=/ read -r GENERATED_RUN_TARGET_OS GENERATED_RUN_TARGET_ARCH <<<"$GENERATED_RUN_TARGET" printf 'GENERATED_RUN_TARGET_OS=%s\nGENERATED_RUN_TARGET_ARCH=%s\n' "$GENERATED_RUN_TARGET_OS" "$GENERATED_RUN_TARGET_ARCH" >>"$WORK_DIR/run-build-config.env" cat >"$WORK_DIR/create-server.request.json" <"$WORK_DIR/server-runtime-binding.request.json" <"$WORK_DIR/create-log-session-server.request.json" <"$WORK_DIR/log-session-runtime-binding.request.json" <"$WORK_DIR/log-session-deployment.request.json" <"$WORK_DIR/server-run-generate.request.json" <"$WORK_DIR/create-scum-alpha.request.json" <"$WORK_DIR/scum-alpha-runtime-binding.request.json" <"$WORK_DIR/create-scum-beta.request.json" <"$WORK_DIR/create-scum-dynamic.request.json" <"$WORK_DIR/scum-alpha-run-generate.request.json" <&2 exit 1 fi node - "$WORK_DIR/create-server.request.json" <<'NODE' const fs = require("fs"); const request = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); for (const forbidden of ["deploymentTargetId", "runEndpointId", "profileKey", "bindings", "deployment"]) { if (Object.hasOwn(request, forbidden)) { console.error(`minimal server creation unexpectedly included ${forbidden}`); process.exit(1); } } if (!request.pluginId || !request.name) { console.error("minimal server creation omitted pluginId or name"); process.exit(1); } NODE printf 'configuring example server runtime binding after minimal creation\n' curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \ --data-binary "@$WORK_DIR/server-runtime-binding.request.json" \ "$API_URL/server-instances/$SERVER_ID/runtime-binding" >"$WORK_DIR/server-runtime-binding.response.json" reject_forbidden_fragments "$WORK_DIR/server-runtime-binding.response.json" require_file_contains "$WORK_DIR/server-runtime-binding.response.json" '"status"[[:space:]]*:[[:space:]]*"complete"' if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" != "true" ]]; then printf 'checking example server platform-builder action\n' json_get "$API_URL/server-instances/$SERVER_ID/runtime/actions" "$WORK_DIR/server-runtime-actions.response.json" "${AUTH_HEADER[@]}" node - "$WORK_DIR/server-runtime-actions.response.json" <<'NODE' const fs = require("fs"); const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const action = (response.actions || []).find((candidate) => candidate.key === "generate-run"); if (!action || action.available !== true || (action.reason || "").includes("run endpoint")) { console.error("expected minimal example server to generate through the ready platform builder"); console.error(JSON.stringify(response, null, 2)); process.exit(1); } NODE reject_forbidden_fragments "$WORK_DIR/server-runtime-actions.response.json" fi printf 'configuring current log session fixture on the local Run endpoint\n' curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \ --data-binary "@$WORK_DIR/log-session-runtime-binding.request.json" \ "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/runtime-binding" >"$WORK_DIR/log-session-runtime-binding.response.json" reject_forbidden_fragments "$WORK_DIR/log-session-runtime-binding.response.json" require_file_contains "$WORK_DIR/log-session-runtime-binding.response.json" '"status"[[:space:]]*:[[:space:]]*"complete"' curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \ --data-binary "@$WORK_DIR/log-session-deployment.request.json" \ "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/deployment" >"$WORK_DIR/log-session-deployment.response.json" reject_forbidden_fragments "$WORK_DIR/log-session-deployment.response.json" write_server_lifecycle_request deploy "$WORK_DIR/log-session-deploy.request.json" json_post "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/deploy" "$WORK_DIR/log-session-deploy.request.json" "$WORK_DIR/log-session-deploy.response.json" "${AUTH_HEADER[@]}" wait_for_job_success "$WORK_DIR/log-session-deploy.response.json" "$WORK_DIR/log-session-deploy-job.response.json" "log session deploy" LOG_SESSION_DEPLOY_JOB_ID="$(json_id "$WORK_DIR/log-session-deploy-job.response.json")" LOG_SESSION_REAL_JOB_STREAM_IDS="job.$LOG_SESSION_DEPLOY_JOB_ID.stdout,job.$LOG_SESSION_DEPLOY_JOB_ID.stderr" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-HISTORICAL-JOB-STDOUT" "$WORK_DIR/log-session-job-stdout-history.response.json" history wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-HISTORICAL-JOB-STDERR" "$WORK_DIR/log-session-job-stderr-history.response.json" history printf 'seeding explicit legacy, job, and file-tail history stream metadata\n' create_historical_log_stream "$LOG_SESSION_LEGACY_STREAM_ID" process legacy.stdout create_historical_log_stream "$LOG_SESSION_JOB_STREAM_ID" process job.stdout create_historical_log_stream "$LOG_SESSION_FILE_STREAM_ID" file file.tail printf 'starting generation A without an open terminal or SSE client\n' dispatch_log_session_lifecycle start "$WORK_DIR/log-session-start-a.response.json" "$WORK_DIR/log-session-start-a-job.response.json" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT" "$WORK_DIR/log-session-a-stdout-history.response.json" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-START-STDERR" "$WORK_DIR/log-session-a-stderr-history.response.json" LOG_SESSION_A="$(current_log_session_id)" if [[ -z "$LOG_SESSION_A" ]]; then dump_log_session_diagnostics "generation A did not expose a current session" exit 1 fi printf '%s\n' "$LOG_SESSION_A" >"$WORK_DIR/current-log-session-a.id" printf 'opening one persistent Platform SSE connection after generation A was already uploaded\n' : >"$LOG_SESSION_SSE_FILE" : >"$LOG_SESSION_SSE_ERROR_FILE" curl -fsS --no-buffer --max-time 240 -H "Authorization: Bearer $SESSION_ID" -H 'Accept: text/event-stream' \ "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/logs/events?historyLimit=200" \ >"$LOG_SESSION_SSE_FILE" 2>"$LOG_SESSION_SSE_ERROR_FILE" & LOG_SESSION_SSE_PID="$!" wait_for_file_literal "initial SSE ready event" "$LOG_SESSION_SSE_FILE" 'event: ready' 60 wait_for_file_literal "generation A stdout replay" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT" 60 wait_for_file_literal "generation A stderr replay" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDERR" 60 LOG_SESSION_A_FROM_SSE="$(node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" session-for-marker "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT")" if [[ "$LOG_SESSION_A_FROM_SSE" != "$LOG_SESSION_A" ]]; then dump_log_session_diagnostics "generation A history and SSE selected different sessions" exit 1 fi printf 'restarting Run while generation A remains alive\n' restart_bootstrap_run_preserving_process wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT" "$WORK_DIR/log-session-a-resume-stdout-history.response.json" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDERR" "$WORK_DIR/log-session-a-resume-stderr-history.response.json" wait_for_file_literal "generation A stdout after Run restart" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT" 60 wait_for_file_literal "generation A stderr after Run restart" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDERR" 60 LOG_SESSION_A_AFTER_RUN_RESTART="$(node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" session-for-marker "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT")" if [[ "$LOG_SESSION_A_AFTER_RUN_RESTART" != "$LOG_SESSION_A" || "$(current_log_session_id)" != "$LOG_SESSION_A" ]]; then dump_log_session_diagnostics "Run restart changed the supervised process session" exit 1 fi printf 'stopping generation A and starting generation B on the same SSE connection\n' dispatch_log_session_lifecycle stop "$WORK_DIR/log-session-stop-a.response.json" "$WORK_DIR/log-session-stop-a-job.response.json" printf 'B\n' >"$LOG_SESSION_SCOPE/smoke-generation.next" mv "$LOG_SESSION_SCOPE/smoke-generation.next" "$LOG_SESSION_SCOPE/smoke-generation" rm -f "$LOG_SESSION_SCOPE/smoke-command" "$LOG_SESSION_SCOPE/smoke-last-command" "$LOG_SESSION_SCOPE/smoke-process.pid" dispatch_log_session_lifecycle start "$WORK_DIR/log-session-start-b.response.json" "$WORK_DIR/log-session-start-b-job.response.json" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-B-START-STDOUT" "$WORK_DIR/log-session-b-stdout-history.response.json" wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-B-START-STDERR" "$WORK_DIR/log-session-b-stderr-history.response.json" wait_for_file_literal "generation B stdout" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-B-START-STDOUT" 60 wait_for_file_literal "generation B stderr" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-B-START-STDERR" 60 LOG_SESSION_B="$(current_log_session_id)" if [[ -z "$LOG_SESSION_B" || "$LOG_SESSION_B" == "$LOG_SESSION_A" ]]; then dump_log_session_diagnostics "generation B did not create a new current session" exit 1 fi printf '%s\n' "$LOG_SESSION_B" >"$WORK_DIR/current-log-session-b.id" node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" verify \ "$LOG_SESSION_SSE_FILE" \ "$LOG_SESSION_SERVER_ID" \ "$LOG_SESSION_MARKER_PREFIX" \ "$LOG_SESSION_A" \ "$LOG_SESSION_LEGACY_STREAM_ID,$LOG_SESSION_JOB_STREAM_ID,$LOG_SESSION_FILE_STREAM_ID,$LOG_SESSION_REAL_JOB_STREAM_IDS" \ >"$WORK_DIR/current-log-session-verification.json" reject_forbidden_fragments "$LOG_SESSION_SSE_FILE" reject_forbidden_fragments "$WORK_DIR/current-log-session-verification.json" dispatch_log_session_lifecycle stop "$WORK_DIR/log-session-stop-b.response.json" "$WORK_DIR/log-session-stop-b-job.response.json" if kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then kill "$LOG_SESSION_SSE_PID" 2>/dev/null || true wait "$LOG_SESSION_SSE_PID" 2>/dev/null || true fi LOG_SESSION_SSE_PID="" printf 'current supervised log session smoke passed; evidence: %s\n' "$WORK_DIR/current-log-session-verification.json" if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" == "true" ]]; then printf 'focused current supervised log session smoke passed\n' printf 'evidence directory: %s\n' "$WORK_DIR" exit 0 fi printf 'generating host-native example Run through platform Docker builder\n' curl -fsS -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" --data-binary "@$WORK_DIR/server-run-generate.request.json" "$API_URL/server-instances/$SERVER_ID/run/generate" >"$WORK_DIR/server-run-generate.response.json" reject_forbidden_fragments "$WORK_DIR/server-run-generate.response.json" node - "$WORK_DIR/server-run-generate.response.json" "$SERVER_ID" "$GENERATED_RUN_ENDPOINT_ID" "$GENERATED_RUN_TARGET_OS" "$GENERATED_RUN_TARGET_ARCH" <<'NODE' const fs = require("fs"); const distribution = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const [serverID, endpointID, targetOS, targetArch] = process.argv.slice(3); if (distribution.serverInstanceId !== serverID || distribution.runEndpointId !== endpointID || distribution.targetOs !== targetOS || distribution.targetArch !== targetArch || !distribution.buildJobId || !distribution.artifactId) { console.error("host-native Run distribution identity did not match the minimal server"); console.error(JSON.stringify(distribution, null, 2)); process.exit(1); } NODE wait_for_distribution_build "$WORK_DIR/server-run-generate.response.json" "$WORK_DIR/server-run-build-job.response.json" "$WORK_DIR/server-run-build-artifact.response.json" node - "$WORK_DIR/server-run-build-job.response.json" <<'NODE' const fs = require("fs"); const job = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); if (job.runEndpointId !== "platform-distribution-builder" || job.capability !== "distribution.build" || job.state !== "succeeded") { console.error("host-native Run was not completed by the platform distribution builder"); console.error(JSON.stringify(job, null, 2)); process.exit(1); } NODE reject_forbidden_fragments "$WORK_DIR/server-run-build-job.response.json" reject_forbidden_fragments "$WORK_DIR/server-run-build-artifact.response.json" read_latest_run_download_content "$SERVER_ID" "$WORK_DIR/server-run-download.response.json" "$WORK_DIR/server-run-download-content.bin" "$WORK_DIR/server-run-download-chunks" launch_generated_run "$WORK_DIR/server-run-download-content.bin" wait_for_generated_run_heartbeat cat >"$WORK_DIR/server-deployment.request.json" <"$WORK_DIR/server-deployment.response.json" reject_forbidden_fragments "$WORK_DIR/server-deployment.response.json" json_get "$API_URL/server-instances/$SERVER_ID" "$WORK_DIR/server-before-deploy.response.json" "${AUTH_HEADER[@]}" SERVER_CONFIG_VERSION="$(node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1], "utf8")).configVersion; if (!Number.isInteger(value) || value < 1) process.exit(2); process.stdout.write(String(value));' "$WORK_DIR/server-before-deploy.response.json")" cat >"$WORK_DIR/server-deploy.request.json" <"$WORK_DIR/scum-alpha-runtime-binding.response.json" reject_forbidden_fragments "$WORK_DIR/scum-alpha-runtime-binding.response.json" require_file_contains "$WORK_DIR/scum-alpha-runtime-binding.response.json" '"status"[[:space:]]*:[[:space:]]*"complete"' printf 'checking SCUM runtime distribution action\n' json_get "$API_URL/server-instances/$SCUM_ALPHA_ID/runtime/actions" "$WORK_DIR/scum-alpha-runtime-actions.response.json" "${AUTH_HEADER[@]}" reject_forbidden_fragments "$WORK_DIR/scum-alpha-runtime-actions.response.json" require_file_contains "$WORK_DIR/scum-alpha-runtime-actions.response.json" '"key"[[:space:]]*:[[:space:]]*"generate-run"' node - "$WORK_DIR/scum-alpha-runtime-actions.response.json" <<'NODE' const fs = require("fs"); const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const action = (response.actions || []).find((candidate) => candidate.key === "generate-run"); if (!action || action.available !== true) { console.error("expected SCUM generate-run action to use the ready platform builder"); console.error(JSON.stringify(response, null, 2)); process.exit(1); } if ((action.reason || "").includes("run endpoint")) { console.error("platform build availability must not reference a Run endpoint capability"); console.error(JSON.stringify(action, null, 2)); process.exit(1); } NODE printf 'generating SCUM run package through platform API\n' curl -fsS -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" --data-binary "@$WORK_DIR/scum-alpha-run-generate.request.json" "$API_URL/server-instances/$SCUM_ALPHA_ID/run/generate" >"$WORK_DIR/scum-alpha-run-generate.response.json" reject_forbidden_fragments "$WORK_DIR/scum-alpha-run-generate.response.json" require_file_contains "$WORK_DIR/scum-alpha-run-generate.response.json" "\"serverInstanceId\"[[:space:]]*:[[:space:]]*\"$SCUM_ALPHA_ID\"" require_file_contains "$WORK_DIR/scum-alpha-run-generate.response.json" "\"artifactId\"[[:space:]]*:[[:space:]]*\"artifact-run-dist-$SCUM_ALPHA_ID" require_file_contains "$WORK_DIR/scum-alpha-run-generate.response.json" '"buildJobId"[[:space:]]*:[[:space:]]*"job-distribution-build' node - "$WORK_DIR/scum-alpha-run-generate.response.json" <<'NODE' const fs = require("fs"); const distribution = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); if (distribution.status !== "building" && distribution.status !== "available") { console.error("expected run distribution to be building or available"); console.error(JSON.stringify(distribution, null, 2)); process.exit(1); } NODE wait_for_distribution_build "$WORK_DIR/scum-alpha-run-generate.response.json" "$WORK_DIR/scum-alpha-run-build-job.response.json" "$WORK_DIR/scum-alpha-run-build-artifact.response.json" reject_forbidden_fragments "$WORK_DIR/scum-alpha-run-build-job.response.json" reject_forbidden_fragments "$WORK_DIR/scum-alpha-run-build-artifact.response.json" read_latest_run_download_content "$SCUM_ALPHA_ID" "$WORK_DIR/scum-alpha-run-download.response.json" "$WORK_DIR/scum-alpha-run-download-content.bin" "$WORK_DIR/scum-alpha-run-download-chunks" reject_forbidden_fragments "$WORK_DIR/scum-alpha-run-download.response.json" printf 'checking jobs, logs, artifacts, and marketplace refs\n' json_get "$API_URL/server-instances" "$WORK_DIR/server-instances.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/jobs?serverInstanceId=$SERVER_ID" "$WORK_DIR/jobs.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/jobs?serverInstanceId=$SCUM_ALPHA_ID" "$WORK_DIR/scum-alpha-jobs.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/log-streams" "$WORK_DIR/log-streams.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/artifacts" "$WORK_DIR/artifacts.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/plugin-marketplace/plugins" "$WORK_DIR/marketplace.response.json" "${AUTH_HEADER[@]}" json_get "$API_URL/plugin-marketplace/plugins?serverType=scum&keyword=scum" "$WORK_DIR/scum-marketplace.response.json" "${AUTH_HEADER[@]}" require_file_contains "$WORK_DIR/scum-marketplace.response.json" '"id"[[:space:]]*:[[:space:]]*"game.scum"' require_file_contains "$WORK_DIR/artifacts.response.json" "\"id\"[[:space:]]*:[[:space:]]*\"artifact-run-dist-$SCUM_ALPHA_ID" require_file_contains "$WORK_DIR/scum-alpha-jobs.response.json" "\"serverInstanceId\"[[:space:]]*:[[:space:]]*\"$SCUM_ALPHA_ID\"" require_file_contains "$WORK_DIR/server-instances.response.json" "\"id\"[[:space:]]*:[[:space:]]*\"$SCUM_BETA_ID\"" require_file_contains "$WORK_DIR/server-instances.response.json" "\"id\"[[:space:]]*:[[:space:]]*\"$SCUM_DYNAMIC_ID\"" for file in \ "$WORK_DIR"/server-instances.response.json \ "$WORK_DIR"/jobs.response.json \ "$WORK_DIR"/server-runtime-binding.response.json \ "$WORK_DIR"/server-runtime-actions.response.json \ "$WORK_DIR"/server-run-generate.response.json \ "$WORK_DIR"/server-run-build-job.response.json \ "$WORK_DIR"/server-run-build-artifact.response.json \ "$WORK_DIR"/server-run-download.response.json \ "$WORK_DIR"/generated-run-registration.response.json \ "$WORK_DIR"/generated-run-heartbeat.response.json \ "$WORK_DIR"/server-deployment.response.json \ "$WORK_DIR"/server-deploy.response.json \ "$WORK_DIR"/scum-alpha-jobs.response.json \ "$WORK_DIR"/scum-beta-jobs.response.json \ "$WORK_DIR"/scum-dynamic-jobs.response.json \ "$WORK_DIR"/log-streams.response.json \ "$WORK_DIR"/artifacts.response.json \ "$WORK_DIR"/marketplace.response.json \ "$WORK_DIR"/scum-marketplace.response.json \ "$WORK_DIR"/scum-alpha-runtime-actions.response.json \ "$WORK_DIR"/scum-alpha-run-generate.response.json \ "$WORK_DIR"/scum-alpha-run-download.response.json; do if [[ -f "$file" ]]; then reject_forbidden_fragments "$file" fi done if [[ "$VITE_PLATFORM_API_BASE_URL" != "/api/v1" ]]; then printf 'VITE_PLATFORM_API_BASE_URL must be /api/v1, got %s\n' "$VITE_PLATFORM_API_BASE_URL" >&2 exit 1 fi if [[ "$PLATFORM_RUN_RELEASE_URL" != "$PLATFORM_URL" ]]; then printf 'PLATFORM_RUN_RELEASE_URL must be %s for executable local Run proof, got %s\n' "$PLATFORM_URL" "$PLATFORM_RUN_RELEASE_URL" >&2 exit 1 fi if [[ "$PLATFORM_API_PROXY" != "$PLATFORM_URL" ]]; then printf 'PLATFORM_API_PROXY must be %s, got %s\n' "$PLATFORM_URL" "$PLATFORM_API_PROXY" >&2 exit 1 fi if [[ "$VITE_ENABLE_LOCAL_AUTH_FALLBACK" != "false" ]]; then printf 'VITE_ENABLE_LOCAL_AUTH_FALLBACK must stay false for local-debug proof, got %s\n' "$VITE_ENABLE_LOCAL_AUTH_FALLBACK" >&2 exit 1 fi cat >"$WORK_DIR/local-ui-checklist.md" <