Implement platform management features
This commit is contained in:
npc0-hue
+471
-30
@@ -10,6 +10,7 @@ API_URL="$PLATFORM_URL/api/v1"
|
||||
WORK_DIR="$LOCAL_DEBUG_ROOT/smoke"
|
||||
SMOKE_INVOCATION_ID="${SMOKE_INVOCATION_ID:-$(date -u +%Y%m%d%H%M%S)-$$}"
|
||||
SERVER_LOCAL_ID="server-local-debug-$SMOKE_INVOCATION_ID"
|
||||
LOG_SESSION_SERVER_ID="server-log-session-$SMOKE_INVOCATION_ID"
|
||||
SCUM_ALPHA_ID="scum-alpha-$SMOKE_INVOCATION_ID"
|
||||
SCUM_BETA_ID="scum-beta-$SMOKE_INVOCATION_ID"
|
||||
SCUM_DYNAMIC_ID="scum-dynamic-$SMOKE_INVOCATION_ID"
|
||||
@@ -17,11 +18,21 @@ GENERATED_RUN_ENDPOINT_ID="server-run-$SERVER_LOCAL_ID"
|
||||
GENERATED_RUN_BIN="$WORK_DIR/generated-$SERVER_LOCAL_ID-run"
|
||||
GENERATED_RUN_LOG="$LOCAL_DEBUG_LOG_DIR/generated-$SERVER_LOCAL_ID-run.log"
|
||||
GENERATED_RUN_PID_FILE="$LOCAL_DEBUG_PID_DIR/generated-$SERVER_LOCAL_ID-run.pid"
|
||||
LOG_SESSION_SCOPE="$RUN_WORKSPACE_ROOT/instances/$LOG_SESSION_SERVER_ID/run-local"
|
||||
LOG_SESSION_MARKER_PREFIX="SMOKE-CURRENT-SESSION-$SMOKE_INVOCATION_ID"
|
||||
LOG_SESSION_SSE_FILE="$WORK_DIR/current-log-session.events.sse"
|
||||
LOG_SESSION_SSE_ERROR_FILE="$WORK_DIR/current-log-session.events.stderr.log"
|
||||
LOG_SESSION_SSE_PID=""
|
||||
LOG_SESSION_LEGACY_STREAM_ID="legacy.$LOG_SESSION_SERVER_ID.stdout"
|
||||
LOG_SESSION_JOB_STREAM_ID="job.smoke-$SMOKE_INVOCATION_ID.stdout"
|
||||
LOG_SESSION_FILE_STREAM_ID="file.$LOG_SESSION_SERVER_ID.backfill"
|
||||
LOG_SESSION_REAL_JOB_STREAM_IDS=""
|
||||
mkdir -p "$WORK_DIR"
|
||||
|
||||
cat >"$WORK_DIR/run-build-config.env" <<EOF
|
||||
SMOKE_INVOCATION_ID=$SMOKE_INVOCATION_ID
|
||||
SERVER_LOCAL_ID=$SERVER_LOCAL_ID
|
||||
LOG_SESSION_SERVER_ID=$LOG_SESSION_SERVER_ID
|
||||
SCUM_ALPHA_ID=$SCUM_ALPHA_ID
|
||||
SCUM_BETA_ID=$SCUM_BETA_ID
|
||||
SCUM_DYNAMIC_ID=$SCUM_DYNAMIC_ID
|
||||
@@ -45,6 +56,7 @@ EOF
|
||||
prepare_run_workspace() {
|
||||
local_debug_prepare_run_lifecycle_templates
|
||||
local_debug_seed_server_lifecycle_workspace "$SERVER_LOCAL_ID" game.example run-local
|
||||
local_debug_seed_server_lifecycle_workspace "$LOG_SESSION_SERVER_ID" game.example run-local
|
||||
local_debug_seed_server_lifecycle_workspace "$SCUM_ALPHA_ID" game.scum run-local
|
||||
local_debug_seed_server_lifecycle_workspace "$SCUM_BETA_ID" game.scum run-local
|
||||
local_debug_seed_server_lifecycle_workspace "$SCUM_DYNAMIC_ID" game.scum run-local
|
||||
@@ -52,6 +64,43 @@ prepare_run_workspace() {
|
||||
|
||||
prepare_run_workspace
|
||||
|
||||
cat >"$WORK_DIR/current-log-session-fixture.sh" <<EOF
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
generation="\$(cat smoke-generation 2>/dev/null || printf A)"
|
||||
printf '%s-%s-START-STDOUT\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation"
|
||||
printf '%s-%s-START-STDERR\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation" >&2
|
||||
printf '%s\n' "\$\$" >smoke-process.pid
|
||||
last_command=""
|
||||
while :; do
|
||||
command="\$(cat smoke-command 2>/dev/null || true)"
|
||||
if [ -n "\$command" ] && [ "\$command" != "\$last_command" ]; then
|
||||
case "\$command" in
|
||||
resume)
|
||||
printf '%s-%s-RUN-RESUME-STDOUT\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation"
|
||||
printf '%s-%s-RUN-RESUME-STDERR\n' '$LOG_SESSION_MARKER_PREFIX' "\$generation" >&2
|
||||
;;
|
||||
esac
|
||||
last_command="\$command"
|
||||
printf '%s\n' "\$command" >smoke-last-command
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
EOF
|
||||
cat >"$WORK_DIR/current-log-session-install-fixture.sh" <<EOF
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
printf '%s-HISTORICAL-JOB-STDOUT\n' '$LOG_SESSION_MARKER_PREFIX'
|
||||
printf '%s-HISTORICAL-JOB-STDERR\n' '$LOG_SESSION_MARKER_PREFIX' >&2
|
||||
EOF
|
||||
cp "$WORK_DIR/current-log-session-fixture.sh" "$LOG_SESSION_SCOPE/bin/game-server"
|
||||
cp "$WORK_DIR/current-log-session-install-fixture.sh" "$LOG_SESSION_SCOPE/bin/install-server"
|
||||
chmod 700 "$LOG_SESSION_SCOPE/bin/game-server"
|
||||
chmod 700 "$LOG_SESSION_SCOPE/bin/install-server"
|
||||
printf 'A\n' >"$LOG_SESSION_SCOPE/smoke-generation"
|
||||
rm -f "$LOG_SESSION_SCOPE/smoke-command" "$LOG_SESSION_SCOPE/smoke-last-command" "$LOG_SESSION_SCOPE/smoke-process.pid"
|
||||
|
||||
SELF_STARTED_PIDS=()
|
||||
|
||||
cleanup_self_started() {
|
||||
@@ -63,6 +112,83 @@ cleanup_self_started() {
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_log_session_fixture() {
|
||||
if [[ -n "${LOG_SESSION_SSE_PID:-}" ]] && kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then
|
||||
kill "$LOG_SESSION_SSE_PID" 2>/dev/null || true
|
||||
wait "$LOG_SESSION_SSE_PID" 2>/dev/null || true
|
||||
fi
|
||||
local fixture_pid_file="$LOG_SESSION_SCOPE/smoke-process.pid"
|
||||
if [[ -f "$fixture_pid_file" ]]; then
|
||||
local fixture_pid
|
||||
fixture_pid="$(<"$fixture_pid_file")"
|
||||
if [[ "$fixture_pid" =~ ^[0-9]+$ ]] && kill -0 "$fixture_pid" 2>/dev/null; then
|
||||
local fixture_command
|
||||
fixture_command="$(ps -p "$fixture_pid" -o command= 2>/dev/null || true)"
|
||||
if [[ "$fixture_command" == *"$LOG_SESSION_SCOPE/bin/game-server"* ]]; then
|
||||
kill "$fixture_pid" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_smoke() {
|
||||
cleanup_log_session_fixture
|
||||
cleanup_self_started
|
||||
}
|
||||
|
||||
trap cleanup_smoke EXIT
|
||||
|
||||
launch_bootstrap_run() {
|
||||
local log_mode="${1:-overwrite}"
|
||||
mkdir -p "$LOCAL_DEBUG_LOG_DIR" "$LOCAL_DEBUG_PID_DIR"
|
||||
if [[ "$log_mode" == "append" ]]; then
|
||||
(
|
||||
cd "$(dirname "$RUN_BOOTSTRAP_BIN")"
|
||||
exec env \
|
||||
GOCACHE="$GOCACHE" \
|
||||
RUN_MODE="$RUN_MODE" \
|
||||
RUN_PLATFORM_URL="$RUN_PLATFORM_URL" \
|
||||
RUN_ENDPOINT_ID="$RUN_ENDPOINT_ID" \
|
||||
RUN_DISPLAY_NAME="$RUN_DISPLAY_NAME" \
|
||||
RUN_VERSION="$RUN_VERSION" \
|
||||
RUN_REGISTRATION_TOKEN="$RUN_REGISTRATION_TOKEN" \
|
||||
RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \
|
||||
RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \
|
||||
RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT" \
|
||||
RUN_MAX_JOBS="$RUN_MAX_JOBS" \
|
||||
RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \
|
||||
RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \
|
||||
RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \
|
||||
"$RUN_BOOTSTRAP_BIN"
|
||||
) >>"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 &
|
||||
else
|
||||
(
|
||||
cd "$(dirname "$RUN_BOOTSTRAP_BIN")"
|
||||
exec env \
|
||||
GOCACHE="$GOCACHE" \
|
||||
RUN_MODE="$RUN_MODE" \
|
||||
RUN_PLATFORM_URL="$RUN_PLATFORM_URL" \
|
||||
RUN_ENDPOINT_ID="$RUN_ENDPOINT_ID" \
|
||||
RUN_DISPLAY_NAME="$RUN_DISPLAY_NAME" \
|
||||
RUN_VERSION="$RUN_VERSION" \
|
||||
RUN_REGISTRATION_TOKEN="$RUN_REGISTRATION_TOKEN" \
|
||||
RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \
|
||||
RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \
|
||||
RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT" \
|
||||
RUN_MAX_JOBS="$RUN_MAX_JOBS" \
|
||||
RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \
|
||||
RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \
|
||||
RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \
|
||||
"$RUN_BOOTSTRAP_BIN"
|
||||
) >"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 &
|
||||
fi
|
||||
local run_pid="$!"
|
||||
printf '%s' "$run_pid" >"$LOCAL_DEBUG_PID_DIR/run.pid"
|
||||
if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then
|
||||
SELF_STARTED_PIDS+=("$run_pid")
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
@@ -127,8 +253,9 @@ assert_no_managed_run_process() {
|
||||
|
||||
start_self_hosted_stack() {
|
||||
mkdir -p "$LOCAL_DEBUG_LOG_DIR" "$LOCAL_DEBUG_PID_DIR" "$PLATFORM_DATA_DIR" "$PLATFORM_LOG_DIR" "$PLATFORM_ARTIFACT_DIR" "$RUN_WORKSPACE_ROOT" "$RUN_SPOOL_ROOT" "$RUN_BUILD_BUCKET_ROOT" "$GOCACHE"
|
||||
local_debug_prepare_distribution_builder
|
||||
trap cleanup_self_started EXIT
|
||||
if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" != "true" ]]; then
|
||||
local_debug_prepare_distribution_builder
|
||||
fi
|
||||
|
||||
printf 'self-starting platform for local debug smoke\n'
|
||||
(
|
||||
@@ -172,27 +299,7 @@ start_self_hosted_stack() {
|
||||
|
||||
printf 'self-starting run worker for local debug smoke\n'
|
||||
local_debug_build_bootstrap_run
|
||||
(
|
||||
cd "$(dirname "$RUN_BOOTSTRAP_BIN")"
|
||||
exec env \
|
||||
GOCACHE="$GOCACHE" \
|
||||
RUN_MODE="$RUN_MODE" \
|
||||
RUN_PLATFORM_URL="$RUN_PLATFORM_URL" \
|
||||
RUN_ENDPOINT_ID="$RUN_ENDPOINT_ID" \
|
||||
RUN_DISPLAY_NAME="$RUN_DISPLAY_NAME" \
|
||||
RUN_VERSION="$RUN_VERSION" \
|
||||
RUN_REGISTRATION_TOKEN="$RUN_REGISTRATION_TOKEN" \
|
||||
RUN_WORKSPACE_ROOT="$RUN_WORKSPACE_ROOT" \
|
||||
RUN_BUILD_SOURCE_ROOT="$RUN_BUILD_SOURCE_ROOT" \
|
||||
RUN_SPOOL_ROOT="$RUN_SPOOL_ROOT" \
|
||||
RUN_MAX_JOBS="$RUN_MAX_JOBS" \
|
||||
RUN_HEARTBEAT_INTERVAL_MS="$RUN_HEARTBEAT_INTERVAL_MS" \
|
||||
RUN_POLL_INTERVAL_MS="$RUN_POLL_INTERVAL_MS" \
|
||||
RUN_RETRY_BACKOFF_MS="$RUN_RETRY_BACKOFF_MS" \
|
||||
"$RUN_BOOTSTRAP_BIN"
|
||||
) >"$LOCAL_DEBUG_LOG_DIR/run.log" 2>&1 &
|
||||
SELF_STARTED_PIDS+=("$!")
|
||||
printf '%s' "$!" >"$LOCAL_DEBUG_PID_DIR/run.pid"
|
||||
launch_bootstrap_run overwrite
|
||||
}
|
||||
|
||||
if [[ "${LOCAL_DEBUG_SELF_START:-false}" == "true" ]]; then
|
||||
@@ -329,7 +436,7 @@ if (!logSources.some((source) => source.kind === "process.stderr" && source.stre
|
||||
if (!plugin.gameClientBridge?.commands?.length) {
|
||||
missing.push("game client bridge declarations");
|
||||
}
|
||||
for (const pageKey of ["files-config", "players", "squads", "live-map", "gifts"]) {
|
||||
for (const pageKey of ["players", "squads", "live-map", "gifts"]) {
|
||||
if (!pageKeys.includes(pageKey)) {
|
||||
missing.push(`SCUM page ${pageKey}`);
|
||||
}
|
||||
@@ -636,6 +743,201 @@ NODE
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for_job_success() {
|
||||
local response_file="$1"
|
||||
local output_file="$2"
|
||||
local label="$3"
|
||||
local job_id
|
||||
job_id="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const id=data.job?.id || data.id; if (!id) process.exit(2); process.stdout.write(id);' "$response_file")"
|
||||
rm -f "$output_file"
|
||||
for _ in $(seq 1 60); do
|
||||
json_get "$API_URL/jobs/$job_id" "$output_file" "${AUTH_HEADER[@]}" || true
|
||||
if [[ -s "$output_file" ]]; then
|
||||
local state
|
||||
state="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(data.state || "unknown");' "$output_file")"
|
||||
case "$state" in
|
||||
succeeded)
|
||||
return 0
|
||||
;;
|
||||
failed | cancelled)
|
||||
printf '%s job %s reached %s\n' "$label" "$job_id" "$state" >&2
|
||||
sed -n '1,200p' "$output_file" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
printf '%s job %s did not succeed before timeout\n' "$label" "$job_id" >&2
|
||||
[[ -s "$output_file" ]] && sed -n '1,200p' "$output_file" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
dump_log_session_diagnostics() {
|
||||
local reason="$1"
|
||||
printf 'current log session smoke diagnostics: %s\n' "$reason" >&2
|
||||
for file in \
|
||||
"$WORK_DIR/log-session-streams.response.json" \
|
||||
"$LOG_SESSION_SSE_FILE" \
|
||||
"$LOG_SESSION_SSE_ERROR_FILE" \
|
||||
"$LOCAL_DEBUG_LOG_DIR/run.log"; do
|
||||
if [[ -s "$file" ]]; then
|
||||
printf '%s (last 160 lines):\n' "$file" >&2
|
||||
tail -n 160 "$file" >&2
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_file_literal() {
|
||||
local label="$1"
|
||||
local file="$2"
|
||||
local literal="$3"
|
||||
local attempts="${4:-60}"
|
||||
for _ in $(seq 1 "$attempts"); do
|
||||
if [[ -f "$file" ]] && grep -Fq -- "$literal" "$file"; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${LOG_SESSION_SSE_PID:-}" ]] && ! kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then
|
||||
dump_log_session_diagnostics "$label: SSE client exited"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
dump_log_session_diagnostics "$label: timed out waiting for $literal"
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_persisted_log_marker() {
|
||||
local marker="$1"
|
||||
local evidence_file="$2"
|
||||
local stream_scope="${3:-session}"
|
||||
local streams_file="$WORK_DIR/log-session-streams.response.json"
|
||||
for attempt in $(seq 1 60); do
|
||||
json_get "$API_URL/log-streams?serverInstanceId=$LOG_SESSION_SERVER_ID" "$streams_file" "${AUTH_HEADER[@]}" || true
|
||||
if [[ -s "$streams_file" ]]; then
|
||||
local stream_ids
|
||||
stream_ids="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); const scope=process.argv[2]; for (const item of data.items || []) if (scope === "history" || item.logSessionId) console.log(item.id);' "$streams_file" "$stream_scope" 2>/dev/null || true)"
|
||||
local stream_id
|
||||
for stream_id in $stream_ids; do
|
||||
local request_file="$WORK_DIR/log-session-query-$attempt.request.json"
|
||||
printf '{"logStreamId":"%s","afterSeq":0,"limit":200}\n' "$stream_id" >"$request_file"
|
||||
if json_post "$API_URL/log-streams/query" "$request_file" "$evidence_file" "${AUTH_HEADER[@]}" && node -e 'const fs=require("fs"); const marker=process.argv[2]; const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.exit((data.entries || []).some((entry) => entry.line === marker) ? 0 : 1);' "$evidence_file" "$marker"; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
dump_log_session_diagnostics "persisted marker timeout: $marker"
|
||||
return 1
|
||||
}
|
||||
|
||||
current_log_session_id() {
|
||||
json_get "$API_URL/log-streams?serverInstanceId=$LOG_SESSION_SERVER_ID" "$WORK_DIR/log-session-streams.response.json" "${AUTH_HEADER[@]}"
|
||||
node - "$WORK_DIR/log-session-streams.response.json" <<'NODE'
|
||||
const fs = require("fs");
|
||||
const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const current = (response.items || [])
|
||||
.filter((item) => item.source === "process" && item.logSessionId && item.sessionStartedAt)
|
||||
.sort((left, right) => Date.parse(right.sessionStartedAt) - Date.parse(left.sessionStartedAt) || right.logSessionId.localeCompare(left.logSessionId))[0];
|
||||
if (!current) process.exit(2);
|
||||
process.stdout.write(current.logSessionId);
|
||||
NODE
|
||||
}
|
||||
|
||||
wait_for_run_heartbeat_after() {
|
||||
local previous_heartbeat="$1"
|
||||
local endpoint_file="$WORK_DIR/log-session-run-endpoint.response.json"
|
||||
for _ in $(seq 1 45); do
|
||||
if json_get "$API_URL/run/endpoints/$RUN_ENDPOINT_ID" "$endpoint_file" "${AUTH_HEADER[@]}" 2>/dev/null && node - "$endpoint_file" "$RUN_ENDPOINT_ID" "$previous_heartbeat" <<'NODE'
|
||||
const fs = require("fs");
|
||||
const endpoint = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const current = Date.parse(endpoint.lastHeartbeatAt || endpoint.LastHeartbeatAt || "");
|
||||
const previous = Date.parse(process.argv[4] || "");
|
||||
process.exit(endpoint.id === process.argv[3] && endpoint.status === "online" && Number.isFinite(current) && current > previous ? 0 : 1);
|
||||
NODE
|
||||
then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
dump_log_session_diagnostics "Run did not heartbeat after restart"
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_bootstrap_run_preserving_process() {
|
||||
local endpoint_file="$WORK_DIR/log-session-run-endpoint-before-restart.response.json"
|
||||
json_get "$API_URL/run/endpoints/$RUN_ENDPOINT_ID" "$endpoint_file" "${AUTH_HEADER[@]}"
|
||||
local previous_heartbeat
|
||||
previous_heartbeat="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(data.lastHeartbeatAt || data.LastHeartbeatAt || "");' "$endpoint_file")"
|
||||
local run_pid
|
||||
run_pid="$(<"$LOCAL_DEBUG_PID_DIR/run.pid")"
|
||||
if [[ ! "$run_pid" =~ ^[0-9]+$ ]] || ! kill -0 "$run_pid" 2>/dev/null; then
|
||||
dump_log_session_diagnostics "managed Run pid is unavailable before restart"
|
||||
return 1
|
||||
fi
|
||||
printf 'stopping Run pid %s while leaving its supervised process alive\n' "$run_pid"
|
||||
kill "$run_pid"
|
||||
for _ in $(seq 1 40); do
|
||||
if ! kill -0 "$run_pid" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
if kill -0 "$run_pid" 2>/dev/null; then
|
||||
dump_log_session_diagnostics "Run pid $run_pid did not stop"
|
||||
return 1
|
||||
fi
|
||||
printf 'resume\n' >"$LOG_SESSION_SCOPE/smoke-command.next"
|
||||
mv "$LOG_SESSION_SCOPE/smoke-command.next" "$LOG_SESSION_SCOPE/smoke-command"
|
||||
wait_for_file_literal "supervised process output while Run is down" "$LOG_SESSION_SCOPE/smoke-last-command" "resume" 40
|
||||
launch_bootstrap_run append
|
||||
wait_for_run_heartbeat_after "$previous_heartbeat"
|
||||
}
|
||||
|
||||
write_server_lifecycle_request() {
|
||||
local action="$1"
|
||||
local output_file="$2"
|
||||
json_get "$API_URL/server-instances/$LOG_SESSION_SERVER_ID" "$WORK_DIR/log-session-server-current.response.json" "${AUTH_HEADER[@]}"
|
||||
local config_version
|
||||
config_version="$(node -e 'const fs=require("fs"); const data=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); if (!Number.isInteger(data.configVersion)) process.exit(2); process.stdout.write(String(data.configVersion));' "$WORK_DIR/log-session-server-current.response.json")"
|
||||
cat >"$output_file" <<JSON
|
||||
{
|
||||
"expectedConfigVersion": $config_version,
|
||||
"idempotencyKey": "local-debug-log-session-$action-$SMOKE_INVOCATION_ID-$(date +%s)-$RANDOM"
|
||||
}
|
||||
JSON
|
||||
}
|
||||
|
||||
dispatch_log_session_lifecycle() {
|
||||
local action="$1"
|
||||
local response_file="$2"
|
||||
local job_file="$3"
|
||||
local request_file="$response_file.request.json"
|
||||
write_server_lifecycle_request "$action" "$request_file"
|
||||
json_post "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/$action" "$request_file" "$response_file" "${AUTH_HEADER[@]}"
|
||||
wait_for_job_success "$response_file" "$job_file" "log session $action"
|
||||
}
|
||||
|
||||
create_historical_log_stream() {
|
||||
local stream_id="$1"
|
||||
local source="$2"
|
||||
local stream_key="$3"
|
||||
local request_file="$WORK_DIR/$stream_id.request.json"
|
||||
local response_file="$WORK_DIR/$stream_id.response.json"
|
||||
cat >"$request_file" <<JSON
|
||||
{
|
||||
"id": "$stream_id",
|
||||
"serverInstanceId": "$LOG_SESSION_SERVER_ID",
|
||||
"source": "$source",
|
||||
"streamKey": "$stream_key",
|
||||
"storageBackend": "local-segments",
|
||||
"retentionPolicy": "local-debug-smoke"
|
||||
}
|
||||
JSON
|
||||
json_post "$API_URL/log-streams" "$request_file" "$response_file" "${AUTH_HEADER[@]}"
|
||||
}
|
||||
|
||||
printf 'checking platform health at %s\n' "$PLATFORM_URL"
|
||||
json_get "$PLATFORM_URL/healthz" "$WORK_DIR/health.json"
|
||||
require_file_contains "$WORK_DIR/health.json" '"status"[[:space:]]*:[[:space:]]*"ok"'
|
||||
@@ -709,7 +1011,11 @@ const manifest = {
|
||||
start: "actions/start.json",
|
||||
stop: "actions/stop.json"
|
||||
}
|
||||
}]
|
||||
}],
|
||||
logSources: [
|
||||
{ key: "smoke-stdout", kind: "process.stdout", streamKey: "smoke.console.stdout", cursorKind: "sequence", retentionDays: 1 },
|
||||
{ key: "smoke-stderr", kind: "process.stderr", streamKey: "smoke.console.stderr", cursorKind: "sequence", retentionDays: 1 }
|
||||
]
|
||||
}
|
||||
};
|
||||
fs.writeFileSync(outputPath, JSON.stringify({
|
||||
@@ -833,6 +1139,32 @@ cat >"$WORK_DIR/server-runtime-binding.request.json" <<JSON
|
||||
}
|
||||
JSON
|
||||
|
||||
cat >"$WORK_DIR/create-log-session-server.request.json" <<JSON
|
||||
{
|
||||
"id": "$LOG_SESSION_SERVER_ID",
|
||||
"pluginId": "game.example",
|
||||
"name": "Current Log Session Smoke $SMOKE_INVOCATION_ID",
|
||||
"idempotencyKey": "local-debug-log-session-create-$SMOKE_INVOCATION_ID"
|
||||
}
|
||||
JSON
|
||||
|
||||
cat >"$WORK_DIR/log-session-runtime-binding.request.json" <<JSON
|
||||
{
|
||||
"profileKey": "run-local",
|
||||
"bindings": {}
|
||||
}
|
||||
JSON
|
||||
|
||||
cat >"$WORK_DIR/log-session-deployment.request.json" <<JSON
|
||||
{
|
||||
"runEndpointId": "$RUN_ENDPOINT_ID",
|
||||
"mode": "existing-server",
|
||||
"profileKey": "run-local",
|
||||
"createInputs": {},
|
||||
"serverRoot": "/srv/local-debug/$LOG_SESSION_SERVER_ID"
|
||||
}
|
||||
JSON
|
||||
|
||||
cat >"$WORK_DIR/server-run-generate.request.json" <<JSON
|
||||
{
|
||||
"targetOs": "$GENERATED_RUN_TARGET_OS",
|
||||
@@ -887,6 +1219,10 @@ printf 'creating server lifecycle workflow through platform API\n'
|
||||
create_server_workflow "dev" "$SERVER_LOCAL_ID" "$WORK_DIR/create-server.request.json" "$WORK_DIR/create-server.response.json"
|
||||
reject_forbidden_fragments "$WORK_DIR/create-server.response.json"
|
||||
|
||||
printf 'creating current supervised log session fixture server\n'
|
||||
create_server_workflow "log session fixture" "$LOG_SESSION_SERVER_ID" "$WORK_DIR/create-log-session-server.request.json" "$WORK_DIR/create-log-session-server.response.json"
|
||||
reject_forbidden_fragments "$WORK_DIR/create-log-session-server.response.json"
|
||||
|
||||
printf 'creating SCUM server lifecycle workflows through platform API\n'
|
||||
create_server_workflow "SCUM alpha" "$SCUM_ALPHA_ID" "$WORK_DIR/create-scum-alpha.request.json" "$WORK_DIR/create-scum-alpha.response.json"
|
||||
create_server_workflow "SCUM beta" "$SCUM_BETA_ID" "$WORK_DIR/create-scum-beta.request.json" "$WORK_DIR/create-scum-beta.response.json"
|
||||
@@ -899,10 +1235,11 @@ require_file_contains "$WORK_DIR/create-scum-beta.response.json" '"pluginId"[[:s
|
||||
require_file_contains "$WORK_DIR/create-scum-dynamic.response.json" '"pluginId"[[:space:]]*:[[:space:]]*"game.scum"'
|
||||
|
||||
SERVER_ID="$(json_id "$WORK_DIR/create-server.response.json")"
|
||||
CREATED_LOG_SESSION_SERVER_ID="$(json_id "$WORK_DIR/create-log-session-server.response.json")"
|
||||
CREATED_SCUM_ALPHA_ID="$(json_id "$WORK_DIR/create-scum-alpha.response.json")"
|
||||
CREATED_SCUM_BETA_ID="$(json_id "$WORK_DIR/create-scum-beta.response.json")"
|
||||
CREATED_SCUM_DYNAMIC_ID="$(json_id "$WORK_DIR/create-scum-dynamic.response.json")"
|
||||
if [[ "$SERVER_ID" != "$SERVER_LOCAL_ID" || "$CREATED_SCUM_ALPHA_ID" != "$SCUM_ALPHA_ID" || "$CREATED_SCUM_BETA_ID" != "$SCUM_BETA_ID" || "$CREATED_SCUM_DYNAMIC_ID" != "$SCUM_DYNAMIC_ID" ]]; then
|
||||
if [[ "$SERVER_ID" != "$SERVER_LOCAL_ID" || "$CREATED_LOG_SESSION_SERVER_ID" != "$LOG_SESSION_SERVER_ID" || "$CREATED_SCUM_ALPHA_ID" != "$SCUM_ALPHA_ID" || "$CREATED_SCUM_BETA_ID" != "$SCUM_BETA_ID" || "$CREATED_SCUM_DYNAMIC_ID" != "$SCUM_DYNAMIC_ID" ]]; then
|
||||
printf 'created server IDs do not match invocation-scoped workspace IDs\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -929,9 +1266,10 @@ curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \
|
||||
reject_forbidden_fragments "$WORK_DIR/server-runtime-binding.response.json"
|
||||
require_file_contains "$WORK_DIR/server-runtime-binding.response.json" '"status"[[:space:]]*:[[:space:]]*"complete"'
|
||||
|
||||
printf 'checking example server platform-builder action\n'
|
||||
json_get "$API_URL/server-instances/$SERVER_ID/runtime/actions" "$WORK_DIR/server-runtime-actions.response.json" "${AUTH_HEADER[@]}"
|
||||
node - "$WORK_DIR/server-runtime-actions.response.json" <<'NODE'
|
||||
if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" != "true" ]]; then
|
||||
printf 'checking example server platform-builder action\n'
|
||||
json_get "$API_URL/server-instances/$SERVER_ID/runtime/actions" "$WORK_DIR/server-runtime-actions.response.json" "${AUTH_HEADER[@]}"
|
||||
node - "$WORK_DIR/server-runtime-actions.response.json" <<'NODE'
|
||||
const fs = require("fs");
|
||||
const response = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const action = (response.actions || []).find((candidate) => candidate.key === "generate-run");
|
||||
@@ -941,7 +1279,110 @@ if (!action || action.available !== true || (action.reason || "").includes("run
|
||||
process.exit(1);
|
||||
}
|
||||
NODE
|
||||
reject_forbidden_fragments "$WORK_DIR/server-runtime-actions.response.json"
|
||||
reject_forbidden_fragments "$WORK_DIR/server-runtime-actions.response.json"
|
||||
fi
|
||||
|
||||
printf 'configuring current log session fixture on the local Run endpoint\n'
|
||||
curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \
|
||||
--data-binary "@$WORK_DIR/log-session-runtime-binding.request.json" \
|
||||
"$API_URL/server-instances/$LOG_SESSION_SERVER_ID/runtime-binding" >"$WORK_DIR/log-session-runtime-binding.response.json"
|
||||
reject_forbidden_fragments "$WORK_DIR/log-session-runtime-binding.response.json"
|
||||
require_file_contains "$WORK_DIR/log-session-runtime-binding.response.json" '"status"[[:space:]]*:[[:space:]]*"complete"'
|
||||
curl -fsS -X PUT -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" \
|
||||
--data-binary "@$WORK_DIR/log-session-deployment.request.json" \
|
||||
"$API_URL/server-instances/$LOG_SESSION_SERVER_ID/deployment" >"$WORK_DIR/log-session-deployment.response.json"
|
||||
reject_forbidden_fragments "$WORK_DIR/log-session-deployment.response.json"
|
||||
|
||||
write_server_lifecycle_request deploy "$WORK_DIR/log-session-deploy.request.json"
|
||||
json_post "$API_URL/server-instances/$LOG_SESSION_SERVER_ID/deploy" "$WORK_DIR/log-session-deploy.request.json" "$WORK_DIR/log-session-deploy.response.json" "${AUTH_HEADER[@]}"
|
||||
wait_for_job_success "$WORK_DIR/log-session-deploy.response.json" "$WORK_DIR/log-session-deploy-job.response.json" "log session deploy"
|
||||
LOG_SESSION_DEPLOY_JOB_ID="$(json_id "$WORK_DIR/log-session-deploy-job.response.json")"
|
||||
LOG_SESSION_REAL_JOB_STREAM_IDS="job.$LOG_SESSION_DEPLOY_JOB_ID.stdout,job.$LOG_SESSION_DEPLOY_JOB_ID.stderr"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-HISTORICAL-JOB-STDOUT" "$WORK_DIR/log-session-job-stdout-history.response.json" history
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-HISTORICAL-JOB-STDERR" "$WORK_DIR/log-session-job-stderr-history.response.json" history
|
||||
|
||||
printf 'seeding explicit legacy, job, and file-tail history stream metadata\n'
|
||||
create_historical_log_stream "$LOG_SESSION_LEGACY_STREAM_ID" process legacy.stdout
|
||||
create_historical_log_stream "$LOG_SESSION_JOB_STREAM_ID" process job.stdout
|
||||
create_historical_log_stream "$LOG_SESSION_FILE_STREAM_ID" file file.tail
|
||||
|
||||
printf 'starting generation A without an open terminal or SSE client\n'
|
||||
dispatch_log_session_lifecycle start "$WORK_DIR/log-session-start-a.response.json" "$WORK_DIR/log-session-start-a-job.response.json"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT" "$WORK_DIR/log-session-a-stdout-history.response.json"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-START-STDERR" "$WORK_DIR/log-session-a-stderr-history.response.json"
|
||||
LOG_SESSION_A="$(current_log_session_id)"
|
||||
if [[ -z "$LOG_SESSION_A" ]]; then
|
||||
dump_log_session_diagnostics "generation A did not expose a current session"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$LOG_SESSION_A" >"$WORK_DIR/current-log-session-a.id"
|
||||
|
||||
printf 'opening one persistent Platform SSE connection after generation A was already uploaded\n'
|
||||
: >"$LOG_SESSION_SSE_FILE"
|
||||
: >"$LOG_SESSION_SSE_ERROR_FILE"
|
||||
curl -fsS --no-buffer --max-time 240 -H "Authorization: Bearer $SESSION_ID" -H 'Accept: text/event-stream' \
|
||||
"$API_URL/server-instances/$LOG_SESSION_SERVER_ID/logs/events?historyLimit=200" \
|
||||
>"$LOG_SESSION_SSE_FILE" 2>"$LOG_SESSION_SSE_ERROR_FILE" &
|
||||
LOG_SESSION_SSE_PID="$!"
|
||||
wait_for_file_literal "initial SSE ready event" "$LOG_SESSION_SSE_FILE" 'event: ready' 60
|
||||
wait_for_file_literal "generation A stdout replay" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT" 60
|
||||
wait_for_file_literal "generation A stderr replay" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDERR" 60
|
||||
LOG_SESSION_A_FROM_SSE="$(node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" session-for-marker "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-START-STDOUT")"
|
||||
if [[ "$LOG_SESSION_A_FROM_SSE" != "$LOG_SESSION_A" ]]; then
|
||||
dump_log_session_diagnostics "generation A history and SSE selected different sessions"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'restarting Run while generation A remains alive\n'
|
||||
restart_bootstrap_run_preserving_process
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT" "$WORK_DIR/log-session-a-resume-stdout-history.response.json"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDERR" "$WORK_DIR/log-session-a-resume-stderr-history.response.json"
|
||||
wait_for_file_literal "generation A stdout after Run restart" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT" 60
|
||||
wait_for_file_literal "generation A stderr after Run restart" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDERR" 60
|
||||
LOG_SESSION_A_AFTER_RUN_RESTART="$(node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" session-for-marker "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-A-RUN-RESUME-STDOUT")"
|
||||
if [[ "$LOG_SESSION_A_AFTER_RUN_RESTART" != "$LOG_SESSION_A" || "$(current_log_session_id)" != "$LOG_SESSION_A" ]]; then
|
||||
dump_log_session_diagnostics "Run restart changed the supervised process session"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'stopping generation A and starting generation B on the same SSE connection\n'
|
||||
dispatch_log_session_lifecycle stop "$WORK_DIR/log-session-stop-a.response.json" "$WORK_DIR/log-session-stop-a-job.response.json"
|
||||
printf 'B\n' >"$LOG_SESSION_SCOPE/smoke-generation.next"
|
||||
mv "$LOG_SESSION_SCOPE/smoke-generation.next" "$LOG_SESSION_SCOPE/smoke-generation"
|
||||
rm -f "$LOG_SESSION_SCOPE/smoke-command" "$LOG_SESSION_SCOPE/smoke-last-command" "$LOG_SESSION_SCOPE/smoke-process.pid"
|
||||
dispatch_log_session_lifecycle start "$WORK_DIR/log-session-start-b.response.json" "$WORK_DIR/log-session-start-b-job.response.json"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-B-START-STDOUT" "$WORK_DIR/log-session-b-stdout-history.response.json"
|
||||
wait_for_persisted_log_marker "$LOG_SESSION_MARKER_PREFIX-B-START-STDERR" "$WORK_DIR/log-session-b-stderr-history.response.json"
|
||||
wait_for_file_literal "generation B stdout" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-B-START-STDOUT" 60
|
||||
wait_for_file_literal "generation B stderr" "$LOG_SESSION_SSE_FILE" "$LOG_SESSION_MARKER_PREFIX-B-START-STDERR" 60
|
||||
LOG_SESSION_B="$(current_log_session_id)"
|
||||
if [[ -z "$LOG_SESSION_B" || "$LOG_SESSION_B" == "$LOG_SESSION_A" ]]; then
|
||||
dump_log_session_diagnostics "generation B did not create a new current session"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$LOG_SESSION_B" >"$WORK_DIR/current-log-session-b.id"
|
||||
|
||||
node "$ROOT_DIR/scripts/local-debug/verify-current-log-session-sse.mjs" verify \
|
||||
"$LOG_SESSION_SSE_FILE" \
|
||||
"$LOG_SESSION_SERVER_ID" \
|
||||
"$LOG_SESSION_MARKER_PREFIX" \
|
||||
"$LOG_SESSION_A" \
|
||||
"$LOG_SESSION_LEGACY_STREAM_ID,$LOG_SESSION_JOB_STREAM_ID,$LOG_SESSION_FILE_STREAM_ID,$LOG_SESSION_REAL_JOB_STREAM_IDS" \
|
||||
>"$WORK_DIR/current-log-session-verification.json"
|
||||
reject_forbidden_fragments "$LOG_SESSION_SSE_FILE"
|
||||
reject_forbidden_fragments "$WORK_DIR/current-log-session-verification.json"
|
||||
dispatch_log_session_lifecycle stop "$WORK_DIR/log-session-stop-b.response.json" "$WORK_DIR/log-session-stop-b-job.response.json"
|
||||
if kill -0 "$LOG_SESSION_SSE_PID" 2>/dev/null; then
|
||||
kill "$LOG_SESSION_SSE_PID" 2>/dev/null || true
|
||||
wait "$LOG_SESSION_SSE_PID" 2>/dev/null || true
|
||||
fi
|
||||
LOG_SESSION_SSE_PID=""
|
||||
printf 'current supervised log session smoke passed; evidence: %s\n' "$WORK_DIR/current-log-session-verification.json"
|
||||
if [[ "${LOCAL_DEBUG_LOG_SESSION_SMOKE_ONLY:-false}" == "true" ]]; then
|
||||
printf 'focused current supervised log session smoke passed\n'
|
||||
printf 'evidence directory: %s\n' "$WORK_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printf 'generating host-native example Run through platform Docker builder\n'
|
||||
curl -fsS -H 'Content-Type: application/json' "${AUTH_HEADER[@]}" --data-binary "@$WORK_DIR/server-run-generate.request.json" "$API_URL/server-instances/$SERVER_ID/run/generate" >"$WORK_DIR/server-run-generate.response.json"
|
||||
|
||||
Reference in New Issue
Block a user