admin343/browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed up artifact downloads

Browse Source
This commit is contained in:
npc0-hue
2026-09-03 11:27:19 +08:00
parent a82f1ff01a
commit 6311eb72ba
10 changed files with 142 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/api/artifact_download_handlers_test.go
+22
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
@@ -55,6 +56,27 @@ func TestArtifactDownloadAPIWorkflowIsPlatformMediated(t *testing.T) {
}
}
func TestArtifactContentWithoutRangeReturnsWholePayload(t *testing.T) {
router := newTestRouter()
adminSession := createAdminSession(t, router)
postJSON[dto.GamePluginResponse](t, router, "/api/v1/game-plugins", validGamePluginRequest())
hello := decodeBody[dto.RunControlHelloResponse](t, performRunControlHello(t, router, artifactDownloadHelloRequest()))
postJSONWithAuth[dto.ServerInstanceResponse](t, router, "/api/v1/server-instances", dto.ServerInstanceCreateRequest{ID: "server-download-full", PluginID: "server.scum", RunEndpointID: "run-local", Name: "Download Full"}, adminSession)
postJSON[dto.JobResponse](t, router, "/api/v1/jobs", dto.JobCreateRequest{ID: "job-download-full", ServerInstanceID: "server-download-full", RunEndpointID: "run-local", Capability: "process.start", IdempotencyKey: "idem-download-full"})
payload := bytes.Repeat([]byte("full-download-payload-"), (validator.MaxArtifactDownloadBytes/len("full-download-payload-"))+2)
uploadCompletedArtifact(t, router, hello.SessionToken, "artifact-download-full", "job-download-full", payload, validator.MaxArtifactChunkBytes)
contentRecorder := requestWithAuth(t, router, http.MethodGet, "/api/v1/artifacts/artifact-download-full/content", "", adminSession)
assertStatus(t, contentRecorder, http.StatusOK)
if got := contentRecorder.Body.Bytes(); !bytes.Equal(got, payload) {
t.Fatalf("expected whole payload length %d, got %d", len(payload), len(got))
}
if contentRecorder.Header().Get("Content-Range") != "" || contentRecorder.Header().Get("Content-Length") != strconv.Itoa(len(payload)) {
t.Fatalf("expected full content headers, got %+v", contentRecorder.Header())
}
}
func TestArtifactDownloadAPIDeniesUnavailableAndUnauthorizedArtifacts(t *testing.T) {
router := newTestRouter()
adminSession := createAdminSession(t, router)
platform/api/routes.md
+1 -1
View File
@@ -220,7 +220,7 @@ Artifact/file transfer is lower priority than control, job lifecycle metadata, a
- `GET /api/v1/artifacts/{id}`: returns authorized artifact metadata for the current bearer session.
- `POST /api/v1/artifacts/{id}/download`: returns `ArtifactDownloadReferenceResponse` with filename, content type, size, checksum, expiry, supported chunk size, and a platform-owned `downloadUrl`.
- `GET /api/v1/artifacts/{id}/content`: returns a bounded byte range using `offset`/`limit` query parameters or a `Range: bytes=start-end` header. Responses include `Content-Length`, `Accept-Ranges`, optional `Content-Range`, `X-Artifact-Checksum`, `X-Artifact-Content-Checksum`, and `X-Artifact-Storage` headers.
- `GET /api/v1/artifacts/{id}/content`: streams the full artifact when no range is supplied, or returns a bounded byte range using `offset`/`limit` query parameters or a `Range: bytes=start-end` header. Responses include `Content-Length`, `Accept-Ranges`, optional `Content-Range`, `X-Artifact-Checksum`, `X-Artifact-Content-Checksum`, and `X-Artifact-Storage` headers.
Browser artifact downloads require an available artifact plus user access to the owning job/server context. Platform/plugin-owned artifacts are limited to platform administrators until a future storage policy adds narrower ownership. Current content reads use the private durable artifact body store; external object storage adapters are deferred behind the same service contract. Browser and plugin pages receive only platform routes and integrity metadata, never raw storage backend URLs, host paths, direct run sockets, run tokens, bearer tokens, or storage credentials.
platform/service/artifact_download.go
+4
View File
@@ -76,8 +76,12 @@ func (svc *CoreService) ReadArtifactContentForSession(sessionID string, request
}
limit := request.Limit
if limit == 0 {
if request.Offset == 0 {
limit = int(artifact.SizeBytes)
} else {
limit = validator.MaxArtifactDownloadBytes
}
}
if request.Offset > artifact.SizeBytes {
return domain.ArtifactContent{}, validationError("artifact range exceeds metadata")
}
platform/validator/artifact_download.go
+6 -2
View File
@@ -94,8 +94,12 @@ func ValidateArtifactContent(content domain.ArtifactContent) error {
if content.TotalSizeBytes <= 0 {
violations = append(violations, "totalSizeBytes must be positive")
}
if content.SizeBytes > MaxArtifactDownloadBytes {
violations = append(violations, fmt.Sprintf("sizeBytes must not exceed %d", MaxArtifactDownloadBytes))
maxContentBytes := int64(MaxArtifactDownloadBytes)
if !content.Partial {
maxContentBytes = MaxArtifactBytes
}
if content.SizeBytes > maxContentBytes {
violations = append(violations, fmt.Sprintf("sizeBytes must not exceed %d", maxContentBytes))
}
if int64(len(content.Payload)) != content.SizeBytes {
violations = append(violations, "payload size must match sizeBytes")
platform_web/api/client.test.ts
+15 -1
View File
@@ -352,6 +352,19 @@ describe("PlatformApiClient AI providers", () => {
}
});
}
if (url.endsWith("/api/v1/artifacts/artifact-1/content")) {
return new Response(new TextEncoder().encode("artifact full body").buffer, {
status: 200,
headers: {
"Content-Type": "application/octet-stream",
"Content-Length": "18",
"X-Artifact-Id": artifact.id,
"X-Artifact-Checksum": artifact.checksum,
"X-Artifact-Content-Checksum": "sha256:fullchecksum",
"X-Artifact-Storage": "platform-memory-transfer-session"
}
});
}
if (url.endsWith("/api/v1/server-instances/workflows/create") && init?.method === "POST") {
return jsonResponse({ accepted: true, action: "create", instance: { ...server, state: "installing" }, job: { ...job, capability: "process.install" } });
}
@@ -611,6 +624,7 @@ describe("PlatformApiClient AI providers", () => {
await expect(client.listArtifacts({ ownerKind: "job", ownerId: job.id, state: "available" })).resolves.toMatchObject({ count: 1, items: [{ id: artifact.id }] });
await expect(client.openArtifactDownload(artifact.id)).resolves.toMatchObject({ downloadUrl: "/api/v1/artifacts/artifact-1/content", rangeSupported: true });
await expect(client.readArtifactContent(artifact.id, 0, 8)).resolves.toMatchObject({ contentLength: 8, contentRange: "bytes 0-7/18", checksum: artifact.checksum });
await expect(client.downloadArtifactContent(artifact.id).then((response) => response.arrayBuffer())).resolves.toHaveProperty("byteLength", 18);
await expect(client.createServerWorkflow({ id: "server-2", pluginId: plugin.id, name: "Server 2", idempotencyKey: "idem-create" })).resolves.toMatchObject({
action: "create"
});
@@ -659,7 +673,7 @@ describe("PlatformApiClient AI providers", () => {
client.invokeAI({ requestId: "ai-1", serverInstanceId: server.id, purpose: "config.suggest", prompt: "Tune PVP safely", currentConfig: "server.name=Example Survival #1\n" })
).resolves.toMatchObject({ status: "ok", usage: { mocked: true }, configRecommendation: { diffSummary: "review required" } });
expect(fetchMock).toHaveBeenCalledTimes(45);
expect(fetchMock).toHaveBeenCalledTimes(46);
});
it("normalizes server file workspace null arrays from older platform responses", async () => {
platform_web/api/client.ts
+13
View File
@@ -298,6 +298,19 @@ export class PlatformApiClient {
};
}
async downloadArtifactContent(id: string): Promise<Response> {
const headers = new Headers();
const sessionToken = this.sessionTokenProvider();
if (sessionToken) {
headers.set("Authorization", `Bearer ${sessionToken}`);
}
const response = await fetch(`${this.baseUrl}/artifacts/${encodeURIComponent(id)}/content`, { headers, credentials: "same-origin" });
if (!response.ok) {
throw await responseError(response);
}
return response;
}
async getJob(id: string): Promise<JobResponse> {
return this.request<JobResponse>(`/jobs/${encodeURIComponent(id)}`);
}
platform_web/api/contracts.md
+1 -1
View File
@@ -29,7 +29,7 @@ Normal browser login uses the platform's HttpOnly SameSite cookie and `credentia
- Game-specific pages use the scoped `plugin-data` collection API and declared plugin bridge machine actions; Platform does not expose game-specific projection or workflow clients.
- `dispatchFileOperation` posts `FileOperationDispatchRequest` to `/file-operations/dispatch` using logical file keys and scoped refs rather than raw host paths; it remains the low-level compatibility dispatch for file work.
- `getServerFileWorkspace`, `browseServerFiles`, `listServerFiles`, `refreshServerFiles`, `readServerFile`, `getServerFileReadSnapshot`, `writeServerFile`, `uploadServerFile`, and `prepareServerFileDownload` power the first-party server-detail file manager. The page renders a generic server-root entry and uses `browseServerFiles` as the live directory path for open, refresh, and search actions; compatibility `list`/`refresh` clients remain available for older flows. File reads use `files.read`, saves use `files.write`, and browser uploads stage server-instance artifacts before Run pulls input chunks on the dedicated file-transfer channel. Server detail may call only these server-file APIs plus the encapsulated download helper; it must not call raw artifact-transfer methods directly.
- `listArtifacts`, `openArtifactDownload`, and `readArtifactContent` use platform artifact routes for available job/server artifacts. Browser reads are chunked through `/artifacts/{id}/content` and must render only safe filenames, checksums, progress, and platform storage behavior.
- `listArtifacts`, `openArtifactDownload`, `downloadArtifactContent`, and `readArtifactContent` use platform artifact routes for available job/server artifacts. Browser downloads stream the full body through `/artifacts/{id}/content`; explicit range reads may still use bounded `offset`/`limit` chunks and must render only safe filenames, checksums, progress, and platform storage behavior.
- `authorizePluginBridge` posts `PluginBridgeAuthorizeRequest` to `/plugin-bridge/authorize` for preflight decisions.
- `executePluginBridge` posts `PluginBridgeExecuteRequest` to `/plugin-bridge/execute` from host-owned bridge dispatch utilities only. Plugin pages receive typed `PluginBridgeExecuteResponse` envelopes and never receive the platform API client, bearer token, raw provider key, run socket, host path, or storage credential.
- `invokeAI` posts `AIInvocationRequest` to `/ai/invocations` for platform-mediated AI assistance. Responses carry redacted recommendations, usage metadata, optional reviewable config suggestions, and safe errors; they must not include provider base URLs, key refs, raw keys, or direct provider transport details.
platform_web/pages/ServersPage.tsx
+2 -2
View File
@@ -307,7 +307,7 @@ export function ServersPage({ session, operations, onNavigate }: PageComponentPr
execute: async () => {
if (action === "download-run") {
const reference = await platformApiClient.downloadLatestRunDistribution(instance.id);
await downloadArtifactReference(reference, (artifactId, offset, limit) => platformApiClient.readArtifactContent(artifactId, offset, limit));
await downloadArtifactReference(reference, (artifactId) => platformApiClient.downloadArtifactContent(artifactId));
return `run 下载已开始，artifact ${reference.artifactId}，文件 ${safeArtifactFilename(reference.filename)}`;
}
if (action === "push-run-update") {
@@ -354,7 +354,7 @@ export function ServersPage({ session, operations, onNavigate }: PageComponentPr
executeStageIndex: 1,
execute: async () => {
const reference = await platformApiClient.openArtifactDownload(artifact.artifactId);
await downloadArtifactReference(reference, (artifactId, offset, limit) => platformApiClient.readArtifactContent(artifactId, offset, limit));
await downloadArtifactReference(reference, (artifactId) => platformApiClient.downloadArtifactContent(artifactId));
return `run 下载已开始，文件 ${safeArtifactFilename(reference.filename)}`;
}
});
platform_web/utils/artifactTransfer.ts
+73 -17
View File
@@ -1,49 +1,105 @@
import type { ArtifactContentChunk, ArtifactDownloadReferenceResponse } from "../api/types";
import type { ArtifactDownloadReferenceResponse } from "../api/types";
type ArtifactContentLoader = (artifactId: string) => Promise<Response>;
type DownloadChunk = ArrayBuffer;
export async function downloadArtifactReference(
reference: ArtifactDownloadReferenceResponse,
readContent: (artifactId: string, offset: number, limit?: number) => Promise<ArtifactContentChunk>,
openContent: ArtifactContentLoader,
onProgress?: (progress: number) => void
) {
const chunks: ArrayBuffer[] = [];
let offset = 0;
const response = await openContent(reference.artifactId);
const writer = await createArtifactFileWriter(reference.filename, reference.contentType);
const totalBytes = Number(response.headers.get("Content-Length") ?? reference.sizeBytes);
let downloadedBytes = 0;
onProgress?.(0);
while (offset < reference.sizeBytes) {
const chunk = await readContent(reference.artifactId, offset, reference.chunkSizeBytes);
chunks.push(chunk.payload);
offset += chunk.payload.byteLength;
onProgress?.(Math.min(100, Math.round((offset / reference.sizeBytes) * 100)));
if (chunk.payload.byteLength === 0) {
break;
try {
const reader = response.body?.getReader();
if (!reader) {
const payload = await response.arrayBuffer();
await writer.write(payload);
downloadedBytes = payload.byteLength;
onProgress?.(progressPercent(downloadedBytes, totalBytes));
} else {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const chunk = copyStreamChunk(value);
await writer.write(chunk);
downloadedBytes += chunk.byteLength;
onProgress?.(progressPercent(downloadedBytes, totalBytes));
}
}
await writer.close();
} catch (error) {
await writer.abort();
throw error;
}
openArtifactBlob(reference, chunks);
onProgress?.(100);
}
export function openArtifactBlob(reference: ArtifactDownloadReferenceResponse, chunks: ArrayBuffer[]) {
export function openArtifactBlob(reference: ArtifactDownloadReferenceResponse, chunks: DownloadChunk[]) {
saveArtifactBlob(new Blob(chunks, { type: reference.contentType }), reference.filename);
}
export function saveArtifactBlob(blob: Blob, filename: string) {
if (typeof document === "undefined" || typeof URL === "undefined") {
return;
}
const blob = new Blob(chunks, { type: reference.contentType });
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = safeArtifactFilename(reference.filename);
anchor.download = safeArtifactFilename(filename);
anchor.rel = "noopener";
document.body.append(anchor);
anchor.click();
anchor.remove();
URL.revokeObjectURL(url);
window.setTimeout(() => URL.revokeObjectURL(url), 1000);
}
export function safeArtifactFilename(filename: string): string {
const cleaned = filename.replace(/[\\/]/g, "").trim();
const cleaned = filename.replace(/[\\/:*?"<>|]+/g, "-").trim();
return cleaned || "artifact.bin";
}
interface ArtifactFileWriter {
write(data: DownloadChunk): Promise<void>;
close(): Promise<void>;
abort(): Promise<void>;
}
async function createArtifactFileWriter(filename: string, contentType: string): Promise<ArtifactFileWriter> {
const picker = typeof window === "undefined" ? undefined : (window as Window & { showSaveFilePicker?: (options?: { suggestedName?: string }) => Promise<{ createWritable(): Promise<FileSystemWritableFileStream> }> }).showSaveFilePicker;
if (picker) {
const handle = await picker({ suggestedName: safeArtifactFilename(filename) });
const writable = await handle.createWritable();
return {
write: async (data) => { await writable.write(data); },
close: async () => { await writable.close(); },
abort: async () => { await writable.abort(); }
};
}
const chunks: DownloadChunk[] = [];
return {
write: async (data) => { chunks.push(data); },
close: async () => { saveArtifactBlob(new Blob(chunks, { type: contentType }), filename); },
abort: async () => { chunks.length = 0; }
};
}
function progressPercent(downloadedBytes: number, totalBytes: number): number {
if (!Number.isFinite(totalBytes) || totalBytes <= 0) return 0;
return Math.min(100, Math.round((downloadedBytes / totalBytes) * 100));
}
function copyStreamChunk(value: Uint8Array): ArrayBuffer {
const copy = new Uint8Array(value.byteLength);
copy.set(value);
return copy.buffer;
}
export function safeArtifactError(error: unknown): string {
const message = error instanceof Error ? error.message : "制品传输失败";
return message.replace(/\/Users\/[^\s]+/g, "[path]").replace(/Bearer\s+[^\s]+/gi, "[token]").replace(/sk-[A-Za-z0-9_-]+/g, "[secret]");
platform_web/utils/serverFileTransfer.ts
+4 -58
View File
@@ -1,72 +1,18 @@
import type { PlatformApiClient } from "../api/client";
import type { ServerFileDownloadResponse } from "../api/types";
import { downloadArtifactReference, safeArtifactFilename, saveArtifactBlob } from "./artifactTransfer";
export async function downloadServerFileResult(client: PlatformApiClient, result: ServerFileDownloadResponse): Promise<string> {
if (result.status !== "ready") {
return result.reason ?? "文件读取任务已派发，完成后可再次下载。";
}
if (result.content !== undefined) {
saveBlob(new Blob([result.content], { type: result.contentType ?? "text/plain;charset=utf-8" }), result.filename || "server-file.txt");
saveArtifactBlob(new Blob([result.content], { type: result.contentType ?? "text/plain;charset=utf-8" }), result.filename || "server-file.txt");
return `下载已开始：${result.filename || result.key}`;
}
if (!result.artifact) {
return "文件内容尚未可用。";
}
const writer = await createFileWriter(result.filename || result.artifact.filename);
let offset = 0;
const chunkSize = Math.max(1, result.artifact.chunkSizeBytes || 1024 * 1024);
try {
while (offset < result.artifact.sizeBytes) {
const chunk = await client.readArtifactContent(result.artifact.artifactId, offset, Math.min(chunkSize, result.artifact.sizeBytes - offset));
if (chunk.contentLength <= 0) throw new Error("文件下载返回空数据");
await writer.write(chunk.payload);
offset += chunk.contentLength;
}
await writer.close();
} catch (error) {
await writer.abort();
throw error;
}
return `下载已开始：${result.filename || result.artifact.filename}`;
}
interface FileWriter {
write(data: ArrayBuffer): Promise<void>;
close(): Promise<void>;
abort(): Promise<void>;
}
async function createFileWriter(filename: string): Promise<FileWriter> {
const picker = (window as Window & { showSaveFilePicker?: (options?: { suggestedName?: string }) => Promise<{ createWritable(): Promise<FileSystemWritableFileStream> }> }).showSaveFilePicker;
if (picker) {
const handle = await picker({ suggestedName: safeFilename(filename) });
const writable = await handle.createWritable();
return {
write: async (data) => { await writable.write(data); },
close: async () => { await writable.close(); },
abort: async () => { await writable.abort(); }
};
}
const chunks: ArrayBuffer[] = [];
return {
write: async (data) => { chunks.push(data); },
close: async () => { saveBlob(new Blob(chunks, { type: "application/octet-stream" }), safeFilename(filename)); },
abort: async () => { chunks.length = 0; }
};
}
function saveBlob(blob: Blob, filename: string) {
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = safeFilename(filename);
document.body.appendChild(anchor);
anchor.click();
anchor.remove();
window.setTimeout(() => URL.revokeObjectURL(url), 1000);
}
function safeFilename(filename: string): string {
const cleaned = filename.replace(/[\\/:*?"<>|]+/g, "-").trim();
return cleaned || "server-file.txt";
await downloadArtifactReference(result.artifact, (artifactId) => client.downloadArtifactContent(artifactId));
return `下载已开始：${safeArtifactFilename(result.filename || result.artifact.filename)}`;
}