Skip to content

Write HTML. Render video. Built for agents.

fix(cli): resume a cloud download part when its connection drops (#5873)

Miguel Ángelcea0bad79311
diff --git a/packages/cli/src/cloud/download.test.ts b/packages/cli/src/cloud/download.test.ts
index ff614546c3..73b50f40cc 100644
--- a/packages/cli/src/cloud/download.test.ts
+++ b/packages/cli/src/cloud/download.test.ts
@@ -40,7 +40,8 @@ interface RangeServer {
async function serveBytes(
body: Buffer,
- options: { honourRange: boolean; failRangeStart?: number },
+ // `drop` cuts the first `times` responses whose range covers byte `at`, right before it.
+ options: { honourRange: boolean; drop?: { at: number; times: number } },
): Promise<RangeServer> {
const ranges: (string | undefined)[] = [];
const server: Server = createServer((req, res) => {
@@ -60,12 +61,17 @@ async function serveBytes(
"content-length": end - start + 1,
"content-range": `bytes ${start}-${end}/${body.length}`,
});
- if (start === options.failRangeStart) {
- res.write(body.subarray(start, start + 1024), () => res.destroy());
+ const drop = options.drop;
+ if (drop && drop.times > 0 && start <= drop.at && drop.at <= end) {
+ drop.times--;
+ res.write(body.subarray(start, drop.at), () => res.destroy());
return;
}
res.end(body.subarray(start, end + 1));
});
+ // Windows completes a large loopback write at once, so the 5 s keep-alive idle clock
+ // started before a slow reader finished and closed the socket under it.
+ server.keepAliveTimeout = 0;
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;
return {
@@ -319,13 +325,26 @@ describe("cloud/download", () => {
expect(server.ranges).toEqual([`bytes=0-${8 * MiB - 1}`]);
});
- it("keeps the previous output and leaves no staged file when one part fails", async () => {
+ it("resumes a range whose connection drops mid-body from the bytes it already has", async () => {
const body = patternBytes(50 * MiB);
- const server = await serve(body, { honourRange: true, failRangeStart: 8 * MiB });
+ const at = 8 * MiB + 1024;
+ const server = await serve(body, { honourRange: true, drop: { at, times: 2 } });
+ const dest = join(dir, "out.mp4");
+ const result = await downloadToFile(server.url, dest);
+ expect(result.bytes).toBe(body.length);
+ expect(readFileSync(dest).equals(body)).toBe(true);
+ expect(server.ranges.filter((range) => range?.startsWith(`bytes=${at}-`))).toHaveLength(2);
+ });
+
+ it("keeps the previous output and leaves no staged file when one part keeps dropping", async () => {
+ const body = patternBytes(50 * MiB);
+ const server = await serve(body, {
+ honourRange: true,
+ drop: { at: 8 * MiB + 1024, times: Infinity },
+ });
const dest = join(dir, "out.mp4");
writeFileSync(dest, "previous render");
- await expect(downloadToFile(server.url, dest)).rejects.toThrow();
- expect(server.ranges.length).toBeGreaterThan(1);
+ await expect(downloadToFile(server.url, dest)).rejects.toThrow(/dropped 4 times/);
expect(readFileSync(dest, "utf8")).toBe("previous render");
expect(readdirSync(dir)).toEqual(["out.mp4"]);
});
diff --git a/packages/cli/src/cloud/download.ts b/packages/cli/src/cloud/download.ts
index 07acc176df..3aa60ae624 100644
--- a/packages/cli/src/cloud/download.ts
+++ b/packages/cli/src/cloud/download.ts
@@ -16,12 +16,15 @@
import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync, statSync } from "node:fs";
import { open } from "node:fs/promises";
import { dirname, join } from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
import { resolveWritePath } from "@hyperframes/core/atomic-file";
// The first request asks for PART_BYTES; a larger file is split into up to
// CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.
const PART_BYTES = 8 * 1024 * 1024;
const CONNECTIONS = 8;
+// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s.
+const RETRIES = 3;
export interface DownloadOptions {
signal?: AbortSignal;
@@ -79,39 +82,61 @@ export async function downloadToFile(
let firstError: unknown;
try {
const file = await open(stagedFile, "wx");
- const writePart = async (res: Response, start: number, length: number | undefined) => {
- let written = 0;
+ // Writes the body at `part.at` and advances it, so a dropped range resumes there.
+ const pump = async (res: Response, part: { at: number }) => {
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
- await file.write(chunk, 0, chunk.byteLength, start + written);
- written += chunk.byteLength;
+ await file.write(chunk, 0, chunk.byteLength, part.at);
+ part.at += chunk.byteLength;
bytes += chunk.byteLength;
signal.throwIfAborted();
options.onProgress?.(bytes, total);
}
- if (length !== undefined && written !== length) {
- throw new Error(
- `Truncated download: got ${written} bytes, expected ${length} (content-length). ` +
- `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
- );
+ };
+ const fetchRange = async (start: number, end: number, res?: Response) => {
+ const part = { at: start };
+ for (let attempt = 0; ; attempt++) {
+ try {
+ res ??= await get(`bytes=${part.at}-${end}`);
+ assertOk(res, url);
+ if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
+ await pump(res, part);
+ if (part.at > end) return;
+ throw new Error(`connection closed at byte ${part.at}`);
+ } catch (error) {
+ if (signal.aborted) throw error;
+ if (attempt === RETRIES) {
+ throw new Error(
+ `Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`,
+ { cause: error },
+ );
+ }
+ res?.body?.cancel().catch(() => {});
+ res = undefined;
+ await sleep(250 * 2 ** attempt, undefined, { signal });
+ }
}
};
const parts: Promise<void>[] = [];
if (ranged && total !== undefined) {
- parts.push(writePart(first, 0, Math.min(PART_BYTES, total)));
+ parts.push(fetchRange(0, Math.min(PART_BYTES, total) - 1, first));
const rest = total - PART_BYTES;
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / PART_BYTES)));
for (let start = PART_BYTES; start < total; start += size) {
- const end = Math.min(start + size, total) - 1;
- parts.push(
- get(`bytes=${start}-${end}`).then((res) => {
- assertOk(res, url);
- if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
- return writePart(res, start, end - start + 1);
- }),
- );
+ parts.push(fetchRange(start, Math.min(start + size, total) - 1));
}
} else {
- parts.push(writePart(first, 0, total));
+ // No Range support means nothing to resume from: fail fast.
+ const part = { at: 0 };
+ parts.push(
+ pump(first, part).then(() => {
+ if (total !== undefined && part.at !== total) {
+ throw new Error(
+ `Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` +
+ `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
+ );
+ }
+ }),
+ );
}
await Promise.all(
parts.map((part) =>