Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): resume a cloud download part when its connection drops (#5873)
Miguel Ángelcea0bad79311
diff --git a/packages/cli/src/cloud/download.test.ts b/packages/cli/src/cloud/download.test.tsindex ff614546c3..73b50f40cc 100644--- a/packages/cli/src/cloud/download.test.ts+++ b/packages/cli/src/cloud/download.test.ts@@ -40,7 +40,8 @@ interface RangeServer {async function serveBytes(body: Buffer,- options: { honourRange: boolean; failRangeStart?: number },+ // `drop` cuts the first `times` responses whose range covers byte `at`, right before it.+ options: { honourRange: boolean; drop?: { at: number; times: number } },): Promise<RangeServer> {const ranges: (string | undefined)[] = [];const server: Server = createServer((req, res) => {@@ -60,12 +61,17 @@ async function serveBytes("content-length": end - start + 1,"content-range": `bytes ${start}-${end}/${body.length}`,});- if (start === options.failRangeStart) {- res.write(body.subarray(start, start + 1024), () => res.destroy());+ const drop = options.drop;+ if (drop && drop.times > 0 && start <= drop.at && drop.at <= end) {+ drop.times--;+ res.write(body.subarray(start, drop.at), () => res.destroy());return;}res.end(body.subarray(start, end + 1));});+ // Windows completes a large loopback write at once, so the 5 s keep-alive idle clock+ // started before a slow reader finished and closed the socket under it.+ server.keepAliveTimeout = 0;await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));const { port } = server.address() as AddressInfo;return {@@ -319,13 +325,26 @@ describe("cloud/download", () => {expect(server.ranges).toEqual([`bytes=0-${8 * MiB - 1}`]);});- it("keeps the previous output and leaves no staged file when one part fails", async () => {+ it("resumes a range whose connection drops mid-body from the bytes it already has", async () => {const body = patternBytes(50 * MiB);- const server = await serve(body, { honourRange: true, failRangeStart: 8 * MiB });+ const at = 8 * MiB + 1024;+ const server = await serve(body, { honourRange: true, drop: { at, times: 2 } });+ const dest = join(dir, "out.mp4");+ const result = await downloadToFile(server.url, dest);+ expect(result.bytes).toBe(body.length);+ expect(readFileSync(dest).equals(body)).toBe(true);+ expect(server.ranges.filter((range) => range?.startsWith(`bytes=${at}-`))).toHaveLength(2);+ });++ it("keeps the previous output and leaves no staged file when one part keeps dropping", async () => {+ const body = patternBytes(50 * MiB);+ const server = await serve(body, {+ honourRange: true,+ drop: { at: 8 * MiB + 1024, times: Infinity },+ });const dest = join(dir, "out.mp4");writeFileSync(dest, "previous render");- await expect(downloadToFile(server.url, dest)).rejects.toThrow();- expect(server.ranges.length).toBeGreaterThan(1);+ await expect(downloadToFile(server.url, dest)).rejects.toThrow(/dropped 4 times/);expect(readFileSync(dest, "utf8")).toBe("previous render");expect(readdirSync(dir)).toEqual(["out.mp4"]);});diff --git a/packages/cli/src/cloud/download.ts b/packages/cli/src/cloud/download.tsindex 07acc176df..3aa60ae624 100644--- a/packages/cli/src/cloud/download.ts+++ b/packages/cli/src/cloud/download.ts@@ -16,12 +16,15 @@import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync, statSync } from "node:fs";import { open } from "node:fs/promises";import { dirname, join } from "node:path";+import { setTimeout as sleep } from "node:timers/promises";import { resolveWritePath } from "@hyperframes/core/atomic-file";// The first request asks for PART_BYTES; a larger file is split into up to// CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.const PART_BYTES = 8 * 1024 * 1024;const CONNECTIONS = 8;+// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s.+const RETRIES = 3;export interface DownloadOptions {signal?: AbortSignal;@@ -79,39 +82,61 @@ export async function downloadToFile(let firstError: unknown;try {const file = await open(stagedFile, "wx");- const writePart = async (res: Response, start: number, length: number | undefined) => {- let written = 0;+ // Writes the body at `part.at` and advances it, so a dropped range resumes there.+ const pump = async (res: Response, part: { at: number }) => {for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {- await file.write(chunk, 0, chunk.byteLength, start + written);- written += chunk.byteLength;+ await file.write(chunk, 0, chunk.byteLength, part.at);+ part.at += chunk.byteLength;bytes += chunk.byteLength;signal.throwIfAborted();options.onProgress?.(bytes, total);}- if (length !== undefined && written !== length) {- throw new Error(- `Truncated download: got ${written} bytes, expected ${length} (content-length). ` +- `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,- );+ };+ const fetchRange = async (start: number, end: number, res?: Response) => {+ const part = { at: start };+ for (let attempt = 0; ; attempt++) {+ try {+ res ??= await get(`bytes=${part.at}-${end}`);+ assertOk(res, url);+ if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);+ await pump(res, part);+ if (part.at > end) return;+ throw new Error(`connection closed at byte ${part.at}`);+ } catch (error) {+ if (signal.aborted) throw error;+ if (attempt === RETRIES) {+ throw new Error(+ `Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`,+ { cause: error },+ );+ }+ res?.body?.cancel().catch(() => {});+ res = undefined;+ await sleep(250 * 2 ** attempt, undefined, { signal });+ }}};const parts: Promise<void>[] = [];if (ranged && total !== undefined) {- parts.push(writePart(first, 0, Math.min(PART_BYTES, total)));+ parts.push(fetchRange(0, Math.min(PART_BYTES, total) - 1, first));const rest = total - PART_BYTES;const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / PART_BYTES)));for (let start = PART_BYTES; start < total; start += size) {- const end = Math.min(start + size, total) - 1;- parts.push(- get(`bytes=${start}-${end}`).then((res) => {- assertOk(res, url);- if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);- return writePart(res, start, end - start + 1);- }),- );+ parts.push(fetchRange(start, Math.min(start + size, total) - 1));}} else {- parts.push(writePart(first, 0, total));+ // No Range support means nothing to resume from: fail fast.+ const part = { at: 0 };+ parts.push(+ pump(first, part).then(() => {+ if (total !== undefined && part.at !== total) {+ throw new Error(+ `Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` ++ `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,+ );+ }+ }),+ );}await Promise.all(parts.map((part) =>