Skip to content

Write HTML. Render video. Built for agents.

feat(cli): cloud render uploads heavy media as HeyGen assets (#5931)

Miguel Ángelb306438e8b5e
diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc
index 29a8930702..05953890e9 100644
--- a/.fallowrc.jsonc
+++ b/.fallowrc.jsonc
@@ -5,7 +5,7 @@
"packages/studio/src/components/ui/index.ts",
"packages/producer/src/**/*.test.ts",
// Studio's browser-journey helpers are plain .mjs under tests/e2e; their unit tests live beside them.
- "packages/studio/tests/e2e/*.test.mjs",
+ "packages/studio/tests/e2e/**/*.test.mjs",
"packages/aws-lambda/src/**/*.test.ts",
"packages/gcp-cloud-run/src/**/*.test.ts",
"packages/gcp-cloud-run/terraform/*.test.ts",
@@ -67,7 +67,7 @@
// Test fixture worker, spawned by path via the pools' workerEntryPath
// option from the crash-recovery tests; has no import-graph referrer.
"packages/producer/src/services/__fixtures__/crashOnMessageWorker.mjs",
- "scripts/*.{ts,mjs,js}",
+ "scripts/**/*.{ts,mjs,js}",
"scripts/*/run.mjs",
// Keyframe UI components — wired dynamically via EaseCurveSection/MotionPanel.
"packages/studio/src/components/editor/KeyframeDiamond.tsx",
@@ -255,7 +255,7 @@
// CLI command files: every command exports a const `examples` per the
// convention documented in CLAUDE.md. This is a namespace barrel, not a
// collision.
- { "file": "packages/cli/src/commands/*.ts", "exports": ["examples"] },
+ { "file": "packages/cli/src/commands/**/*.ts", "exports": ["examples"] },
// Options type for the render-queue hook's public startRender callback —
// consumed structurally by callers (StudioRightPanels), not by import.
{
diff --git a/docs/deploy/cloud.mdx b/docs/deploy/cloud.mdx
index 734559cd41..e0e93cb735 100644
--- a/docs/deploy/cloud.mdx
+++ b/docs/deploy/cloud.mdx
@@ -59,7 +59,7 @@ hyperframes cloud render --resolution 4k
## Check what will upload
-Cloud project archives have a 200 MB limit. Inspect the archive before starting a render:
+Media files of 5 MB or more that the project names by path upload as HeyGen assets and are referenced by public, unsigned URL, so large footage and audio do not count against the 200 MB project archive limit. Uploads are cached, so re-rendering uploads only new or changed media. Media is streamed from disk as it uploads, so a large file never has to fit in memory; editing one while the command is preparing it stops the run. Inspect what will upload before starting a render:
```bash
hyperframes cloud render --dry-run
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index dcfce67382..6377777684 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -1803,11 +1803,28 @@ match. Those rules use gitignore syntax and apply to `hyperframes publish` too.
Keep them narrow — a dynamically selected asset may have no static reference to
protect it.
-The zip has to come in under 200 MB — the direct-upload cap. Use `--dry-run`
-first to see the compressed size and the largest included files, without
+Media files of 5 MB or more that the project names by path go up as HeyGen
+assets before zipping, and the zip's copy of each HTML, CSS, script and JSON
+file points at the asset URL instead; your project files are not changed.
+Asset URLs are public and unsigned: anyone who has one can fetch the
+media. Uploads are cached by content, so a re-render uploads only what changed.
+Media the asset API does not take as is is converted on your machine first,
+which needs ffmpeg: video (for example MKV, or over 200 MB) and audio (for
+example FLAC or M4A) at visually lossless quality, still images such as AVIF to
+PNG, and animated GIFs of 5 MB or more in the entry composition to the same
+seekable video the renderer makes from them. A video or audio file that has to
+be re-encoded is first cut to the parts the entry composition plays (plus half
+a second either side), and the composition's media starts shift to match, so a
+long source fits. Media inside sub-compositions, looping clips, clips with
+`data-var-src` and files a script names keep their full length. A file no
+plain string names, such as one a script builds a path to or one picked through
+`--variables`, stays in the zip, as do animated GIFs inside sub-compositions.
+
+The zip has to come in under 200 MB — the direct-upload cap — or the command
+stops before uploading it and names the largest files left. Use `--dry-run`
+first to see which media would upload and the compressed size, without
authenticating, uploading, or starting anything. If you are over, add only
-paths you have verified are unneeded to `.hyperframesignore`, or pre-host the
-large media and reference it by URL.
+paths you have verified are unneeded to `.hyperframesignore`.
Render parameters mirror local `hyperframes render` where they overlap:
diff --git a/packages/cli/src/cloud/errors.test.ts b/packages/cli/src/cloud/errors.test.ts
index 6155de8ccb..7507e4aabe 100644
--- a/packages/cli/src/cloud/errors.test.ts
+++ b/packages/cli/src/cloud/errors.test.ts
@@ -49,7 +49,21 @@ describe("cloud/errors reportApiError", () => {
expect(errorBox).toHaveBeenCalledWith(
"Upload failed (HTTP 413)",
"project too large",
- "The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run`, then add only verified-unneeded paths to `.hyperframesignore` or pre-host required large media.",
+ "The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run` to see what stays in it, then add only verified-unneeded paths to `.hyperframesignore`.",
+ );
+ });
+
+ it("points an upload the server refuses as too large to dry-run, not to --help", () => {
+ const err = new HyperframesApiError({
+ status: 400,
+ message: "File too large. Maximum is 200 MiB.",
+ code: "invalid_parameter",
+ });
+ expect(() => reportApiError("Upload failed", err)).toThrow(CliRuntimeError);
+ expect(errorBox).toHaveBeenCalledWith(
+ "Upload failed (HTTP 400)",
+ "File too large. Maximum is 200 MiB.",
+ expect.stringContaining("hyperframes cloud render --dry-run"),
);
});
diff --git a/packages/cli/src/cloud/errors.ts b/packages/cli/src/cloud/errors.ts
index 6a96275658..b9b551e456 100644
--- a/packages/cli/src/cloud/errors.ts
+++ b/packages/cli/src/cloud/errors.ts
@@ -20,7 +20,7 @@ import { HyperframesApiError } from "./_gen/client.js";
*/
const ERROR_CODE_HINTS: Record<string, string> = {
hyperframes_project_too_large:
- "The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run`, then add only verified-unneeded paths to `.hyperframesignore` or pre-host required large media.",
+ "The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run` to see what stays in it, then add only verified-unneeded paths to `.hyperframesignore`.",
hyperframes_render_not_found:
"The render_id no longer exists — either soft-deleted or never created.",
invalid_parameter:
@@ -30,6 +30,14 @@ const ERROR_CODE_HINTS: Record<string, string> = {
rate_limit_exceeded: "Retry after the duration in the Retry-After header.",
};
+// The asset upload API reports an over-limit file as invalid_parameter; only its message says why.
+function hintCode(err: HyperframesApiError): string | undefined {
+ if (err.code === "invalid_parameter" && /^File too large\b/.test(err.message)) {
+ return "hyperframes_project_too_large";
+ }
+ return err.code;
+}
+
/**
* Print an errorBox and `process.exit(1)` for any unknown error from
* the cloud subverbs. The `stage` is the human-readable name of the
@@ -64,7 +72,8 @@ export function reportApiError(
errorBox("Not found", options.notFound);
failCommand();
}
- const hint = err.code ? hints[err.code] : undefined;
+ const code = hintCode(err);
+ const hint = code ? hints[code] : undefined;
const title = `${stage} (HTTP ${err.status})`;
// Priority: code-specific hint > caller suggestion > bare code
// label > no third line. Code-specific hints win because they
diff --git a/packages/cli/src/cloud/offload.test.ts b/packages/cli/src/cloud/offload.test.ts
new file mode 100644
index 0000000000..8b7bef7fab
--- /dev/null
+++ b/packages/cli/src/cloud/offload.test.ts
@@ -0,0 +1,787 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ truncateSync,
+ utimesSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { buildPublishFileMap, stagePublishFiles } from "../utils/publishProject.js";
+import {
+ type GifPrep,
+ OFFLOAD_MIN_BYTES,
+ offloadHeavyMedia,
+ type Reencoder,
+ type UploadableFile,
+} from "./offload.js";
+
+const HEAVY = Buffer.alloc(OFFLOAD_MIN_BYTES, 1);
+const dirs: string[] = [];
+
+afterEach(() => {
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function project(files: Record<string, string | Buffer>): string {
+ const dir = mkdtempSync(join(tmpdir(), "hf-offload-test-"));
+ dirs.push(dir);
+ for (const [path, content] of Object.entries(files)) {
+ mkdirSync(dirname(join(dir, path)), { recursive: true });
+ writeFileSync(join(dir, path), content);
+ }
+ return dir;
+}
+
+function cachePath(): string {
+ const dir = mkdtempSync(join(tmpdir(), "hf-offload-cache-"));
+ dirs.push(dir);
+ return join(dir, "cloud-assets.json");
+}
+
+function fakeUpload() {
+ let n = 0;
+ return vi.fn(async (file: UploadableFile) => {
+ n += 1;
+ return { asset_id: `asset_${n}`, url: `https://assets.example/${n}/${file.filename}` };
+ });
+}
+
+// HEAD answers like the asset CDN: the stored size, or 404 once the asset is gone.
+function fakeHead(status: number, size: number): typeof fetch {
+ return vi.fn(
+ async () => new Response(null, { status, headers: { "content-length": String(size) } }),
+ ) as unknown as typeof fetch;
+}
+
+// Encoders read a temp snapshot of the staged bytes, never the project file.
+const staged = (name: string) =>
+ expect.stringMatching(new RegExp(`hf-offload-src-[^/\\\\]+[/\\\\]${name}$`));
+
+const html = (body: string) => `<!doctype html><html><head></head><body>${body}</body></html>`;
+
+describe("offloadHeavyMedia", () => {
+ it("rewrites every reference in the staged copy and drops the file from the zip", async () => {
+ const indexHtml = html(
+ `<style>.bg { background: url("media/bg.png"); }</style>` +
+ `<video src="media/clip.mp4" poster="media/bg.png"><source src="./media/clip.mp4" /></video>` +
+ `<img srcset="media/bg.png 1x, media/small.png 2x" />` +
+ `<div style="background-image: url(media/bg.png)"></div>` +
+ `<audio src="/media/voice.mp3#t=2"></audio>` +
+ `<div data-composition-src="compositions/scene.html"></div>`,
+ );
+ const dir = project({
+ "index.html": indexHtml,
+ // Sub-compositions are inlined into the root, so a root-relative src resolves from the project root.
+ "compositions/scene.html": html(
+ `<video src="media/clip.mp4"></video><img src="../media/bg.png" />`,
+ ),
+ "styles/main.css": `.x { background: url('../media/bg.png'); }`,
+ "media/clip.mp4": HEAVY,
+ "media/bg.png": HEAVY,
+ "media/voice.mp3": HEAVY,
+ "media/small.png": "tiny",
+ });
+ const fileMap = buildPublishFileMap(dir);
+ const upload = fakeUpload();
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ cachePath: cachePath(),
+ });
+
+ const url = (path: string) => report.find((item) => item.path === path)!.url!;
+ expect(report.map((item) => [item.path, item.action])).toEqual([
+ ["media/bg.png", "upload"],
+ ["media/clip.mp4", "upload"],
+ ["media/voice.mp3", "upload"],
+ ]);
+ const index = fileMap.get("index.html")!.toString();
+ expect(index).toContain(`url("${url("media/bg.png")}")`);
+ expect(index).toContain(`src="${url("media/clip.mp4")}" poster="${url("media/bg.png")}"`);
+ expect(index).toContain(`<source src="${url("media/clip.mp4")}"`);
+ expect(index).toContain(`srcset="${url("media/bg.png")} 1x, media/small.png 2x"`);
+ expect(index).toContain(`url(${url("media/bg.png")})`);
+ expect(index).toContain(`<audio src="${url("media/voice.mp3")}#t=2"`);
+ const scene = fileMap.get("compositions/scene.html")!.toString();
+ expect(scene).toContain(`<video src="${url("media/clip.mp4")}"`);
+ expect(scene).toContain(`<img src="${url("media/bg.png")}"`);
+ expect(fileMap.get("styles/main.css")!.toString()).toContain(`url('${url("media/bg.png")}')`);
+ expect([...fileMap.keys()].sort()).toEqual([
+ "compositions/scene.html",
+ "index.html",
+ "media/small.png",
+ "styles/main.css",
+ ]);
+ expect(readFileSync(join(dir, "index.html"), "utf-8")).toBe(indexHtml);
+ });
+
+ it("reuses a cached asset that is still served, without uploading again", async () => {
+ const dir = project({
+ "index.html": html(`<video src="clip.mp4"></video>`),
+ "clip.mp4": HEAVY,
+ });
+ const cache = cachePath();
+ const upload = fakeUpload();
+ const opts = { apiBaseUrl: "https://api.example", upload, cachePath: cache };
+
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), opts);
+ const second = buildPublishFileMap(dir);
+ const report = await offloadHeavyMedia(dir, second, {
+ ...opts,
+ fetchImpl: fakeHead(200, HEAVY.byteLength),
+ });
+
+ expect(upload).toHaveBeenCalledOnce();
+ expect(report).toEqual([
+ expect.objectContaining({
+ path: "clip.mp4",
+ action: "cached",
+ url: "https://assets.example/1/clip.mp4",
+ }),
+ ]);
+ expect(second.get("index.html")!.toString()).toContain("https://assets.example/1/clip.mp4");
+ });
+
+ it("treats a cache file holding null as empty", async () => {
+ const dir = project({
+ "index.html": html(`<video src="clip.mp4"></video>`),
+ "clip.mp4": HEAVY,
+ });
+ const cache = cachePath();
+ writeFileSync(cache, "null");
+
+ await expect(
+ offloadHeavyMedia(dir, buildPublishFileMap(dir), {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ cachePath: cache,
+ }),
+ ).resolves.toHaveLength(1);
+ });
+
+ it("uploads again when the cached asset URL no longer serves the file", async () => {
+ const dir = project({
+ "index.html": html(`<video src="clip.mp4"></video>`),
+ "clip.mp4": HEAVY,
+ });
+ const opts = {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ cachePath: cachePath(),
+ };
+
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), opts);
+ const report = await offloadHeavyMedia(dir, buildPublishFileMap(dir), {
+ ...opts,
+ fetchImpl: fakeHead(404, 0),
+ });
+ // Served, but at a size other than the one uploaded.
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), {
+ ...opts,
+ fetchImpl: fakeHead(200, HEAVY.byteLength - 1),
+ });
+
+ expect(opts.upload).toHaveBeenCalledTimes(3);
+ expect(report[0]).toMatchObject({ action: "upload", url: "https://assets.example/2/clip.mp4" });
+ });
+
+ it("re-encodes media the asset API refuses and uploads the accepted format", async () => {
+ const dir = project({
+ "index.html": html(`<video src="take.mkv"></video>`),
+ "take.mkv": HEAVY,
+ });
+ const reencoder: Reencoder = {
+ choose: vi.fn(async () => "video" as const),
+ run: vi.fn(async () => ({
+ bytes: Buffer.from("h264"),
+ filename: "take.mp4",
+ contentType: "video/mp4",
+ })),
+ };
+ const upload = fakeUpload();
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ reencoder,
+ cachePath: cachePath(),
+ });
+
+ expect(reencoder.run).toHaveBeenCalledWith(staged("take.mkv"), "video", undefined);
+ expect(upload.mock.calls[0]![0]).toMatchObject({
+ filename: "take.mp4",
+ contentType: "video/mp4",
+ });
+ expect(report[0]).toMatchObject({ path: "take.mkv", action: "reencode" });
+ expect(fileMap.get("index.html")!.toString()).toContain("https://assets.example/1/take.mp4");
+ expect(fileMap.has("take.mkv")).toBe(false);
+ });
+
+ it("fails naming the file when a re-encode is still over the asset cap", async () => {
+ const dir = project({
+ "index.html": html(`<video src="long.mkv"></video>`),
+ "long.mkv": HEAVY,
+ });
+ const reencoder: Reencoder = {
+ choose: async () => "video",
+ run: async () => ({
+ bytes: { byteLength: 201 * 1024 * 1024 } as Uint8Array,
+ filename: "long.mp4",
+ contentType: "video/mp4",
+ }),
+ };
+ const upload = fakeUpload();
+
+ await expect(
+ offloadHeavyMedia(dir, buildPublishFileMap(dir), {
+ apiBaseUrl: "https://api.example",
+ upload,
+ reencoder,
+ cachePath: cachePath(),
+ }),
+ ).rejects.toThrow(/^long\.mkv is 201(\.0)? MB after re-encoding/);
+ expect(upload).not.toHaveBeenCalled();
+ });
+
+ it("rewrites plain string literals in scripts and data, keeping a copy of a path a script builds", async () => {
+ const dir = project({
+ "index.html": html(
+ `<script src="main.js"></script>` +
+ `<script>fetch('./media/clip.mp4'); video.src = \`media/clip.mp4\`;</script>`,
+ ),
+ "main.js": `new Audio("music.mp3").play(); const voice = \`media/\${name}\`; // voice.mp3`,
+ "data/cues.json": `{"track": "music.mp3"}`,
+ "media/clip.mp4": HEAVY,
+ "music.mp3": HEAVY,
+ "voice.mp3": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ cachePath: cachePath(),
+ });
+
+ const url = (path: string) => report.find((item) => item.path === path)!.url!;
+ const index = fileMap.get("index.html")!.toString();
+ expect(index).toContain(`fetch('${url("media/clip.mp4")}')`);
+ expect(index).toContain(`video.src = \`${url("media/clip.mp4")}\``);
+ expect(fileMap.get("main.js")!.toString()).toContain(`new Audio("${url("music.mp3")}")`);
+ expect(fileMap.get("data/cues.json")!.toString()).toBe(`{"track": "${url("music.mp3")}"}`);
+ expect(report.find((item) => item.path === "voice.mp3")).toBeUndefined();
+ expect(fileMap.has("voice.mp3")).toBe(true);
+ expect(fileMap.has("music.mp3")).toBe(false);
+ expect(fileMap.has("media/clip.mp4")).toBe(false);
+ });
+
+ it("converts a still image the API refuses and leaves an animated one in the zip", async () => {
+ const dir = project({
+ "index.html": html(`<img src="still.avif" /><img src="moving.avif" />`),
+ "still.avif": HEAVY,
+ "moving.avif": Buffer.alloc(OFFLOAD_MIN_BYTES, 2),
+ });
+ const reencoder: Reencoder = {
+ choose: async (sourcePath) => (sourcePath.endsWith("still.avif") ? "image" : null),
+ run: async () => ({
+ bytes: Buffer.from("png"),
+ filename: "still.png",
+ contentType: "image/png",
+ }),
+ };
+ const upload = fakeUpload();
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ reencoder,
+ cachePath: cachePath(),
+ });
+
+ expect(report).toEqual([expect.objectContaining({ path: "still.avif", action: "reencode" })]);
+ expect(upload.mock.calls[0]![0]).toMatchObject({ contentType: "image/png" });
+ expect(fileMap.has("still.avif")).toBe(false);
+ expect(fileMap.has("moving.avif")).toBe(true);
+ expect(fileMap.get("index.html")!.toString()).toContain(`<img src="moving.avif">`);
+ });
+
+ it("offloads the video the renderer's GIF prep makes from an entry GIF, and drops the GIF", async () => {
+ const dir = project({
+ "index.html": html(`<img src="media/loop.gif" data-loop="true" data-duration="4" />`),
+ "compositions/scene.html": html(`<img src="media/other.gif" />`),
+ "media/loop.gif": HEAVY,
+ "media/other.gif": HEAVY,
+ });
+ const gifPrep = vi.fn<GifPrep>(async (input, options) => {
+ const outputPath = join(options.outputDir, "hfgif-v1-abc.webm");
+ writeFileSync(outputPath, HEAVY);
+ return {
+ html: input.replace(
+ `<img src="media/loop.gif" data-loop="true" data-duration="4" />`,
+ `<video src="${options.outputSrcPrefix}/hfgif-v1-abc.webm" loop></video>`,
+ ),
+ preparedGifs: [{ sourceSrc: "media/loop.gif", outputPath }],
+ };
+ });
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ gifPrep,
+ reencoder: { choose: async () => null, run: vi.fn() },
+ cachePath: cachePath(),
+ });
+
+ expect(gifPrep.mock.calls[0]![1]).toMatchObject({
+ projectDir: dir,
+ outputSrcPrefix: "_animated_gif",
+ });
+ expect(report).toEqual([
+ expect.objectContaining({ path: "_animated_gif/hfgif-v1-abc.webm", action: "upload" }),
+ ]);
+ expect(fileMap.get("index.html")!.toString()).toContain(
+ `<video src="https://assets.example/1/hfgif-v1-abc.webm" loop>`,
+ );
+ expect(fileMap.has("media/loop.gif")).toBe(false);
+ expect(fileMap.has("_animated_gif/hfgif-v1-abc.webm")).toBe(false);
+ // Sub-compositions are prepared by the renderer after inlining, so their GIFs stay in the zip.
+ expect(fileMap.has("media/other.gif")).toBe(true);
+ });
+
+ it("prepares a nested entry's GIF from the file next to it, as the renderer resolves it", async () => {
+ const nested = Buffer.alloc(OFFLOAD_MIN_BYTES, 2);
+ const dir = project({
+ "index.html": html(""),
+ "compositions/scene.html": html(`<img src="logo.gif" />`),
+ "compositions/logo.gif": nested,
+ "logo.gif": HEAVY,
+ });
+ let prepared: Buffer | undefined;
+ const gifPrep = vi.fn<GifPrep>(async (input, options) => {
+ prepared = readFileSync(options.sourceAssets!.get("logo.gif")!);
+ const outputPath = join(options.outputDir, "hfgif-v1-abc.webm");
+ writeFileSync(outputPath, HEAVY);
+ return {
+ html: input.replace(
+ `<img src="logo.gif" />`,
+ `<video src="../_animated_gif/hfgif-v1-abc.webm"></video>`,
+ ),
+ preparedGifs: [{ sourceSrc: "logo.gif", outputPath }],
+ };
+ });
+ const fileMap = buildPublishFileMap(dir);
+
+ await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ entryPath: "compositions/scene.html",
+ upload: fakeUpload(),
+ gifPrep,
+ cachePath: cachePath(),
+ });
+
+ expect(prepared?.equals(nested)).toBe(true);
+ expect(fileMap.has("compositions/logo.gif")).toBe(false);
+ expect(fileMap.has("logo.gif")).toBe(true);
+ });
+
+ it("hashes and encodes media over 4 GiB from disk, never reading it whole", async () => {
+ const dir = project({ "index.html": html(`<video src="take.mov"></video>`), "take.mov": "" });
+ truncateSync(join(dir, "take.mov"), 2 ** 32 + 1);
+ const fileMap = stagePublishFiles(dir, OFFLOAD_MIN_BYTES);
+ const run = vi.fn(async () => ({
+ bytes: Buffer.from("mp4"),
+ filename: "take.mp4",
+ contentType: "video/mp4",
+ }));
+
+ const [item] = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "video", run },
+ cachePath: cachePath(),
+ });
+
+ expect(run).toHaveBeenCalledWith(join(dir, "take.mov"), "video", undefined);
+ expect(item).toMatchObject({ path: "take.mov", action: "reencode", size_bytes: 2 ** 32 + 1 });
+ expect(fileMap.has("take.mov")).toBe(false);
+ }, 60_000);
+
+ it("uploads media staged on disk as a stream of the file", async () => {
+ const dir = project({
+ "index.html": html(`<video src="clip.mp4"></video>`),
+ "clip.mp4": HEAVY,
+ });
+ const upload = fakeUpload();
+
+ await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
+ apiBaseUrl: "https://api.example",
+ upload,
+ cachePath: cachePath(),
+ });
+
+ expect(upload).toHaveBeenCalledWith({
+ path: join(dir, "clip.mp4"),
+ size: HEAVY.byteLength,
+ filename: "clip.mp4",
+ contentType: "video/mp4",
+ });
+ });
+
+ it("stops without caching when media staged on disk changes during its encode", async () => {
+ const dir = project({
+ "index.html": html(`<audio src="voice.flac"></audio>`),
+ "voice.flac": HEAVY,
+ });
+ const fileMap = stagePublishFiles(dir, OFFLOAD_MIN_BYTES);
+ const cache = cachePath();
+ const run = vi.fn(async (sourcePath: string) => {
+ utimesSync(sourcePath, new Date(), new Date(Date.now() + 5_000));
+ return { bytes: Buffer.from("wav"), filename: "voice.wav", contentType: "audio/wav" };
+ });
+
+ await expect(
+ offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "audio", run },
+ cachePath: cache,
+ }),
+ ).rejects.toThrow(/changed on disk/);
+ expect(existsSync(cache)).toBe(false);
+ });
+
+ it("re-encodes the staged bytes, not a file saved on disk after staging", async () => {
+ const dir = project({
+ "index.html": html(`<audio src="voice.flac"></audio>`),
+ "voice.flac": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+ writeFileSync(join(dir, "voice.flac"), Buffer.alloc(OFFLOAD_MIN_BYTES, 9));
+ let encoded: Buffer | undefined;
+ const run = vi.fn(async (sourcePath: string) => {
+ encoded = readFileSync(sourcePath);
+ return { bytes: Buffer.from("wav"), filename: "voice.wav", contentType: "audio/wav" };
+ });
+
+ await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "audio", run },
+ cachePath: cachePath(),
+ });
+
+ expect(encoded?.equals(HEAVY)).toBe(true);
+ });
+
+ it("cuts a re-encoded video to the windows its clips play and shifts their starts to match", async () => {
+ const dir = project({
+ "index.html": html(
+ `<video id="a" src="long.mkv" data-start="0" data-duration="2" data-media-start="10"></video>` +
+ `<video id="b" src="long.mkv" data-start="2" data-duration="2" data-media-start="100.5" data-playback-rate="2"></video>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const run = vi.fn(async () => ({
+ bytes: Buffer.from("cut"),
+ filename: "long.mp4",
+ contentType: "video/mp4",
+ }));
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "video", trimGrid: async () => 30, run },
+ cachePath: cachePath(),
+ });
+
+ expect(run).toHaveBeenCalledWith(staged("long.mkv"), "video", [
+ { start: 9.5, end: 12.5 },
+ { start: 100, end: 105 },
+ ]);
+ expect(report[0]).toMatchObject({ path: "long.mkv", trimmed_to_seconds: 8 });
+ const index = fileMap.get("index.html")!.toString();
+ expect(index).toMatch(/id="a"[^>]*data-media-start="0.5"/);
+ expect(index).toMatch(/id="b"[^>]*data-media-start="3.5"/);
+ });
+
+ it("cuts on the source's own frames at 29.97 fps and re-encodes when the cut changes", async () => {
+ const clip = (start: number) =>
+ html(`<video src="long.mkv" data-duration="2" data-media-start="${start}"></video>`);
+ const dir = project({ "index.html": clip(10), "long.mkv": HEAVY });
+ const fps = 30000 / 1001;
+ const run = vi.fn(async () => ({
+ bytes: Buffer.from("cut"),
+ filename: "long.mp4",
+ contentType: "video/mp4",
+ }));
+ const opts = {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "video" as const, trimGrid: async () => fps, run },
+ // Any cached URL is still served, so only the key decides reuse.
+ fetchImpl: fakeHead(200, 3),
+ cachePath: cachePath(),
+ };
+
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), opts);
+ writeFileSync(join(dir, "index.html"), clip(20));
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), opts);
+ await offloadHeavyMedia(dir, buildPublishFileMap(dir), opts);
+
+ expect(run).toHaveBeenCalledTimes(2);
+ const calls = run.mock.calls as unknown as Array<[string, string, Array<{ start: number }>]>;
+ for (const [, , segments] of calls) {
+ const frames = segments[0]!.start * fps;
+ expect(Math.abs(frames - Math.round(frames))).toBeLessThan(1e-6);
+ }
+ });
+
+ it("keeps the whole file when a use is not a bounded clip or a script names it", async () => {
+ const looped = project({
+ "index.html": html(
+ `<video src="long.mkv" data-duration="2" data-media-start="10" loop></video>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const scripted = project({
+ "index.html": html(`<video src="long.mkv" data-duration="2" data-media-start="10"></video>`),
+ "main.js": "const name = `long.mkv`.replace('.mkv', '');",
+ "long.mkv": HEAVY,
+ });
+ const variableBound = project({
+ "index.html": html(
+ `<video data-var-src="clip" src="long.mkv" data-duration="2" data-media-start="10"></video>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const fragment = project({
+ "index.html": html(
+ `<video src="long.mkv#t=10,12" data-duration="2" data-media-start="10"></video>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const automated = project({
+ "index.html": html(
+ `<video src="long.mkv" data-duration="2" data-media-start="10" data-automation="x"></video>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const offsetParent = project({
+ "index.html": html(
+ `<div data-media-start="3"><video src="long.mkv" data-duration="2" data-media-start="10"></video></div>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const nested = project({
+ "index.html": html(
+ `<div data-composition-id="root"><div data-composition-id="inner">` +
+ `<video src="long.mkv" data-duration="2" data-media-start="10"></video></div></div>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const retimedById = project({
+ "index.html": html(
+ `<video id="a" src="long.mkv" data-start="0" data-duration="2" data-media-start="10"></video>` +
+ `<script>const a = document.getElementById("a"); a.setAttribute("data-media-start", "100"); a.volume = 0.5;</script>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ const retimedByDataset = project({
+ "index.html": html(
+ `<video id="a" src="long.mkv" data-duration="2" data-media-start="10"></video><script src="main.js"></script>`,
+ ),
+ "main.js": `document.getElementById("a").dataset.mediaStart = "100";`,
+ "long.mkv": HEAVY,
+ });
+ const namedInPage = project({
+ "index.html": html(
+ `<video src="long.mkv" data-duration="2" data-media-start="10"></video><p data-note="long.mkv"></p>`,
+ ),
+ "long.mkv": HEAVY,
+ });
+ for (const dir of [
+ looped,
+ scripted,
+ variableBound,
+ fragment,
+ automated,
+ offsetParent,
+ nested,
+ retimedById,
+ retimedByDataset,
+ namedInPage,
+ ]) {
+ const run = vi.fn(async () => ({
+ bytes: Buffer.from("full"),
+ filename: "long.mp4",
+ contentType: "video/mp4",
+ }));
+ const fileMap = buildPublishFileMap(dir);
+ await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ reencoder: { choose: async () => "video", trimGrid: async () => 30, run },
+ cachePath: cachePath(),
+ });
+ expect(run).toHaveBeenCalledWith(staged("long.mkv"), "video", undefined);
+ expect(fileMap.get("index.html")!.toString()).toContain(`data-media-start="10"`);
+ }
+ });
+
+ it("keeps a heavy file no reference names in the zip, since --variables can pick it at render time", async () => {
+ const dir = project({
+ "index.html": html(`<video data-var-src="clip" src="media/a.mp4"></video>`),
+ "media/a.mp4": HEAVY,
+ "media/b.mp4": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+ const upload = fakeUpload();
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ cachePath: cachePath(),
+ });
+
+ expect(report.map((item) => item.path)).toEqual(["media/a.mp4"]);
+ expect(upload).toHaveBeenCalledTimes(1);
+ expect(fileMap.has("media/b.mp4")).toBe(true);
+ });
+
+ it("keeps a heavy file a script builds from split strings in the zip", async () => {
+ const dir = project({
+ "index.html": html(
+ `<video id="v" data-start="0" data-duration="2"></video><script>v.src = "media/" + "clip" + ".mp4";</script>`,
+ ),
+ "media/clip.mp4": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+ const upload = fakeUpload();
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ cachePath: cachePath(),
+ });
+
+ expect(report).toEqual([]);
+ expect(upload).not.toHaveBeenCalled();
+ expect(fileMap.has("media/clip.mp4")).toBe(true);
+ });
+
+ it("keeps a zip copy of an offloaded file a script can also build from split strings", async () => {
+ const dir = project({
+ "index.html": html(
+ `<video id="v" src="media/clip.mp4"></video>` +
+ `<script>document.getElementById("v").src = "media/" + "clip" + ".mp4";</script>`,
+ ),
+ "media/clip.mp4": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ cachePath: cachePath(),
+ });
+
+ expect(report).toEqual([
+ expect.objectContaining({ path: "media/clip.mp4", kept_in_archive: true }),
+ ]);
+ expect(fileMap.has("media/clip.mp4")).toBe(true);
+ });
+
+ it("prepares an entry GIF staged on disk from a checked private copy", async () => {
+ const files = {
+ "index.html": html(`<img src="media/loop.gif" />`),
+ "media/loop.gif": HEAVY,
+ };
+ const prepared: Buffer[] = [];
+ const gifPrep = vi.fn<GifPrep>(async (input, options) => {
+ const source = options.sourceAssets.get("media/loop.gif")!;
+ // An edit while the renderer hashes and encodes must not reach what it caches.
+ writeFileSync(join(options.projectDir, "media/loop.gif"), Buffer.alloc(9));
+ prepared.push(readFileSync(source));
+ const outputPath = join(options.outputDir, "hfgif-v1-abc.webm");
+ writeFileSync(outputPath, "webm");
+ return {
+ html: input.replace(
+ `<img src="media/loop.gif" />`,
+ `<video src="_animated_gif/hfgif-v1-abc.webm"></video>`,
+ ),
+ preparedGifs: [{ sourceSrc: "media/loop.gif", outputPath }],
+ };
+ });
+ const opts = { apiBaseUrl: "https://api.example", upload: fakeUpload(), gifPrep };
+
+ const dir = project(files);
+ await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
+ ...opts,
+ cachePath: cachePath(),
+ });
+ expect(prepared[0]?.equals(HEAVY)).toBe(true);
+
+ const edited = project(files);
+ const fileMap = stagePublishFiles(edited, OFFLOAD_MIN_BYTES);
+ writeFileSync(join(edited, "media/loop.gif"), Buffer.alloc(9));
+ await expect(
+ offloadHeavyMedia(edited, fileMap, { ...opts, cachePath: cachePath() }),
+ ).rejects.toThrow(/changed on disk/);
+ });
+
+ it("never decodes a media file as text, since one can exceed the longest string allowed", async () => {
+ const dir = project({
+ "index.html": html(`<video src="clip.mp4"></video>`),
+ "clip.mp4": HEAVY,
+ });
+ const fileMap = buildPublishFileMap(dir);
+ const binary = Buffer.alloc(16);
+ binary.toString = () => {
+ throw new Error("decoded a binary entry");
+ };
+ fileMap.set("media/raw.bin", binary);
+
+ await expect(
+ offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload: fakeUpload(),
+ cachePath: cachePath(),
+ }),
+ ).resolves.toHaveLength(1);
+ });
+
+ it("leaves small media and remote URLs alone", async () => {
+ const dir = project({
+ "index.html": html(
+ `<img src="logo.png" /><img src="spin.gif" /><video src="https://cdn.example/a.mp4"></video>`,
+ ),
+ "logo.png": "small",
+ "spin.gif": "small",
+ });
+ const fileMap = buildPublishFileMap(dir);
+ const upload = fakeUpload();
+ const gifPrep = vi.fn<GifPrep>();
+
+ const report = await offloadHeavyMedia(dir, fileMap, {
+ apiBaseUrl: "https://api.example",
+ upload,
+ gifPrep,
+ cachePath: cachePath(),
+ });
+
+ expect(report).toEqual([]);
+ expect(upload).not.toHaveBeenCalled();
+ expect(gifPrep).not.toHaveBeenCalled();
+ expect(fileMap.has("logo.png")).toBe(true);
+ });
+});
diff --git a/packages/cli/src/cloud/offload.ts b/packages/cli/src/cloud/offload.ts
new file mode 100644
index 0000000000..dfe9e7a73b
--- /dev/null
+++ b/packages/cli/src/cloud/offload.ts
@@ -0,0 +1,917 @@
+/**
+ * Moves a cloud render's heavy media out of the project zip into HeyGen assets and points the
+ * staged copy's references at the asset URLs. The project on disk is never touched.
+ */
+
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+ copyFileSync,
+ createReadStream,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ renameSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, dirname, extname, join, posix, relative, resolve } from "node:path";
+import { promisify } from "node:util";
+import { parseHTML } from "linkedom";
+import { parseAnimatedGifMetadata } from "@hyperframes/core";
+import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths";
+import {
+ cleanAssetUrl,
+ isRemoteOrInlineUrl,
+ resolveLocalAssetCandidates,
+} from "@hyperframes/parsers/asset-resolution";
+import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
+import { formatBytes } from "../ui/format.js";
+import { type PlayedSegment, planTrim } from "./offloadTrim.js";
+import {
+ assertUnchanged,
+ type HtmlAttributeResolver,
+ readChecked,
+ rewriteCssUrls,
+ rewriteHtmlAttributes,
+ rewriteStyleBlocks,
+ type StagedFile,
+ stagedSize,
+ TEXT_FILE_EXTS,
+} from "../utils/publishProject.js";
+
+const execFileAsync = promisify(execFile);
+
+/** Media at or above this size leaves the zip as a HeyGen asset. */
+export const OFFLOAD_MIN_BYTES = 5 * 1024 * 1024;
+
+const MiB = 1024 * 1024;
+const AUDIO_MAX_BYTES = 1_500_000_000;
+
+// What POST /v3/assets/direct-uploads takes as-is, and its per-type cap.
+const ACCEPTED: Record<string, { contentType: string; maxBytes: number }> = {
+ ".mp4": { contentType: "video/mp4", maxBytes: 200 * MiB },
+ ".m4v": { contentType: "video/mp4", maxBytes: 200 * MiB },
+ ".mov": { contentType: "video/quicktime", maxBytes: 200 * MiB },
+ ".webm": { contentType: "video/webm", maxBytes: 200 * MiB },
+ ".png": { contentType: "image/png", maxBytes: 200 * MiB },
+ ".jpg": { contentType: "image/jpeg", maxBytes: 200 * MiB },
+ ".jpeg": { contentType: "image/jpeg", maxBytes: 200 * MiB },
+ ".webp": { contentType: "image/webp", maxBytes: 200 * MiB },
+ ".wav": { contentType: "audio/wav", maxBytes: AUDIO_MAX_BYTES },
+ ".mp3": { contentType: "audio/mpeg", maxBytes: AUDIO_MAX_BYTES },
+};
+
+const VIDEO_EXTS = new Set([
+ ".mp4",
+ ".m4v",
+ ".mov",
+ ".webm",
+ ".mkv",
+ ".avi",
+ ".mpg",
+ ".mpeg",
+ ".ts",
+ ".m2ts",
+ ".mts",
+ ".flv",
+ ".wmv",
+ ".ogv",
+ ".3gp",
+]);
+const AUDIO_EXTS = new Set([
+ ".wav",
+ ".mp3",
+ ".m4a",
+ ".aac",
+ ".ogg",
+ ".oga",
+ ".opus",
+ ".flac",
+ ".aif",
+ ".aiff",
+ ".wma",
+]);
+// Still images the API refuses; converted to PNG unless animated.
+const CONVERTIBLE_IMAGE_EXTS = new Set([".gif", ".avif", ".bmp", ".tif", ".tiff"]);
+
+/** Visually lossless re-encodes for media the asset API refuses or caps. Quality lives here. */
+const REENCODE_PRESETS = {
+ video: {
+ ext: ".mp4",
+ contentType: "video/mp4",
+ args: ["-c:v", "libx264", "-preset", "slow", "-crf", "18", "-pix_fmt", "yuv420p"],
+ audio: ["-c:a", "aac", "-b:a", "320k", "-movflags", "+faststart"],
+ },
+ hdrVideo: {
+ ext: ".mp4",
+ contentType: "video/mp4",
+ args: ["-c:v", "libx265", "-preset", "slow", "-crf", "18", "-pix_fmt", "yuv420p10le"],
+ audio: ["-tag:v", "hvc1", "-c:a", "aac", "-b:a", "320k", "-movflags", "+faststart"],
+ },
+ alphaVideo: {
+ ext: ".webm",
+ contentType: "video/webm",
+ args: ["-c:v", "libvpx-vp9", "-crf", "18", "-b:v", "0", "-pix_fmt", "yuva420p"],
+ audio: ["-c:a", "libopus", "-b:a", "256k"],
+ },
+ audio: {
+ ext: ".wav",
+ contentType: "audio/wav",
+ args: ["-vn"],
+ audio: ["-c:a", "pcm_s16le"],
+ },
+ image: {
+ ext: ".png",
+ contentType: "image/png",
+ args: ["-frames:v", "1"],
+ audio: [],
+ },
+} as const;
+
+export type PresetName = keyof typeof REENCODE_PRESETS;
+
+export interface OffloadedMedia {
+ path: string;
+ size_bytes: number;
+ /** What happened (or, on a dry run, would happen) to the file. */
+ action: "upload" | "reencode" | "cached";
+ url?: string;
+ /** Seconds of the source kept when only the parts the composition plays are uploaded. */
+ trimmed_to_seconds?: number;
+ /** Still referenced from a script or data file, so a copy also stays in the zip. */
+ kept_in_archive?: boolean;
+}
+
+/** In memory, or a file on disk that the upload streams. */
+export type UploadableFile = { filename: string; contentType: string } & (
+ | { bytes: Uint8Array }
+ | { path: string; size: number }
+);
+
+export interface OffloadOptions {
+ /** Cache key namespace, so assets from one API host are never reused on another. */
+ apiBaseUrl: string;
+ /** Absent on a dry run: nothing is encoded, uploaded or fetched. */
+ upload?: (file: UploadableFile) => Promise<{ asset_id: string; url: string }>;
+ fetchImpl?: typeof fetch;
+ cachePath?: string;
+ /** The composition the render starts from; its animated GIFs are prepared as video. */
+ entryPath?: string;
+ /** Overrides ffprobe/ffmpeg for tests. */
+ reencoder?: Reencoder;
+ /** Overrides the renderer's GIF preparation for tests. */
+ gifPrep?: GifPrep;
+ onFile?: (item: OffloadedMedia) => void;
+}
+
+type MediaKind = "video" | "audio" | "image";
+
+export interface Reencoder {
+ /** `null` keeps the file in the zip as authored. */
+ choose: (sourcePath: string, kind: MediaKind) => Promise<PresetName | null>;
+ /** Positions per second a cut may land on (the frame rate), or null when the file cannot be cut. */
+ trimGrid?: (sourcePath: string, kind: MediaKind) => Promise<number | null>;
+ run: (
+ sourcePath: string,
+ preset: PresetName,
+ segments?: PlayedSegment[],
+ ) => Promise<UploadableFile>;
+}
+
+export type GifPrep = (
+ html: string,
+ options: {
+ projectDir: string;
+ downloadDir: string;
+ outputDir: string;
+ outputSrcPrefix: string;
+ cacheDir: string;
+ sourceAssets: Map<string, string>;
+ },
+) => Promise<{ html: string; preparedGifs: Array<{ sourceSrc: string; outputPath: string }> }>;
+
+interface CacheEntry {
+ asset_id: string;
+ url: string;
+ size_bytes: number;
+}
+
+function defaultOffloadCachePath(): string {
+ return join(homedir(), ".cache", "hyperframes", "cloud-assets.json");
+}
+
+function readCache(path: string): Record<string, CacheEntry> {
+ try {
+ return (JSON.parse(readFileSync(path, "utf-8")) as Record<string, CacheEntry> | null) ?? {};
+ } catch {
+ return {};
+ }
+}
+
+function writeCache(path: string, cache: Record<string, CacheEntry>): void {
+ mkdirSync(dirname(path), { recursive: true });
+ const tmp = `${path}.${process.pid}.tmp`;
+ writeFileSync(tmp, JSON.stringify(cache, null, 2));
+ renameSync(tmp, path);
+}
+
+// Asset URLs are public and unsigned, so HEAD needs no auth; a deleted or replaced asset fails it.
+async function stillServed(entry: CacheEntry, fetchImpl: typeof fetch): Promise<boolean> {
+ try {
+ const res = await fetchImpl(entry.url, { method: "HEAD" });
+ return res.ok && Number(res.headers.get("content-length")) === entry.size_bytes;
+ } catch {
+ return false;
+ }
+}
+
+function mediaKind(path: string): MediaKind | null {
+ const ext = extname(path).toLowerCase();
+ if (VIDEO_EXTS.has(ext)) return "video";
+ if (AUDIO_EXTS.has(ext)) return "audio";
+ if (ACCEPTED[ext] || CONVERTIBLE_IMAGE_EXTS.has(ext)) return "image";
+ return null;
+}
+
+interface VideoProbe {
+ alpha: boolean;
+ hdr: boolean;
+ codec?: string;
+ color: { primaries?: string; transfer?: string; space?: string };
+}
+
+async function probeVideo(path: string): Promise<VideoProbe> {
+ const ffprobe = findFFprobe();
+ if (!ffprobe)
+ throw new Error(`Re-encoding needs ffprobe. Install ffmpeg: ${getFFmpegInstallHint()}`);
+ const { stdout } = await execFileAsync(ffprobe, [
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-show_entries",
+ "stream=codec_name,pix_fmt,color_primaries,color_transfer,color_space:stream_tags",
+ "-of",
+ "json",
+ "--",
+ path,
+ ]);
+ const stream = (JSON.parse(stdout).streams?.[0] ?? {}) as Record<string, unknown>;
+ const tags = (stream.tags ?? {}) as Record<string, string>;
+ const alphaTag = Object.entries(tags).some(
+ ([k, v]) => k.toLowerCase() === "alpha_mode" && v === "1",
+ );
+ const transfer = stream.color_transfer as string | undefined;
+ return {
+ // VP9 alpha keeps pix_fmt yuv420p and flags the plane only in the ALPHA_MODE tag.
+ alpha: alphaTag || /^(yuva|rgba|argb|bgra|abgr|gbrap|ya)/.test(String(stream.pix_fmt ?? "")),
+ hdr: transfer === "smpte2084" || transfer === "arib-std-b67",
+ codec: stream.codec_name as string | undefined,
+ color: {
+ primaries: stream.color_primaries as string | undefined,
+ transfer,
+ space: stream.color_space as string | undefined,
+ },
+ };
+}
+
+// ffmpeg's native VP8/VP9 decoders drop the alpha plane; libvpx keeps it.
+function alphaDecoderArgs(probe: VideoProbe): string[] {
+ if (!probe.alpha) return [];
+ if (probe.codec === "vp9") return ["-c:v", "libvpx-vp9"];
+ if (probe.codec === "vp8") return ["-c:v", "libvpx"];
+ return [];
+}
+
+function hdrColorArgs({ color }: VideoProbe): string[] {
+ return [
+ ...(color.primaries ? ["-color_primaries", color.primaries] : []),
+ ...(color.transfer ? ["-color_trc", color.transfer] : []),
+ ...(color.space ? ["-colorspace", color.space] : []),
+ ];
+}
+
+async function hasAudioStream(ffprobe: string, path: string): Promise<boolean> {
+ const { stdout } = await execFileAsync(ffprobe, [
+ "-v",
+ "error",
+ "-select_streams",
+ "a",
+ "-show_entries",
+ "stream=index",
+ "-of",
+ "csv=p=0",
+ "--",
+ path,
+ ]);
+ return stdout.trim() !== "";
+}
+
+// Seconds floored to the microsecond, so a frame-aligned cut never rounds past its first frame.
+const seconds = (t: number) => (Math.floor(t * 1e6) / 1e6).toFixed(6);
+
+/** Inputs and mapping for a cut: one accurately seeked input per segment, joined in order. */
+async function segmentArgs(
+ sourcePath: string,
+ decoder: string[],
+ segments: PlayedSegment[] | undefined,
+ audioOnly: boolean,
+): Promise<string[]> {
+ if (!segments?.length) return [...decoder, "-i", sourcePath];
+ const inputs = segments.flatMap((s) => [
+ ...decoder,
+ "-ss",
+ seconds(s.start),
+ "-t",
+ seconds(s.end - s.start),
+ "-i",
+ sourcePath,
+ ]);
+ if (segments.length === 1) return inputs;
+ const audio = audioOnly || (await hasAudioStream(findFFprobe()!, sourcePath));
+ return [...inputs, ...concatArgs(segments.length, !audioOnly, audio)];
+}
+
+function concatArgs(count: number, video: boolean, audio: boolean): string[] {
+ const pad = (i: number) => (video ? `[${i}:v]` : "") + (audio ? `[${i}:a]` : "");
+ const pads = Array.from({ length: count }, (_, i) => pad(i)).join("");
+ const flags = `n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`;
+ return [
+ "-filter_complex",
+ `${pads}concat=${flags}${video ? "[v]" : ""}${audio ? "[a]" : ""}`,
+ ...(video ? ["-map", "[v]"] : []),
+ ...(audio ? ["-map", "[a]"] : []),
+ ];
+}
+
+async function ffmpegReencode(
+ sourcePath: string,
+ presetName: PresetName,
+ segments?: PlayedSegment[],
+): Promise<UploadableFile> {
+ const ffmpeg = findFFmpeg();
+ if (!ffmpeg) throw new Error(`Re-encoding needs ffmpeg. Install it: ${getFFmpegInstallHint()}`);
+ const preset = REENCODE_PRESETS[presetName];
+ const probe = preset.contentType.startsWith("video/") ? await probeVideo(sourcePath) : undefined;
+ const decoder = probe ? alphaDecoderArgs(probe) : [];
+ const color = presetName === "hdrVideo" && probe ? hdrColorArgs(probe) : [];
+ const workDir = mkdtempSync(join(tmpdir(), "hf-offload-"));
+ const outPath = join(workDir, `${basename(sourcePath, extname(sourcePath))}${preset.ext}`);
+ try {
+ await execFileAsync(
+ ffmpeg,
+ [
+ "-v",
+ "error",
+ "-y",
+ ...(await segmentArgs(sourcePath, decoder, segments, presetName === "audio")),
+ ...preset.args,
+ ...color,
+ ...preset.audio,
+ outPath,
+ ],
+ { maxBuffer: 16 * MiB },
+ );
+ // Never read an output the asset API would refuse: it can exceed what one read can hold.
+ return {
+ bytes: readChecked(outPath, ({ size }) => assertFitsAsset(sourcePath, outPath, size)),
+ filename: basename(outPath),
+ contentType: preset.contentType,
+ };
+ } finally {
+ rmSync(workDir, { recursive: true, force: true });
+ }
+}
+
+async function choosePreset(sourcePath: string, kind: MediaKind): Promise<PresetName | null> {
+ if (kind === "audio") return "audio";
+ // An animated image's first frame is not the image; such files stay in the zip.
+ if (kind === "image") return (await isAnimatedImage(sourcePath)) ? null : "image";
+ const probe = await probeVideo(sourcePath);
+ return probe.alpha ? "alphaVideo" : probe.hdr ? "hdrVideo" : "video";
+}
+
+async function isAnimatedImage(path: string): Promise<boolean> {
+ if (extname(path).toLowerCase() === ".gif") {
+ return parseAnimatedGifMetadata(readFileSync(path))?.animated ?? true;
+ }
+ const ffprobe = findFFprobe();
+ if (!ffprobe) return true;
+ try {
+ const { stdout } = await execFileAsync(ffprobe, [
+ "-v",
+ "error",
+ "-show_entries",
+ "stream=nb_frames",
+ "-of",
+ "csv=p=0",
+ "--",
+ path,
+ ]);
+ return stdout.split("\n").some((line) => Number(line) > 1);
+ } catch {
+ return true;
+ }
+}
+
+// Audio cuts land on the millisecond; video only on a constant frame rate's frames.
+async function trimGrid(sourcePath: string, kind: MediaKind): Promise<number | null> {
+ if (kind === "audio") return 1000;
+ if (kind !== "video") return null;
+ const ffprobe = findFFprobe();
+ if (!ffprobe) return null;
+ const { stdout } = await execFileAsync(ffprobe, [
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-show_entries",
+ "stream=r_frame_rate,avg_frame_rate",
+ "-of",
+ "csv=p=0",
+ "--",
+ sourcePath,
+ ]);
+ const rate = (fraction: string | undefined) => {
+ const [num, den] = (fraction ?? "").split("/").map(Number);
+ return num && den ? num / den : NaN;
+ };
+ const [real, average] = stdout.trim().split(",");
+ const fps = rate(real);
+ return Number.isFinite(fps) && Math.abs(fps - rate(average)) < 0.01 ? fps : null;
+}
+
+const ffmpegReencoder: Reencoder = { choose: choosePreset, trimGrid, run: ffmpegReencode };
+
+async function stagedSha256(file: StagedFile): Promise<string> {
+ if (Buffer.isBuffer(file)) return createHash("sha256").update(file).digest("hex");
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(file.absPath)) hash.update(chunk as Buffer);
+ assertUnchanged(file);
+ return hash.digest("hex");
+}
+
+/**
+ * A path holding the staged bytes, so an encode reads what its cache key hashed: a file left on disk
+ * is used in place (checked unchanged after use), one staged in memory gets a temp copy.
+ */
+function snapshotPath(path: string, file: StagedFile, tempFiles: string[]): string {
+ return Buffer.isBuffer(file) ? stagedCopy(path, file, tempFiles) : file.absPath;
+}
+
+/** A private copy of the staged bytes, for readers that cache what they read before any check runs. */
+function stagedCopy(path: string, file: StagedFile, tempFiles: string[]): string {
+ const dir = mkdtempSync(join(tmpdir(), "hf-offload-src-"));
+ tempFiles.push(dir);
+ const copy = join(dir, basename(path));
+ if (Buffer.isBuffer(file)) writeFileSync(copy, file);
+ else {
+ copyFileSync(file.absPath, copy);
+ assertUnchanged(file);
+ }
+ return copy;
+}
+
+/** A text file's contents; media is never decoded, since it can exceed the longest string V8 allows. */
+function textOf(path: string, file: StagedFile | undefined): string | undefined {
+ return TEXT_FILE_EXTS.test(path) && Buffer.isBuffer(file) ? file.toString("utf-8") : undefined;
+}
+
+/** What the page's scripts can read: inline scripts of HTML files, and script and data files. */
+function scriptCode(fileMap: Map<string, StagedFile>): string {
+ const parts: string[] = [];
+ for (const [path, content] of fileMap) {
+ const text = textOf(path, content);
+ if (text === undefined) continue;
+ if (/\.html?$/i.test(path)) {
+ for (const script of parseHTML(text).document.querySelectorAll("script:not([src])")) {
+ parts.push(script.textContent ?? "");
+ }
+ } else if (/\.(m?js|cjs|json)$/i.test(path)) parts.push(text);
+ }
+ return parts.join("\n");
+}
+
+function assertFitsAsset(path: string, filename: string, size: number): void {
+ const maxBytes = ACCEPTED[extname(filename).toLowerCase()]?.maxBytes ?? 0;
+ if (size > maxBytes) {
+ throw new Error(
+ `${path} is ${formatBytes(size)} after re-encoding at visually lossless quality, over the ${formatBytes(maxBytes)} a HeyGen asset can hold.`,
+ );
+ }
+}
+
+interface OffloadContext {
+ absDir: string;
+ opts: OffloadOptions;
+ cachePath: string;
+ cache: Record<string, CacheEntry>;
+ fetchImpl: typeof fetch;
+ reencoder: Reencoder;
+ tempFiles: string[];
+ /** Staged GIFs the entry composition no longer shows as images. */
+ preparedGifSources: string[];
+ fileMap: Map<string, StagedFile>;
+}
+
+interface FilePlan {
+ key: string;
+ preset?: PresetName;
+ sourcePath: string;
+ contentType?: string;
+ segments?: PlayedSegment[];
+}
+
+async function planFile(
+ ctx: OffloadContext,
+ path: string,
+ file: StagedFile,
+): Promise<FilePlan | null> {
+ const kind = mediaKind(path);
+ const size = stagedSize(file);
+ if (!kind || size < OFFLOAD_MIN_BYTES) return null;
+ const sha256 = await stagedSha256(file);
+ const keyFor = (variant: string) => `${ctx.opts.apiBaseUrl} ${sha256} ${variant}`;
+ const accepted = ACCEPTED[extname(path).toLowerCase()];
+ if (accepted && size <= accepted.maxBytes) {
+ return { key: keyFor("as-is"), sourcePath: "", contentType: accepted.contentType };
+ }
+ const sourcePath = snapshotPath(path, file, ctx.tempFiles);
+ const preset = await ctx.reencoder.choose(sourcePath, kind);
+ // Files left as authored stay in the zip; the archive size check names them.
+ if (!preset) return null;
+ const segments = kind === "image" ? null : await planFileTrim(ctx, path, sourcePath, kind);
+ // Keyed by encode settings and cut too, so either changing re-encodes instead of reusing old output.
+ const variant = JSON.stringify([REENCODE_PRESETS[preset], segments]);
+ return { key: keyFor(variant), preset, sourcePath, segments: segments ?? undefined };
+}
+
+/** Cuts a re-encoded file to what the entry composition plays, shifting its clips to match. */
+async function planFileTrim(
+ ctx: OffloadContext,
+ path: string,
+ sourcePath: string,
+ kind: MediaKind,
+): Promise<PlayedSegment[] | null> {
+ const entryPath = ctx.opts.entryPath ?? "index.html";
+ const entryHtml = textOf(entryPath, ctx.fileMap.get(entryPath));
+ const gridPerSecond = entryHtml ? await ctx.reencoder.trimGrid?.(sourcePath, kind) : null;
+ if (!entryHtml || !gridPerSecond) return null;
+ const name = basename(path);
+ const plan = planTrim({
+ entryHtml,
+ path,
+ gridPerSecond,
+ resolveKey: (raw) => htmlSrcKey(ctx.absDir, entryPath, raw, ctx.fileMap),
+ mentionedElsewhere: [...ctx.fileMap].some(
+ ([other, content]) =>
+ other !== entryPath && other !== path && !!textOf(other, content)?.includes(name),
+ ),
+ scriptCode: scriptCode(ctx.fileMap),
+ });
+ if (!plan) return null;
+ ctx.fileMap.set(entryPath, Buffer.from(plan.html, "utf-8"));
+ return plan.segments;
+}
+
+async function offloadFile(
+ ctx: OffloadContext,
+ path: string,
+ file: StagedFile,
+): Promise<OffloadedMedia | null> {
+ const plan = await planFile(ctx, path, file);
+ if (!plan) return null;
+ const item: OffloadedMedia = {
+ path,
+ size_bytes: stagedSize(file),
+ action: plan.preset ? "reencode" : "upload",
+ ...(plan.segments && {
+ trimmed_to_seconds: plan.segments.reduce((sum, s) => sum + s.end - s.start, 0),
+ }),
+ };
+ const hit = ctx.cache[plan.key];
+ if (hit && (!ctx.opts.upload || (await stillServed(hit, ctx.fetchImpl)))) {
+ return { ...item, action: "cached", url: hit.url };
+ }
+ if (!ctx.opts.upload) return item;
+ return { ...item, url: await uploadPlanned(ctx, ctx.opts.upload, path, file, plan) };
+}
+
+async function uploadPlanned(
+ ctx: OffloadContext,
+ upload: NonNullable<OffloadOptions["upload"]>,
+ path: string,
+ staged: StagedFile,
+ plan: FilePlan,
+): Promise<string> {
+ const filename = basename(path);
+ const file: UploadableFile = plan.preset
+ ? await ctx.reencoder.run(plan.sourcePath, plan.preset, plan.segments)
+ : Buffer.isBuffer(staged)
+ ? { bytes: staged, filename, contentType: plan.contentType! }
+ : { path: staged.absPath, size: staged.size, filename, contentType: plan.contentType! };
+ if (plan.preset && !Buffer.isBuffer(staged)) assertUnchanged(staged);
+ const size = "bytes" in file ? file.bytes.byteLength : file.size;
+ assertFitsAsset(path, file.filename, size);
+ const asset = await upload(file);
+ if (!Buffer.isBuffer(staged)) assertUnchanged(staged);
+ ctx.cache[plan.key] = {
+ asset_id: asset.asset_id,
+ url: asset.url,
+ size_bytes: size,
+ };
+ writeCache(ctx.cachePath, ctx.cache);
+ return asset.url;
+}
+
+const GIF_PREFIX = "_animated_gif";
+
+// The renderer's own GIF prep turns each animated <img> into a seek-synced WebM <video>, which then
+// offloads like any video. Sub-compositions are prepared by the renderer after inlining, so not here.
+async function prepareEntryGifs(
+ ctx: OffloadContext,
+ fileMap: Map<string, StagedFile>,
+): Promise<void> {
+ const entryPath = ctx.opts.entryPath ?? "index.html";
+ const html = textOf(entryPath, fileMap.get(entryPath));
+ if (!html || !/\.gif\b/i.test(html)) return;
+ const sourceAssets = entryGifSources(ctx, entryPath, html);
+ if (!sourceAssets) return;
+ const prep = ctx.opts.gifPrep ?? (await import("@hyperframes/producer")).prepareAnimatedGifInputs;
+ const outputDir = mkdtempSync(join(tmpdir(), "hf-offload-gif-"));
+ ctx.tempFiles.push(outputDir);
+ const result = await prep(html, {
+ projectDir: ctx.absDir,
+ downloadDir: outputDir,
+ outputDir,
+ outputSrcPrefix: posix.relative(posix.dirname(entryPath), GIF_PREFIX),
+ cacheDir: join(homedir(), ".cache", "hyperframes", "animated-gif"),
+ sourceAssets,
+ });
+ if (result.preparedGifs.length === 0) return;
+ fileMap.set(entryPath, Buffer.from(result.html, "utf-8"));
+ for (const gif of result.preparedGifs) {
+ fileMap.set(`${GIF_PREFIX}/${basename(gif.outputPath)}`, readFileSync(gif.outputPath));
+ const source = htmlSrcKey(ctx.absDir, entryPath, gif.sourceSrc.trim(), fileMap);
+ if (source) ctx.preparedGifSources.push(source);
+ }
+}
+
+/**
+ * Each entry GIF's staged bytes, resolved the way the renderer rebases a nested entry; `null` when none
+ * is heavy, since the renderer's own GIF prep then suffices and needs no local ffmpeg.
+ */
+function entryGifSources(
+ ctx: OffloadContext,
+ entryPath: string,
+ html: string,
+): Map<string, string> | null {
+ const sourceAssets = new Map<string, string>();
+ let heavy = false;
+ for (const img of parseHTML(html).document.querySelectorAll("img[src]")) {
+ const src = img.getAttribute("src")!.trim();
+ const key = htmlSrcKey(ctx.absDir, entryPath, src, ctx.fileMap);
+ if (!key?.toLowerCase().endsWith(".gif")) continue;
+ const file = ctx.fileMap.get(key)!;
+ sourceAssets.set(src, stagedCopy(key, file, ctx.tempFiles));
+ heavy ||= stagedSize(file) >= OFFLOAD_MIN_BYTES;
+ }
+ return heavy ? sourceAssets : null;
+}
+
+/**
+ * Upload each heavy media file as an asset (or reuse a cached one still served), point its references at the
+ * asset URL and drop it from `fileMap`. Without `upload`, nothing uploads and only animated GIFs are encoded.
+ */
+export async function offloadHeavyMedia(
+ projectDir: string,
+ fileMap: Map<string, StagedFile>,
+ opts: OffloadOptions,
+): Promise<OffloadedMedia[]> {
+ const cachePath = opts.cachePath ?? defaultOffloadCachePath();
+ const ctx: OffloadContext = {
+ absDir: resolve(projectDir),
+ opts,
+ cachePath,
+ cache: readCache(cachePath),
+ fetchImpl: opts.fetchImpl ?? fetch,
+ reencoder: opts.reencoder ?? ffmpegReencoder,
+ tempFiles: [],
+ preparedGifSources: [],
+ fileMap,
+ };
+ const urls = new Map<string, string>();
+ const report: OffloadedMedia[] = [];
+ try {
+ await prepareEntryGifs(ctx, fileMap);
+ // Only files a reference points at: one used through render-time --variables must stay in the zip.
+ const referenced = referencedFiles(ctx.absDir, fileMap);
+ for (const [path, file] of [...fileMap].sort(([a], [b]) => a.localeCompare(b))) {
+ if (!referenced.has(path)) continue;
+ const item = await offloadFile(ctx, path, file);
+ if (!item) continue;
+ // A dry run rewrites to a stand-in URL so the reported archive size matches the real one.
+ urls.set(path, item.url ?? `https://assets.invalid/${report.length}`);
+ report.push(item);
+ opts.onFile?.(item);
+ }
+ } finally {
+ for (const dir of ctx.tempFiles) rmSync(dir, { recursive: true, force: true });
+ }
+
+ if (urls.size > 0) rewriteReferences(ctx.absDir, fileMap, (key) => urls.get(key) ?? null);
+ dropReplacedFiles(fileMap, report, ctx.preparedGifSources, [...urls.values()]);
+ return report;
+}
+
+function dropReplacedFiles(
+ fileMap: Map<string, StagedFile>,
+ report: OffloadedMedia[],
+ preparedGifSources: string[],
+ assetUrls: string[],
+): void {
+ // Asset URLs end in names like original.mp4 that can collide with a project file's name.
+ const withoutUrls = (text: string) =>
+ assetUrls.reduce((acc, url) => acc.replaceAll(url, ""), text);
+ const code = withoutUrls(scriptCode(fileMap));
+ for (const item of report) {
+ if (isStillMentioned(fileMap, item.path, code, withoutUrls)) item.kept_in_archive = true;
+ else fileMap.delete(item.path);
+ }
+ for (const gif of preparedGifSources) {
+ if (!isStillMentioned(fileMap, gif, code, withoutUrls)) fileMap.delete(gif);
+ }
+}
+
+/**
+ * Whether a reference rewriting could not replace may still load the file: its name anywhere, or its
+ * stem and extension both in script code, which can join them at runtime ("clip" + ".mp4").
+ */
+function isStillMentioned(
+ fileMap: Map<string, StagedFile>,
+ path: string,
+ code: string,
+ withoutUrls: (text: string) => string,
+): boolean {
+ const ext = extname(path);
+ if (code.includes(basename(path, ext)) && code.includes(ext.slice(1))) return true;
+ const name = basename(path);
+ for (const [entryPath, content] of fileMap) {
+ const contentText = textOf(entryPath, content);
+ if (entryPath === path || !contentText?.includes(name)) continue;
+ if (withoutUrls(contentText).includes(name)) return true;
+ }
+ return false;
+}
+
+/** The staged file an HTML src loads, resolved the way the renderer inlines sub-compositions. */
+function htmlSrcKey(
+ absDir: string,
+ entryPath: string,
+ raw: string,
+ fileMap: Map<string, StagedFile>,
+): string | null {
+ const clean = cleanAssetUrl(raw);
+ if (!clean || isRemoteOrInlineUrl(clean)) return null;
+ const rootRelative = clean.startsWith("/")
+ ? clean
+ : rewriteAssetPath(entryPath, clean, (p) => fileMap.has(p));
+ for (const candidate of resolveLocalAssetCandidates(absDir, rootRelative)) {
+ const key = relative(absDir, candidate).replaceAll("\\", "/");
+ if (fileMap.has(key)) return key;
+ }
+ return null;
+}
+
+type UrlFor = (rootRelative: string) => string | null;
+
+/** The staged files that references rewriteReferences would replace point at. */
+function referencedFiles(absDir: string, fileMap: Map<string, StagedFile>): Set<string> {
+ const keys = new Set<string>();
+ rewriteReferences(absDir, fileMap, (key) => {
+ if (fileMap.has(key)) keys.add(key);
+ return null;
+ });
+ return keys;
+}
+
+function rewriteReferences(
+ absDir: string,
+ fileMap: Map<string, StagedFile>,
+ urlForKey: (key: string) => string | null,
+): void {
+ const urlFor: UrlFor = (rootRelative) => {
+ for (const candidate of resolveLocalAssetCandidates(absDir, rootRelative)) {
+ const url = urlForKey(relative(absDir, candidate).replaceAll("\\", "/"));
+ if (url) return url;
+ }
+ return null;
+ };
+
+ for (const [entryPath, content] of [...fileMap]) {
+ const rewrite = rewriterFor(absDir, entryPath, urlFor, fileMap);
+ const text = rewrite ? textOf(entryPath, content) : undefined;
+ const next = rewrite && text !== undefined ? rewrite(text) : null;
+ if (next !== null) fileMap.set(entryPath, Buffer.from(next, "utf-8"));
+ }
+}
+
+function rewriterFor(
+ absDir: string,
+ entryPath: string,
+ urlFor: UrlFor,
+ fileMap: Map<string, StagedFile>,
+): ((text: string) => string | null) | null {
+ if (/\.html?$/i.test(entryPath)) {
+ return (text) => rewriteHtmlEntry(absDir, entryPath, text, urlFor, fileMap);
+ }
+ if (/\.css$/i.test(entryPath)) return (text) => rewriteCssEntry(absDir, entryPath, text, urlFor);
+ if (/\.(m?js|cjs|json)$/i.test(entryPath)) return (text) => rewriteStringLiterals(text, urlFor);
+ return null;
+}
+
+function localTarget(raw: string, toRootRelative: (clean: string) => string, urlFor: UrlFor) {
+ const clean = cleanAssetUrl(raw);
+ if (!clean || isRemoteOrInlineUrl(clean)) return null;
+ const url = urlFor(clean.startsWith("/") ? clean : toRootRelative(clean));
+ // Keep a query or media fragment such as #t=5,8.
+ return url && url + raw.trim().slice(clean.length);
+}
+
+function rewriteCssEntry(absDir: string, entryPath: string, css: string, urlFor: UrlFor) {
+ // A stylesheet's url() resolves against the stylesheet itself.
+ const result = rewriteCssUrls(css, resolve(absDir, dirname(entryPath)), entryPath, (raw) =>
+ localTarget(raw, (clean) => posix.join(posix.dirname(entryPath), clean), urlFor),
+ );
+ return result.modified ? result.css : null;
+}
+
+function rewriteHtmlEntry(
+ absDir: string,
+ entryPath: string,
+ html: string,
+ urlFor: UrlFor,
+ fileMap: Map<string, StagedFile>,
+): string | null {
+ const referrerAbsDir = resolve(absDir, dirname(entryPath));
+ // Sub-composition srcs resolve the way the renderer inlines them.
+ const resolveTarget: HtmlAttributeResolver = (raw) =>
+ localTarget(raw, (clean) => rewriteAssetPath(entryPath, clean, (p) => fileMap.has(p)), urlFor);
+ const { document } = parseHTML(html);
+ const attrs = rewriteHtmlAttributes(document, referrerAbsDir, entryPath, resolveTarget, {
+ attrs: ["src", "poster", "href"],
+ });
+ const srcsets = rewriteSrcsets(document, resolveTarget);
+ const styles = rewriteStyleBlocks(document, referrerAbsDir, entryPath, resolveTarget);
+ let scripts = false;
+ for (const script of document.querySelectorAll("script")) {
+ if (script.hasAttribute("src")) continue;
+ const next = rewriteStringLiterals(script.textContent ?? "", urlFor);
+ if (next === null) continue;
+ script.textContent = next;
+ scripts = true;
+ }
+ return attrs || srcsets || styles || scripts ? document.toString() : null;
+}
+
+// A plain string literal: no escapes, interpolation or line breaks.
+const STRING_LITERAL_RE = /(["'`])([^"'`\\\n$]+)\1/g;
+
+/** Script URLs resolve against the page, which the renderer roots at the project. */
+function rewriteStringLiterals(code: string, urlFor: UrlFor): string | null {
+ let modified = false;
+ const next = code.replace(STRING_LITERAL_RE, (full, quote: string, value: string) => {
+ const url = localTarget(value, (clean) => clean, urlFor);
+ if (!url) return full;
+ modified = true;
+ return `${quote}${url}${quote}`;
+ });
+ return modified ? next : null;
+}
+
+function rewriteSrcsets(document: Document, resolveTarget: HtmlAttributeResolver): boolean {
+ let modified = false;
+ for (const el of document.querySelectorAll("[srcset]")) {
+ const srcset = el.getAttribute("srcset") ?? "";
+ // A data: candidate carries its own commas; leave such a srcset as authored.
+ if (srcset.includes("data:")) continue;
+ let changed = false;
+ const next = srcset
+ .split(",")
+ .map((part) => {
+ const [url, ...descriptors] = part.trim().split(/\s+/);
+ const target = url ? resolveTarget(url, "") : null;
+ if (!target) return part.trim();
+ changed = true;
+ return [target, ...descriptors].join(" ");
+ })
+ .join(", ");
+ if (changed) {
+ el.setAttribute("srcset", next);
+ modified = true;
+ }
+ }
+ return modified;
+}
diff --git a/packages/cli/src/cloud/offloadTrim.ts b/packages/cli/src/cloud/offloadTrim.ts
new file mode 100644
index 0000000000..4db38b2c71
--- /dev/null
+++ b/packages/cli/src/cloud/offloadTrim.ts
@@ -0,0 +1,128 @@
+import { basename } from "node:path";
+import { parseHTML } from "linkedom";
+import {
+ readAuthoredDurationSeconds,
+ readMediaOffsetSeconds,
+ readPlaybackRate,
+} from "@hyperframes/parsers/media-duration";
+
+/** Kept either side of a played window, so a decoder has frames to seek from. */
+const TRIM_MARGIN_SECONDS = 0.5;
+
+export interface PlayedSegment {
+ start: number;
+ end: number;
+}
+
+// Script that can write a clip's in-point, length, rate, loop or source moves what it plays.
+const RETIMING_SCRIPT_RE =
+ /data-(?:playback-start|media-start|start|duration|end|playback-rate)|\b(?:dataset|setAttribute|currentTime|playbackRate)\b|\.(?:src|loop)\b/;
+
+type AttrReader = (name: string) => string | null;
+
+function mediaSources(el: Element): string[] {
+ const own = el.getAttribute("src");
+ const nested = [...el.querySelectorAll("source[src]")].map((s) => s.getAttribute("src")!);
+ return [...(own ? [own] : []), ...nested];
+}
+
+// A host composition or in-point above the element moves what it plays; only root-level uses are bounded here.
+function hasTimedAncestor(el: Element): boolean {
+ for (let p = el.parentElement; p; p = p.parentElement) {
+ const get: AttrReader = (name) => p.getAttribute(name);
+ if (readMediaOffsetSeconds(get) > 0) return true;
+ if (
+ p.hasAttribute("data-composition-id") &&
+ p.parentElement?.closest("[data-composition-id]")
+ ) {
+ return true;
+ }
+ }
+ return false;
+}
+
+function playedWindow(el: Element): PlayedSegment | null {
+ // A data-var-src clip's source is only known at render time, so neither a cut nor a shift holds.
+ if (
+ el.hasAttribute("loop") ||
+ el.hasAttribute("data-automation") ||
+ el.hasAttribute("data-var-src") ||
+ hasTimedAncestor(el)
+ ) {
+ return null;
+ }
+ const get: AttrReader = (name) => el.getAttribute(name);
+ const start = readMediaOffsetSeconds(get);
+ const duration = readAuthoredDurationSeconds(get, Number(get("data-start") ?? 0));
+ if (duration === null || !(duration > 0)) return null;
+ return { start, end: start + duration * readPlaybackRate(get) };
+}
+
+/** Merged windows with margin, on the source's frame grid so shifted starts land on the same frames. */
+function mergeWindows(windows: PlayedSegment[], gridPerSecond: number): PlayedSegment[] {
+ const down = (t: number) => Math.floor(t * gridPerSecond) / gridPerSecond;
+ const up = (t: number) => Math.ceil(t * gridPerSecond) / gridPerSecond;
+ const merged: PlayedSegment[] = [];
+ for (const w of [...windows].sort((a, b) => a.start - b.start)) {
+ const start = down(Math.max(0, w.start - TRIM_MARGIN_SECONDS));
+ const end = up(w.end + TRIM_MARGIN_SECONDS);
+ const last = merged.at(-1);
+ if (last && start <= last.end) last.end = Math.max(last.end, end);
+ else merged.push({ start, end });
+ }
+ return merged;
+}
+
+function shiftMediaStart(el: Element, segments: PlayedSegment[]): void {
+ const get: AttrReader = (name) => el.getAttribute(name);
+ const start = readMediaOffsetSeconds(get);
+ let offset = 0;
+ for (const segment of segments) {
+ if (start >= segment.start && start < segment.end) {
+ // Write back to whichever attribute readMediaOffsetSeconds read.
+ const attr =
+ Number.parseFloat(get("data-playback-start") ?? "") >= 0
+ ? "data-playback-start"
+ : "data-media-start";
+ el.setAttribute(attr, String(offset + (start - segment.start)));
+ return;
+ }
+ offset += segment.end - segment.start;
+ }
+}
+
+/**
+ * The parts of `path` the entry composition plays, with the composition shifted to match a file cut
+ * to those parts. `null` when any use is not a bounded <video>/<audio> clip, so the whole file is kept.
+ */
+export function planTrim(input: {
+ entryHtml: string;
+ path: string;
+ resolveKey: (rawSrc: string) => string | null;
+ gridPerSecond: number;
+ mentionedElsewhere: boolean;
+ /** Every script the page runs, entry inline scripts included. */
+ scriptCode: string;
+}): { segments: PlayedSegment[]; html: string } | null {
+ if (input.mentionedElsewhere || RETIMING_SCRIPT_RE.test(input.scriptCode)) return null;
+ const { document } = parseHTML(input.entryHtml);
+ const uses: Element[] = [];
+ let sources = 0;
+ for (const el of document.querySelectorAll("video, audio")) {
+ const matching = mediaSources(el).filter((src) => input.resolveKey(src) === input.path);
+ if (matching.length === 0) continue;
+ // A media fragment (#t=) names times in the uncut file.
+ if (matching.some((src) => src.includes("#"))) return null;
+ uses.push(el);
+ sources += matching.length;
+ }
+ // Every mention of the file in the page must be one of these clips' sources.
+ if (uses.length === 0 || input.entryHtml.split(basename(input.path)).length - 1 !== sources) {
+ return null;
+ }
+ const windows = uses.map(playedWindow);
+ if (windows.some((w) => w === null)) return null;
+ const segments = mergeWindows(windows as PlayedSegment[], input.gridPerSecond);
+ for (const el of uses) shiftMediaStart(el, segments);
+ return { segments, html: document.toString() };
+}
diff --git a/packages/cli/src/cloud/upload.test.ts b/packages/cli/src/cloud/upload.test.ts
index 89713f83a5..9b121b104c 100644
--- a/packages/cli/src/cloud/upload.test.ts
+++ b/packages/cli/src/cloud/upload.test.ts
@@ -1,8 +1,13 @@
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect, onTestFinished, vi } from "vitest";
import { createHash } from "node:crypto";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { createServer, type IncomingHttpHeaders } from "node:http";
+import type { AddressInfo } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import { HyperframesApiError } from "./_gen/client.js";
import type { HyperframesCloudClient } from "./_gen/client.js";
-import { uploadZipViaDirectUpload } from "./upload.js";
+import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js";
function sha256Hex(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
@@ -232,3 +237,81 @@ describe("uploadZipViaDirectUpload", () => {
expect(events.map((e) => e.phase)).toEqual(["initialize", "upload", "upload", "complete"]);
});
});
+
+describe("uploadAssetViaDirectUpload from a file on disk", () => {
+ async function withServer(
+ status: number,
+ run: (url: string, got: { headers?: IncomingHttpHeaders; body?: Buffer }) => Promise<void>,
+ ): Promise<void> {
+ const got: { headers?: IncomingHttpHeaders; body?: Buffer } = {};
+ const server = createServer(async (req, res) => {
+ const chunks: Buffer[] = [];
+ for await (const chunk of req) chunks.push(chunk as Buffer);
+ Object.assign(got, { headers: req.headers, body: Buffer.concat(chunks) });
+ res.writeHead(status).end(status === 200 ? "" : "SignatureDoesNotMatch");
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ try {
+ await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}/put`, got);
+ } finally {
+ server.close();
+ }
+ }
+
+ function fileOnDisk(bytes: Buffer): string {
+ const dir = mkdtempSync(join(tmpdir(), "hf-upload-test-"));
+ onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
+ writeFileSync(join(dir, "clip.mp4"), bytes);
+ return join(dir, "clip.mp4");
+ }
+
+ const clientFor = (url: string) =>
+ makeClient({
+ createAssetUpload: vi.fn(async () => ({
+ asset_id: "asset_xyz",
+ upload_url: url,
+ upload_headers: {},
+ expires_in_seconds: 3600,
+ max_bytes: 200 * 1024 * 1024,
+ status: "pending_upload" as const,
+ })),
+ } as Partial<HyperframesCloudClient>);
+
+ it("hashes the file and PUTs it as a stream with its exact length", async () => {
+ const bytes = Buffer.from("a file far too big to read whole, in miniature");
+ const path = fileOnDisk(bytes);
+ await withServer(200, async (url, got) => {
+ const client = clientFor(url);
+ await uploadAssetViaDirectUpload({
+ client,
+ path,
+ size: bytes.byteLength,
+ filename: "clip.mp4",
+ contentType: "video/mp4",
+ });
+ expect(client.createAssetUpload).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({ checksum_sha256: sha256Hex(bytes) }),
+ }),
+ );
+ expect(got.body?.equals(bytes)).toBe(true);
+ expect(got.headers?.["content-length"]).toBe(String(bytes.byteLength));
+ expect(got.headers?.["content-type"]).toBe("video/mp4");
+ });
+ });
+
+ it("reports the status of a refused PUT", async () => {
+ const path = fileOnDisk(Buffer.from("bytes"));
+ await withServer(403, async (url) => {
+ await expect(
+ uploadAssetViaDirectUpload({
+ client: clientFor(url),
+ path,
+ size: 5,
+ filename: "clip.mp4",
+ contentType: "video/mp4",
+ }),
+ ).rejects.toThrow(/Direct upload PUT failed: 403.*SignatureDoesNotMatch/);
+ });
+ });
+});
diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts
index 703b83e697..2598ab5305 100644
--- a/packages/cli/src/cloud/upload.ts
+++ b/packages/cli/src/cloud/upload.ts
@@ -24,11 +24,16 @@
*/
import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { type IncomingMessage, request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { pipeline } from "node:stream/promises";
import type { HyperframesCloudClient } from "./_gen/client.js";
import { HyperframesApiError } from "./_gen/client.js";
export interface UploadZipViaDirectResult {
asset_id: string;
+ url: string;
size_bytes: number;
duration_ms: number;
}
@@ -38,6 +43,12 @@ export type UploadProgressEvent =
| { phase: "upload"; percent: number }
| { phase: "complete" };
+/** A file on disk, hashed and uploaded as a stream so it is never held in memory. */
+export interface UploadFileSource {
+ path: string;
+ size: number;
+}
+
export interface UploadZipViaDirectOptions {
client: HyperframesCloudClient;
bytes: Uint8Array;
@@ -58,6 +69,12 @@ function sha256Hex(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
}
+async function sha256OfFile(path: string): Promise<string> {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
+ return hash.digest("hex");
+}
+
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
@@ -84,20 +101,21 @@ async function putBytesToPresignedUrl(
fetchImpl: typeof fetch,
uploadUrl: string,
uploadHeaders: Record<string, unknown>,
- bytes: Uint8Array,
+ body: Uint8Array | UploadFileSource,
contentType: string,
): Promise<void> {
const headers: Record<string, string> = {
"content-type": contentType,
...normalizeUploadHeaders(uploadHeaders),
};
+ if (!(body instanceof Uint8Array)) return putFileToPresignedUrl(uploadUrl, headers, body);
// `Uint8Array<ArrayBufferLike>` is a valid `BodyInit` at runtime but
// not strictly assignable per lib.dom.d.ts — cast rather than copy,
// since a 200MB buffer copy would be wasteful.
const res = await fetchImpl(uploadUrl, {
method: "PUT",
headers,
- body: bytes as unknown as BodyInit,
+ body: body as unknown as BodyInit,
});
if (!res.ok) {
const detail = await res.text().catch(() => "");
@@ -109,6 +127,45 @@ async function putBytesToPresignedUrl(
}
}
+// fetch holds a streamed request body in memory whole, so a file is piped through node:http(s).
+async function putFileToPresignedUrl(
+ uploadUrl: string,
+ headers: Record<string, string>,
+ file: UploadFileSource,
+): Promise<void> {
+ const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
+ const req = send(uploadUrl, {
+ method: "PUT",
+ headers: { ...headers, "content-length": String(file.size) },
+ });
+ const response = new Promise<IncomingMessage>((resolve, reject) => {
+ req.once("response", resolve).once("error", reject);
+ });
+ const sent = pipeline(createReadStream(file.path), req);
+ let res: IncomingMessage;
+ try {
+ res = await response;
+ } catch (err) {
+ await sent.catch(() => undefined);
+ throw err;
+ }
+ const status = res.statusCode ?? 0;
+ const ok = status >= 200 && status < 300;
+ // A refused upload can close the connection mid-body; its status explains that write error.
+ await sent.catch((err: unknown) => {
+ if (ok) throw err;
+ });
+ let detail = "";
+ for await (const chunk of res) detail += String(chunk);
+ if (!ok) {
+ throw new Error(
+ `Direct upload PUT failed: ${status} ${res.statusMessage ?? ""}${
+ detail ? ` — ${detail.slice(0, 300)}` : ""
+ }`,
+ );
+ }
+}
+
// Complete with retry-on-409. Retry ONLY on the documented "PUT not
// visible yet" race between S3 write consistency and finalize; any other
// error surfaces immediately. `completeAssetUpload` itself is idempotent,
@@ -117,7 +174,7 @@ async function completeWithRetry(
client: HyperframesCloudClient,
asset_id: string,
checksum_sha256: string,
-): Promise<{ asset_id: string }> {
+): Promise<{ asset_id: string; url: string }> {
let lastErr: unknown;
for (let attempt = 0; attempt < COMPLETE_MAX_RETRIES; attempt++) {
try {
@@ -138,12 +195,17 @@ async function completeWithRetry(
}
export async function uploadAssetViaDirectUpload(
- opts: UploadZipViaDirectOptions & { contentType: string },
+ opts: Omit<UploadZipViaDirectOptions, "bytes"> & { contentType: string } & (
+ | { bytes: Uint8Array }
+ | UploadFileSource
+ ),
): Promise<UploadZipViaDirectResult> {
const start = Date.now();
- const { bytes, filename, idempotencyKey } = opts;
- const size_bytes = bytes.byteLength;
- const checksum_sha256 = sha256Hex(bytes);
+ const { filename, idempotencyKey } = opts;
+ const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size };
+ const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
+ const checksum_sha256 =
+ body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
const fetchImpl = opts.fetchImpl ?? fetch;
opts.onProgress?.({ phase: "initialize" });
@@ -157,7 +219,7 @@ export async function uploadAssetViaDirectUpload(
fetchImpl,
initialize.upload_url,
initialize.upload_headers,
- bytes,
+ body,
opts.contentType,
);
opts.onProgress?.({ phase: "upload", percent: 100 });
@@ -166,6 +228,7 @@ export async function uploadAssetViaDirectUpload(
const completed = await completeWithRetry(opts.client, initialize.asset_id, checksum_sha256);
return {
asset_id: completed.asset_id,
+ url: completed.url,
size_bytes,
duration_ms: Date.now() - start,
};
diff --git a/packages/cli/src/commands/cloud/render.test.ts b/packages/cli/src/commands/cloud/render.test.ts
index 95a097b7bc..52eb0cb561 100644
--- a/packages/cli/src/commands/cloud/render.test.ts
+++ b/packages/cli/src/commands/cloud/render.test.ts
@@ -1,11 +1,12 @@
import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest";
import { spawnSync } from "node:child_process";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import {
+ assertArchiveFits,
resolveAspectRatioForSubmit,
validateDryRunSource,
validateResolutionFormatCombo,
@@ -116,6 +117,97 @@ describe("cloud render --dry-run", () => {
});
});
+describe("cloud render --dry-run with heavy media", () => {
+ it("plans the media as HeyGen asset uploads and leaves it out of the archive", () => {
+ const dir = writeComposition(1920, 1080);
+ writeFileSync(
+ join(dir, "index.html"),
+ `<!doctype html><html><body><div data-composition-id="main" data-width="1920" data-height="1080"><video src="clip.mp4"></video></div></body></html>`,
+ "utf-8",
+ );
+ writeFileSync(join(dir, "clip.mp4"), Buffer.alloc(6 * 1024 * 1024, 1));
+ const result = spawnSync(
+ "bun",
+ ["run", cliEntry, "cloud", "render", dir, "--dry-run", "--json"],
+ {
+ encoding: "utf-8",
+ timeout: 30_000,
+ env: {
+ ...process.env,
+ CI: "1",
+ HEYGEN_API_URL: "http://127.0.0.1:1",
+ HYPERFRAMES_NO_UPDATE_CHECK: "1",
+ HOME: dir,
+ },
+ },
+ );
+
+ expect(result.status).toBe(0);
+ const { archive } = JSON.parse(result.stdout) as {
+ archive: {
+ file_count: number;
+ size_bytes: number;
+ offloaded_media: Array<{ path: string; action: string; size_bytes: number }>;
+ };
+ };
+ expect(archive.offloaded_media).toEqual([
+ { path: "clip.mp4", action: "upload", size_bytes: 6 * 1024 * 1024 },
+ ]);
+ expect(archive.file_count).toBe(1);
+ expect(archive.size_bytes).toBeLessThan(64 * 1024);
+ });
+});
+
+describe("cloud render --dry-run with a file over 4 GiB left in the zip", () => {
+ it("reports it as over the limit without reading it", () => {
+ const dir = writeComposition(1920, 1080);
+ mkdirSync(join(dir, "raw"));
+ writeFileSync(join(dir, "raw", "take.mov"), "");
+ truncateSync(join(dir, "raw", "take.mov"), 2 ** 32 + 1);
+ const result = spawnSync(
+ "bun",
+ ["run", cliEntry, "cloud", "render", dir, "--dry-run", "--json"],
+ {
+ encoding: "utf-8",
+ timeout: 60_000,
+ env: {
+ ...process.env,
+ CI: "1",
+ HEYGEN_API_URL: "http://127.0.0.1:1",
+ HYPERFRAMES_NO_UPDATE_CHECK: "1",
+ HOME: dir,
+ },
+ },
+ );
+
+ expect(result.status).toBe(0);
+ const { archive } = JSON.parse(result.stdout) as {
+ archive: {
+ exceeds_upload_limit: boolean;
+ largest_files: Array<{ path: string; size_bytes: number }>;
+ };
+ };
+ expect(archive.exceeds_upload_limit).toBe(true);
+ expect(archive.largest_files[0]).toEqual({ path: "raw/take.mov", size_bytes: 2 ** 32 + 1 });
+ });
+});
+
+describe("assertArchiveFits", () => {
+ it("passes an archive at the limit and fails one over it, naming its largest files", () => {
+ expect(() => assertArchiveFits(200 * 1024 * 1024, [])).not.toThrow();
+ const errors = vi.mocked(console.error);
+ const logs = vi.mocked(console.log);
+ expect(() =>
+ assertArchiveFits(200 * 1024 * 1024 + 1, [
+ { path: "data/scan.tiff", size_bytes: 190 * 1024 * 1024 },
+ ]),
+ ).toThrow(CliRuntimeError);
+ expect([...errors.mock.calls, ...logs.mock.calls].flat().join("\n")).toContain(
+ "data/scan.tiff",
+ );
+ });
+});
+
describe("resolveAspectRatioForSubmit — non-local sources", () => {
it("trusts an explicit flag for asset_id / url", () => {
const asset: ProjectInputSource = { kind: "asset_id", assetId: "a" };
diff --git a/packages/cli/src/commands/cloud/render.ts b/packages/cli/src/commands/cloud/render.ts
index facaae7500..9b0e96b64d 100644
--- a/packages/cli/src/commands/cloud/render.ts
+++ b/packages/cli/src/commands/cloud/render.ts
@@ -25,7 +25,7 @@ import { failCommand } from "../../utils/commandResult.js";
*
* Auth comes from the existing `cli/src/auth/` chain via `cloud/auth.ts`.
* The cloud HTTP client (`cloud/_gen/client.ts`) is generated from
- * `experiment-framework/openapi/external-api.json`; never hand-edit it.
+ * HeyGen's external API OpenAPI spec; never hand-edit it.
*/
import { defineCommand } from "citty";
@@ -39,7 +39,10 @@ import { errorBox, formatBytes, formatDuration } from "../../ui/format.js";
import { resolveProject, type ProjectDir } from "../../utils/project.js";
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
import {
- buildPublishFileMap,
+ readStaged,
+ type StagedFile,
+ stagedSize,
+ stagePublishFiles,
type PublishArchiveResult,
zipPublishFileMap,
} from "../../utils/publishProject.js";
@@ -61,7 +64,14 @@ import {
} from "../../cloud/index.js";
import { reportApiError } from "../../cloud/errors.js";
import { parseEnumFlag, parseIntFlag, parseNumericFlag } from "../../cloud/parsing.js";
-import { uploadZipViaDirectUpload } from "../../cloud/upload.js";
+import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "../../cloud/upload.js";
+import {
+ OFFLOAD_MIN_BYTES,
+ offloadHeavyMedia,
+ type OffloadedMedia,
+ type OffloadOptions,
+} from "../../cloud/offload.js";
+import { resolveCloudBaseUrl } from "../../cloud/auth.js";
import { colorStatus } from "../../cloud/statusColor.js";
import type {
CreateHyperframesRenderRequest,
@@ -244,7 +254,7 @@ export default defineCommand({
if (args["dry-run"]) {
if (project.kind !== "dir") throw new Error("Dry-run project must be a local directory");
- reportDryRun(prepareLocalArchive(project, args.composition, asJson), asJson);
+ reportDryRun(await prepareLocalArchive(project, args.composition, asJson), asJson);
return;
}
@@ -257,7 +267,11 @@ export default defineCommand({
const client = await createCloudClient();
const preparedArchive =
- project.kind === "dir" ? prepareLocalArchive(project, args.composition, asJson) : undefined;
+ project.kind === "dir"
+ ? await prepareLocalArchive(project, args.composition, asJson, (file) =>
+ uploadAssetViaDirectUpload({ client, ...file }),
+ )
+ : undefined;
const upload = await maybeUploadProject(
client,
@@ -562,37 +576,76 @@ interface UploadResult {
projectInput: CreateHyperframesRenderRequest["project"];
}
-interface ArchiveFileSummary {
+export interface ArchiveFileSummary {
path: string;
size_bytes: number;
}
interface PreparedLocalArchive {
project: ProjectDir;
- archive: PublishArchiveResult;
+ /** Absent when the files left on disk alone are over the upload limit: they are never read. */
+ archive?: PublishArchiveResult;
+ sizeBytes: number;
+ fileCount: number;
largestFiles: ArchiveFileSummary[];
+ offloaded: OffloadedMedia[];
}
-function summarizeLargestFiles(fileMap: Map<string, Buffer>): ArchiveFileSummary[] {
+function summarizeLargestFiles(fileMap: Map<string, StagedFile>): ArchiveFileSummary[] {
return [...fileMap.entries()]
- .map(([path, content]) => ({ path, size_bytes: content.byteLength }))
+ .map(([path, content]) => ({ path, size_bytes: stagedSize(content) }))
.sort((a, b) => b.size_bytes - a.size_bytes || a.path.localeCompare(b.path))
.slice(0, LARGEST_FILE_COUNT);
}
-function prepareLocalArchive(
+async function prepareLocalArchive(
source: ProjectInputSource,
composition: string | undefined,
asJson: boolean,
-): PreparedLocalArchive {
+ upload?: OffloadOptions["upload"],
+): Promise<PreparedLocalArchive> {
const project = resolveProject(source.dir);
if (!asJson) {
console.log("");
console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`);
}
+ const { fileMap, entryPath } = stageProjectFiles(project, composition);
+ const offloaded = await offloadMedia(project, fileMap, entryPath, asJson, upload);
+
+ // Media left on disk barely compresses: past the limit, its size is the report and nothing is read.
+ const onDiskBytes = [...fileMap.values()]
+ .filter((file) => !Buffer.isBuffer(file))
+ .reduce((sum, file) => sum + stagedSize(file), 0);
+ const archive =
+ onDiskBytes > DIRECT_UPLOAD_LIMIT_BYTES
+ ? undefined
+ : zipPublishFileMap(new Map([...fileMap].map(([path, file]) => [path, readStaged(file)])));
+ const sizeBytes = archive?.buffer.byteLength ?? onDiskBytes;
+ if (!asJson) {
+ console.log(c.dim(` ${fileMap.size} files · ${formatBytes(sizeBytes)}`));
+ }
+ const largestFiles = summarizeLargestFiles(fileMap);
+ const prepared = {
+ project,
+ archive,
+ sizeBytes,
+ fileCount: fileMap.size,
+ largestFiles,
+ offloaded,
+ };
+ if (upload) assertArchiveFits(sizeBytes, largestFiles);
+ if (!asJson && sizeBytes >= ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES) {
+ reportLargestFiles(prepared);
+ }
+ return prepared;
+}
+function stageProjectFiles(
+ project: ProjectDir,
+ composition: string | undefined,
+): { fileMap: Map<string, StagedFile>; entryPath: string } {
try {
- const fileMap = buildPublishFileMap(project.dir);
+ const fileMap = stagePublishFiles(project.dir, OFFLOAD_MIN_BYTES);
const entryPath = relative(
project.dir,
resolvePath(project.dir, composition ?? "index.html"),
@@ -602,17 +655,7 @@ function prepareLocalArchive(
`Composition "${composition ?? "index.html"}" is excluded from the archive. Check .hyperframesignore.`,
);
}
- const archive = zipPublishFileMap(fileMap);
- if (!asJson) {
- console.log(
- c.dim(` ${archive.fileCount} files · ${formatBytes(archive.buffer.byteLength)}`),
- );
- }
- const prepared = { project, archive, largestFiles: summarizeLargestFiles(fileMap) };
- if (!asJson && archive.buffer.byteLength >= ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES) {
- reportLargestFiles(prepared);
- }
- return prepared;
+ return { fileMap, entryPath };
} catch (err) {
const msg = normalizeErrorMessage(err);
errorBox("Zip failed", msg, "Check the project and .hyperframesignore for missing files.");
@@ -620,6 +663,64 @@ function prepareLocalArchive(
}
}
+async function offloadMedia(
+ project: ProjectDir,
+ fileMap: Map<string, StagedFile>,
+ entryPath: string,
+ asJson: boolean,
+ upload: OffloadOptions["upload"],
+): Promise<OffloadedMedia[]> {
+ let offloaded: OffloadedMedia[];
+ try {
+ offloaded = await offloadHeavyMedia(project.dir, fileMap, {
+ apiBaseUrl: resolveCloudBaseUrl(),
+ upload,
+ entryPath,
+ onFile: asJson
+ ? undefined
+ : (item) => console.log(c.dim(` ${describeOffload(item, !upload)}`)),
+ });
+ } catch (err) {
+ reportApiError("Media upload failed", err);
+ }
+ if (!asJson) {
+ for (const item of offloaded.filter((entry) => entry.kept_in_archive)) {
+ console.log(
+ c.dim(
+ ` ${item.path} also stays in the zip: a script builds its path in a way that cannot be rewritten`,
+ ),
+ );
+ }
+ }
+ return offloaded;
+}
+
+export function assertArchiveFits(sizeBytes: number, largestFiles: ArchiveFileSummary[]): void {
+ if (sizeBytes <= DIRECT_UPLOAD_LIMIT_BYTES) return;
+ errorBox(
+ "Project too large for cloud render",
+ `The zip is ${formatBytes(sizeBytes)} after moving media to HeyGen assets, over the ${formatBytes(DIRECT_UPLOAD_LIMIT_BYTES)} upload limit. Largest files left in it:\n${largestFiles
+ .map((file) => ` ${formatBytes(file.size_bytes).padStart(9)} ${file.path}`)
+ .join("\n")}`,
+ "Exclude files the render does not need with .hyperframesignore.",
+ );
+ failCommand();
+}
+
+function describeOffload(item: OffloadedMedia, dryRun: boolean): string {
+ const verb =
+ item.action === "cached"
+ ? "already uploaded"
+ : item.action === "reencode"
+ ? dryRun
+ ? "would re-encode and upload"
+ : "re-encoded and uploaded"
+ : dryRun
+ ? "would upload"
+ : "uploaded";
+ return `${formatBytes(item.size_bytes).padStart(9)} ${item.path} · ${verb}`;
+}
+
function reportLargestFiles(prepared: PreparedLocalArchive): void {
console.log(c.dim(" Largest included files:"));
for (const file of prepared.largestFiles) {
@@ -629,18 +730,19 @@ function reportLargestFiles(prepared: PreparedLocalArchive): void {
}
function reportDryRun(prepared: PreparedLocalArchive, asJson: boolean): void {
- const sizeBytes = prepared.archive.buffer.byteLength;
+ const { sizeBytes } = prepared;
if (asJson) {
console.log(
JSON.stringify(
withMeta({
archive: {
project: prepared.project.name,
- file_count: prepared.archive.fileCount,
+ file_count: prepared.fileCount,
size_bytes: sizeBytes,
upload_limit_bytes: DIRECT_UPLOAD_LIMIT_BYTES,
exceeds_upload_limit: sizeBytes > DIRECT_UPLOAD_LIMIT_BYTES,
largest_files: prepared.largestFiles,
+ offloaded_media: prepared.offloaded,
},
}),
null,
@@ -675,6 +777,7 @@ async function maybeUploadProject(
if (!preparedArchive) throw new Error("Local project archive was not prepared");
const { project, archive } = preparedArchive;
+ if (!archive) throw new Error("Local project archive is over the upload limit");
if (!asJson) {
console.log("");
diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts
index 2679a3915f..f9ed417278 100644
--- a/packages/cli/src/utils/publishProject.test.ts
+++ b/packages/cli/src/utils/publishProject.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
-import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { mkdtempSync, mkdirSync, renameSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, relative } from "node:path";
import AdmZip from "adm-zip";
@@ -38,6 +38,8 @@ import {
getPublishApiBaseUrl,
localizeExternalAssets,
publishProjectArchive,
+ readStaged,
+ stagePublishFiles,
uploadTimeoutMs,
zipPublishFileMap,
} from "./publishProject.js";
@@ -686,6 +688,30 @@ describe("publishProjectArchive", () => {
});
});
+describe("publishProjectArchive", () => {
+ it("keeps server-sent upload headers in any case without adding duplicates", async () => {
+ const dir = makeProjectDir();
+ const serverHeaders = {
+ "Content-Type": "application/zip",
+ "Content-Length": "123",
+ "X-Amz-Server-Side-Encryption": "AES256",
+ };
+ const fetchMock = stagedFetch(
+ { file_count: 1 },
+ { upload_url: signedStagedS3Url, upload_headers: serverHeaders },
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ try {
+ writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+ await publishProjectArchive(dir);
+ expect(fetchMock.mock.calls[1]![1].headers).toEqual(serverHeaders);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
+
describe("publishProjectArchive", () => {
it("authenticates staged metadata requests but not the presigned S3 upload", async () => {
const dir = makeProjectDir();
@@ -1329,3 +1355,24 @@ describe("publishProjectArchive with an expired login and no refresh token", ()
}
});
});
+
+describe("readStaged", () => {
+ let dir: string;
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), "hf-staged-"));
+ writeFileSync(join(dir, "index.html"), "<html></html>");
+ writeFileSync(join(dir, "clip.bin"), "staged bytes");
+ });
+ afterEach(() => rmSync(dir, { recursive: true, force: true }));
+
+ it("reads a file left on disk only while it is the file that was staged", () => {
+ const staged = stagePublishFiles(dir, 4).get("clip.bin")!;
+ expect(Buffer.isBuffer(staged)).toBe(false);
+ expect(readStaged(staged).toString()).toBe("staged bytes");
+
+ writeFileSync(join(dir, "swap.bin"), "other bytes");
+ utimesSync(join(dir, "swap.bin"), 1, 1);
+ renameSync(join(dir, "swap.bin"), join(dir, "clip.bin"));
+ expect(() => readStaged(staged)).toThrow(/changed on disk/);
+ });
+});
diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts
index c8ad5f958b..ac39bcfb53 100644
--- a/packages/cli/src/utils/publishProject.ts
+++ b/packages/cli/src/utils/publishProject.ts
@@ -1,5 +1,14 @@
import { basename, dirname, join, posix, relative, resolve } from "node:path";
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import {
+ closeSync,
+ existsSync,
+ fstatSync,
+ openSync,
+ readdirSync,
+ readFileSync,
+ type Stats,
+ statSync,
+} from "node:fs";
import { parseHTML } from "linkedom";
import AdmZip from "adm-zip";
import ignore, { type Ignore } from "ignore";
@@ -130,27 +139,22 @@ function getUploadHeaders(
}
}
- if (!Object.keys(headers).some((key) => key.toLowerCase() === "content-type")) {
- headers["content-type"] = contentType;
+ setHeaderIfAbsent(headers, "content-type", contentType);
+ const signedHeaders =
+ new URL(uploadUrl).searchParams.get("X-Amz-SignedHeaders")?.split(";") ?? [];
+ if (signedHeaders.includes("x-amz-server-side-encryption")) {
+ setHeaderIfAbsent(headers, "x-amz-server-side-encryption", "AES256");
}
-
- const signedHeaders = new URL(uploadUrl).searchParams.get("X-Amz-SignedHeaders");
- if (
- signedHeaders?.split(";").includes("x-amz-server-side-encryption") &&
- !Object.keys(headers).some((key) => key.toLowerCase() === "x-amz-server-side-encryption")
- ) {
- headers["x-amz-server-side-encryption"] = "AES256";
- }
- if (
- signedHeaders?.split(";").includes("content-length") &&
- !Object.keys(headers).some((key) => key.toLowerCase() === "content-length")
- ) {
- headers["content-length"] = String(archiveByteLength);
+ if (signedHeaders.includes("content-length")) {
+ setHeaderIfAbsent(headers, "content-length", String(archiveByteLength));
}
-
return headers;
}
+function setHeaderIfAbsent(headers: Record<string, string>, name: string, value: string): void {
+ if (!Object.keys(headers).some((key) => key.toLowerCase() === name)) headers[name] = value;
+}
+
async function readJson(response: Response): Promise<unknown> {
return response
.clone()
@@ -338,9 +342,66 @@ function collectProjectFiles(
const EXT_ASSETS_PREFIX = "_ext";
+/** Files whose text a reference rewrite or scan reads; they are always staged in memory. */
+export const TEXT_FILE_EXTS = /\.(html?|css|m?js|cjs|json|svg|xml|txt)$/i;
+
+/** A large non-text project file left on disk; `size` and `mtimeMs` detect an edit after staging. */
+export interface DiskFile {
+ absPath: string;
+ size: number;
+ mtimeMs: number;
+}
+
+export type StagedFile = Buffer | DiskFile;
+
+export function stagedSize(file: StagedFile): number {
+ return Buffer.isBuffer(file) ? file.byteLength : file.size;
+}
+
+function assertSame(file: DiskFile, now: Stats | undefined): void {
+ if (now?.size !== file.size || now.mtimeMs !== file.mtimeMs) {
+ throw new Error(`${file.absPath} changed on disk while it was being prepared. Run it again.`);
+ }
+}
+
+export function assertUnchanged(file: DiskFile): void {
+ assertSame(file, statSync(file.absPath, { throwIfNoEntry: false }));
+}
+
+/** Reads through one descriptor, so `check` (run before and after) sees the stat of the bytes returned. */
+export function readChecked(path: string, check: (stat: Stats) => void): Buffer {
+ const fd = openSync(path, "r");
+ try {
+ check(fstatSync(fd));
+ const bytes = readFileSync(fd);
+ check(fstatSync(fd));
+ return bytes;
+ } finally {
+ closeSync(fd);
+ }
+}
+
+/** The staged bytes; a file left on disk is read now and must not have changed since staging. */
+export function readStaged(file: StagedFile): Buffer {
+ if (Buffer.isBuffer(file)) return file;
+ return readChecked(file.absPath, (stat) => assertSame(file, stat));
+}
+
+function stageFile(absPath: string, onDiskMinBytes: number): StagedFile {
+ const fd = openSync(absPath, "r");
+ try {
+ const { size, mtimeMs } = fstatSync(fd);
+ if (size < onDiskMinBytes || TEXT_FILE_EXTS.test(absPath)) return readFileSync(fd);
+ return { absPath, size, mtimeMs };
+ } finally {
+ closeSync(fd);
+ }
+}
+
interface ExternalAssetContext {
absProjectDir: string;
- fileContents: Map<string, Buffer>;
+ fileContents: Map<string, StagedFile>;
+ onDiskMinBytes: number;
externalMap: Map<string, string>;
usedArchivePaths: Set<string>;
}
@@ -361,7 +422,7 @@ function addExternalAsset(ctx: ExternalAssetContext, absPath: string): string {
archivePath = `${base}_${i}${ext}`;
}
- ctx.fileContents.set(archivePath, readFileSync(absPath));
+ ctx.fileContents.set(archivePath, stageFile(absPath, ctx.onDiskMinBytes));
ctx.externalMap.set(absPath, archivePath);
ctx.usedArchivePaths.add(archivePath);
return archivePath;
@@ -383,26 +444,30 @@ function tryResolveExternal(
return addExternalAsset(ctx, absPath);
}
-function rewriteCssUrls(
- ctx: ExternalAssetContext,
+/** Resolves a raw attribute or CSS `url()` value to the archive path or absolute
+ * URL it should point at, or `null` to leave it untouched. */
+export type HtmlAttributeResolver = (rawValue: string, referrerAbsDir: string) => string | null;
+
+function referenceFrom(entryPath: string, target: string): string {
+ return isNonRelativeUrl(target) ? target : posix.relative(posix.dirname(entryPath), target);
+}
+
+export function rewriteCssUrls(
css: string,
referrerAbsDir: string,
entryPath: string,
+ resolveTarget: HtmlAttributeResolver,
): { css: string; modified: boolean } {
let modified = false;
const rewritten = css.replace(CSS_URL_RE, (full, quote: string, rawUrl: string) => {
- const archivePath = tryResolveExternal(ctx, (rawUrl || "").trim(), referrerAbsDir);
- if (!archivePath) return full;
+ const target = resolveTarget((rawUrl || "").trim(), referrerAbsDir);
+ if (!target) return full;
modified = true;
- return `url(${quote || ""}${posix.relative(posix.dirname(entryPath), archivePath)}${quote || ""})`;
+ return `url(${quote || ""}${referenceFrom(entryPath, target)}${quote || ""})`;
});
return { css: rewritten, modified };
}
-/** Resolves a raw attribute value (plus the referrer's absolute directory) to
- * the archive path it should point at, or `null` to leave it untouched. */
-export type HtmlAttributeResolver = (rawValue: string, referrerAbsDir: string) => string | null;
-
interface RewriteHtmlAttributesOptions {
/** Attributes to inspect (default: src + href, matching the external-asset
* localization use case below). */
@@ -436,26 +501,26 @@ export function rewriteHtmlAttributes(
for (const attr of attrs) {
const val = (el.getAttribute(attr) || "").trim();
if (!val) continue;
- const archivePath = resolveTarget(val, referrerAbsDir);
- if (!archivePath) continue;
- el.setAttribute(attr, posix.relative(posix.dirname(entryPath), archivePath));
+ const target = resolveTarget(val, referrerAbsDir);
+ if (!target) continue;
+ el.setAttribute(attr, referenceFrom(entryPath, target));
modified = true;
}
}
return modified;
}
-function rewriteStyleBlocks(
- ctx: ExternalAssetContext,
+export function rewriteStyleBlocks(
document: Document,
referrerAbsDir: string,
entryPath: string,
+ resolveTarget: HtmlAttributeResolver,
): boolean {
let modified = false;
for (const styleEl of document.querySelectorAll("style")) {
const css = styleEl.textContent || "";
if (!css.includes("url(")) continue;
- const result = rewriteCssUrls(ctx, css, referrerAbsDir, entryPath);
+ const result = rewriteCssUrls(css, referrerAbsDir, entryPath, resolveTarget);
if (result.modified) {
styleEl.textContent = result.css;
modified = true;
@@ -464,7 +529,7 @@ function rewriteStyleBlocks(
for (const el of document.querySelectorAll("[style]")) {
const style = el.getAttribute("style") || "";
if (!style.includes("url(")) continue;
- const result = rewriteCssUrls(ctx, style, referrerAbsDir, entryPath);
+ const result = rewriteCssUrls(style, referrerAbsDir, entryPath, resolveTarget);
if (result.modified) {
el.setAttribute("style", result.css);
modified = true;
@@ -476,10 +541,9 @@ function rewriteStyleBlocks(
function localizeHtmlEntry(ctx: ExternalAssetContext, entryPath: string, content: Buffer): void {
const referrerAbsDir = resolve(ctx.absProjectDir, dirname(entryPath));
const { document } = parseHTML(content.toString("utf-8"));
- const attrsChanged = rewriteHtmlAttributes(document, referrerAbsDir, entryPath, (val, dir) =>
- tryResolveExternal(ctx, val, dir),
- );
- const stylesChanged = rewriteStyleBlocks(ctx, document, referrerAbsDir, entryPath);
+ const resolveTarget: HtmlAttributeResolver = (val, dir) => tryResolveExternal(ctx, val, dir);
+ const attrsChanged = rewriteHtmlAttributes(document, referrerAbsDir, entryPath, resolveTarget);
+ const stylesChanged = rewriteStyleBlocks(document, referrerAbsDir, entryPath, resolveTarget);
if (attrsChanged || stylesChanged) {
ctx.fileContents.set(entryPath, Buffer.from(document.toString(), "utf-8"));
}
@@ -489,7 +553,9 @@ function localizeCssEntry(ctx: ExternalAssetContext, entryPath: string, content:
const referrerAbsDir = resolve(ctx.absProjectDir, dirname(entryPath));
const css = content.toString("utf-8");
if (!css.includes("url(")) return;
- const result = rewriteCssUrls(ctx, css, referrerAbsDir, entryPath);
+ const result = rewriteCssUrls(css, referrerAbsDir, entryPath, (val, dir) =>
+ tryResolveExternal(ctx, val, dir),
+ );
if (result.modified) {
ctx.fileContents.set(entryPath, Buffer.from(result.css, "utf-8"));
}
@@ -502,17 +568,19 @@ function localizeCssEntry(ctx: ExternalAssetContext, entryPath: string, content:
*/
export function localizeExternalAssets(
absProjectDir: string,
- fileContents: Map<string, Buffer>,
+ fileContents: Map<string, StagedFile>,
+ onDiskMinBytes = Infinity,
): number {
const ctx: ExternalAssetContext = {
absProjectDir,
fileContents,
+ onDiskMinBytes,
externalMap: new Map(),
usedArchivePaths: new Set(),
};
for (const [entryPath, content] of [...fileContents.entries()]) {
- if (entryPath.startsWith(EXT_ASSETS_PREFIX + "/")) continue;
+ if (entryPath.startsWith(EXT_ASSETS_PREFIX + "/") || !Buffer.isBuffer(content)) continue;
if (entryPath.endsWith(".html") || entryPath.endsWith(".htm")) {
localizeHtmlEntry(ctx, entryPath, content);
} else if (entryPath.endsWith(".css")) {
@@ -531,6 +599,14 @@ export function localizeExternalAssets(
* keeps calling `createPublishArchive` directly.
*/
export function buildPublishFileMap(projectDir: string): Map<string, Buffer> {
+ return stagePublishFiles(projectDir, Infinity) as Map<string, Buffer>;
+}
+
+/** {@link buildPublishFileMap}, but non-text files of `onDiskMinBytes` or more stay on disk. */
+export function stagePublishFiles(
+ projectDir: string,
+ onDiskMinBytes: number,
+): Map<string, StagedFile> {
const absProjectDir = resolve(projectDir);
const filePaths: string[] = [];
collectProjectFiles(absProjectDir, absProjectDir, filePaths, createProjectIgnore(absProjectDir));
@@ -540,12 +616,12 @@ export function buildPublishFileMap(projectDir: string): Map<string, Buffer> {
);
}
- const fileContents = new Map<string, Buffer>();
+ const fileContents = new Map<string, StagedFile>();
for (const filePath of filePaths) {
- fileContents.set(filePath, readFileSync(join(absProjectDir, filePath)));
+ fileContents.set(filePath, stageFile(join(absProjectDir, filePath), onDiskMinBytes));
}
- localizeExternalAssets(absProjectDir, fileContents);
+ localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes);
return fileContents;
}
@@ -561,7 +637,7 @@ export function buildPublishFileMap(projectDir: string): Map<string, Buffer> {
* rather than of its contents, which is the opposite of what a
* content-addressed artifact needs.
*
- * Built from local components on purpose: `fromDate2DOS` reads `getFullYear`,
+ * Built from local components on purpose: adm-zip's DOS date encoding reads `getFullYear`,
* `getMonth`, `getHours` and friends, so a fixed *instant* would still encode
* differently in different timezones. Fixing the wall-clock reading is what
* makes the bytes match across machines. 1980-01-01 is the earliest a DOS
diff --git a/packages/producer/src/index.ts b/packages/producer/src/index.ts
index e9b900e997..76300ab239 100644
--- a/packages/producer/src/index.ts
+++ b/packages/producer/src/index.ts
@@ -88,6 +88,8 @@ export {
type FileServerHandle,
} from "./services/fileServer.js";
+export { prepareAnimatedGifInputs } from "./services/animatedGifPrep.js";
+
// ── Video frame injection (Hyperframes-specific hook) ───────────────────────
export { createVideoFrameInjector } from "@hyperframes/engine";