Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): overlap the plan step's audio work and hash files once (#5777)
Miguel Ángelfb17f4119620
diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.tsindex 4e5487a108..9160725c19 100644--- a/packages/aws-lambda/src/handler.test.ts+++ b/packages/aws-lambda/src/handler.test.ts@@ -15,7 +15,7 @@* the glue's contract.*/-import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";+import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test";import { createHash } from "node:crypto";import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";@@ -658,6 +658,10 @@ describe("handler dispatch", () => {skipChromeResolution: true,};+ const logged: string[] = [];+ const logSpy = spyOn(console, "log").mockImplementation((line: string) => {+ logged.push(line);+ });const planned = await handler({Action: "plan",@@ -666,7 +670,12 @@ describe("handler dispatch", () => {Config: { fps: 30, width: 640, height: 360, format: "mp4" },},deps,- );+ ).finally(() => logSpy.mockRestore());+ const stageLines = logged+ .filter((line) => line.includes('"plan_stage"'))+ .map((line) => JSON.parse(line) as { stage?: string; ms?: unknown });+ expect(stageLines.map((line) => line.stage)).toEqual(["download", "untar"]);+ expect(stageLines.every((line) => typeof line.ms === "number")).toBe(true);expect(planned).toMatchObject({PlanProtocol: "v2",PlanV2ManifestS3Uri: "s3://bucket/renders/v2/v2/manifest.json",diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.tsindex a3456ea2fc..4c8cbb5e0a 100644--- a/packages/aws-lambda/src/handler.ts+++ b/packages/aws-lambda/src/handler.ts@@ -427,8 +427,12 @@ async function handlePlanV2(const projectArchive = join(work, "project.tar.gz");const projectDir = join(work, "project");try {+ let stageStarted = Date.now();await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);+ logEvent({ event: "plan_stage", stage: "download", ms: Date.now() - stageStarted });+ stageStarted = Date.now();await untarDirectory(projectArchive, projectDir);+ logEvent({ event: "plan_stage", stage: "untar", ms: Date.now() - stageStarted });const publisher = new S3PlanV2ArtifactPublisher({s3,planOutputS3Prefix: event.PlanOutputS3Prefix,diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts b/packages/aws-lambda/src/s3PlanV2Publisher.test.tsindex 94a35a6e68..9b437f9cf3 100644--- a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts+++ b/packages/aws-lambda/src/s3PlanV2Publisher.test.ts@@ -19,12 +19,14 @@ interface PutOperation {class FakeS3 {readonly objects = new Map<string, StoredObject>();readonly puts: PutOperation[] = [];+ sends = 0;asClient(): import("@aws-sdk/client-s3").S3Client {return this as unknown as import("@aws-sdk/client-s3").S3Client;}async send(command: unknown): Promise<unknown> {+ this.sends += 1;// AWS command inputs are the runtime boundary exercised by this fake.const value = command as unknown as {readonly constructor: { readonly name: string };@@ -139,6 +141,22 @@ describe("S3PlanV2ArtifactPublisher", () => {expect(s3.objects.get(manifestUri)?.bytes.toString("utf8")).toBe(manifest);});+ it("sends nothing for a digest it already published", async () => {+ const source = makeSource("hello");+ const s3 = new FakeS3();+ const publisher = new S3PlanV2ArtifactPublisher({+ s3: s3.asClient(),+ planOutputS3Prefix: "s3://bucket/render",+ temporaryRoot: source.root,+ });+ const blob = { sourcePath: source.path, sha256: source.digest, sizeBytes: source.sizeBytes };++ await publisher.putBlob(blob);+ const sendsAfterFirst = s3.sends;+ await publisher.putBlob(blob);+ expect(s3.sends).toBe(sendsAfterFirst);+ });+it("refuses to expose a manifest that references an unpublished digest", async () => {const source = makeSource("hello");const s3 = new FakeS3();diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.ts b/packages/aws-lambda/src/s3PlanV2Publisher.tsindex afcb45928e..8eee0d8681 100644--- a/packages/aws-lambda/src/s3PlanV2Publisher.ts+++ b/packages/aws-lambda/src/s3PlanV2Publisher.ts@@ -83,6 +83,7 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {async putBlob(blob: Readonly<PlanV2PublishBlob>): Promise<void> {this.#assertOpen("publish a blob");const digest = assertSha256(blob.sha256, "S3 published blob sha256");+ if (this.#publishedDigests.has(digest)) return;const sourceSize = statSync(blob.sourcePath).size;if (sourceSize !== blob.sizeBytes) {throw new PlanV2IntegrityError(diff --git a/packages/aws-lambda/src/s3Transport.test.ts b/packages/aws-lambda/src/s3Transport.test.tsindex cd101ad090..6e5ea6dd58 100644--- a/packages/aws-lambda/src/s3Transport.test.ts+++ b/packages/aws-lambda/src/s3Transport.test.ts@@ -6,6 +6,7 @@*/import { afterAll, beforeAll, describe, expect, it } from "bun:test";+import { createHash } from "node:crypto";import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";@@ -116,6 +117,19 @@ describe("content-addressed v2 artifacts", () => {expect(s3.putCount).toBe(1);});+ it("leaves verifying the local bytes to the S3 checksum instead of hashing them again", async () => {+ const source = join(scratchRoot, "artifact-changed.bin");+ writeFileSync(source, "bytes that changed");+ const digest = createHash("sha256").update("bytes as hashed").digest("hex");+ const uri = `s3://bucket/v2/artifacts/sha256/${digest.slice(0, 2)}/${digest}`;+ const s3 = new ContentAddressedFakeS3();++ await expect(+ uploadContentAddressedFileToS3(s3.asClient(), source, uri, digest),+ ).rejects.toMatchObject({ name: "BadDigest" });+ expect(s3.objects.has(uri)).toBe(false);+ });+it("refuses to overwrite an immutable key with conflicting metadata", async () => {const source = join(scratchRoot, "artifact-conflict.bin");writeFileSync(source, "expected bytes");@@ -199,6 +213,7 @@ class ContentAddressedFakeS3 {Key: string;Body?: NodeJS.ReadableStream;Metadata?: Record<string, string>;+ ChecksumSHA256?: string;};};const uri = `s3://${value.input.Bucket}/${value.input.Key}`;@@ -238,6 +253,12 @@ class ContentAddressedFakeS3 {const chunks: Buffer[] = [];for await (const chunk of value.input.Body ?? []) chunks.push(Buffer.from(chunk));const bytes = Buffer.concat(chunks);+ const checksum = createHash("sha256").update(bytes).digest("base64");+ if (value.input.ChecksumSHA256 !== undefined && value.input.ChecksumSHA256 !== checksum) {+ const error = new Error("checksum mismatch");+ error.name = "BadDigest";+ throw error;+ }this.objects.set(uri, {bytes,sha256: value.input.Metadata?.sha256 ?? "",diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.tsindex 5efbac7be1..379fa147ab 100644--- a/packages/aws-lambda/src/s3Transport.ts+++ b/packages/aws-lambda/src/s3Transport.ts@@ -131,11 +131,9 @@ export async function uploadFileToS3(}/**- * Upload one content-addressed plan-v2 artifact exactly once.- *- * Existing objects are reused only when their immutable digest metadata and- * byte length agree. A conflicting object is never overwritten: doing so- * could change a plan already being consumed by another chunk invocation.+ * Upload one content-addressed plan-v2 artifact exactly once; S3 verifies the bytes against+ * `ChecksumSHA256`. An existing object is reused only when its digest metadata and length agree,+ * and a conflicting one is never overwritten, since a chunk may already be reading that plan.*/export async function uploadContentAddressedFileToS3(client: S3Client,@@ -148,15 +146,6 @@ export async function uploadContentAddressedFileToS3(if (!existsSync(localPath)) {throw new Error(`[s3Transport] upload source missing: ${localPath}`);}- const actualSha256 = await sha256File(localPath);- if (actualSha256 !== expectedSha256) {- const error = new Error(- `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`,- );- error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";- throw error;- }-const { bucket, key } = parseS3Uri(uri);const size = statSync(localPath).size;const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex ad5d5445ad..2a6bedf1f8 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -66,6 +66,8 @@ import {type CompositionMetadataJson,freezePlan,type LockedRenderConfig,+ type PlanFileDigests,+ prehashPlanFiles,} from "../render/stages/freezePlan.js";import {canonicalJsonStringify,@@ -299,6 +301,14 @@ export interface BuildLocalExecutionPlanOptions {* chunks write those frames into their own work directory.*/readonly deferVideoExtraction?: boolean;+ /** Shared with the publisher so each plan file is hashed once. */+ readonly digests?: PlanFileDigests;+ /** Runs while the audio mixes, once `compiled/` and the video trees are final. */+ readonly whileAudioMixes?: () => Promise<void>;+}++export function logPlanStage(log: ProducerLogger, stage: string, startedAt: number): void {+ log.info("[plan] stage", { stage, ms: Date.now() - startedAt });}/**@@ -1064,6 +1074,7 @@ export async function buildLocalExecutionPlan(// overwrites the entry HTML afterwards. `dereference: true` resolves// symlinks so the planDir survives S3 / Lambda /tmp round-trips.mkdirSync(compiledDir, { recursive: true });+ let stageStarted = Date.now();cpSync(projectDir, compiledDir, {recursive: true,dereference: true,@@ -1077,6 +1088,7 @@ export async function buildLocalExecutionPlan(return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);},});+ logPlanStage(log, "copy", stageStarted);// The compiled directory lives at `<planDir>/compiled/` in the final// layout. The stages write under `<planDir>/.plan-work/compiled/`; we@@ -1095,6 +1107,7 @@ export async function buildLocalExecutionPlan(const needsAlpha = outputNeedsAlpha(config.format);// ── Compile ──+ stageStarted = Date.now();const compileResult = await runCompileStage({projectDir,workDir,@@ -1114,6 +1127,7 @@ export async function buildLocalExecutionPlan(allowSystemFontCapture: false,variables: config.variables,});+ logPlanStage(log, "compile", stageStarted);let compiled = compileResult.compiled;const composition = compileResult.composition;const { deviceScaleFactor } = compileResult;@@ -1143,6 +1157,7 @@ export async function buildLocalExecutionPlan(// ── Probe ──// Browser probe runs only when needed. For statically-resolvable durations// this is a near-zero pass.+ stageStarted = Date.now();const probeResult = await runProbeStage({projectDir,workDir,@@ -1160,6 +1175,7 @@ export async function buildLocalExecutionPlan(deviceScaleFactor,subTimelineWaitMemo: {},});+ logPlanStage(log, "probe", stageStarted);compiled = probeResult.compiled;job.duration = probeResult.duration;job.totalFrames = probeResult.totalFrames;@@ -1212,6 +1228,7 @@ export async function buildLocalExecutionPlan(// ── Extract videos ──// `materializeSymlinks: true` recursively copies frames so the planDir is// self-contained (symlinks don't survive S3/GCS round-trips).+ stageStarted = Date.now();const extractResult = await runExtractVideosStage({projectDir,compiledDir,@@ -1223,6 +1240,7 @@ export async function buildLocalExecutionPlan(materializeSymlinks: true,deferRangeExtraction: options.deferVideoExtraction,});+ logPlanStage(log, "extract", stageStarted);if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;if (extractResult.extractionResult) {// Distributed chunks cannot safely fall back to native remote decoding:@@ -1236,23 +1254,6 @@ export async function buildLocalExecutionPlan(// video's outputDir, but in `plan()` those directories ARE the source// material the renames below move into `planDir/video-frames/`.- // ── Audio ──- const audioResult = await runAudioStage({- projectDir,- workDir,- compiledDir,- duration: job.duration,- ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,- audioGain: cfg.audioGain,- audios: composition.audios,- masterFxChain: composition.masterFxChain,- abortSignal,- assertNotAborted,- });- if (audioResult.audioError) {- applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);- }-// Promote staged artifacts from the temp work tree into the final planDir// shape. `workDir` is `<planDir>/.plan-work/` — always the same filesystem// as `planDir`, so `renameSync` succeeds without copying. Video frames@@ -1262,7 +1263,8 @@ export async function buildLocalExecutionPlan(const stagedVideoFrames = join(compiledDir, "__hyperframes_video_frames");const videoFramesDst = join(planDir, "video-frames");if (existsSync(videoFramesDst)) rmSync(videoFramesDst, { recursive: true, force: true });- rmSync(join(planDir, "video-sources"), { recursive: true, force: true });+ const videoSourcesDir = join(planDir, "video-sources");+ rmSync(videoSourcesDir, { recursive: true, force: true });if (existsSync(stagedVideoFrames)) {renameSync(stagedVideoFrames, videoFramesDst);} else {@@ -1306,6 +1308,43 @@ export async function buildLocalExecutionPlan("utf-8",);+ // ── Audio ──+ // Only the assembler reads the mix, so hashing and publishing the final+ // compiled and video trees run beside it instead of after it.+ const digests: PlanFileDigests = options.digests ?? new Map();+ stageStarted = Date.now();+ const [audioOutcome, overlapOutcome] = await Promise.allSettled([+ runAudioStage({+ projectDir,+ workDir,+ compiledDir: finalCompiledDir,+ duration: probeResult.duration,+ ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,+ audioGain: cfg.audioGain,+ audios: composition.audios,+ masterFxChain: composition.masterFxChain,+ abortSignal,+ assertNotAborted,+ }).finally(() => logPlanStage(log, "audio", stageStarted)),+ (async () => {+ for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {+ await prehashPlanFiles(dir, digests);+ }+ logPlanStage(log, "prehash", stageStarted);+ if (options.whileAudioMixes) {+ const prepublishStarted = Date.now();+ await options.whileAudioMixes();+ logPlanStage(log, "prepublish", prepublishStarted);+ }+ })(),+ ]);+ if (audioOutcome.status === "rejected") throw audioOutcome.reason;+ if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;+ const audioResult = audioOutcome.value;+ if (audioResult.audioError) {+ applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);+ }+const planAudioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {renameSync(audioResult.audioOutputPath, planAudioPath);@@ -1382,6 +1421,7 @@ export async function buildLocalExecutionPlan(log,});+ stageStarted = Date.now();const freezeResult = await freezePlan({planDir,composition: compositionJson,@@ -1393,7 +1433,9 @@ export async function buildLocalExecutionPlan(durationSeconds: job.duration ?? 0,totalFrames,hasAudio: audioResult.hasAudio,+ digests,});+ logPlanStage(log, "freeze", stageStarted);const planHash = freezeResult.planHash;// 2 GB hard cap so the planDir fits inside Lambda's 10 GB /tmp budgetdiff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.tsindex 649554462d..f808a5724d 100644--- a/packages/producer/src/services/distributed/planV2.test.ts+++ b/packages/producer/src/services/distributed/planV2.test.ts@@ -1,4 +1,5 @@import { afterEach, describe, expect, it } from "bun:test";+import { spawnSync } from "node:child_process";import {existsSync,mkdirSync,@@ -12,8 +13,9 @@ import {} from "node:fs";import { tmpdir } from "node:os";import { dirname, join } from "node:path";-import { recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";+import { planFileSha256, recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";+import { buildLocalExecutionPlan } from "./plan.js";import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";import { FFMPEG_VERSION_MISMATCH, renderChunk, RenderChunkValidationError } from "./renderChunk.js";import {@@ -26,11 +28,12 @@ import {PlanV2IntegrityError,publishPlanV2FromV1,publishPlanV2FromExecutionPlan,+ planV2WithPublisher,readPlanV2Manifest,validatePlanV2MaterializedTarget,} from "./planV2.js";import { LocalPlanV2ArtifactPublisher, type PlanV2ArtifactPublisher } from "./planV2Publisher.js";-import { buildPlanVideosJson, type PlanVideosJson } from "./shared.js";+import { buildPlanVideosJson, isPlanAudioArtifactPath, type PlanVideosJson } from "./shared.js";const tempDirs: string[] = [];@@ -1003,6 +1006,114 @@ describe("Plan v2 artifact publisher", () => {});});+describe("Plan v2 planning", () => {+ const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;++ it("reuses a digest taken earlier in the plan instead of rehashing the file", () => {+ const planDir = createV1Plan(tempPath("hf-plan-v2-digests-"));+ const asset = join(planDir, "compiled", "asset.txt");+ const digests = new Map([[asset, "f".repeat(64)]]);++ expect(planFileSha256(asset, digests)).toBe("f".repeat(64));+ expect(recomputePlanHashFromPlanDir(planDir, digests)).not.toBe(+ recomputePlanHashFromPlanDir(planDir),+ );+ });++ it.skipIf(!HAS_FFMPEG)(+ "publishes the compiled tree while the audio mixes, with the serial manifest",+ async () => {+ const root = tempPath("hf-plan-v2-overlap-");+ const projectDir = join(root, "project");+ const stagingParentDir = join(root, "staging");+ mkdirSync(projectDir);+ mkdirSync(stagingParentDir);+ const silence = spawnSync("ffmpeg", [+ ...["-nostdin", "-v", "error", "-f", "lavfi"],+ ...["-i", "anullsrc=channel_layout=mono:sample_rate=44100", "-t", "1"],+ ...["-c:a", "pcm_s16le", join(projectDir, "silence.wav")],+ ]);+ expect(silence.status).toBe(0);+ writeFileSync(join(projectDir, "poster.bin"), Buffer.alloc(1 << 20, 7));+ writeFileSync(+ join(projectDir, "index.html"),+ `<!doctype html><html><head><meta charset="utf-8"></head><body>+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1">+ <audio data-composition-audio src="silence.wav"></audio><p>overlap fixture</p>+ </div></body></html>`,+ );+ const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const };++ const stages: string[] = [];+ const puts: Array<{ sha256: string; duringAudio: boolean }> = [];+ const local = new LocalPlanV2ArtifactPublisher(join(root, "overlapped"));+ const manifest = await planV2WithPublisher(+ projectDir,+ {+ ...config,+ logger: {+ info: (message, meta) => {+ if (message === "[plan] stage") stages.push(String(meta?.stage));+ },+ debug() {},+ warn() {},+ error() {},+ },+ },+ {+ async putBlob(blob) {+ const staging = readdirSync(stagingParentDir).find((name) =>+ name.startsWith(".execution-plan-"),+ );+ const workDir = join(stagingParentDir, String(staging), ".plan-work");+ if (!puts.some((put) => put.sha256 === blob.sha256)) {+ puts.push({ sha256: blob.sha256, duringAudio: existsSync(workDir) });+ }+ await local.putBlob(blob);+ },+ commitManifest: (bytes) => local.commitManifest(bytes),+ abort: () => local.abort(),+ },+ { stagingParentDir },+ );++ const serialDir = join(root, "serial-plan");+ mkdirSync(serialDir);+ await buildLocalExecutionPlan(projectDir, config, serialDir, {+ executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,+ deferVideoExtraction: true,+ });+ const serial = await publishPlanV2FromExecutionPlan(+ serialDir,+ new LocalPlanV2ArtifactPublisher(join(root, "serial")),+ );+ expect(canonicalJsonStringify(manifest)).toBe(canonicalJsonStringify(serial));+ expect(manifest.artifacts.some((artifact) => isPlanAudioArtifactPath(artifact.path))).toBe(+ true,+ );++ const compiled = new Set(+ manifest.artifacts+ .filter((artifact) => artifact.path.startsWith("compiled/"))+ .map((artifact) => artifact.sha256),+ );+ const compiledPuts = puts.filter((put) => compiled.has(put.sha256));+ expect(compiled.size).toBeGreaterThan(1);+ expect(compiledPuts).toHaveLength(compiled.size);+ expect(compiledPuts.every((put) => put.duringAudio)).toBe(true);+ const referenced = new Set(manifest.artifacts.map((artifact) => artifact.sha256));+ expect(puts.every((put) => referenced.has(put.sha256))).toBe(true);+ expect(stages).toEqual(+ expect.arrayContaining([+ ...["copy", "compile", "probe", "extract", "audio", "prehash", "prepublish"],+ ...["freeze", "publication", "upload", "manifest"],+ ]),+ );+ },+ 60_000,+ );+});+describe("Plan v2 hash schema", () => {it("does not reuse a raw artifact digest as its manifest hash", () => {const root = tempPath("hf-plan-v2-hash-");diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex 64df925a2a..d5f24c110a 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -12,29 +12,31 @@*/import {- closeSync,copyFileSync,existsSync,lstatSync,mkdirSync,mkdtempSync,- openSync,readFileSync,- readSync,readdirSync,renameSync,rmSync,statSync,writeFileSync,} from "node:fs";-import { createHash } from "node:crypto";import { tmpdir } from "node:os";import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";import { DISTRIBUTED_FPS, type DistributedFps, isDistributedFps } from "@hyperframes/core";import { createFrameLookupTable, type ExtractedFrames } from "@hyperframes/engine";-import { recomputePlanHashFromPlanDir, type ChunkSliceJson } from "../render/stages/freezePlan.js";-import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";-import { buildLocalExecutionPlan, type DistributedRenderConfig } from "./plan.js";+import {+ planFileSha256,+ recomputePlanHashFromPlanDir,+ type ChunkSliceJson,+ type PlanFileDigests,+} from "../render/stages/freezePlan.js";+import { canonicalJsonStringify, sha256File, sha256Hex } from "../render/stages/planHash.js";+import { defaultLogger, type ProducerLogger } from "../../logger.js";+import { buildLocalExecutionPlan, type DistributedRenderConfig, logPlanStage } from "./plan.js";import {PLAN_PROTOCOL_V2,readPlanProtocolV1,@@ -217,25 +219,6 @@ function listFiles(root: string): Array<{ path: string; absolutePath: string }>return files.sort((a, b) => a.path.localeCompare(b.path));}-/** Hash large plan artifacts with bounded memory (important above the legacy 2 GiB cap). */-function sha256File(path: string): string {- const hash = createHash("sha256");- const buffer = Buffer.allocUnsafe(1024 * 1024);- // lgtm[js/insecure-temporary-file] — read-only open of a caller-owned regular file;- // this call never creates a file, temporary or otherwise.- const fd = openSync(path, "r");- try {- let bytesRead = 0;- do {- bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);- if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));- } while (bytesRead > 0);- } finally {- closeSync(fd);- }- return hash.digest("hex");-}-function assertValidExtractionCacheCompleteSentinel(path: string): void {const sentinel = lstatSync(path);if (!sentinel.isFile() || sentinel.size !== 0) {@@ -493,6 +476,35 @@ function writeBlob(sourcePath: string, destinationPath: string): void {}}+function publishBlobFor(absolutePath: string, digests?: PlanFileDigests): PlanV2PublishBlob {+ return {+ sourcePath: absolutePath,+ sha256: planFileSha256(absolutePath, digests),+ sizeBytes: statSync(absolutePath).size,+ };+}++async function putBlobs(+ publisher: PlanV2ArtifactPublisher,+ blobs: readonly Readonly<PlanV2PublishBlob>[],+): Promise<void> {+ const uploadSlots = 32;+ let next = 0;+ let failure: { reason: unknown } | undefined;+ const upload = async () => {+ while (!failure && next < blobs.length) {+ const blob = blobs[next++]!;+ try {+ await publisher.putBlob(blob);+ } catch (reason) {+ failure ??= { reason };+ }+ }+ };+ await Promise.all(Array.from({ length: uploadSlots }, upload));+ if (failure) throw failure.reason;+}+interface PlanV2Publication {readonly manifest: PlanV2Manifest;readonly blobs: readonly Readonly<PlanV2PublishBlob>[];@@ -500,7 +512,10 @@ interface PlanV2Publication {// Artifact classification intentionally keeps every fail-safe branch together.// fallow-ignore-next-line complexity-function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {+function buildPlanV2Publication(+ executionPlanDir: string,+ digests?: PlanFileDigests,+): PlanV2Publication {const executionPlanPath = join(executionPlanDir, "plan.json");if (!existsSync(executionPlanPath)) {throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);@@ -517,7 +532,7 @@ function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {if (!isSha256(executionPlanHash)) {throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");}- const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir);+ const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);if (recomputedExecutionPlanHash !== executionPlanHash) {throw new PlanV2IntegrityError(`execution plan content fingerprint does not match plan.json.planHash: ` +@@ -544,15 +559,12 @@ function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {) {continue;}- const sha256 = sha256File(file.absolutePath);- const sizeBytes = statSync(file.absolutePath).size;- if (!blobs.has(sha256)) {- blobs.set(sha256, { sourcePath: file.absolutePath, sha256, sizeBytes });- }+ const blob = publishBlobFor(file.absolutePath, digests);+ if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);artifacts.push({path: file.path,- sha256,- sizeBytes,+ sha256: blob.sha256,+ sizeBytes: blob.sizeBytes,...targets,});}@@ -623,25 +635,25 @@ export function createPlanV2FromV1(executionPlanDir: string, planV2Dir: string):export async function publishPlanV2FromExecutionPlan(executionPlanDir: string,publisher: PlanV2ArtifactPublisher,+): Promise<PlanV2Manifest> {+ return publishExecutionPlan(executionPlanDir, publisher, {});+}++async function publishExecutionPlan(+ executionPlanDir: string,+ publisher: PlanV2ArtifactPublisher,+ { digests, log = defaultLogger }: { digests?: PlanFileDigests; log?: ProducerLogger },): Promise<PlanV2Manifest> {try {- const publication = buildPlanV2Publication(executionPlanDir);- const uploadSlots = 32;- let next = 0;- let failure: { reason: unknown } | undefined;- const upload = async () => {- while (!failure && next < publication.blobs.length) {- const blob = publication.blobs[next++]!;- try {- await publisher.putBlob(blob);- } catch (reason) {- failure ??= { reason };- }- }- };- await Promise.all(Array.from({ length: uploadSlots }, upload));- if (failure) throw failure.reason;+ let stageStarted = Date.now();+ const publication = buildPlanV2Publication(executionPlanDir, digests);+ logPlanStage(log, "publication", stageStarted);+ stageStarted = Date.now();+ await putBlobs(publisher, publication.blobs);+ logPlanStage(log, "upload", stageStarted);+ stageStarted = Date.now();await publisher.commitManifest(canonicalJsonStringify(publication.manifest));+ logPlanStage(log, "manifest", stageStarted);return publication.manifest;} catch (error) {try {@@ -678,12 +690,22 @@ export async function planV2WithPublisher(options: Readonly<PlanV2WithPublisherOptions> = {},): Promise<PlanV2Manifest> {const stagingRoot = mkdtempSync(join(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));+ const digests: PlanFileDigests = new Map();try {await buildLocalExecutionPlan(projectDir, config, stagingRoot, {executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,deferVideoExtraction: true,+ digests,+ // artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.+ whileAudioMixes: () =>+ putBlobs(+ publisher,+ listFiles(join(stagingRoot, "compiled")).map((file) =>+ publishBlobFor(file.absolutePath, digests),+ ),+ ),});- return await publishPlanV2FromExecutionPlan(stagingRoot, publisher);+ return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });} catch (error) {try {await publisher.abort();diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.tsindex 62842e6238..4b5b40e5da 100644--- a/packages/producer/src/services/render/stages/freezePlan.ts+++ b/packages/producer/src/services/render/stages/freezePlan.ts@@ -20,7 +20,8 @@ import {computePlanHash,type PlanAssetHash,type PlanDimensions,- sha256Hex,+ sha256File,+ sha256FileAsync,} from "./planHash.js";/**@@ -134,6 +135,7 @@ export interface FreezePlanInput {totalFrames: number;/** Whether the plan's mixed-audio artifact was produced. */hasAudio: boolean;+ digests?: PlanFileDigests;}export interface FreezePlanResult {@@ -159,9 +161,29 @@ const HASH_EXCLUDED_PLAN_FILES = new Set<string>([COMPILED_INDEX_RELATIVE_PATH,]);+/** sha256 by absolute path. A staged plan file never changes, so one plan hashes it once. */+export type PlanFileDigests = Map<string, string>;++export function planFileSha256(absolutePath: string, digests?: PlanFileDigests): string {+ let sha256 = digests?.get(absolutePath);+ if (sha256 === undefined) {+ sha256 = sha256File(absolutePath);+ digests?.set(absolutePath, sha256);+ }+ return sha256;+}++/** Hash every file under `dir` into `digests` without blocking the event loop. */+export async function prehashPlanFiles(dir: string, digests: PlanFileDigests): Promise<void> {+ if (!existsSync(dir)) return;+ for (const { absolutePath } of listPlanFiles(dir)) {+ if (!digests.has(absolutePath)) digests.set(absolutePath, await sha256FileAsync(absolutePath));+ }+}+/*** Recursively drop keys whose value is `undefined`. Preserves arrays and- * primitive leaves. Used to sanitize the `LockedRenderConfig` before+ * primitive leaves. Sanitizes the `LockedRenderConfig` before* canonical-JSON serialization so optional fields collapse to "absent"* rather than tripping `canonicalJsonStringify`'s undefined-rejection.*/@@ -219,7 +241,10 @@ function listPlanFiles(planDir: string): Array<{ planRelativePath: string; absol* the compiled HTML bytes (mixed verbatim into `planHash`) and the* sorted-by-path asset hashes.*/-function collectPlanAssetShas(planDir: string): {+function collectPlanAssetShas(+ planDir: string,+ digests?: PlanFileDigests,+): {compositionHtml: Uint8Array;assets: PlanAssetHash[];} {@@ -232,8 +257,10 @@ function collectPlanAssetShas(planDir: string): {continue;}if (HASH_EXCLUDED_PLAN_FILES.has(file.planRelativePath)) continue;- const bytes = readFileSync(file.absolutePath);- assets.push({ path: file.planRelativePath, sha256: sha256Hex(bytes) });+ assets.push({+ path: file.planRelativePath,+ sha256: planFileSha256(file.absolutePath, digests),+ });}if (compositionHtml === null) {throw new Error(@@ -260,7 +287,7 @@ function collectPlanAssetShas(planDir: string): {* callers should catch those as `MISSING_PLAN_ARTIFACT` rather than* lumping them with hash drift.*/-export function recomputePlanHashFromPlanDir(planDir: string): string {+export function recomputePlanHashFromPlanDir(planDir: string, digests?: PlanFileDigests): string {const planJsonPath = join(planDir, "plan.json");const encoderJsonPath = join(planDir, "meta", "encoder.json");if (!existsSync(planJsonPath)) {@@ -283,7 +310,7 @@ export function recomputePlanHashFromPlanDir(planDir: string): string {// normalization differences, and Node version skew on number printing.const encoderConfigCanonicalJson = readFileSync(encoderJsonPath, "utf-8");- const { compositionHtml, assets } = collectPlanAssetShas(planDir);+ const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);return computePlanHash({compositionHtml,@@ -316,6 +343,7 @@ export async function freezePlan(input: FreezePlanInput): Promise<FreezePlanResudurationSeconds,totalFrames,hasAudio,+ digests,} = input;if (!existsSync(planDir)) {@@ -343,7 +371,7 @@ export async function freezePlan(input: FreezePlanInput): Promise<FreezePlanResuwriteFileSync(join(metaDir, "chunks.json"), `${JSON.stringify(chunks, null, 2)}\n`, "utf-8");- const { compositionHtml, assets } = collectPlanAssetShas(planDir);+ const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);const planHash = computePlanHash({compositionHtml,diff --git a/packages/producer/src/services/render/stages/planHash.ts b/packages/producer/src/services/render/stages/planHash.tsindex 127db7c244..03cc3a2167 100644--- a/packages/producer/src/services/render/stages/planHash.ts+++ b/packages/producer/src/services/render/stages/planHash.ts@@ -32,6 +32,7 @@*/import { createHash } from "node:crypto";+import { closeSync, createReadStream, openSync, readSync } from "node:fs";import type { DistributedFormat } from "../../distributed/shared.js";/**@@ -172,3 +173,30 @@ export function sha256Hex(bytes: Uint8Array | string): string {h.update(bytes);return h.digest("hex");}++/** Hash large plan artifacts with bounded memory (important above the legacy 2 GiB cap). */+export function sha256File(path: string): string {+ const hash = createHash("sha256");+ const buffer = Buffer.allocUnsafe(1024 * 1024);+ // lgtm[js/insecure-temporary-file] — read-only open of a caller-owned regular file;+ // this call never creates a file, temporary or otherwise.+ const fd = openSync(path, "r");+ try {+ let bytesRead = 0;+ do {+ bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);+ if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));+ } while (bytesRead > 0);+ } finally {+ closeSync(fd);+ }+ return hash.digest("hex");+}++export async function sha256FileAsync(path: string): Promise<string> {+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(path, { highWaterMark: 1024 * 1024 })) {+ hash.update(chunk as Buffer);+ }+ return hash.digest("hex");+}