Skip to content

Write HTML. Render video. Built for agents.

perf(producer): overlap the plan step's audio work and hash files once (#5777)

Miguel Ángelfb17f4119620
diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts
index 4e5487a108..9160725c19 100644
--- a/packages/aws-lambda/src/handler.test.ts
+++ b/packages/aws-lambda/src/handler.test.ts
@@ -15,7 +15,7 @@
* the glue's contract.
*/
-import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test";
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -658,6 +658,10 @@ describe("handler dispatch", () => {
skipChromeResolution: true,
};
+ const logged: string[] = [];
+ const logSpy = spyOn(console, "log").mockImplementation((line: string) => {
+ logged.push(line);
+ });
const planned = await handler(
{
Action: "plan",
@@ -666,7 +670,12 @@ describe("handler dispatch", () => {
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
},
deps,
- );
+ ).finally(() => logSpy.mockRestore());
+ const stageLines = logged
+ .filter((line) => line.includes('"plan_stage"'))
+ .map((line) => JSON.parse(line) as { stage?: string; ms?: unknown });
+ expect(stageLines.map((line) => line.stage)).toEqual(["download", "untar"]);
+ expect(stageLines.every((line) => typeof line.ms === "number")).toBe(true);
expect(planned).toMatchObject({
PlanProtocol: "v2",
PlanV2ManifestS3Uri: "s3://bucket/renders/v2/v2/manifest.json",
diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts
index a3456ea2fc..4c8cbb5e0a 100644
--- a/packages/aws-lambda/src/handler.ts
+++ b/packages/aws-lambda/src/handler.ts
@@ -427,8 +427,12 @@ async function handlePlanV2(
const projectArchive = join(work, "project.tar.gz");
const projectDir = join(work, "project");
try {
+ let stageStarted = Date.now();
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
+ logEvent({ event: "plan_stage", stage: "download", ms: Date.now() - stageStarted });
+ stageStarted = Date.now();
await untarDirectory(projectArchive, projectDir);
+ logEvent({ event: "plan_stage", stage: "untar", ms: Date.now() - stageStarted });
const publisher = new S3PlanV2ArtifactPublisher({
s3,
planOutputS3Prefix: event.PlanOutputS3Prefix,
diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts b/packages/aws-lambda/src/s3PlanV2Publisher.test.ts
index 94a35a6e68..9b437f9cf3 100644
--- a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts
+++ b/packages/aws-lambda/src/s3PlanV2Publisher.test.ts
@@ -19,12 +19,14 @@ interface PutOperation {
class FakeS3 {
readonly objects = new Map<string, StoredObject>();
readonly puts: PutOperation[] = [];
+ sends = 0;
asClient(): import("@aws-sdk/client-s3").S3Client {
return this as unknown as import("@aws-sdk/client-s3").S3Client;
}
async send(command: unknown): Promise<unknown> {
+ this.sends += 1;
// AWS command inputs are the runtime boundary exercised by this fake.
const value = command as unknown as {
readonly constructor: { readonly name: string };
@@ -139,6 +141,22 @@ describe("S3PlanV2ArtifactPublisher", () => {
expect(s3.objects.get(manifestUri)?.bytes.toString("utf8")).toBe(manifest);
});
+ it("sends nothing for a digest it already published", async () => {
+ const source = makeSource("hello");
+ const s3 = new FakeS3();
+ const publisher = new S3PlanV2ArtifactPublisher({
+ s3: s3.asClient(),
+ planOutputS3Prefix: "s3://bucket/render",
+ temporaryRoot: source.root,
+ });
+ const blob = { sourcePath: source.path, sha256: source.digest, sizeBytes: source.sizeBytes };
+
+ await publisher.putBlob(blob);
+ const sendsAfterFirst = s3.sends;
+ await publisher.putBlob(blob);
+ expect(s3.sends).toBe(sendsAfterFirst);
+ });
+
it("refuses to expose a manifest that references an unpublished digest", async () => {
const source = makeSource("hello");
const s3 = new FakeS3();
diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.ts b/packages/aws-lambda/src/s3PlanV2Publisher.ts
index afcb45928e..8eee0d8681 100644
--- a/packages/aws-lambda/src/s3PlanV2Publisher.ts
+++ b/packages/aws-lambda/src/s3PlanV2Publisher.ts
@@ -83,6 +83,7 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
async putBlob(blob: Readonly<PlanV2PublishBlob>): Promise<void> {
this.#assertOpen("publish a blob");
const digest = assertSha256(blob.sha256, "S3 published blob sha256");
+ if (this.#publishedDigests.has(digest)) return;
const sourceSize = statSync(blob.sourcePath).size;
if (sourceSize !== blob.sizeBytes) {
throw new PlanV2IntegrityError(
diff --git a/packages/aws-lambda/src/s3Transport.test.ts b/packages/aws-lambda/src/s3Transport.test.ts
index cd101ad090..6e5ea6dd58 100644
--- a/packages/aws-lambda/src/s3Transport.test.ts
+++ b/packages/aws-lambda/src/s3Transport.test.ts
@@ -6,6 +6,7 @@
*/
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { createHash } from "node:crypto";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -116,6 +117,19 @@ describe("content-addressed v2 artifacts", () => {
expect(s3.putCount).toBe(1);
});
+ it("leaves verifying the local bytes to the S3 checksum instead of hashing them again", async () => {
+ const source = join(scratchRoot, "artifact-changed.bin");
+ writeFileSync(source, "bytes that changed");
+ const digest = createHash("sha256").update("bytes as hashed").digest("hex");
+ const uri = `s3://bucket/v2/artifacts/sha256/${digest.slice(0, 2)}/${digest}`;
+ const s3 = new ContentAddressedFakeS3();
+
+ await expect(
+ uploadContentAddressedFileToS3(s3.asClient(), source, uri, digest),
+ ).rejects.toMatchObject({ name: "BadDigest" });
+ expect(s3.objects.has(uri)).toBe(false);
+ });
+
it("refuses to overwrite an immutable key with conflicting metadata", async () => {
const source = join(scratchRoot, "artifact-conflict.bin");
writeFileSync(source, "expected bytes");
@@ -199,6 +213,7 @@ class ContentAddressedFakeS3 {
Key: string;
Body?: NodeJS.ReadableStream;
Metadata?: Record<string, string>;
+ ChecksumSHA256?: string;
};
};
const uri = `s3://${value.input.Bucket}/${value.input.Key}`;
@@ -238,6 +253,12 @@ class ContentAddressedFakeS3 {
const chunks: Buffer[] = [];
for await (const chunk of value.input.Body ?? []) chunks.push(Buffer.from(chunk));
const bytes = Buffer.concat(chunks);
+ const checksum = createHash("sha256").update(bytes).digest("base64");
+ if (value.input.ChecksumSHA256 !== undefined && value.input.ChecksumSHA256 !== checksum) {
+ const error = new Error("checksum mismatch");
+ error.name = "BadDigest";
+ throw error;
+ }
this.objects.set(uri, {
bytes,
sha256: value.input.Metadata?.sha256 ?? "",
diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.ts
index 5efbac7be1..379fa147ab 100644
--- a/packages/aws-lambda/src/s3Transport.ts
+++ b/packages/aws-lambda/src/s3Transport.ts
@@ -131,11 +131,9 @@ export async function uploadFileToS3(
}
/**
- * Upload one content-addressed plan-v2 artifact exactly once.
- *
- * Existing objects are reused only when their immutable digest metadata and
- * byte length agree. A conflicting object is never overwritten: doing so
- * could change a plan already being consumed by another chunk invocation.
+ * Upload one content-addressed plan-v2 artifact exactly once; S3 verifies the bytes against
+ * `ChecksumSHA256`. An existing object is reused only when its digest metadata and length agree,
+ * and a conflicting one is never overwritten, since a chunk may already be reading that plan.
*/
export async function uploadContentAddressedFileToS3(
client: S3Client,
@@ -148,15 +146,6 @@ export async function uploadContentAddressedFileToS3(
if (!existsSync(localPath)) {
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
}
- const actualSha256 = await sha256File(localPath);
- if (actualSha256 !== expectedSha256) {
- const error = new Error(
- `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`,
- );
- error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
- throw error;
- }
-
const { bucket, key } = parseS3Uri(uri);
const size = statSync(localPath).size;
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index ad5d5445ad..2a6bedf1f8 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -66,6 +66,8 @@ import {
type CompositionMetadataJson,
freezePlan,
type LockedRenderConfig,
+ type PlanFileDigests,
+ prehashPlanFiles,
} from "../render/stages/freezePlan.js";
import {
canonicalJsonStringify,
@@ -299,6 +301,14 @@ export interface BuildLocalExecutionPlanOptions {
* chunks write those frames into their own work directory.
*/
readonly deferVideoExtraction?: boolean;
+ /** Shared with the publisher so each plan file is hashed once. */
+ readonly digests?: PlanFileDigests;
+ /** Runs while the audio mixes, once `compiled/` and the video trees are final. */
+ readonly whileAudioMixes?: () => Promise<void>;
+}
+
+export function logPlanStage(log: ProducerLogger, stage: string, startedAt: number): void {
+ log.info("[plan] stage", { stage, ms: Date.now() - startedAt });
}
/**
@@ -1064,6 +1074,7 @@ export async function buildLocalExecutionPlan(
// overwrites the entry HTML afterwards. `dereference: true` resolves
// symlinks so the planDir survives S3 / Lambda /tmp round-trips.
mkdirSync(compiledDir, { recursive: true });
+ let stageStarted = Date.now();
cpSync(projectDir, compiledDir, {
recursive: true,
dereference: true,
@@ -1077,6 +1088,7 @@ export async function buildLocalExecutionPlan(
return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
},
});
+ logPlanStage(log, "copy", stageStarted);
// The compiled directory lives at `<planDir>/compiled/` in the final
// layout. The stages write under `<planDir>/.plan-work/compiled/`; we
@@ -1095,6 +1107,7 @@ export async function buildLocalExecutionPlan(
const needsAlpha = outputNeedsAlpha(config.format);
// ── Compile ──
+ stageStarted = Date.now();
const compileResult = await runCompileStage({
projectDir,
workDir,
@@ -1114,6 +1127,7 @@ export async function buildLocalExecutionPlan(
allowSystemFontCapture: false,
variables: config.variables,
});
+ logPlanStage(log, "compile", stageStarted);
let compiled = compileResult.compiled;
const composition = compileResult.composition;
const { deviceScaleFactor } = compileResult;
@@ -1143,6 +1157,7 @@ export async function buildLocalExecutionPlan(
// ── Probe ──
// Browser probe runs only when needed. For statically-resolvable durations
// this is a near-zero pass.
+ stageStarted = Date.now();
const probeResult = await runProbeStage({
projectDir,
workDir,
@@ -1160,6 +1175,7 @@ export async function buildLocalExecutionPlan(
deviceScaleFactor,
subTimelineWaitMemo: {},
});
+ logPlanStage(log, "probe", stageStarted);
compiled = probeResult.compiled;
job.duration = probeResult.duration;
job.totalFrames = probeResult.totalFrames;
@@ -1212,6 +1228,7 @@ export async function buildLocalExecutionPlan(
// ── Extract videos ──
// `materializeSymlinks: true` recursively copies frames so the planDir is
// self-contained (symlinks don't survive S3/GCS round-trips).
+ stageStarted = Date.now();
const extractResult = await runExtractVideosStage({
projectDir,
compiledDir,
@@ -1223,6 +1240,7 @@ export async function buildLocalExecutionPlan(
materializeSymlinks: true,
deferRangeExtraction: options.deferVideoExtraction,
});
+ logPlanStage(log, "extract", stageStarted);
if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
if (extractResult.extractionResult) {
// Distributed chunks cannot safely fall back to native remote decoding:
@@ -1236,23 +1254,6 @@ export async function buildLocalExecutionPlan(
// video's outputDir, but in `plan()` those directories ARE the source
// material the renames below move into `planDir/video-frames/`.
- // ── Audio ──
- const audioResult = await runAudioStage({
- projectDir,
- workDir,
- compiledDir,
- duration: job.duration,
- ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
- audioGain: cfg.audioGain,
- audios: composition.audios,
- masterFxChain: composition.masterFxChain,
- abortSignal,
- assertNotAborted,
- });
- if (audioResult.audioError) {
- applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
- }
-
// Promote staged artifacts from the temp work tree into the final planDir
// shape. `workDir` is `<planDir>/.plan-work/` — always the same filesystem
// as `planDir`, so `renameSync` succeeds without copying. Video frames
@@ -1262,7 +1263,8 @@ export async function buildLocalExecutionPlan(
const stagedVideoFrames = join(compiledDir, "__hyperframes_video_frames");
const videoFramesDst = join(planDir, "video-frames");
if (existsSync(videoFramesDst)) rmSync(videoFramesDst, { recursive: true, force: true });
- rmSync(join(planDir, "video-sources"), { recursive: true, force: true });
+ const videoSourcesDir = join(planDir, "video-sources");
+ rmSync(videoSourcesDir, { recursive: true, force: true });
if (existsSync(stagedVideoFrames)) {
renameSync(stagedVideoFrames, videoFramesDst);
} else {
@@ -1306,6 +1308,43 @@ export async function buildLocalExecutionPlan(
"utf-8",
);
+ // ── Audio ──
+ // Only the assembler reads the mix, so hashing and publishing the final
+ // compiled and video trees run beside it instead of after it.
+ const digests: PlanFileDigests = options.digests ?? new Map();
+ stageStarted = Date.now();
+ const [audioOutcome, overlapOutcome] = await Promise.allSettled([
+ runAudioStage({
+ projectDir,
+ workDir,
+ compiledDir: finalCompiledDir,
+ duration: probeResult.duration,
+ ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
+ audioGain: cfg.audioGain,
+ audios: composition.audios,
+ masterFxChain: composition.masterFxChain,
+ abortSignal,
+ assertNotAborted,
+ }).finally(() => logPlanStage(log, "audio", stageStarted)),
+ (async () => {
+ for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {
+ await prehashPlanFiles(dir, digests);
+ }
+ logPlanStage(log, "prehash", stageStarted);
+ if (options.whileAudioMixes) {
+ const prepublishStarted = Date.now();
+ await options.whileAudioMixes();
+ logPlanStage(log, "prepublish", prepublishStarted);
+ }
+ })(),
+ ]);
+ if (audioOutcome.status === "rejected") throw audioOutcome.reason;
+ if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;
+ const audioResult = audioOutcome.value;
+ if (audioResult.audioError) {
+ applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
+ }
+
const planAudioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {
renameSync(audioResult.audioOutputPath, planAudioPath);
@@ -1382,6 +1421,7 @@ export async function buildLocalExecutionPlan(
log,
});
+ stageStarted = Date.now();
const freezeResult = await freezePlan({
planDir,
composition: compositionJson,
@@ -1393,7 +1433,9 @@ export async function buildLocalExecutionPlan(
durationSeconds: job.duration ?? 0,
totalFrames,
hasAudio: audioResult.hasAudio,
+ digests,
});
+ logPlanStage(log, "freeze", stageStarted);
const planHash = freezeResult.planHash;
// 2 GB hard cap so the planDir fits inside Lambda's 10 GB /tmp budget
diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.ts
index 649554462d..f808a5724d 100644
--- a/packages/producer/src/services/distributed/planV2.test.ts
+++ b/packages/producer/src/services/distributed/planV2.test.ts
@@ -1,4 +1,5 @@
import { afterEach, describe, expect, it } from "bun:test";
+import { spawnSync } from "node:child_process";
import {
existsSync,
mkdirSync,
@@ -12,8 +13,9 @@ import {
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
-import { recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";
+import { planFileSha256, recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";
import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";
+import { buildLocalExecutionPlan } from "./plan.js";
import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
import { FFMPEG_VERSION_MISMATCH, renderChunk, RenderChunkValidationError } from "./renderChunk.js";
import {
@@ -26,11 +28,12 @@ import {
PlanV2IntegrityError,
publishPlanV2FromV1,
publishPlanV2FromExecutionPlan,
+ planV2WithPublisher,
readPlanV2Manifest,
validatePlanV2MaterializedTarget,
} from "./planV2.js";
import { LocalPlanV2ArtifactPublisher, type PlanV2ArtifactPublisher } from "./planV2Publisher.js";
-import { buildPlanVideosJson, type PlanVideosJson } from "./shared.js";
+import { buildPlanVideosJson, isPlanAudioArtifactPath, type PlanVideosJson } from "./shared.js";
const tempDirs: string[] = [];
@@ -1003,6 +1006,114 @@ describe("Plan v2 artifact publisher", () => {
});
});
+describe("Plan v2 planning", () => {
+ const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
+
+ it("reuses a digest taken earlier in the plan instead of rehashing the file", () => {
+ const planDir = createV1Plan(tempPath("hf-plan-v2-digests-"));
+ const asset = join(planDir, "compiled", "asset.txt");
+ const digests = new Map([[asset, "f".repeat(64)]]);
+
+ expect(planFileSha256(asset, digests)).toBe("f".repeat(64));
+ expect(recomputePlanHashFromPlanDir(planDir, digests)).not.toBe(
+ recomputePlanHashFromPlanDir(planDir),
+ );
+ });
+
+ it.skipIf(!HAS_FFMPEG)(
+ "publishes the compiled tree while the audio mixes, with the serial manifest",
+ async () => {
+ const root = tempPath("hf-plan-v2-overlap-");
+ const projectDir = join(root, "project");
+ const stagingParentDir = join(root, "staging");
+ mkdirSync(projectDir);
+ mkdirSync(stagingParentDir);
+ const silence = spawnSync("ffmpeg", [
+ ...["-nostdin", "-v", "error", "-f", "lavfi"],
+ ...["-i", "anullsrc=channel_layout=mono:sample_rate=44100", "-t", "1"],
+ ...["-c:a", "pcm_s16le", join(projectDir, "silence.wav")],
+ ]);
+ expect(silence.status).toBe(0);
+ writeFileSync(join(projectDir, "poster.bin"), Buffer.alloc(1 << 20, 7));
+ writeFileSync(
+ join(projectDir, "index.html"),
+ `<!doctype html><html><head><meta charset="utf-8"></head><body>
+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1">
+ <audio data-composition-audio src="silence.wav"></audio><p>overlap fixture</p>
+ </div></body></html>`,
+ );
+ const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const };
+
+ const stages: string[] = [];
+ const puts: Array<{ sha256: string; duringAudio: boolean }> = [];
+ const local = new LocalPlanV2ArtifactPublisher(join(root, "overlapped"));
+ const manifest = await planV2WithPublisher(
+ projectDir,
+ {
+ ...config,
+ logger: {
+ info: (message, meta) => {
+ if (message === "[plan] stage") stages.push(String(meta?.stage));
+ },
+ debug() {},
+ warn() {},
+ error() {},
+ },
+ },
+ {
+ async putBlob(blob) {
+ const staging = readdirSync(stagingParentDir).find((name) =>
+ name.startsWith(".execution-plan-"),
+ );
+ const workDir = join(stagingParentDir, String(staging), ".plan-work");
+ if (!puts.some((put) => put.sha256 === blob.sha256)) {
+ puts.push({ sha256: blob.sha256, duringAudio: existsSync(workDir) });
+ }
+ await local.putBlob(blob);
+ },
+ commitManifest: (bytes) => local.commitManifest(bytes),
+ abort: () => local.abort(),
+ },
+ { stagingParentDir },
+ );
+
+ const serialDir = join(root, "serial-plan");
+ mkdirSync(serialDir);
+ await buildLocalExecutionPlan(projectDir, config, serialDir, {
+ executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
+ deferVideoExtraction: true,
+ });
+ const serial = await publishPlanV2FromExecutionPlan(
+ serialDir,
+ new LocalPlanV2ArtifactPublisher(join(root, "serial")),
+ );
+ expect(canonicalJsonStringify(manifest)).toBe(canonicalJsonStringify(serial));
+ expect(manifest.artifacts.some((artifact) => isPlanAudioArtifactPath(artifact.path))).toBe(
+ true,
+ );
+
+ const compiled = new Set(
+ manifest.artifacts
+ .filter((artifact) => artifact.path.startsWith("compiled/"))
+ .map((artifact) => artifact.sha256),
+ );
+ const compiledPuts = puts.filter((put) => compiled.has(put.sha256));
+ expect(compiled.size).toBeGreaterThan(1);
+ expect(compiledPuts).toHaveLength(compiled.size);
+ expect(compiledPuts.every((put) => put.duringAudio)).toBe(true);
+ const referenced = new Set(manifest.artifacts.map((artifact) => artifact.sha256));
+ expect(puts.every((put) => referenced.has(put.sha256))).toBe(true);
+ expect(stages).toEqual(
+ expect.arrayContaining([
+ ...["copy", "compile", "probe", "extract", "audio", "prehash", "prepublish"],
+ ...["freeze", "publication", "upload", "manifest"],
+ ]),
+ );
+ },
+ 60_000,
+ );
+});
+
describe("Plan v2 hash schema", () => {
it("does not reuse a raw artifact digest as its manifest hash", () => {
const root = tempPath("hf-plan-v2-hash-");
diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.ts
index 64df925a2a..d5f24c110a 100644
--- a/packages/producer/src/services/distributed/planV2.ts
+++ b/packages/producer/src/services/distributed/planV2.ts
@@ -12,29 +12,31 @@
*/
import {
- closeSync,
copyFileSync,
existsSync,
lstatSync,
mkdirSync,
mkdtempSync,
- openSync,
readFileSync,
- readSync,
readdirSync,
renameSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
-import { createHash } from "node:crypto";
import { tmpdir } from "node:os";
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import { DISTRIBUTED_FPS, type DistributedFps, isDistributedFps } from "@hyperframes/core";
import { createFrameLookupTable, type ExtractedFrames } from "@hyperframes/engine";
-import { recomputePlanHashFromPlanDir, type ChunkSliceJson } from "../render/stages/freezePlan.js";
-import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";
-import { buildLocalExecutionPlan, type DistributedRenderConfig } from "./plan.js";
+import {
+ planFileSha256,
+ recomputePlanHashFromPlanDir,
+ type ChunkSliceJson,
+ type PlanFileDigests,
+} from "../render/stages/freezePlan.js";
+import { canonicalJsonStringify, sha256File, sha256Hex } from "../render/stages/planHash.js";
+import { defaultLogger, type ProducerLogger } from "../../logger.js";
+import { buildLocalExecutionPlan, type DistributedRenderConfig, logPlanStage } from "./plan.js";
import {
PLAN_PROTOCOL_V2,
readPlanProtocolV1,
@@ -217,25 +219,6 @@ function listFiles(root: string): Array<{ path: string; absolutePath: string }>
return files.sort((a, b) => a.path.localeCompare(b.path));
}
-/** Hash large plan artifacts with bounded memory (important above the legacy 2 GiB cap). */
-function sha256File(path: string): string {
- const hash = createHash("sha256");
- const buffer = Buffer.allocUnsafe(1024 * 1024);
- // lgtm[js/insecure-temporary-file] — read-only open of a caller-owned regular file;
- // this call never creates a file, temporary or otherwise.
- const fd = openSync(path, "r");
- try {
- let bytesRead = 0;
- do {
- bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);
- if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));
- } while (bytesRead > 0);
- } finally {
- closeSync(fd);
- }
- return hash.digest("hex");
-}
-
function assertValidExtractionCacheCompleteSentinel(path: string): void {
const sentinel = lstatSync(path);
if (!sentinel.isFile() || sentinel.size !== 0) {
@@ -493,6 +476,35 @@ function writeBlob(sourcePath: string, destinationPath: string): void {
}
}
+function publishBlobFor(absolutePath: string, digests?: PlanFileDigests): PlanV2PublishBlob {
+ return {
+ sourcePath: absolutePath,
+ sha256: planFileSha256(absolutePath, digests),
+ sizeBytes: statSync(absolutePath).size,
+ };
+}
+
+async function putBlobs(
+ publisher: PlanV2ArtifactPublisher,
+ blobs: readonly Readonly<PlanV2PublishBlob>[],
+): Promise<void> {
+ const uploadSlots = 32;
+ let next = 0;
+ let failure: { reason: unknown } | undefined;
+ const upload = async () => {
+ while (!failure && next < blobs.length) {
+ const blob = blobs[next++]!;
+ try {
+ await publisher.putBlob(blob);
+ } catch (reason) {
+ failure ??= { reason };
+ }
+ }
+ };
+ await Promise.all(Array.from({ length: uploadSlots }, upload));
+ if (failure) throw failure.reason;
+}
+
interface PlanV2Publication {
readonly manifest: PlanV2Manifest;
readonly blobs: readonly Readonly<PlanV2PublishBlob>[];
@@ -500,7 +512,10 @@ interface PlanV2Publication {
// Artifact classification intentionally keeps every fail-safe branch together.
// fallow-ignore-next-line complexity
-function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {
+function buildPlanV2Publication(
+ executionPlanDir: string,
+ digests?: PlanFileDigests,
+): PlanV2Publication {
const executionPlanPath = join(executionPlanDir, "plan.json");
if (!existsSync(executionPlanPath)) {
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
@@ -517,7 +532,7 @@ function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {
if (!isSha256(executionPlanHash)) {
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
}
- const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir);
+ const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);
if (recomputedExecutionPlanHash !== executionPlanHash) {
throw new PlanV2IntegrityError(
`execution plan content fingerprint does not match plan.json.planHash: ` +
@@ -544,15 +559,12 @@ function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {
) {
continue;
}
- const sha256 = sha256File(file.absolutePath);
- const sizeBytes = statSync(file.absolutePath).size;
- if (!blobs.has(sha256)) {
- blobs.set(sha256, { sourcePath: file.absolutePath, sha256, sizeBytes });
- }
+ const blob = publishBlobFor(file.absolutePath, digests);
+ if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
artifacts.push({
path: file.path,
- sha256,
- sizeBytes,
+ sha256: blob.sha256,
+ sizeBytes: blob.sizeBytes,
...targets,
});
}
@@ -623,25 +635,25 @@ export function createPlanV2FromV1(executionPlanDir: string, planV2Dir: string):
export async function publishPlanV2FromExecutionPlan(
executionPlanDir: string,
publisher: PlanV2ArtifactPublisher,
+): Promise<PlanV2Manifest> {
+ return publishExecutionPlan(executionPlanDir, publisher, {});
+}
+
+async function publishExecutionPlan(
+ executionPlanDir: string,
+ publisher: PlanV2ArtifactPublisher,
+ { digests, log = defaultLogger }: { digests?: PlanFileDigests; log?: ProducerLogger },
): Promise<PlanV2Manifest> {
try {
- const publication = buildPlanV2Publication(executionPlanDir);
- const uploadSlots = 32;
- let next = 0;
- let failure: { reason: unknown } | undefined;
- const upload = async () => {
- while (!failure && next < publication.blobs.length) {
- const blob = publication.blobs[next++]!;
- try {
- await publisher.putBlob(blob);
- } catch (reason) {
- failure ??= { reason };
- }
- }
- };
- await Promise.all(Array.from({ length: uploadSlots }, upload));
- if (failure) throw failure.reason;
+ let stageStarted = Date.now();
+ const publication = buildPlanV2Publication(executionPlanDir, digests);
+ logPlanStage(log, "publication", stageStarted);
+ stageStarted = Date.now();
+ await putBlobs(publisher, publication.blobs);
+ logPlanStage(log, "upload", stageStarted);
+ stageStarted = Date.now();
await publisher.commitManifest(canonicalJsonStringify(publication.manifest));
+ logPlanStage(log, "manifest", stageStarted);
return publication.manifest;
} catch (error) {
try {
@@ -678,12 +690,22 @@ export async function planV2WithPublisher(
options: Readonly<PlanV2WithPublisherOptions> = {},
): Promise<PlanV2Manifest> {
const stagingRoot = mkdtempSync(join(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
+ const digests: PlanFileDigests = new Map();
try {
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
deferVideoExtraction: true,
+ digests,
+ // artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
+ whileAudioMixes: () =>
+ putBlobs(
+ publisher,
+ listFiles(join(stagingRoot, "compiled")).map((file) =>
+ publishBlobFor(file.absolutePath, digests),
+ ),
+ ),
});
- return await publishPlanV2FromExecutionPlan(stagingRoot, publisher);
+ return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
} catch (error) {
try {
await publisher.abort();
diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.ts
index 62842e6238..4b5b40e5da 100644
--- a/packages/producer/src/services/render/stages/freezePlan.ts
+++ b/packages/producer/src/services/render/stages/freezePlan.ts
@@ -20,7 +20,8 @@ import {
computePlanHash,
type PlanAssetHash,
type PlanDimensions,
- sha256Hex,
+ sha256File,
+ sha256FileAsync,
} from "./planHash.js";
/**
@@ -134,6 +135,7 @@ export interface FreezePlanInput {
totalFrames: number;
/** Whether the plan's mixed-audio artifact was produced. */
hasAudio: boolean;
+ digests?: PlanFileDigests;
}
export interface FreezePlanResult {
@@ -159,9 +161,29 @@ const HASH_EXCLUDED_PLAN_FILES = new Set<string>([
COMPILED_INDEX_RELATIVE_PATH,
]);
+/** sha256 by absolute path. A staged plan file never changes, so one plan hashes it once. */
+export type PlanFileDigests = Map<string, string>;
+
+export function planFileSha256(absolutePath: string, digests?: PlanFileDigests): string {
+ let sha256 = digests?.get(absolutePath);
+ if (sha256 === undefined) {
+ sha256 = sha256File(absolutePath);
+ digests?.set(absolutePath, sha256);
+ }
+ return sha256;
+}
+
+/** Hash every file under `dir` into `digests` without blocking the event loop. */
+export async function prehashPlanFiles(dir: string, digests: PlanFileDigests): Promise<void> {
+ if (!existsSync(dir)) return;
+ for (const { absolutePath } of listPlanFiles(dir)) {
+ if (!digests.has(absolutePath)) digests.set(absolutePath, await sha256FileAsync(absolutePath));
+ }
+}
+
/**
* Recursively drop keys whose value is `undefined`. Preserves arrays and
- * primitive leaves. Used to sanitize the `LockedRenderConfig` before
+ * primitive leaves. Sanitizes the `LockedRenderConfig` before
* canonical-JSON serialization so optional fields collapse to "absent"
* rather than tripping `canonicalJsonStringify`'s undefined-rejection.
*/
@@ -219,7 +241,10 @@ function listPlanFiles(planDir: string): Array<{ planRelativePath: string; absol
* the compiled HTML bytes (mixed verbatim into `planHash`) and the
* sorted-by-path asset hashes.
*/
-function collectPlanAssetShas(planDir: string): {
+function collectPlanAssetShas(
+ planDir: string,
+ digests?: PlanFileDigests,
+): {
compositionHtml: Uint8Array;
assets: PlanAssetHash[];
} {
@@ -232,8 +257,10 @@ function collectPlanAssetShas(planDir: string): {
continue;
}
if (HASH_EXCLUDED_PLAN_FILES.has(file.planRelativePath)) continue;
- const bytes = readFileSync(file.absolutePath);
- assets.push({ path: file.planRelativePath, sha256: sha256Hex(bytes) });
+ assets.push({
+ path: file.planRelativePath,
+ sha256: planFileSha256(file.absolutePath, digests),
+ });
}
if (compositionHtml === null) {
throw new Error(
@@ -260,7 +287,7 @@ function collectPlanAssetShas(planDir: string): {
* callers should catch those as `MISSING_PLAN_ARTIFACT` rather than
* lumping them with hash drift.
*/
-export function recomputePlanHashFromPlanDir(planDir: string): string {
+export function recomputePlanHashFromPlanDir(planDir: string, digests?: PlanFileDigests): string {
const planJsonPath = join(planDir, "plan.json");
const encoderJsonPath = join(planDir, "meta", "encoder.json");
if (!existsSync(planJsonPath)) {
@@ -283,7 +310,7 @@ export function recomputePlanHashFromPlanDir(planDir: string): string {
// normalization differences, and Node version skew on number printing.
const encoderConfigCanonicalJson = readFileSync(encoderJsonPath, "utf-8");
- const { compositionHtml, assets } = collectPlanAssetShas(planDir);
+ const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);
return computePlanHash({
compositionHtml,
@@ -316,6 +343,7 @@ export async function freezePlan(input: FreezePlanInput): Promise<FreezePlanResu
durationSeconds,
totalFrames,
hasAudio,
+ digests,
} = input;
if (!existsSync(planDir)) {
@@ -343,7 +371,7 @@ export async function freezePlan(input: FreezePlanInput): Promise<FreezePlanResu
writeFileSync(join(metaDir, "chunks.json"), `${JSON.stringify(chunks, null, 2)}\n`, "utf-8");
- const { compositionHtml, assets } = collectPlanAssetShas(planDir);
+ const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);
const planHash = computePlanHash({
compositionHtml,
diff --git a/packages/producer/src/services/render/stages/planHash.ts b/packages/producer/src/services/render/stages/planHash.ts
index 127db7c244..03cc3a2167 100644
--- a/packages/producer/src/services/render/stages/planHash.ts
+++ b/packages/producer/src/services/render/stages/planHash.ts
@@ -32,6 +32,7 @@
*/
import { createHash } from "node:crypto";
+import { closeSync, createReadStream, openSync, readSync } from "node:fs";
import type { DistributedFormat } from "../../distributed/shared.js";
/**
@@ -172,3 +173,30 @@ export function sha256Hex(bytes: Uint8Array | string): string {
h.update(bytes);
return h.digest("hex");
}
+
+/** Hash large plan artifacts with bounded memory (important above the legacy 2 GiB cap). */
+export function sha256File(path: string): string {
+ const hash = createHash("sha256");
+ const buffer = Buffer.allocUnsafe(1024 * 1024);
+ // lgtm[js/insecure-temporary-file] — read-only open of a caller-owned regular file;
+ // this call never creates a file, temporary or otherwise.
+ const fd = openSync(path, "r");
+ try {
+ let bytesRead = 0;
+ do {
+ bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);
+ if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));
+ } while (bytesRead > 0);
+ } finally {
+ closeSync(fd);
+ }
+ return hash.digest("hex");
+}
+
+export async function sha256FileAsync(path: string): Promise<string> {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(path, { highWaterMark: 1024 * 1024 })) {
+ hash.update(chunk as Buffer);
+ }
+ return hash.digest("hex");
+}