Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(cloud): move project bytes once through shared content storage (#5772)
Miguel Ángel992df8b20313
diff --git a/bun.lock b/bun.lockindex 1317e1e49f..45be98330f 100644--- a/bun.lock+++ b/bun.lock@@ -32,6 +32,7 @@"@aws-sdk/client-lambda": "^3.1146.0","@aws-sdk/client-s3": "^3.1146.0","@aws-sdk/client-sfn": "^3.1146.0",+ "@aws-sdk/s3-request-presigner": "3.1146.0","@hyperframes/producer": "workspace:^","@sparticuz/chromium": "148.0.0","ffmpeg-static": "^5.2.0",@@ -467,6 +468,8 @@"@aws-sdk/nested-clients": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/signature-v4-multi-region": "^3.996.47", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ=="],+ "@aws-sdk/s3-request-presigner": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/signature-v4-multi-region": "^3.996.47", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-5BEfVr2OYX2bwB45NSWAXPoTN5M2NPLOHohgKV+XfMNcL5TF7UQN4hAOVF4S6DwrA4BrI3MsAry7vqdZ2aJEjg=="],+"@aws-sdk/signature-v4-multi-region": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.974.6", "@smithy/signature-v4": "^5.7.3", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g=="],"@aws-sdk/token-providers": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/nested-clients": "^3.997.46", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw=="],diff --git a/docs/deploy/aws-lambda.mdx b/docs/deploy/aws-lambda.mdxindex dd4a0e00c2..d6f769431c 100644--- a/docs/deploy/aws-lambda.mdx+++ b/docs/deploy/aws-lambda.mdx@@ -87,7 +87,7 @@ See the [CLI reference](/packages/cli#hyperframes-lambda) for full flag document#### Pre-staging a project with `sites create`-Re-rendering the same project tree on every `lambda render` call re-tars and re-uploads it each time. For tight inner loops (CI smoke jobs, prompt iteration in a demo flow), pre-stage the project once and reuse the upload:+Every `lambda render` call without `--site-id` hashes the project tree and checks each file against the bucket. For tight inner loops (CI smoke jobs, prompt iteration in a demo flow), pre-stage the project once and reuse the upload:```bashhyperframes lambda sites create ./my-project@@ -97,7 +97,15 @@ hyperframes lambda render ./my-project --site-id=a1b2c3d4e5f6g7h8 \--width 1920 --height 1080 --wait```-The `siteId` is content-addressed via a SHA-256 of the project tree; re-running `sites create` on an unchanged tree skips the upload via a `HeadObject` short-circuit. Pass the same `--site-id` to as many `lambda render` calls as you like — they all reuse the one S3 PUT.+The `siteId` is content-addressed via a SHA-256 of the project tree. A site is a small `sites/<siteId>/files.json` that lists each file's path and SHA-256. The files themselves live once per bucket under `cas/sha256/`, shared by every site and render, so a file another project already uploaded is never uploaded again. Pass the same `--site-id` to as many `lambda render` calls as you like.++The stack's bucket expires objects under `cas/sha256/` 30 days after they were last written (the `ExpireSharedContent` lifecycle rule). Each render or `sites create` that reuses a file at least 7 days old rewrites it in place, so files in use never expire. If a site's files did expire, `lambda render --site-id` re-uploads them from the project directory. `renderToLambda` given a `siteHandle` but no `projectDir` stops with `SITE_FILES_MISSING` before starting; run `sites create` again to restore them.++#### Upgrading from a version before shared content storage++- **Redeploy the stack before upgrading the CLI or SDK that starts renders.** A stack from an earlier version cannot read `files.json` sites. The CLI and `renderToLambda` check the state machine first and stop with `STACK_TOO_OLD` before uploading anything; run `hyperframes lambda deploy` (or redeploy your SAM template or CDK construct from the same version) and render again.+- **Existing site ids keep working.** When `sites/<siteId>/files.json` is missing, the render uses that site's `project.tar.gz` from the earlier version. If neither exists the render stops with `SITE_NOT_FOUND`, naming both keys; run `sites create` again.+- **Every project uploads once more.** The site id scheme changed (it now also covers symlinked files), so `sites create` or the first `lambda render` of each project after the upgrade returns a new site id and uploads its files once.### Path 2 — Direct SAM deploy@@ -166,6 +174,8 @@ hyperframes lambda policies rolehyperframes lambda policies validate ./infra/iam/hyperframes-deploy.json```+Code that calls the SDK's `renderToLambda` with its own role needs `states:DescribeStateMachine` on the state machine besides `states:StartExecution` and S3 read/write on the bucket: each render reads the definition to confirm the stack is new enough.+The generated documents grant `Resource: "*"` for the CLI's required action set. After your first successful deploy you can narrow `Resource` to the deployed ARNs — predictable per the CloudFormation outputs above. Adopters running the CLI in CI typically check the policy doc into source control and run `policies validate` as a pre-deploy step to catch drift.## Cost shape@@ -220,6 +230,14 @@ If progress doesn't advance for >10 minutes, check the Step Functions executionThe render bucket is created with CloudFormation `Retain` on delete — `hyperframes lambda destroy` (or `sam delete`) tears the function + state machine down but the bucket survives. This is intentional: it protects final-rendered MP4s from being lost when you re-deploy. To fully reclaim storage, empty + delete the bucket via the AWS console / `aws s3 rb`.+### `STACK_TOO_OLD`++The stack was deployed by an earlier version that cannot read this version's sites. Redeploy it (`hyperframes lambda deploy`), then render again. See [Upgrading](#upgrading-from-a-version-before-shared-content-storage).++### `SITE_FILES_MISSING` or `SITE_NOT_FOUND`++`SITE_FILES_MISSING`: some of the site's files expired from `cas/sha256/` after 30 days unused. Pass the project directory to `lambda render`, or run `sites create` again. `SITE_NOT_FOUND`: the bucket has no site under that id in either format; run `sites create` again and use the id it prints.+## Current limits- There is no completion webhook. Poll with `hyperframes lambda progress` or@@ -231,8 +249,8 @@ The render bucket is created with CloudFormation `Retain` on delete — `hyperfr- The distributed AWS path is SDR-only.- Each render chunk extracts only the video frames it shows, so the plan stepno longer grows with video length. On a 10,240 MB function, a 26-minute- 576×1024 24 fps H.264 clip plans in about 20 seconds. Each chunk downloads- the whole source file of every video it shows. Variable-frame-rate and HDR+ 576×1024 24 fps H.264 clip plans in about 20 seconds. Each chunk reads+ only the byte ranges of each video it shows, straight from the bucket. Variable-frame-rate and HDRsources are the exception: the plan step still extracts every frame of those,inside Lambda's 15-minute limit, in 2-minute segments, so`FFMPEG_PROCESS_TIMEOUT_MS` applies to each segment, not to the whole clip.diff --git a/docs/deploy/gcp-cloud-run.mdx b/docs/deploy/gcp-cloud-run.mdxindex 3baa640219..72f655aa59 100644--- a/docs/deploy/gcp-cloud-run.mdx+++ b/docs/deploy/gcp-cloud-run.mdx@@ -72,7 +72,8 @@ use H.264 or H.265. Distributed rendering is currently SDR-only.## Reuse an uploadProjects are content-addressed. Upload an unchanged project once, then reuse-its site ID for later renders:+its site ID for later renders. Site IDs from earlier versions keep working, but+the ID scheme changed, so each project uploads once more under a new ID:```bashhyperframes cloudrun sites create ./my-projectdiff --git a/docs/deploy/migrating-to-hyperframes-lambda.mdx b/docs/deploy/migrating-to-hyperframes-lambda.mdxindex 7301df8033..cba939ee59 100644--- a/docs/deploy/migrating-to-hyperframes-lambda.mdx+++ b/docs/deploy/migrating-to-hyperframes-lambda.mdx@@ -10,7 +10,7 @@ If you're already running a different framework that deploys a serverless video| In your current framework you call... | In HyperFrames you call... | Notes || ------------------------------------- | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------- || One-shot deploy command | `hyperframes lambda deploy` | Builds `packages/aws-lambda/dist/handler.zip` and runs `sam deploy`. Idempotent. |-| One-shot site upload | `hyperframes lambda sites create ./project` | Content-addressed S3 key — re-uploads of an unchanged tree are skipped via a HeadObject 200. |+| One-shot site upload | `hyperframes lambda sites create ./project` | Content-addressed file list; files already in the bucket's shared store are skipped. || Trigger a render | `hyperframes lambda render ./project --width 1920 --height 1080` | Returns immediately with a `renderId`; add `--wait` to stream per-chunk progress. || Poll render progress | `hyperframes lambda progress <renderId>` | Includes accrued cost in the same response. || Tear down | `hyperframes lambda destroy` | The S3 bucket is `Retain`'d — documented in the deploy guide. |diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex 2c4b74a486..684a0a8515 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -1983,8 +1983,9 @@ hyperframes lambda deploy \### `lambda sites create <projectDir>`-Tars and uploads the project to S3 under a content-addressed key, and returns a-`siteId` you can reuse. A re-render of the same tree then skips the upload.+Uploads the project's files to the bucket's shared content store, skipping any+file already there, and returns a `siteId` you can reuse. A re-render of the+same tree then skips the upload.```bashhyperframes lambda sites create ./my-projectdiff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yamlindex d44d00d437..bc8e38cd3f 100644--- a/examples/aws-lambda/template.yaml+++ b/examples/aws-lambda/template.yaml@@ -130,6 +130,11 @@ Resources:# Users keep the final mp4 (different key prefix); the rest# can age out after a week to keep storage costs flat.ExpirationInDays: 7+ - Id: ExpireSharedContent+ Status: Enabled+ Prefix: cas/sha256/+ # Project files and plan artifacts by SHA-256, shared across renders.+ ExpirationInDays: 30Tags:- Key: ProjectValue: !Ref ProjectName@@ -283,6 +288,7 @@ Resources:- PLAN_ARTIFACT_DIGEST_MISMATCH- FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED- CHUNK_EXCEEDS_WORKER_TIMEOUT+ - SITE_MANIFEST_INVALIDMaxAttempts: 0# Another render on the same function can still take the reserved slots; wait minutes.- ErrorEquals: [Lambda.TooManyRequestsException]@@ -340,6 +346,7 @@ Resources:- FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED- CHUNK_EXCEEDS_WORKER_TIMEOUT- ChromeBinaryUnavailableError+ - SITE_MANIFEST_INVALIDMaxAttempts: 0- ErrorEquals: [Lambda.TooManyRequestsException]IntervalSeconds: 5diff --git a/packages/aws-lambda/README.md b/packages/aws-lambda/README.mdindex 0ef9a34cb6..0ae0cf54e8 100644--- a/packages/aws-lambda/README.md+++ b/packages/aws-lambda/README.md@@ -77,10 +77,15 @@ infrastructure upgrade. Before upgrading an application that callssame new package version.3. Resume renders, then upgrade the application/SDK dependency.-Older state machines can default missing protocol fields to v1 or lack v2-branches, while the new SDK sends explicit v2. If infrastructure cannot be-redeployed first, keep the application on its previous package version or-pass `planProtocol: "v1"` explicitly until the redeploy is complete.+A site is a `sites/<id>/files.json` file list that an older stack cannot+read, under either plan protocol. Before it uploads anything,+`renderToLambda` reads the state machine definition+(`states:DescribeStateMachine`) and fails with `STACK_TOO_OLD` when the+stack predates that format. If infrastructure cannot be redeployed first,+keep the application on its previous package version until the redeploy is+complete. Site ids from earlier versions keep working after the redeploy:+when `sites/<id>/files.json` is missing, the render uses that site's+`project.tar.gz`.## Chrome runtimediff --git a/packages/aws-lambda/build.mjs b/packages/aws-lambda/build.mjsindex d639aca5c5..5472558b20 100644--- a/packages/aws-lambda/build.mjs+++ b/packages/aws-lambda/build.mjs@@ -2,10 +2,11 @@/*** Build script for @hyperframes/aws-lambda (public OSS package).*- * Bundles each subpath barrel via esbuild → dist/, then emits .d.ts via tsc.+ * Bundles the subpath barrels in one esbuild pass → dist/, then emits .d.ts via tsc.** Subpaths (each gets its own dist entry so adopters that import one path- * don't load the others' transitive graphs at module-load time):+ * don't load the others' transitive graphs at module-load time; code two+ * entries share is emitted once, in dist/_chunks/):** . (the umbrella barrel: handler + sdk + cdk types re-exported)* ./handler (the Lambda runtime entry — what `scripts/build-zip.ts` ZIPs)@@ -33,6 +34,7 @@ const sharedOpts = {"@aws-sdk/client-lambda","@aws-sdk/client-s3","@aws-sdk/client-sfn",+ "@aws-sdk/s3-request-presigner","@hyperframes/producer","@hyperframes/producer/distributed","@sparticuz/chromium",@@ -45,12 +47,18 @@ const sharedOpts = {],};-await Promise.all([- build({ ...sharedOpts, entryPoints: ["src/index.ts"], outfile: "dist/index.js" }),- build({ ...sharedOpts, entryPoints: ["src/handler.ts"], outfile: "dist/handler.js" }),- build({ ...sharedOpts, entryPoints: ["src/sdk/index.ts"], outfile: "dist/sdk/index.js" }),- build({ ...sharedOpts, entryPoints: ["src/cdk/index.ts"], outfile: "dist/cdk/index.js" }),-]);+await build({+ ...sharedOpts,+ entryPoints: {+ index: "src/index.ts",+ handler: "src/handler.ts",+ "sdk/index": "src/sdk/index.ts",+ "cdk/index": "src/cdk/index.ts",+ },+ outdir: "dist",+ splitting: true,+ chunkNames: "_chunks/[name]-[hash]",+});// esbuild doesn't emit .d.ts. tsc does, with a build-only tsconfig that// drops the workspace `paths` overrides so `@hyperframes/producer` resolves@@ -62,4 +70,4 @@ execSync({ stdio: "inherit" },);-console.log("[Build] Complete: dist/{index,handler,sdk/index,cdk/index}.js + .d.ts");+console.log("[Build] Complete: dist/{index,handler,sdk/index,cdk/index}.js + dist/_chunks + .d.ts");diff --git a/packages/aws-lambda/package.json b/packages/aws-lambda/package.jsonindex a42cde95a2..a45dd3ca4e 100644--- a/packages/aws-lambda/package.json+++ b/packages/aws-lambda/package.json@@ -72,6 +72,7 @@"@aws-sdk/client-lambda": "^3.1146.0","@aws-sdk/client-s3": "^3.1146.0","@aws-sdk/client-sfn": "^3.1146.0",+ "@aws-sdk/s3-request-presigner": "3.1146.0","@hyperframes/producer": "workspace:^","@sparticuz/chromium": "148.0.0","ffmpeg-static": "^5.2.0",diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.contract.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.contract.test.tsindex 9db00548df..80313afb66 100644--- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.contract.test.ts+++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.contract.test.ts@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { App, Stack } from "aws-cdk-lib";import { Template } from "aws-cdk-lib/assertions";+import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";import { HyperframesRenderStack } from "./HyperframesRenderStack.js";// CDK synth is slow on cold start and reached 32s under full-workspace CI@@ -61,7 +62,7 @@ describe("HyperframesRenderStack — contract", () => {});});- it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates lifecycle", () => {+ it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates and 30-day shared content lifecycle", () => {const t = DEFAULT_TEMPLATE;t.resourceCountIs("AWS::S3::Bucket", 1);t.hasResourceProperties("AWS::S3::Bucket", {@@ -79,6 +80,12 @@ describe("HyperframesRenderStack — contract", () => {Prefix: "renders/",ExpirationInDays: 7,},+ {+ Id: "ExpireSharedContent",+ Status: "Enabled",+ Prefix: `${CONTENT_STORE_PREFIX}/`,+ ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,+ },],},});diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.tsindex e1493ae7b4..8179f9c0cf 100644--- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts+++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts@@ -26,6 +26,12 @@ import { join } from "node:path";import { App, Stack } from "aws-cdk-lib";import { Template } from "aws-cdk-lib/assertions";import { parse as parseYaml } from "yaml";+import {+ CONTENT_STORE_PREFIX,+ CONTENT_STORE_REFRESH_AFTER_DAYS,+ CONTENT_STORE_RETENTION_DAYS,+} from "../contentStore.js";+import { SITE_MANIFEST_INVALID } from "../siteManifest.js";import { HyperframesRenderStack } from "./HyperframesRenderStack.js";// CDK synth + Template.fromStack is slow on cold start in CI (~5-8s on@@ -84,6 +90,7 @@ const EXPECTED_NON_RETRYABLE_ERRORS = new Set(["PLAN_ARTIFACT_DIGEST_MISMATCH","FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED","ChromeBinaryUnavailableError",+ "SITE_MANIFEST_INVALID",]);function doSynth(reservedConcurrency?: number): {@@ -213,6 +220,16 @@ describe("HyperframesRenderStack — snapshot", () => {}});+ it("lists the error the SDK checks for before rendering in every SAM and CDK plan task", () => {+ for (const definition of [SYNTHED.definition, readSamDefinition()]) {+ for (const plan of [getV1TaskStates(definition).Plan, getV2TaskStates(definition).PlanV2]) {+ const errors = new Set<string>();+ collectNonRetryableErrors(plan, errors);+ expect(errors.has(SITE_MANIFEST_INVALID)).toBe(true);+ }+ }+ });+it("keeps SAM and CDK terminal classifiers identical for every v2 Lambda task", () => {const cdkTasks = getV2TaskStates(SYNTHED.definition);const samTasks = getV2TaskStates(readSamDefinition());@@ -267,6 +284,26 @@ describe("HyperframesRenderStack — snapshot", () => {expect(v2).not.toContain("PlanS3Uri");});+ it("keeps the SAM bucket lifecycle identical to CDK's, shared content on the transport's terms", () => {+ const cdkBucket = Object.values(SYNTHED.template.findResources("AWS::S3::Bucket"))[0] as {+ Properties: { LifecycleConfiguration: unknown };+ };+ const samBucket = requireRecordProperty(readSamResources(), "RenderBucket", "SAM RenderBucket");+ const samProperties = requireRecordProperty(samBucket, "Properties", "SAM bucket properties");+ expect(samProperties.LifecycleConfiguration).toEqual(+ cdkBucket.Properties.LifecycleConfiguration,+ );+ expect(CONTENT_STORE_REFRESH_AFTER_DAYS * 2).toBeLessThan(CONTENT_STORE_RETENTION_DAYS);+ expect(cdkBucket.Properties.LifecycleConfiguration).toMatchObject({+ Rules: expect.arrayContaining([+ expect.objectContaining({+ Prefix: `${CONTENT_STORE_PREFIX}/`,+ ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,+ }),+ ]),+ });+ });+it("sends the same payload from every Lambda task as the SAM template", () => {// CDK drops null-valued payload fields when it renders the definition (#5259), so compare// what each deployed task sends rather than the construct source.diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.tsindex 5e6031a0c1..08038d1432 100644--- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts+++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts@@ -30,6 +30,7 @@ import * as s3 from "aws-cdk-lib/aws-s3";import * as sfn from "aws-cdk-lib/aws-stepfunctions";import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks";import { Construct } from "constructs";+import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";/** Construction-time props for {@link HyperframesRenderStack}. */export interface HyperframesRenderStackProps {@@ -93,6 +94,12 @@ export class HyperframesRenderStack extends Construct {prefix: "renders/",expiration: Duration.days(7),},+ {+ id: "ExpireSharedContent",+ enabled: true,+ prefix: `${CONTENT_STORE_PREFIX}/`,+ expiration: Duration.days(CONTENT_STORE_RETENTION_DAYS),+ },],});@@ -218,6 +225,7 @@ export class HyperframesRenderStack extends Construct {"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED","CHUNK_EXCEEDS_WORKER_TIMEOUT","ChromeBinaryUnavailableError",+ "SITE_MANIFEST_INVALID",];const NON_RETRYABLE_CHUNK = ["FFMPEG_VERSION_MISMATCH",diff --git a/packages/aws-lambda/src/contentStore.ts b/packages/aws-lambda/src/contentStore.tsnew file mode 100644index 0000000000..bcccd5f8a5--- /dev/null+++ b/packages/aws-lambda/src/contentStore.ts@@ -0,0 +1,15 @@+// Shared by every render: project files and plan-v2 artifacts by SHA-256. Dependency-free for CDK.++export const CONTENT_STORE_PREFIX = "cas/sha256";+export const CONTENT_STORE_RETENTION_DAYS = 30;+/** Reusing an object at least this old rewrites it, so a plan's objects outlive it by 23+ days. */+export const CONTENT_STORE_REFRESH_AFTER_DAYS = 7;++export function contentStoreUri(bucket: string, sha256: string): string {+ if (!/^[a-f0-9]{64}$/.test(sha256)) {+ throw new Error(+ `[contentStore] expected lowercase SHA-256 digest, got ${JSON.stringify(sha256)}`,+ );+ }+ return `s3://${bucket}/${CONTENT_STORE_PREFIX}/${sha256.slice(0, 2)}/${sha256}`;+}diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.tsindex 2e4e22a19d..4731384d0c 100644--- a/packages/aws-lambda/src/handler.test.ts+++ b/packages/aws-lambda/src/handler.test.ts@@ -25,6 +25,7 @@ import { LambdaClient } from "@aws-sdk/client-lambda";import { S3Client } from "@aws-sdk/client-s3";import {CURRENT_PLAN_PROTOCOL,+ PLAN_V2_MATERIALIZATION_MARKER,PlanVideosMetadataError,type AssembleResult,type ChunkResult,@@ -32,12 +33,14 @@ import {type PlanResult,type PlanV2ArtifactPublisher,type PlanV2Manifest,+ type PlanV2WithPublisherOptions,publishPlanV2FromExecutionPlan,} from "@hyperframes/producer/distributed";import { resolveMaxParallelChunks } from "../../producer/src/services/distributed/plan.js";import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js";import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./events.js";import { handler as handleEvent, unwrapEvent } from "./handler.js";+import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js";interface FakeS3Op {kind: "download" | "upload" | "copy";@@ -55,6 +58,7 @@ class FakeS3Client {// Map S3 URIs → byte buffers the fake serves.objects = new Map<string, Buffer>();metadata = new Map<string, Record<string, string>>();+ checksums = new Map<string, string>();// Methods called by the real S3 transport — minimal surface so the// handler's call sites don't need rewriting under test.@@ -95,6 +99,7 @@ class FakeS3Client {return {ContentLength: bytes.length,Metadata: this.metadata.get(uri),+ ChecksumSHA256: this.checksums.get(uri),};}if (cmdName === "PutObjectCommand") {@@ -114,6 +119,8 @@ class FakeS3Client {this.objects.set(uri, bytes);const metadata = (command as { input: { Metadata?: Record<string, string> } }).input.Metadata;if (metadata) this.metadata.set(uri, metadata);+ const checksum = (command as { input: { ChecksumSHA256?: string } }).input.ChecksumSHA256;+ if (checksum) this.checksums.set(uri, checksum);return {};}if (cmdName === "CreateMultipartUploadCommand") return { UploadId: "upload-1" };@@ -157,6 +164,15 @@ const accountUnreadable = {} as unknown as LambdaClient;const handler: typeof handleEvent = (event, deps) =>handleEvent(event, { lambda: accountUnreadable, ...deps });+/** A real client (so it can pre-sign offline) whose requests go to the fake. */+function presigningClient(fake: FakeS3Client): S3Client {+ const client = new S3Client({+ region: "us-east-2",+ credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" },+ });+ client.send = fake.send.bind(fake) as S3Client["send"];+ return client;+}const tmpDirs: string[] = [];@@ -866,7 +882,7 @@ describe("handler dispatch", () => {expect(planned).toMatchObject({PlanProtocol: "v2",PlanV2ManifestS3Uri: "s3://bucket/renders/v2/v2/manifest.json",- PlanV2ArtifactS3Prefix: "s3://bucket/renders/v2/v2/artifacts/sha256",+ PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256",});expect("PlanS3Uri" in planned).toBe(false);if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {@@ -917,6 +933,85 @@ describe("handler dispatch", () => {).toBe(true);});+ it("hands a chunk its deferred video as a pre-signed URL and downloads the rest", async () => {+ const tmpRoot = makeTmpRoot();+ const fake = new FakeS3Client();+ const s3 = presigningClient(fake);+ const v1Dir = join(tmpRoot, "v1");+ makeMinimalV1PlanDir(v1Dir, true, { withDeferredVideo: true });+ const publisher = new S3PlanV2ArtifactPublisher({+ s3,+ planOutputS3Prefix: "s3://bucket/renders/deferred/",+ temporaryRoot: tmpRoot,+ });+ const manifest = await publishPlanV2FromExecutionPlan(v1Dir, publisher);+ const video = manifest.artifacts.find((artifact) => artifact.path === "compiled/clip.mp4")!;+ expect(video.chunks).toEqual([0]);+ const blobUri = (digest: string) =>+ `${publisher.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;++ let marker: { remoteSources?: Record<string, string> } = {};+ const renderChunkMock = mock(+ async (planDir: string, _chunkIndex: number, outputPath: string): Promise<ChunkResult> => {+ expect(existsSync(join(planDir, "compiled", "clip.mp4"))).toBe(false);+ marker = JSON.parse(readFileSync(join(planDir, PLAN_V2_MATERIALIZATION_MARKER), "utf-8"));+ writeFileSync(outputPath, "CHUNK");+ return {+ outputPath,+ outputKind: "file",+ framesEncoded: 30,+ sha256: "c".repeat(64),+ durationMs: 1,+ planHashMs: 0,+ sessionBootMs: 0,+ captureStageMs: 1,+ encodeStageMs: 0,+ workers: 1,+ captureMode: "beginframe",+ perfPath: `${outputPath}.perf.json`,+ };+ },+ );+ const beforeChunk = fake.ops.length;+ await handler(+ {+ Action: "renderChunk",+ PlanV2ManifestS3Uri: publisher.manifestUri,+ PlanV2ArtifactS3Prefix: publisher.artifactPrefix,+ PlanHash: manifest.planHash,+ ChunkIndex: 0,+ ChunkOutputS3Prefix: "s3://bucket/renders/deferred/",+ Format: "mp4",+ },+ {+ s3,+ primitives: {+ plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,+ renderChunk:+ renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,+ assemble:+ mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,+ },+ tmpRoot,+ skipChromeResolution: true,+ },+ );++ const downloads = fake.ops+ .slice(beforeChunk)+ .filter((operation) => operation.kind === "download")+ .map((operation) => operation.uri);+ const expectedDownloads = manifest.artifacts+ .filter((artifact) => artifact.chunks === "all")+ .map((artifact) => blobUri(artifact.sha256));+ expect(new Set(downloads)).toEqual(new Set([publisher.manifestUri, ...expectedDownloads]));+ const url = new URL(marker.remoteSources?.["compiled/clip.mp4"] ?? "");+ expect(url.protocol).toBe("https:");+ expect(url.pathname).toEndWith(`/${video.sha256}`);+ expect(url.searchParams.get("X-Amz-Expires")).toBe("3600");+ expect(url.searchParams.get("X-Amz-Signature")).toMatch(/^[a-f0-9]{64}$/);+ });+async function planStitched(chunkCount: number) {const tmpRoot = makeTmpRoot();const s3 = new FakeS3Client();@@ -1063,6 +1158,132 @@ describe("handler dispatch", () => {});});+/** A real client (so it can pre-sign offline) whose requests go to the fake. */+function presigningClient(fake: FakeS3Client): import("@aws-sdk/client-s3").S3Client {+ const client = new S3Client({+ region: "us-east-2",+ credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" },+ });+ client.send = fake.send.bind(fake) as S3Client["send"];+ return client;+}++describe("handler — plan from a deployed site", () => {+ function storeSiteFile(s3: FakeS3Client, bytes: string): string {+ const sha256 = createHash("sha256").update(bytes).digest("hex");+ const uri = `s3://bucket/cas/sha256/${sha256.slice(0, 2)}/${sha256}`;+ s3.objects.set(uri, Buffer.from(bytes));+ s3.metadata.set(uri, { sha256 });+ return sha256;+ }++ function planEvent(): PlanEvent {+ return {+ Action: "plan",+ ProjectS3Uri: "s3://bucket/sites/site1/files.json",+ PlanOutputS3Prefix: "s3://bucket/renders/r1/",+ Config: { fps: 30, width: 640, height: 360, format: "mp4" },+ };+ }++ it("plans a site's video in place: never downloads, never uploads its bytes", async () => {+ const s3 = new FakeS3Client();+ const html = storeSiteFile(s3, "<html></html>");+ const media = storeSiteFile(s3, "VIDEO-BYTES");+ s3.objects.set(+ "s3://bucket/sites/site1/files.json",+ Buffer.from(+ JSON.stringify({+ files: [+ { path: "assets/bg.mp4", sha256: media, sizeBytes: 11 },+ { path: "index.html", sha256: html, sizeBytes: 13 },+ ],+ }),+ ),+ );+ const planV2WithPublisher = mock(+ async (+ projectDir: string,+ _config: unknown,+ publisher: PlanV2ArtifactPublisher,+ options: PlanV2WithPublisherOptions,+ ) => {+ expect(readFileSync(join(projectDir, "index.html"), "utf-8")).toBe("<html></html>");+ expect(existsSync(join(projectDir, "assets", "bg.mp4"))).toBe(false);+ const remote = options.remoteFiles?.["assets/bg.mp4"];+ expect(remote).toMatchObject({ sha256: media, sizeBytes: 11 });+ expect(remote?.url).toContain(`/cas/sha256/${media.slice(0, 2)}/${media}?`);+ expect(remote?.url).toContain("X-Amz-Signature=");+ await publisher.referenceStoredBlob!({ sha256: media, sizeBytes: 11 });+ const manifest = {+ planHash: "c".repeat(64),+ chunkCount: 1,+ totalFrames: 30,+ fps: 30,+ width: 640,+ height: 360,+ format: "mp4",+ ffmpegVersion: "test",+ producerVersion: "test",+ artifacts: [{ path: "compiled/assets/bg.mp4", sha256: media, sizeBytes: 11 }],+ };+ await publisher.commitManifest(JSON.stringify(manifest));+ return manifest as unknown as PlanV2Manifest;+ },+ );++ const result = await handler(planEvent(), {+ s3: presigningClient(s3),+ primitives: {+ plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,+ planV2WithPublisher:+ planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,+ renderChunk:+ mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,+ assemble: mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,+ },+ tmpRoot: makeTmpRoot(),+ skipChromeResolution: true,+ });++ expect(result).toMatchObject({ PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256" });+ expect(planV2WithPublisher).toHaveBeenCalledTimes(1);+ expect(s3.ops.filter((op) => op.kind === "upload").map((op) => op.uri)).toEqual([+ "s3://bucket/renders/r1/v2/manifest.json",+ ]);+ expect(s3.ops.some((op) => op.uri.endsWith(".tar.gz"))).toBe(false);+ const mediaKey = `s3://bucket/cas/sha256/${media.slice(0, 2)}/${media}`;+ expect(s3.ops.filter((op) => op.uri === mediaKey && op.kind === "download")).toEqual([]);+ });++ it("refuses a site manifest path that escapes the project", async () => {+ const s3 = new FakeS3Client();+ const sha256 = storeSiteFile(s3, "x");+ s3.objects.set(+ "s3://bucket/sites/site1/files.json",+ Buffer.from(JSON.stringify({ files: [{ path: "../escape", sha256, sizeBytes: 1 }] })),+ );+ const planV2WithPublisher = mock();+ await expect(+ handler(planEvent(), {+ s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,+ primitives: {+ plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,+ planV2WithPublisher:+ planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,+ renderChunk:+ mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,+ assemble:+ mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,+ },+ tmpRoot: makeTmpRoot(),+ skipChromeResolution: true,+ }),+ ).rejects.toMatchObject({ name: "SITE_MANIFEST_INVALID" });+ expect(planV2WithPublisher).not.toHaveBeenCalled();+ });+});+describe("handler — S3 URI allowlist (security: F-004)", () => {let prevBucket: string | undefined;@@ -1177,7 +1398,7 @@ async function makeMinimalPlanTar(): Promise<Buffer> {function makeMinimalV1PlanDir(dir: string,withAudio: boolean,- options: { chunkCount?: number; chunksCarryAudio?: boolean } = {},+ options: { chunkCount?: number; chunksCarryAudio?: boolean; withDeferredVideo?: boolean } = {},): void {const chunkCount = options.chunkCount ?? 1;mkdirSync(join(dir, "meta"), { recursive: true });@@ -1208,6 +1429,54 @@ function makeMinimalV1PlanDir(options.chunksCarryAudio ? JSON.stringify({ chunksCarryAudio: true }) : "{}",);if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC");+ if (options.withDeferredVideo) writeDeferredVideo(dir);planJson.planHash = recomputePlanHashFromPlanDir(dir);writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));}++/** One clip the chunk extracts from `compiled/clip.mp4` itself. */+function writeDeferredVideo(dir: string): void {+ writeFileSync(join(dir, "compiled", "clip.mp4"), "VIDEO-BYTES");+ const metadata = {+ durationSeconds: 1,+ videoStreamDurationSeconds: 1,+ width: 640,+ height: 360,+ fps: 30,+ videoCodec: "h264",+ hasAudio: false,+ isVFR: false,+ hasAlpha: false,+ colorSpace: null,+ };+ const videos = {+ videos: [+ {+ id: "clip",+ src: "clip.mp4",+ start: 0,+ end: 1,+ mediaStart: 0,+ loop: false,+ hasAudio: false,+ },+ ],+ extracted: [+ {+ videoId: "clip",+ srcPath: "/planner/clip.mp4",+ framePattern: "frame_%05d.jpg",+ fps: 30,+ totalFrames: 30,+ metadata,+ deferredRange: {+ sourcePath: "compiled/clip.mp4",+ startTime: 0,+ durationSeconds: 1,+ format: "jpg",+ },+ },+ ],+ };+ writeFileSync(join(dir, "meta", "videos.json"), JSON.stringify(videos));+}diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.tsindex f0fa4831fe..5e2bbc06be 100644--- a/packages/aws-lambda/src/handler.ts+++ b/packages/aws-lambda/src/handler.ts@@ -32,10 +32,12 @@ import {resolvePlanAudioPath,planV2WithPublisher,type PlanResult,+ isPlanV2DeferredSourceArtifact,type PlanV2Artifact,type PlanV2Manifest,type PlanV2MaterializationTarget,readPlanV2Manifest,+ type RemoteProjectFile,renderChunk,} from "@hyperframes/producer/distributed";import { resolveChromeExecutablePath } from "./chromium.js";@@ -58,12 +60,15 @@ import type {import {downloadS3ObjectToFile,downloadS3ObjectToFileVerified,+ mapConcurrent,parseS3Uri,+ presignVerifiedS3GetUrl,tarDirectory,untarDirectory,uploadFileToS3,} from "./s3Transport.js";import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js";+import { downloadSiteFiles, isSiteManifestUri } from "./siteManifest.js";/*** Lazily-constructed S3 client. Cached at module scope so warm Lambda@@ -401,17 +406,10 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam}const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-"));- // We use `.tar.gz` (not `.zip`) as the project archive's on-the-wire- // format because Lambda's Amazon Linux base image ships GNU `tar` but- // not `unzip` in `/usr/bin`. The smoke script + future CLI both- // produce tar.gz uploads.- const projectArchive = join(work, "project.tar.gz");- const projectDir = join(work, "project");const planDir = join(work, "plan");try {- await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);- await untarDirectory(projectArchive, projectDir);+ const { projectDir } = await fetchProject(s3, event.ProjectS3Uri, work);// Same chunk layout as v2, so both protocols produce the same chunks and output.const result: PlanResult = await primitive(projectDir, config, planDir, {@@ -460,6 +458,31 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam}}+/**+ * `ProjectS3Uri` is a site manifest from `deploySite`, or a `.tar.gz` of the project+ * (hand-built uploads such as `examples/aws-lambda/scripts/smoke.sh`).+ */+async function fetchProject(+ s3: S3Client,+ projectS3Uri: string,+ work: string,+ leaveVideosRemote = false,+): Promise<{ projectDir: string; remoteFiles: Record<string, RemoteProjectFile> }> {+ const projectDir = join(work, "project");+ if (isSiteManifestUri(projectS3Uri)) {+ const remoteFiles = await downloadSiteFiles(s3, projectS3Uri, projectDir, leaveVideosRemote);+ return { projectDir, remoteFiles };+ }+ const projectArchive = join(work, "project.tar.gz");+ let stageStarted = Date.now();+ await downloadS3ObjectToFile(s3, projectS3Uri, projectArchive);+ logEvent({ event: "plan_stage", stage: "download", ms: Date.now() - stageStarted });+ stageStarted = Date.now();+ await untarDirectory(projectArchive, projectDir);+ logEvent({ event: "plan_stage", stage: "untar", ms: Date.now() - stageStarted });+ return { projectDir, remoteFiles: {} };+}+// Plan v2 orchestration is kept as one transactional boundary: stage, CAS upload,// and manifest-last publication must stay ordered and fail together.// fallow-ignore-next-line complexity@@ -476,15 +499,8 @@ async function handlePlanV2(}const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));- const projectArchive = join(work, "project.tar.gz");- const projectDir = join(work, "project");try {- let stageStarted = Date.now();- await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);- logEvent({ event: "plan_stage", stage: "download", ms: Date.now() - stageStarted });- stageStarted = Date.now();- await untarDirectory(projectArchive, projectDir);- logEvent({ event: "plan_stage", stage: "untar", ms: Date.now() - stageStarted });+ const { projectDir, remoteFiles } = await fetchProject(s3, event.ProjectS3Uri, work, true);const publisher = new S3PlanV2ArtifactPublisher({s3,planOutputS3Prefix: event.PlanOutputS3Prefix,@@ -492,6 +508,7 @@ async function handlePlanV2(});const manifest: PlanV2Manifest = await primitive(projectDir, config, publisher, {stagingParentDir: work,+ remoteFiles,chunksCarryAudio: true,});@@ -828,17 +845,54 @@ async function downloadAndMaterializePlanV2(throwPlanHashMismatch(event.PlanHash, manifest.planHash);}const artifacts = listPlanV2ArtifactsForTarget(manifest, target);+ const remoteSources = await presignDeferredSources(s3, event.PlanV2ArtifactS3Prefix, artifacts);const uniqueArtifacts = [- ...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values(),+ ...new Map(+ artifacts+ .filter((artifact) => !Object.hasOwn(remoteSources, artifact.path))+ .map((artifact) => [artifact.sha256, artifact]),+ ).values(),];await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);});const planDir = join(work, "plan");- materializePlanV2Target(transportDir, target, planDir);+ materializePlanV2Target(transportDir, target, planDir, { remoteSources });return { planDir, manifest };}+/** Outlives the longest chunk (Lambda caps an invocation at 900 s) with margin. */+const DEFERRED_SOURCE_URL_TTL_SECONDS = 3600;++/**+ * A chunk's ffmpeg reads a deferred video by byte range from a pre-signed URL+ * instead of downloading it whole. The key is the content hash, written once.+ */+async function presignDeferredSources(+ s3: S3Client,+ artifactPrefix: string,+ artifacts: readonly Readonly<PlanV2Artifact>[],+): Promise<Record<string, string>> {+ const entries = await Promise.all(+ artifacts+ .filter(isPlanV2DeferredSourceArtifact)+ .map(+ async (artifact) =>+ [+ artifact.path,+ await presignVerifiedS3GetUrl(+ s3,+ planV2BlobUri(artifactPrefix, artifact.sha256),+ artifact.sha256,+ artifact.sizeBytes,+ DEFERRED_SOURCE_URL_TTL_SECONDS,+ ),+ ] as const,+ ),+ );+ return Object.fromEntries(entries);+}+async function downloadPlanV2Artifact(s3: S3Client,artifactPrefix: string,@@ -869,30 +923,6 @@ function throwPlanHashMismatch(expected: string, actual: string): never {throw error;}-async function mapConcurrent<T>(- values: readonly T[],- concurrency: number,- fn: (value: T) => Promise<void>,-): Promise<void> {- let cursor = 0;- async function worker(): Promise<void> {- while (cursor < values.length) {- const index = cursor++;- await fn(values[index]!);- }- }- const results = await Promise.allSettled(- Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),- );- const failure = results.find(- (result): result is PromiseRejectedResult => result.status === "rejected",- );- // Do not reject while sibling workers may still be writing into invocation- // scratch. The caller removes that directory in `finally`; draining the pool- // first prevents late S3 streams from racing cleanup after another GET fails.- if (failure) throw failure.reason;-}-async function downloadChunkObjects(s3: S3Client,uris: string[],diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts b/packages/aws-lambda/src/s3PlanV2Publisher.test.tsindex 9b437f9cf3..23ecde2279 100644--- a/packages/aws-lambda/src/s3PlanV2Publisher.test.ts+++ b/packages/aws-lambda/src/s3PlanV2Publisher.test.ts@@ -112,14 +112,61 @@ describe("S3PlanV2ArtifactPublisher", () => {planOutputS3Prefix: `s3://bucket/render${"/".repeat(10_000)}`,});- expect(publisher.artifactPrefix).toBe("s3://bucket/render/v2/artifacts/sha256");+ expect(publisher.artifactPrefix).toBe("s3://bucket/cas/sha256");expect(publisher.manifestUri).toBe("s3://bucket/render/v2/manifest.json");});+ it("references a remote project file already in the store, and refuses one that is not", async () => {+ const digest = createHash("sha256").update("remote video").digest("hex");+ const s3 = new FakeS3();+ s3.objects.set(`s3://bucket/cas/sha256/${digest.slice(0, 2)}/${digest}`, {+ bytes: Buffer.from("remote video"),+ sha256: digest,+ });+ const publisher = new S3PlanV2ArtifactPublisher({+ s3: s3.asClient(),+ planOutputS3Prefix: "s3://bucket/renders/remote",+ temporaryRoot: makeSource("x").root,+ });++ await publisher.referenceStoredBlob({ sha256: digest, sizeBytes: 12 });+ await expect(+ publisher.referenceStoredBlob({ sha256: "e".repeat(64), sizeBytes: 12 }),+ ).rejects.toMatchObject({ name: "PLAN_ARTIFACT_MISSING" });+ await publisher.commitManifest(manifestFor(digest));++ expect(s3.puts.map((operation) => operation.uri)).toEqual([+ "s3://bucket/renders/remote/v2/manifest.json",+ ]);+ });++ it("reuses a blob the SDK or another render already stored instead of uploading it", async () => {+ const source = makeSource("shared media");+ const s3 = new FakeS3();+ const blobUri = `s3://bucket/cas/sha256/${source.digest.slice(0, 2)}/${source.digest}`;+ s3.objects.set(blobUri, { bytes: Buffer.from("shared media"), sha256: source.digest });+ const publisher = new S3PlanV2ArtifactPublisher({+ s3: s3.asClient(),+ planOutputS3Prefix: "s3://bucket/renders/second",+ temporaryRoot: source.root,+ });++ await publisher.putBlob({+ sourcePath: source.path,+ sha256: source.digest,+ sizeBytes: source.sizeBytes,+ });+ await publisher.commitManifest(manifestFor(source.digest));++ expect(s3.puts.map((operation) => operation.uri)).toEqual([+ "s3://bucket/renders/second/v2/manifest.json",+ ]);+ });+it("publishes immutable blobs before the fixed-key manifest", async () => {const source = makeSource("hello");const s3 = new FakeS3();- const artifactPrefix = "s3://bucket/render/v2/artifacts/sha256";+ const artifactPrefix = "s3://bucket/cas/sha256";const manifestUri = "s3://bucket/render/v2/manifest.json";const publisher = new S3PlanV2ArtifactPublisher({s3: s3.asClient(),diff --git a/packages/aws-lambda/src/s3PlanV2Publisher.ts b/packages/aws-lambda/src/s3PlanV2Publisher.tsindex 8eee0d8681..b3f1d6814e 100644--- a/packages/aws-lambda/src/s3PlanV2Publisher.ts+++ b/packages/aws-lambda/src/s3PlanV2Publisher.ts@@ -8,7 +8,14 @@ import {type PlanV2ArtifactPublisher,type PlanV2PublishBlob,} from "@hyperframes/producer/distributed";-import { parseS3Uri, uploadContentAddressedFileToS3 } from "./s3Transport.js";+import { CONTENT_STORE_PREFIX } from "./contentStore.js";+import {+ ensureContentStoreObject,+ formatS3Uri,+ parseS3Uri,+ putContentStoreFile,+ uploadContentAddressedFileToS3,+} from "./s3Transport.js";export interface S3PlanV2ArtifactPublisherOptions {readonly s3: S3Client;@@ -66,15 +73,18 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {readonly artifactPrefix: string;readonly manifestUri: string;readonly #s3: S3Client;+ readonly #bucket: string;readonly #temporaryRoot: string;readonly #publishedDigests = new Set<string>();#state: "open" | "committed" | "aborted" = "open";constructor(options: Readonly<S3PlanV2ArtifactPublisherOptions>) {const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;- parseS3Uri(outputPrefix);+ const { bucket } = parseS3Uri(outputPrefix);this.#s3 = options.s3;- this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;+ this.#bucket = bucket;+ // Shared by every render, so a blob the SDK or an earlier render uploaded is reused, not re-sent.+ this.artifactPrefix = formatS3Uri({ bucket, key: CONTENT_STORE_PREFIX });this.manifestUri = `${outputPrefix}/manifest.json`;this.#temporaryRoot = options.temporaryRoot ?? tmpdir();mkdirSync(this.#temporaryRoot, { recursive: true });@@ -90,8 +100,14 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`,);}- const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;- await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);+ await putContentStoreFile(this.#s3, blob.sourcePath, this.#bucket, digest);+ this.#publishedDigests.add(digest);+ }++ async referenceStoredBlob(blob: Readonly<Omit<PlanV2PublishBlob, "sourcePath">>): Promise<void> {+ this.#assertOpen("reference a stored blob");+ const digest = assertSha256(blob.sha256, "S3 referenced blob sha256");+ await ensureContentStoreObject(this.#s3, this.#bucket, digest, blob.sizeBytes);this.#publishedDigests.add(digest);}@@ -125,9 +141,8 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {async abort(): Promise<void> {if (this.#state === "open") this.#state = "aborted";- // Remote CAS blobs are immutable and may already be reused by a retry.- // Without a committed manifest they are unreachable and expire under the- // render bucket's intermediate-object lifecycle policy.+ // Remote CAS blobs are immutable and may already be reused by a retry or+ // another render. They expire under the shared content store's lifecycle.}#assertOpen(operation: string): void {diff --git a/packages/aws-lambda/src/s3Transport.test.ts b/packages/aws-lambda/src/s3Transport.test.tsindex 6e5ea6dd58..888bb48017 100644--- a/packages/aws-lambda/src/s3Transport.test.ts+++ b/packages/aws-lambda/src/s3Transport.test.ts@@ -10,13 +10,17 @@ import { createHash } from "node:crypto";import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";+import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";import {downloadS3ObjectToFileVerified,formatS3Uri,parseS3Uri,+ presignVerifiedS3GetUrl,sha256File,tarDirectory,untarDirectory,+ putContentStoreFile,uploadContentAddressedFileToS3,} from "./s3Transport.js";@@ -176,6 +180,37 @@ describe("content-addressed v2 artifacts", () => {expect(s3.putCount).toBe(0);});+ it("rewrites a reused object at the refresh age so its lifecycle expiry restarts", async () => {+ const source = join(scratchRoot, "artifact-aging.bin");+ writeFileSync(source, "aging bytes");+ const digest = await sha256File(source);+ const uri = contentStoreUri("bucket", digest);+ const s3 = new ContentAddressedFakeS3();+ const days = (n: number) => new Date(Date.now() - n * 86_400_000);+ s3.objects.set(uri, {+ bytes: Buffer.from("aging bytes"),+ sha256: digest,+ lastModified: days(CONTENT_STORE_REFRESH_AFTER_DAYS - 1),+ });++ expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");+ expect(s3.copies).toEqual([]);++ s3.objects.get(uri)!.lastModified = days(CONTENT_STORE_REFRESH_AFTER_DAYS);+ expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");+ expect(s3.copies).toEqual([+ {+ uri,+ CopySource: `bucket/cas/sha256/${digest.slice(0, 2)}/${digest}`,+ MetadataDirective: "REPLACE",+ Metadata: { sha256: digest },+ ChecksumAlgorithm: "SHA256",+ },+ ]);+ expect(s3.objects.get(uri)).toMatchObject({ sha256: digest });+ expect(s3.putCount).toBe(0);+ });+it("deletes a downloaded artifact when digest verification fails", async () => {const expectedSource = join(scratchRoot, "artifact-expected.bin");const destination = join(scratchRoot, "artifact-download.bin");@@ -193,8 +228,57 @@ describe("content-addressed v2 artifacts", () => {});});+describe("presignVerifiedS3GetUrl", () => {+ const digest = "ab".repeat(32);+ const uri = `s3://bucket/v2/artifacts/sha256/ab/${digest}`;+ function clientHolding(head: { ChecksumSHA256?: string; ContentLength: number }): S3Client {+ const client = new S3Client({+ region: "us-east-2",+ credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" },+ });+ client.send = (async (command: unknown) => {+ expect(command).toBeInstanceOf(HeadObjectCommand);+ expect((command as HeadObjectCommand).input.ChecksumMode).toBe("ENABLED");+ return head;+ }) as S3Client["send"];+ return client;+ }++ it.each([+ { uri, key: `/v2/artifacts/sha256/ab/${digest}` },+ { uri: contentStoreUri("bucket", digest), key: `/cas/sha256/ab/${digest}` },+ ])("signs an attested legacy or shared-content source at $key", async (source) => {+ const checksum = Buffer.from(digest, "hex").toString("base64");+ const url = new URL(+ await presignVerifiedS3GetUrl(+ clientHolding({ ChecksumSHA256: checksum, ContentLength: 42 }),+ source.uri,+ digest,+ 42,+ 600,+ ),+ );+ expect(url.pathname).toBe(source.key);+ expect(url.searchParams.get("X-Amz-Expires")).toBe("600");+ });++ it("refuses an object whose stored checksum is not the pinned digest", async () => {+ const other = Buffer.from("cd".repeat(32), "hex").toString("base64");+ await expect(+ presignVerifiedS3GetUrl(+ clientHolding({ ChecksumSHA256: other, ContentLength: 42 }),+ uri,+ digest,+ 42,+ 600,+ ),+ ).rejects.toMatchObject({ name: "PLAN_ARTIFACT_DIGEST_MISMATCH" });+ });+});+class ContentAddressedFakeS3 {- readonly objects = new Map<string, { bytes: Buffer; sha256: string }>();+ readonly objects = new Map<string, { bytes: Buffer; sha256: string; lastModified?: Date }>();+ readonly copies: Record<string, unknown>[] = [];putCount = 0;raceOnNextPut: { bytes: Buffer; sha256: string } | undefined;@@ -213,10 +297,25 @@ class ContentAddressedFakeS3 {Key: string;Body?: NodeJS.ReadableStream;Metadata?: Record<string, string>;+ CopySource?: string;+ MetadataDirective?: string;+ ChecksumAlgorithm?: string;ChecksumSHA256?: string;};};const uri = `s3://${value.input.Bucket}/${value.input.Key}`;+ if (value.constructor.name === "CopyObjectCommand") {+ const { CopySource, MetadataDirective, Metadata, ChecksumAlgorithm } = value.input;+ this.copies.push({ uri, CopySource, MetadataDirective, Metadata, ChecksumAlgorithm });+ const source = this.objects.get(`s3://${CopySource}`);+ if (!source) throw new Error("missing copy source");+ this.objects.set(uri, {+ bytes: source.bytes,+ sha256: Metadata?.sha256 ?? "",+ lastModified: new Date(),+ });+ return {};+ }if (value.constructor.name === "HeadObjectCommand") {const object = this.objects.get(uri);if (!object) {@@ -230,6 +329,7 @@ class ContentAddressedFakeS3 {return {ContentLength: object.bytes.length,Metadata: { sha256: object.sha256 },+ LastModified: object.lastModified,};}if (value.constructor.name === "GetObjectCommand") {diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.tsindex 379fa147ab..3469749b80 100644--- a/packages/aws-lambda/src/s3Transport.ts+++ b/packages/aws-lambda/src/s3Transport.ts@@ -29,12 +29,15 @@ import { createHash } from "node:crypto";import { dirname } from "node:path";import { pipeline } from "node:stream/promises";import {+ CopyObjectCommand,GetObjectCommand,HeadObjectCommand,PutObjectCommand,type S3Client,} from "@aws-sdk/client-s3";+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";import * as tar from "tar";+import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";/** Parsed `s3://bucket/key` URI. */export interface S3Location {@@ -101,6 +104,35 @@ export async function downloadS3ObjectToFileVerified(}}+/**+ * Pre-sign a GET for a plan-v2 artifact, after S3 attests the object's stored+ * SHA-256 and size are the ones the manifest pins.+ */+export async function presignVerifiedS3GetUrl(+ client: S3Client,+ uri: string,+ expectedSha256: string,+ expectedSize: number,+ expiresInSeconds: number,+): Promise<string> {+ assertSha256(expectedSha256);+ const { bucket, key } = parseS3Uri(uri);+ const head = await client.send(+ new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" }),+ );+ const expectedChecksum = Buffer.from(expectedSha256, "hex").toString("base64");+ if (head.ChecksumSHA256 !== expectedChecksum || head.ContentLength !== expectedSize) {+ const error = new Error(+ `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256} (${expectedSize} bytes), S3 holds checksum ${String(head.ChecksumSHA256)} (${String(head.ContentLength)} bytes)`,+ );+ error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";+ throw error;+ }+ return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {+ expiresIn: expiresInSeconds,+ });+}+/*** Upload a local file's contents to an S3 URI using a streaming* `PutObjectCommand`. PutObject's 5 GB cap comfortably exceeds the@@ -142,6 +174,16 @@ export async function uploadContentAddressedFileToS3(expectedSha256: string,contentType?: string,): Promise<"uploaded" | "reused"> {+ return (await putImmutableFile(client, localPath, uri, expectedSha256, contentType)).outcome;+}++async function putImmutableFile(+ client: S3Client,+ localPath: string,+ uri: string,+ expectedSha256: string,+ contentType?: string,+): Promise<{ outcome: "uploaded" | "reused"; lastModified?: Date }> {assertSha256(expectedSha256);if (!existsSync(localPath)) {throw new Error(`[s3Transport] upload source missing: ${localPath}`);@@ -149,8 +191,9 @@ export async function uploadContentAddressedFileToS3(const { bucket, key } = parseS3Uri(uri);const size = statSync(localPath).size;const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);- if (existing === "matching") return "reused";- if (existing === "conflict") throwImmutableObjectConflict(uri);+ if (existing.state === "matching")+ return { outcome: "reused", lastModified: existing.lastModified };+ if (existing.state === "conflict") throwImmutableObjectConflict(uri);const body = createReadStream(localPath);try {@@ -169,12 +212,12 @@ export async function uploadContentAddressedFileToS3(IfNoneMatch: "*",}),);- return "uploaded";+ return { outcome: "uploaded" };} catch (error) {if (!isS3PreconditionFailed(error)) throw error;const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);- if (raced === "matching") return "reused";- if (raced === "conflict") throwImmutableObjectConflict(uri);+ if (raced.state === "matching") return { outcome: "reused" };+ if (raced.state === "conflict") throwImmutableObjectConflict(uri);// The winning object was deleted between the conditional failure and// verification. Preserve the service error so the orchestrator may retry.throw error;@@ -186,6 +229,30 @@ export async function uploadContentAddressedFileToS3(}}+export async function mapConcurrent<T>(+ values: readonly T[],+ concurrency: number,+ fn: (value: T) => Promise<void>,+): Promise<void> {+ let cursor = 0;+ async function worker(): Promise<void> {+ while (cursor < values.length) {+ const index = cursor++;+ await fn(values[index]!);+ }+ }+ const results = await Promise.allSettled(+ Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),+ );+ const failure = results.find(+ (result): result is PromiseRejectedResult => result.status === "rejected",+ );+ // Do not reject while sibling workers may still be writing into invocation+ // scratch. The caller removes that directory in `finally`; draining the pool+ // first prevents late S3 streams from racing cleanup after another GET fails.+ if (failure) throw failure.reason;+}+export async function sha256File(path: string): Promise<string> {const hash = createHash("sha256");for await (const chunk of createReadStream(path)) {@@ -202,7 +269,9 @@ function assertSha256(value: string): void {}}-type ContentAddressedObjectState = "missing" | "matching" | "conflict";+type ContentAddressedObjectState =+ | { state: "missing" | "conflict" }+ | { state: "matching"; lastModified?: Date };async function inspectContentAddressedObject(client: S3Client,@@ -216,14 +285,82 @@ async function inspectContentAddressedObject(new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" }),);return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256- ? "matching"- : "conflict";+ ? { state: "matching", lastModified: existing.LastModified }+ : { state: "conflict" };} catch (error) {- if (isS3NotFound(error)) return "missing";+ if (isS3NotFound(error)) return { state: "missing" };throw error;}}+/** Put a file into the shared content store once; reusing an aging object restarts its expiry. */+export async function putContentStoreFile(+ client: S3Client,+ localPath: string,+ bucket: string,+ sha256: string,+): Promise<"uploaded" | "reused"> {+ const uri = contentStoreUri(bucket, sha256);+ const { outcome, lastModified } = await putImmutableFile(client, localPath, uri, sha256);+ if (lastModified) await refreshAgingObject(client, uri, sha256, lastModified);+ return outcome;+}++/** Confirm an object the caller never uploaded is in the content store, restarting an aging one. */+export async function ensureContentStoreObject(+ client: S3Client,+ bucket: string,+ sha256: string,+ sizeBytes: number,+): Promise<void> {+ const uri = contentStoreUri(bucket, sha256);+ const { key } = parseS3Uri(uri);+ const existing = await inspectContentAddressedObject(client, bucket, key, sizeBytes, sha256);+ if (existing.state === "matching") {+ if (existing.lastModified) await refreshAgingObject(client, uri, sha256, existing.lastModified);+ return;+ }+ if (existing.state === "conflict") throwImmutableObjectConflict(uri);+ const error = new Error(+ `[s3Transport] PLAN_ARTIFACT_MISSING: ${uri} is not in the content store`,+ );+ error.name = "PLAN_ARTIFACT_MISSING";+ throw error;+}++export async function presignS3GetUrl(+ client: S3Client,+ uri: string,+ expiresInSeconds: number,+): Promise<string> {+ const { bucket, key } = parseS3Uri(uri);+ return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {+ expiresIn: expiresInSeconds,+ });+}++// S3 lifecycle expiry counts from the last write, not the last read, so reuse has to write.+async function refreshAgingObject(+ client: S3Client,+ uri: string,+ sha256: string,+ lastModified: Date,+): Promise<void> {+ const { bucket, key } = parseS3Uri(uri);+ const ageMs = Date.now() - lastModified.getTime();+ if (ageMs < CONTENT_STORE_REFRESH_AFTER_DAYS * 86_400_000) return;+ await client.send(+ new CopyObjectCommand({+ Bucket: bucket,+ Key: key,+ CopySource: `${bucket}/${key}`,+ MetadataDirective: "REPLACE",+ Metadata: { sha256 },+ ChecksumAlgorithm: "SHA256",+ }),+ );+}+function throwImmutableObjectConflict(uri: string): never {const error = new Error(`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`,@@ -232,7 +369,18 @@ function throwImmutableObjectConflict(uri: string): never {throw error;}-function isS3NotFound(error: unknown): boolean {+export async function s3ObjectExists(client: S3Client, uri: string): Promise<boolean> {+ const { bucket, key } = parseS3Uri(uri);+ try {+ await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));+ return true;+ } catch (error) {+ if (isS3NotFound(error)) return false;+ throw error;+ }+}++export function isS3NotFound(error: unknown): boolean {if (!isRecord(error)) return false;const metadata = isRecord(error.$metadata) ? error.$metadata : undefined;return (@@ -256,12 +404,20 @@ function isRecord(value: unknown): value is Record<string, unknown> {* binary — the AWS Lambda Node 22 base image ships a minimal set of* userland tools and does NOT include `tar` in `/usr/bin`.*/-export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {+export async function tarDirectory(+ sourceDir: string,+ destTarball: string,+ files?: string[],+): Promise<void> {if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);}mkdirSync(dirname(destTarball), { recursive: true });- await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);+ // `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.+ await tar.create(+ { gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },+ files ?? ["."],+ );}/**diff --git a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.tsindex 8221b9c797..f47b27f4af 100644--- a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts+++ b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts@@ -8,6 +8,8 @@* `deploySite`'s `finally` block has rmSync'd it.*/+import { Readable } from "node:stream";+import { buffer } from "node:stream/consumers";import type { S3Client } from "@aws-sdk/client-s3";export interface FakeS3Op {@@ -19,33 +21,51 @@ export interface FakeS3Op {export class FakeS3 {ops: FakeS3Op[] = [];existing = new Set<string>();+ bodies = new Map<string, Buffer>();+ metadata = new Map<string, Record<string, string> | undefined>();async send(command: unknown): Promise<unknown> {const cmdName = (command as { constructor: { name: string } }).constructor.name;- const input = (command as { input: { Bucket: string; Key: string } }).input;- if (cmdName === "HeadObjectCommand") {- this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key });- if (this.existing.has(`${input.Bucket}/${input.Key}`)) {- return { ContentLength: 1, LastModified: new Date("2026-05-16T00:00:00Z") };+ const input = (+ command as {+ input: { Bucket: string; Key: string; Metadata?: Record<string, string>; Body?: unknown };}- const err = new Error("Not Found") as Error & {- $metadata: { httpStatusCode: number };- name: string;- };- err.name = "NotFound";- err.$metadata = { httpStatusCode: 404 };- throw err;- }- if (cmdName === "PutObjectCommand") {- this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });- this.existing.add(`${input.Bucket}/${input.Key}`);- await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body);- return {};+ ).input;+ const id = `${input.Bucket}/${input.Key}`;+ const handlers: Record<string, () => Promise<unknown>> = {+ HeadObjectCommand: async () => {+ this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key });+ return this.head(id);+ },+ GetObjectCommand: async () => {+ if (!this.existing.has(id)) throw notFound("NoSuchKey");+ return { Body: Readable.from([this.bodies.get(id) ?? Buffer.alloc(0)]) };+ },+ PutObjectCommand: async () => {+ this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });+ this.existing.add(id);+ this.metadata.set(id, input.Metadata);+ this.bodies.set(id, await drainBody(input.Body as NodeJS.ReadableStream | Buffer));+ return {};+ },+ };+ const handler = handlers[cmdName];+ if (!handler) throw new Error(`FakeS3: unexpected command ${cmdName}`);+ return handler();+ }++ private head(id: string): unknown {+ if (this.existing.has(id)) {+ return { ContentLength: this.bodies.get(id)?.length, Metadata: this.metadata.get(id) };}- throw new Error(`FakeS3: unexpected command ${cmdName}`);+ throw notFound("NotFound");}}+function notFound(name: string): Error {+ return Object.assign(new Error(name), { name, $metadata: { httpStatusCode: 404 } });+}+/** Convenience cast so `deploySite({ s3: makeFakeS3() })` reads cleanly. */export function asS3Client(fake: FakeS3): S3Client {return fake as unknown as S3Client;@@ -58,12 +78,6 @@ export function asS3Client(fake: FakeS3): S3Client {* the workdir — surfacing as an "Unhandled error between tests"* teardown noise that bun's runner attributes to the next test.*/-async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<void> {- if (Buffer.isBuffer(body)) return;- await new Promise<void>((resolve, reject) => {- body.on("data", () => {});- body.on("end", () => resolve());- body.on("close", () => resolve());- body.on("error", reject);- });+async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<Buffer> {+ return Buffer.isBuffer(body) ? body : buffer(body);}diff --git a/packages/aws-lambda/src/sdk/deploySite.test.ts b/packages/aws-lambda/src/sdk/deploySite.test.tsindex 9863b05187..125d54c15e 100644--- a/packages/aws-lambda/src/sdk/deploySite.test.ts+++ b/packages/aws-lambda/src/sdk/deploySite.test.ts@@ -1,15 +1,19 @@import { afterEach, beforeEach, describe, expect, it } from "bun:test";-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import { createHash } from "node:crypto";import type { S3Client } from "@aws-sdk/client-s3";+import { parseSiteManifest } from "../siteManifest.js";import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";import { deploySite } from "./deploySite.js";let projectDir: string;+let scratch: string;beforeEach(() => {projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));+ scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-"));mkdirSync(join(projectDir, "assets"));writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");@@ -17,10 +21,28 @@ beforeEach(() => {afterEach(() => {rmSync(projectDir, { recursive: true, force: true });+ rmSync(scratch, { recursive: true, force: true });});+function sha256(text: string): string {+ return createHash("sha256").update(text).digest("hex");+}++function storeKey(text: string): string {+ const digest = sha256(text);+ return `cas/sha256/${digest.slice(0, 2)}/${digest}`;+}++function sitePaths(s3: FakeS3, siteId: string): string[] {+ const manifest = s3.bodies.get(`test-bucket/sites/${siteId}/files.json`)?.toString() ?? "";+ return parseSiteManifest(manifest).map((file) => file.path);+}++const HTML = "<html><body>hi</body></html>";+const CSS = "body { color: red; }";+describe("deploySite", () => {- it("uploads the tarball when no matching object exists", async () => {+ it("uploads each file once under its content hash, then the site manifest", async () => {const s3 = new FakeS3();const result = await deploySite({projectDir,@@ -31,14 +53,33 @@ describe("deploySite", () => {expect(result.uploaded).toBe(true);expect(result.siteId).toMatch(/^[0-9a-f]{16}$/);expect(result.bucketName).toBe("test-bucket");- expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/project.tar.gz`);- expect(result.bytes).toBeGreaterThan(0);- expect(s3.ops).toEqual([- { kind: "head", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },- { kind: "put", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },+ expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/files.json`);+ expect(result.bytes).toBe(HTML.length + CSS.length);+ const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);+ expect(puts.slice(0, 2).sort()).toEqual([storeKey(HTML), storeKey(CSS)].sort());+ expect(puts[2]).toBe(`sites/${result.siteId}/files.json`);+ expect(s3.bodies.get(`test-bucket/${storeKey(CSS)}`)?.toString()).toBe(CSS);+ expect(parseSiteManifest(s3.bodies.get(`test-bucket/${puts[2]}`)?.toString() ?? "")).toEqual([+ { path: "assets/style.css", sha256: sha256(CSS), sizeBytes: CSS.length },+ { path: "index.html", sha256: sha256(HTML), sizeBytes: HTML.length },]);});+ it("skips a file another site already stored", async () => {+ const s3 = new FakeS3();+ const cssId = `test-bucket/${storeKey(CSS)}`;+ s3.existing.add(cssId);+ s3.bodies.set(cssId, Buffer.from(CSS));+ s3.metadata.set(cssId, { sha256: sha256(CSS) });++ await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });++ const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);+ expect(puts).toHaveLength(2);+ expect(puts).not.toContain(storeKey(CSS));+ expect(puts).toContain(storeKey(HTML));+ });+it("yields a stable siteId across re-runs of the same tree", async () => {const s3a = new FakeS3();const a = await deploySite({@@ -73,7 +114,7 @@ describe("deploySite", () => {expect(after.siteId).not.toBe(before.siteId);});- it("short-circuits on HEAD 200 (skips PUT)", async () => {+ it("uploads nothing when the same tree is deployed again", async () => {const s3 = new FakeS3();const first = await deploySite({projectDir,@@ -89,9 +130,9 @@ describe("deploySite", () => {expect(second.uploaded).toBe(false);expect(second.siteId).toBe(first.siteId);- // Only one PUT total, plus two HEADs.- expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(1);- expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(2);+ // Two files plus the manifest, each put once and HEAD-checked on both deploys.+ expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(3);+ expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(6);});it("honours a caller-supplied siteId", async () => {@@ -103,7 +144,7 @@ describe("deploySite", () => {s3: asS3Client(s3),});expect(result.siteId).toBe("release-v1.2.3");- expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/project.tar.gz");+ expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/files.json");});it("propagates non-404 S3 errors", async () => {@@ -145,4 +186,25 @@ describe("deploySite", () => {// node_modules contents shouldn't move the hash.expect(after.siteId).toBe(before.siteId);});++ it("ships a symlinked project and its symlinked files and folders as content", async () => {+ mkdirSync(join(scratch, "media"));+ writeFileSync(join(scratch, "media", "clip.txt"), "clip");+ writeFileSync(join(scratch, "logo.txt"), "logo");+ symlinkSync(join(scratch, "media"), join(projectDir, "media"));+ symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));+ symlinkSync(projectDir, join(scratch, "fx"));+ const s3 = new FakeS3();+ const result = await deploySite({+ projectDir: join(scratch, "fx"),+ bucketName: "test-bucket",+ s3: asS3Client(s3),+ });+ expect(sitePaths(s3, result.siteId)).toEqual([+ "assets/style.css",+ "index.html",+ "logo.txt",+ "media/clip.txt",+ ]);+ });});diff --git a/packages/aws-lambda/src/sdk/deploySite.ts b/packages/aws-lambda/src/sdk/deploySite.tsindex c325a0b6a1..338b43556a 100644--- a/packages/aws-lambda/src/sdk/deploySite.ts+++ b/packages/aws-lambda/src/sdk/deploySite.ts@@ -1,23 +1,41 @@/**- * `deploySite` — upload a project directory to S3 once per content hash- * and return a reusable handle.+ * `deploySite` — upload a project directory to S3 and return a reusable handle.** `renderToLambda` calls this implicitly when no `siteHandle` is passed,- * but exposing it as a standalone verb lets adopters bundle a project- * ahead of time and reuse the handle across many renders without- * re-tarring the project tree on every call.+ * but exposing it as a standalone verb lets adopters upload a project+ * ahead of time and reuse the handle across many renders.*- * The handle is **content-addressed**: `siteId` is derived from a SHA-256- * over the project files. Two `deploySite` calls on an unchanged tree- * produce the same `siteId` and `HeadObject`-short-circuit the upload.+ * Every file goes into the bucket's shared content store under its SHA-256,+ * once: a file any earlier site or render already stored is skipped. The+ * site itself is a small `sites/<siteId>/files.json` listing path → hash,+ * where `siteId` is derived from a SHA-256 over the project files.*/-import { mkdtempSync, rmSync, statSync } from "node:fs";+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";-import { hashProjectDir } from "@hyperframes/producer/distributed";-import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";+import { S3Client } from "@aws-sdk/client-s3";+import { CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";+import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";+import {+ downloadS3ObjectToFile,+ ensureContentStoreObject,+ isS3NotFound,+ mapConcurrent,+ parseS3Uri,+ putContentStoreFile,+ s3ObjectExists,+ sha256File,+ uploadContentAddressedFileToS3,+} from "../s3Transport.js";+import {+ isSiteManifestUri,+ parseSiteManifest,+ serializeSiteManifest,+ siteManifestUri,+ siteTarballUri,+ type SiteFile,+} from "../siteManifest.js";/** Options for {@link deploySite}. */export interface DeploySiteOptions {@@ -43,94 +61,122 @@ export interface SiteHandle {siteId: string;/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectS3Uri`. */bucketName: string;- /** Full `s3://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToLambda`. */+ /** Full `s3://bucket/sites/<siteId>/files.json` URI; pass through to `renderToLambda`. */projectS3Uri: string;- /** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */+ /** Total size of the project's files in bytes. */bytes: number;- /** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */+ /** ISO timestamp of this `deploySite` call. */uploadedAt: string;- /** `false` if the object already existed and we skipped the PUT. */+ /** `false` if every file and the site manifest were already in the bucket. */uploaded: boolean;}+/** Handle for a site an earlier {@link deploySite} uploaded; `bytes` and `uploadedAt` are unknown. */+export function siteHandleFromId(bucketName: string, siteId: string): SiteHandle {+ const projectS3Uri = siteManifestUri(bucketName, siteId);+ return { siteId, bucketName, projectS3Uri, bytes: 0, uploadedAt: "", uploaded: false };+}+/**- * Upload `projectDir` to `s3://bucketName/sites/<siteId>/project.tar.gz`.- *- * Short-circuits when an object with the same key already exists in the- * bucket — `siteId` derives from the project's content hash, so the same- * bytes produce the same key, and re-uploading would be redundant.+ * Before a render reuses `site`, re-store any file the content store expired, from `projectDir`.+ * Returns the handle to render: a site uploaded before site manifests resolves to its tarball.*/+export async function ensureSiteStored(+ site: SiteHandle,+ s3: S3Client,+ projectDir?: string,+): Promise<SiteHandle> {+ if (!isSiteManifestUri(site.projectS3Uri)) return site;+ const { bucket } = parseS3Uri(site.projectS3Uri);+ const workdir = mkdtempSync(join(tmpdir(), "hf-site-check-"));+ try {+ const manifestPath = join(workdir, "files.json");+ try {+ await downloadS3ObjectToFile(s3, site.projectS3Uri, manifestPath);+ } catch (error) {+ if (!isS3NotFound(error)) throw error;+ return await tarballSite(site, s3, bucket);+ }+ const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));+ const missing: string[] = [];+ await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {+ try {+ await ensureContentStoreObject(s3, bucket, sha256, sizeBytes);+ } catch (error) {+ if ((error as Error).name !== "PLAN_ARTIFACT_MISSING") throw error;+ if (!projectDir) return void missing.push(path);+ await putContentStoreFile(s3, join(projectDir, path), bucket, sha256);+ }+ });+ if (missing.length > 0) {+ const error = new Error(+ `[deploySite] SITE_FILES_MISSING: ${missing.length} of ${files.length} files of site ` ++ `${site.siteId} are no longer stored (unused for ${CONTENT_STORE_RETENTION_DAYS} days). ` ++ "Run `hyperframes lambda sites create`, or pass projectDir to re-upload them.",+ );+ error.name = "SITE_FILES_MISSING";+ throw error;+ }+ return site;+ } finally {+ rmSync(workdir, { recursive: true, force: true });+ }+}++async function tarballSite(site: SiteHandle, s3: S3Client, bucket: string): Promise<SiteHandle> {+ const projectS3Uri = siteTarballUri(bucket, site.siteId);+ if (await s3ObjectExists(s3, projectS3Uri)) return { ...site, projectS3Uri };+ const error = new Error(+ `[deploySite] SITE_NOT_FOUND: site ${site.siteId} has neither ${site.projectS3Uri} nor ` ++ `${projectS3Uri}. Run \`hyperframes lambda sites create\` again and use the new site id.`,+ );+ error.name = "SITE_NOT_FOUND";+ throw error;+}++/** Upload `projectDir`'s files to the content store and write the site manifest. */export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {+ const s3 = opts.s3 ?? new S3Client({ region: opts.region });if (!statSync(opts.projectDir).isDirectory()) {throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);}+ const files = listSiteFiles(opts.projectDir);+ const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);+ const projectS3Uri = siteManifestUri(opts.bucketName, siteId);- const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);- const key = `sites/${siteId}/project.tar.gz`;- const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });- const s3 = opts.s3 ?? new S3Client({ region: opts.region });-- // HeadObject short-circuit. Adopters re-rendering the same project on- // a tight inner loop (CI smoke, demo flows) save the tar+gzip+PUT pass- // on every iteration.- const existing = await headObject(s3, opts.bucketName, key);- if (existing) {- return {- siteId,- bucketName: opts.bucketName,- projectS3Uri,- bytes: existing.bytes,- uploadedAt: existing.lastModified,- uploaded: false,- };- }+ const stored: SiteFile[] = [];+ let uploaded = false;+ await mapConcurrent(files, 8, async (path) => {+ const localPath = join(opts.projectDir, path);+ const sha256 = await sha256File(localPath);+ stored.push({ path, sha256, sizeBytes: statSync(localPath).size });+ const outcome = await putContentStoreFile(s3, localPath, opts.bucketName, sha256);+ uploaded ||= outcome === "uploaded";+ });+ stored.sort((a, b) => (a.path < b.path ? -1 : 1));const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));try {- const tarball = join(workdir, "project.tar.gz");- await tarDirectory(opts.projectDir, tarball);- // Note: tarDirectory packs *everything* under `cwd`. We don't need to- // re-implement the skip list inside the tar pack because the- // producer's plan stage applies the same skip during its copy; the- // archive is slightly bigger than the planDir's compiled/ subtree- // but the cost is bounded by the project's user-authored content.- const size = statSync(tarball).size;- await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");- return {- siteId,- bucketName: opts.bucketName,+ const manifestPath = join(workdir, "files.json");+ const manifest = serializeSiteManifest(stored);+ writeFileSync(manifestPath, manifest);+ const outcome = await uploadContentAddressedFileToS3(+ s3,+ manifestPath,projectS3Uri,- bytes: size,- uploadedAt: new Date().toISOString(),- uploaded: true,- };+ await sha256File(manifestPath),+ "application/json",+ );+ uploaded ||= outcome === "uploaded";} finally {rmSync(workdir, { recursive: true, force: true });}-}--async function headObject(- s3: S3Client,- bucket: string,- key: string,-): Promise<{ bytes: number; lastModified: string } | null> {- try {- const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));- return {- bytes: typeof res.ContentLength === "number" ? res.ContentLength : 0,- lastModified:- res.LastModified instanceof Date- ? res.LastModified.toISOString()- : new Date().toISOString(),- };- } catch (err) {- // The SDK throws different error shapes for 404 vs 403 vs network;- // a 404 means "needs upload" and is the most common case. Anything- // else propagates so callers see auth / network failures.- const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;- if (status === 404) return null;- const name = (err as { name?: string }).name;- if (name === "NotFound" || name === "NoSuchKey") return null;- throw err;- }+ return {+ siteId,+ bucketName: opts.bucketName,+ projectS3Uri,+ bytes: stored.reduce((sum, file) => sum + file.sizeBytes, 0),+ uploadedAt: new Date().toISOString(),+ uploaded,+ };}diff --git a/packages/aws-lambda/src/sdk/index.ts b/packages/aws-lambda/src/sdk/index.tsindex f0e600afae..ad7f842151 100644--- a/packages/aws-lambda/src/sdk/index.ts+++ b/packages/aws-lambda/src/sdk/index.ts@@ -8,7 +8,12 @@* AWS-SDK only — safe to load in any Node environment.*/-export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";+export {+ deploySite,+ type DeploySiteOptions,+ type SiteHandle,+ siteHandleFromId,+} from "./deploySite.js";export { renderToLambda, type RenderHandle, type RenderToLambdaOptions } from "./renderToLambda.js";export {getRenderProgress,diff --git a/packages/aws-lambda/src/sdk/renderToLambda.test.ts b/packages/aws-lambda/src/sdk/renderToLambda.test.tsindex 9e36eeaf15..0b3292938d 100644--- a/packages/aws-lambda/src/sdk/renderToLambda.test.ts+++ b/packages/aws-lambda/src/sdk/renderToLambda.test.ts@@ -1,11 +1,12 @@import { afterEach, beforeEach, describe, expect, it } from "bun:test";-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import type { SFNClient } from "@aws-sdk/client-sfn";import type { SerializableDistributedRenderConfig } from "../events.js";+import { SITE_MANIFEST_INVALID } from "../siteManifest.js";import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";-import type { SiteHandle } from "./deploySite.js";+import { deploySite, type SiteHandle, siteHandleFromId } from "./deploySite.js";import { renderToLambda } from "./renderToLambda.js";import { InvalidConfigError } from "./validateConfig.js";@@ -15,10 +16,16 @@ interface CapturedStart {input: unknown;}+const CURRENT_DEFINITION = JSON.stringify({+ States: { PlanV2: { Retry: [{ ErrorEquals: [SITE_MANIFEST_INVALID], MaxAttempts: 0 }] } },+});+class FakeSFN {starts: CapturedStart[] = [];+ definition = CURRENT_DEFINITION;async send(command: unknown): Promise<unknown> {const cmdName = (command as { constructor: { name: string } }).constructor.name;+ if (cmdName === "DescribeStateMachineCommand") return { definition: this.definition };if (cmdName === "StartExecutionCommand") {const input = (command as { input: { stateMachineArn: string; name: string; input: string } }).input;@@ -76,9 +83,7 @@ describe("renderToLambda", () => {expect(handle.executionArn).toContain("smoke-1");expect(handle.bucketName).toBe("test-bucket");expect(handle.outputS3Uri).toBe("s3://test-bucket/renders/smoke-1/output.mp4");- expect(handle.projectS3Uri).toMatch(- /^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/project\.tar\.gz$/,- );+ expect(handle.projectS3Uri).toMatch(/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/files\.json$/);expect(Object.keys(handle)).toEqual(["renderId","executionArn",@@ -165,6 +170,102 @@ describe("renderToLambda", () => {expect(s3.existing.size).toBe(0);});+ it("refuses a stack deployed before files.json sites before uploading or starting", async () => {+ const sfn = new FakeSFN();+ sfn.definition = JSON.stringify({ States: { PlanV2: { Retry: [] } } });+ const s3 = new FakeS3();+ await expect(+ renderToLambda({+ projectDir,+ bucketName: "test-bucket",+ stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",+ config: baseConfig,+ sfn: asSFNClient(sfn),+ s3: asS3Client(s3),+ }),+ ).rejects.toMatchObject({+ name: "STACK_TOO_OLD",+ message: expect.stringContaining("hyperframes lambda deploy"),+ });+ expect(s3.ops).toHaveLength(0);+ expect(sfn.starts).toHaveLength(0);+ });++ describe("with a site id from before files.json", () => {+ const renderSite = (s3: FakeS3, sfn: FakeSFN, siteId: string) =>+ renderToLambda({+ siteHandle: siteHandleFromId("test-bucket", siteId),+ bucketName: "test-bucket",+ stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",+ config: baseConfig,+ sfn: asSFNClient(sfn),+ s3: asS3Client(s3),+ });++ it("renders its project.tar.gz", async () => {+ const s3 = new FakeS3();+ const sfn = new FakeSFN();+ s3.existing.add("test-bucket/sites/oldsite/project.tar.gz");+ const handle = await renderSite(s3, sfn, "oldsite");+ expect(handle.projectS3Uri).toBe("s3://test-bucket/sites/oldsite/project.tar.gz");+ expect(sfn.starts[0]?.input).toMatchObject({ ProjectS3Uri: handle.projectS3Uri });+ });++ it("names both keys and the fix when neither exists, before starting", async () => {+ const s3 = new FakeS3();+ const sfn = new FakeSFN();+ await expect(renderSite(s3, sfn, "gone")).rejects.toMatchObject({+ name: "SITE_NOT_FOUND",+ message: expect.stringMatching(+ /sites\/gone\/files\.json.*sites\/gone\/project\.tar\.gz.*sites create/,+ ),+ });+ expect(sfn.starts).toHaveLength(0);+ });+ });++ describe("with a site whose files expired from the content store", () => {+ async function deployThenExpire(s3: FakeS3): Promise<{ site: SiteHandle; expiredKey: string }> {+ const site = await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });+ const expiredKey = [...s3.existing].find((id) => id.includes("/cas/sha256/"))!;+ s3.existing.delete(expiredKey);+ s3.ops.length = 0;+ return { site, expiredKey };+ }+ const renderWith = (s3: FakeS3, sfn: FakeSFN, site: SiteHandle, dir?: string) =>+ renderToLambda({+ siteHandle: site,+ projectDir: dir,+ bucketName: "test-bucket",+ stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",+ config: baseConfig,+ sfn: asSFNClient(sfn),+ s3: asS3Client(s3),+ });++ it("re-uploads the expired file from the project dir, then renders", async () => {+ const s3 = new FakeS3();+ const sfn = new FakeSFN();+ const { site, expiredKey } = await deployThenExpire(s3);+ await renderWith(s3, sfn, site, projectDir);+ expect(+ s3.ops.filter((op) => op.kind === "put").map((op) => `${op.bucket}/${op.key}`),+ ).toEqual([expiredKey]);+ expect(sfn.starts).toHaveLength(1);+ });++ it("without a project dir, refuses with the missing-file count before starting", async () => {+ const s3 = new FakeS3();+ const sfn = new FakeSFN();+ const { site } = await deployThenExpire(s3);+ await expect(renderWith(s3, sfn, site)).rejects.toMatchObject({+ name: "SITE_FILES_MISSING",+ message: expect.stringContaining("1 of 1 files"),+ });+ expect(sfn.starts).toHaveLength(0);+ });+ });+it("rejects invalid configs synchronously before any AWS call", async () => {const sfn = new FakeSFN();const s3 = new FakeS3();@@ -277,10 +378,23 @@ describe("renderToLambda", () => {expect(sfn.starts).toHaveLength(0);});+ it("has a troubleshooting entry for every error it raises before starting", () => {+ const docs = readFileSync(+ new URL("../../../../docs/deploy/aws-lambda.mdx", import.meta.url),+ "utf8",+ );+ for (const name of ["STACK_TOO_OLD", "SITE_FILES_MISSING", "SITE_NOT_FOUND"]) {+ expect({ name, documented: new RegExp(`^### .*\`${name}\``, "m").test(docs) }).toEqual({+ name,+ documented: true,+ });+ }+ });+it("propagates a missing executionArn as an error", async () => {const sfn = {async send(_cmd: unknown): Promise<unknown> {- return { executionArn: undefined };+ return { executionArn: undefined, definition: CURRENT_DEFINITION };},};const s3 = new FakeS3();diff --git a/packages/aws-lambda/src/sdk/renderToLambda.ts b/packages/aws-lambda/src/sdk/renderToLambda.tsindex 1232e68761..a0c80ebb65 100644--- a/packages/aws-lambda/src/sdk/renderToLambda.ts+++ b/packages/aws-lambda/src/sdk/renderToLambda.ts@@ -18,12 +18,13 @@*/import { randomUUID } from "node:crypto";-import { SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";-import type { S3Client } from "@aws-sdk/client-s3";+import { DescribeStateMachineCommand, SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";+import { S3Client } from "@aws-sdk/client-s3";import type { LambdaPlanProtocol, SerializableDistributedRenderConfig } from "../events.js";import { formatExtension } from "../formatExtension.js";import { formatS3Uri } from "../s3Transport.js";-import { deploySite, type SiteHandle } from "./deploySite.js";+import { SITE_MANIFEST_INVALID } from "../siteManifest.js";+import { deploySite, ensureSiteStored, type SiteHandle } from "./deploySite.js";import {validateDistributedRenderConfig,validateStepFunctionsInputSize,@@ -31,7 +32,7 @@ import {/** Options for {@link renderToLambda}. */export interface RenderToLambdaOptions {- /** Local project directory. Required when `siteHandle` is not supplied. */+ /** Local project directory. Required without `siteHandle`; with one, re-uploads expired files. */projectDir?: string;/** Re-use an existing `deploySite` upload (skips tar+S3 PUT). */siteHandle?: SiteHandle;@@ -92,6 +93,9 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise<Rendethrow new Error("[renderToLambda] either siteHandle or projectDir must be supplied");}+ const sfn = opts.sfn ?? new SFNClient({ region: opts.region });+ await assertStackReadsSites(sfn, opts.stateMachineArn);+const executionName = opts.executionName ?? `hf-render-${randomUUID()}`;const ext = formatExtension(opts.config.format);const outputKey = opts.outputKey ?? `renders/${executionName}/output${ext}`;@@ -101,14 +105,18 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise<Rende});const outputS3Uri = formatS3Uri({ bucket: opts.bucketName, key: outputKey });- const site =- opts.siteHandle ??- (await deploySite({- projectDir: opts.projectDir as string,- bucketName: opts.bucketName,- region: opts.region,- s3: opts.s3,- }));+ const site = opts.siteHandle+ ? await ensureSiteStored(+ opts.siteHandle,+ opts.s3 ?? new S3Client({ region: opts.region }),+ opts.projectDir,+ )+ : await deploySite({+ projectDir: opts.projectDir as string,+ bucketName: opts.bucketName,+ region: opts.region,+ s3: opts.s3,+ });const input = {ProjectS3Uri: site.projectS3Uri,@@ -127,7 +135,6 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise<Rende// check should be against the actual wire payload).validateStepFunctionsInputSize(input);- const sfn = opts.sfn ?? new SFNClient({ region: opts.region });const startedAt = new Date().toISOString();const response = await sfn.send(new StartExecutionCommand({@@ -151,3 +158,16 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise<RendestartedAt,};}++/** An older stack's plan step would try to untar a site manifest and fail after retries. */+async function assertStackReadsSites(sfn: SFNClient, stateMachineArn: string): Promise<void> {+ const { definition } = await sfn.send(new DescribeStateMachineCommand({ stateMachineArn }));+ if (definition?.includes(`"${SITE_MANIFEST_INVALID}"`)) return;+ const error = new Error(+ `[renderToLambda] STACK_TOO_OLD: ${stateMachineArn} was deployed by an earlier ` ++ "@hyperframes/aws-lambda that cannot read this version's sites. Redeploy the stack first " ++ "(`hyperframes lambda deploy`, or the SAM template / CDK construct from this version).",+ );+ error.name = "STACK_TOO_OLD";+ throw error;+}diff --git a/packages/aws-lambda/src/siteManifest.ts b/packages/aws-lambda/src/siteManifest.tsnew file mode 100644index 0000000000..36e632e2a6--- /dev/null+++ b/packages/aws-lambda/src/siteManifest.ts@@ -0,0 +1,106 @@+// A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once+// in the shared content store under its SHA-256 (see `contentStoreUri`).++import { readFileSync } from "node:fs";+import { extname, join } from "node:path";+import type { S3Client } from "@aws-sdk/client-s3";+import type { RemoteProjectFile } from "@hyperframes/producer/distributed";+import { contentStoreUri } from "./contentStore.js";+import {+ downloadS3ObjectToFile,+ downloadS3ObjectToFileVerified,+ formatS3Uri,+ mapConcurrent,+ parseS3Uri,+ presignS3GetUrl,+} from "./s3Transport.js";++const SITE_MANIFEST_NAME = "files.json";+/** Stacks that read site manifests list this as a terminal plan error; the SDK checks for it. */+export const SITE_MANIFEST_INVALID = "SITE_MANIFEST_INVALID";+const VIDEO_EXTENSIONS_PLANNED_IN_PLACE = new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv"]);+const REMOTE_URL_TTL_SECONDS = 3600;++export interface SiteFile {+ /** `/`-separated, relative to the project root. */+ readonly path: string;+ readonly sha256: string;+ readonly sizeBytes: number;+}++export function siteManifestUri(bucket: string, siteId: string): string {+ return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });+}++/** Sites uploaded before site manifests are one tarball; the plan step still reads them. */+export function siteTarballUri(bucket: string, siteId: string): string {+ return formatS3Uri({ bucket, key: `sites/${siteId}/project.tar.gz` });+}++export function isSiteManifestUri(uri: string): boolean {+ return uri.endsWith(`/${SITE_MANIFEST_NAME}`);+}++export function serializeSiteManifest(files: readonly SiteFile[]): string {+ return JSON.stringify({ files });+}++export function parseSiteManifest(text: string): SiteFile[] {+ const value = JSON.parse(text) as { files?: unknown };+ if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");+ const seen = new Set<string>();+ return value.files.map((entry: Partial<SiteFile>, index) => {+ const { path, sha256, sizeBytes } = entry ?? {};+ if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {+ throw siteManifestError(`files[${index}].path is not a unique relative path`);+ }+ if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {+ throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);+ }+ if (!Number.isSafeInteger(sizeBytes) || (sizeBytes as number) < 0) {+ throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);+ }+ seen.add(path);+ return { path, sha256, sizeBytes: sizeBytes as number };+ });+}++/** Rebuild the project under `projectDir`, SHA-256 verified; videos can stay remote, as URLs. */+export async function downloadSiteFiles(+ s3: S3Client,+ manifestUri: string,+ projectDir: string,+ leaveVideosRemote: boolean,+): Promise<Record<string, RemoteProjectFile>> {+ const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;+ await downloadS3ObjectToFile(s3, manifestUri, manifestPath);+ const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));+ const { bucket } = parseS3Uri(manifestUri);+ const remoteFiles: Record<string, RemoteProjectFile> = {};+ await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {+ const uri = contentStoreUri(bucket, sha256);+ if (leaveVideosRemote && VIDEO_EXTENSIONS_PLANNED_IN_PLACE.has(extname(path).toLowerCase())) {+ remoteFiles[path] = {+ url: await presignS3GetUrl(s3, uri, REMOTE_URL_TTL_SECONDS),+ sha256,+ sizeBytes,+ };+ return;+ }+ await downloadS3ObjectToFileVerified(s3, uri, join(projectDir, ...path.split("/")), sha256);+ });+ return remoteFiles;+}++function isSafeRelativePath(path: string): boolean {+ return (+ path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") &&+ !path.includes("\\")+ );+}++function siteManifestError(detail: string): Error {+ const error = new Error(`[siteManifest] ${SITE_MANIFEST_INVALID}: ${detail}`);+ error.name = SITE_MANIFEST_INVALID;+ return error;+}diff --git a/packages/cli/src/commands/lambda/render-batch.ts b/packages/cli/src/commands/lambda/render-batch.tsindex 6296fb6801..c5bff7f846 100644--- a/packages/cli/src/commands/lambda/render-batch.ts+++ b/packages/cli/src/commands/lambda/render-batch.ts@@ -226,12 +226,12 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {// Deploy the site once and reuse it across every entry. --site-id and// --dry-run both skip the deploy via a synthesised handle.let siteHandle: SiteHandle;+ const { deploySite, siteHandleFromId } = await loadSDK();if (args.siteId) {- siteHandle = makePlaceholderSiteHandle(args.siteId, stack.bucketName);+ siteHandle = siteHandleFromId(stack.bucketName, args.siteId);} else if (args.dryRun) {- siteHandle = makePlaceholderSiteHandle("dry-run-site", stack.bucketName);+ siteHandle = siteHandleFromId(stack.bucketName, "dry-run-site");} else {- const { deploySite } = await loadSDK();siteHandle = await deploySite({projectDir,bucketName: stack.bucketName,@@ -275,6 +275,7 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {}try {const handle = await renderToLambda({+ projectDir,siteHandle,bucketName: stack.bucketName,stateMachineArn: stack.stateMachineArn,@@ -333,23 +334,6 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {if (failed > 0) setCommandExitCode(1);}-/**- * Synthesise a SiteHandle from a known siteId for paths that skip the- * `deploySite` upload (`--site-id` and `--dry-run`). The SDK reads only- * `siteId` + `projectS3Uri` when `uploaded: false`, so the byte / time- * fields are intentional placeholders.- */-function makePlaceholderSiteHandle(siteId: string, bucketName: string): SiteHandle {- return {- siteId,- bucketName,- projectS3Uri: `s3://${bucketName}/sites/${siteId}/project.tar.gz`,- bytes: 0,- uploadedAt: "",- uploaded: false,- };-}-/*** Build the shared wire {@link SerializableDistributedRenderConfig} used* for every entry in a `hyperframes lambda render-batch` run. Extracted fordiff --git a/packages/cli/src/commands/lambda/render.ts b/packages/cli/src/commands/lambda/render.tsindex f4075ae58b..3d936b5e64 100644--- a/packages/cli/src/commands/lambda/render.ts+++ b/packages/cli/src/commands/lambda/render.ts@@ -130,24 +130,12 @@ export async function runRender(args: RenderArgs): Promise<void> {stack.reservedConcurrency,);- // When the caller passes only --site-id, synthesise the minimum-shape- // SiteHandle pointing at the deterministic content-addressed key. The- // `bytes` / `uploadedAt` fields are intentionally placeholders — the- // SDK reads only `siteId` + `projectS3Uri` when `uploaded: false`.- const siteHandle = args.siteId- ? {- siteId: args.siteId,- bucketName: stack.bucketName,- projectS3Uri: `s3://${stack.bucketName}/sites/${args.siteId}/project.tar.gz`,- bytes: 0,- uploadedAt: "",- uploaded: false,- }- : undefined;+ // --site-id renders a site an earlier `lambda sites create` uploaded.+ const { renderToLambda, siteHandleFromId } = await loadSDK();+ const siteHandle = args.siteId ? siteHandleFromId(stack.bucketName, args.siteId) : undefined;- const { renderToLambda } = await loadSDK();const handle = await renderToLambda({- projectDir: siteHandle ? undefined : projectDir,+ projectDir,siteHandle,bucketName: stack.bucketName,stateMachineArn: stack.stateMachineArn,diff --git a/packages/cli/src/commands/lambda/sites.ts b/packages/cli/src/commands/lambda/sites.tsindex 3f949afa48..f033b1810f 100644--- a/packages/cli/src/commands/lambda/sites.ts+++ b/packages/cli/src/commands/lambda/sites.ts@@ -1,12 +1,8 @@/**- * `hyperframes lambda sites create <projectDir>` — tar + upload a project- * to the deployed render bucket at a content-addressed S3 prefix. Used- * by adopters who want to pre-stage a project so multiple subsequent- * renders share the upload.- *- * Without this verb, every `render` call re-tars the same tree on every- * invocation. With it, the same `siteId` resolves to a `HeadObject`-204- * short-circuit inside `deploySite`.+ * `hyperframes lambda sites create <projectDir>` — upload a project's files+ * to the deployed render bucket's shared content store. Used by adopters+ * who want to pre-stage a project so multiple subsequent renders share the+ * upload; files already in the bucket are skipped.*/import { resolve as resolvePath } from "node:path";diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex ee90c67f79..50c3c5bcdf 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -299,7 +299,11 @@ export {quantizeTimeToFrame,type MediaVisualStyleProperty,} from "./inline-scripts/parityContract";-export { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction";+export {+ redactKnownPaths,+ redactTelemetryString,+ redactUrlQueryStrings,+} from "./telemetryRedaction";export {folderGone,isProjectRootMissing,diff --git a/packages/core/src/telemetryRedaction.test.ts b/packages/core/src/telemetryRedaction.test.tsindex ee98b4b0df..6a6a9a5daf 100644--- a/packages/core/src/telemetryRedaction.test.ts+++ b/packages/core/src/telemetryRedaction.test.ts@@ -17,6 +17,14 @@ describe("redactTelemetryString", () => {).toBe("[path] [path] [path] [path] [file-url] https://example.com/video.mp4?…");});+ it("redacts a signing parameter cut off from its URL", () => {+ expect(+ redactTelemetryString(+ "z-Signature=supersecret&X-Amz-Credential=AKID x-goog-signature=abc -t 1",+ ),+ ).toBe("z-Signature=…&X-Amz-Credential=… x-goog-signature=… -t 1");+ });+// The redactor used to enumerate roots (/Users, /home, /opt, /tmp, …). Any// root outside that list reached telemetry verbatim, which is most of them.it.each([diff --git a/packages/core/src/telemetryRedaction.ts b/packages/core/src/telemetryRedaction.tsindex 423a9d29e9..84b12f90b4 100644--- a/packages/core/src/telemetryRedaction.ts+++ b/packages/core/src/telemetryRedaction.ts@@ -5,8 +5,11 @@ function truncateTelemetryString(value: string, maxLength: number): string {return `${value.slice(0, maxLength - 1)}…`;}-function redactUrlQueryStrings(value: string): string {- return value.replace(/\b(https?:\/\/[^\s?]+)\?[^\s]*/g, "$1?…");+/** Strips http(s) URL queries, and any signing parameter left without its URL (e.g. after a cut). */+export function redactUrlQueryStrings(value: string): string {+ return value+ .replace(/\b(https?:\/\/[^\s?]+)\?[^\s]*/g, "$1?…")+ .replace(/((?:X-(?:Amz|Goog)-[\w-]*|Signature)=)[^&\s]*/gi, "$1…");}/**diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex e641fd6a6c..e33baaf377 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -1112,6 +1112,7 @@ export async function processCompositionAudio(config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">> & {/** Serialised master bus chain, from `parseMasterFxChain`. */masterFxChain?: string;+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;},compiledDir?: string,): Promise<MixResult> {@@ -1166,6 +1167,7 @@ export async function processCompositionAudio(if (!isHttpUrl(srcPath)) {// Same browser-URL path semantics as videos.srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);+ srcPath = config?.urlByAbsentLocalPath?.get(srcPath) ?? srcPath;}if (isHttpUrl(srcPath)) {diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 03038a5f5d..282a6959d4 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -45,6 +45,7 @@ import {type VideoMetadata,} from "../utils/ffprobe.js";import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";+import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";import {analyzeCompositionHdr,isHdrColorSpace as isHdrColorSpaceUtil,@@ -247,6 +248,7 @@ export interface ExtractionOptions {deferRangeExtraction?: boolean;/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */contentKeyedDirs?: readonly string[];+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;}const DOWNLOAD_SUBDIR = "_downloads";@@ -815,7 +817,9 @@ export function parseImageElements(html: string): ImageElement[] {/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) {- const result = await runFfmpeg(firstFrameColourArgs(videoPath), { signal });+ const result = await runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)], {+ signal,+ });return parseFirstFrameColour(result.stderr);}@@ -936,6 +940,16 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp}export async function extractVideoFramesRange(+ ...args: Parameters<typeof extractVideoFramesRangeUnredacted>+): Promise<ExtractedFrames> {+ try {+ return await extractVideoFramesRangeUnredacted(...args);+ } catch (error) {+ throw redactUrlQueriesInError(error);+ }+}++async function extractVideoFramesRangeUnredacted(videoPath: string,videoId: string,startTime: number,@@ -995,7 +1009,7 @@ export async function extractVideoFramesRange(throw new Error("frameRanges extraction needs a constant-frame-rate source");}- const args: string[] = [];+ const args: string[] = urlInputArgs(videoPath);if (decodeWithVideoToolbox) {args.push("-hwaccel", "videotoolbox");}@@ -1816,6 +1830,11 @@ export async function extractAllVideoFrames(if (!isHttpUrl(videoPath)) {videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);}+ const remoteUrl = options.urlByAbsentLocalPath?.get(videoPath);+ if (remoteUrl) {+ resolvedVideos.push({ video, videoPath: remoteUrl });+ continue;+ }if (isHttpUrl(videoPath)) {const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);diff --git a/packages/engine/src/utils/ffprobe.test.ts b/packages/engine/src/utils/ffprobe.test.tsindex 33e0cc3797..969028ad2f 100644--- a/packages/engine/src/utils/ffprobe.test.ts+++ b/packages/engine/src/utils/ffprobe.test.ts@@ -82,6 +82,38 @@ describe("extractMediaMetadata nb_frames duration cross-check", () => {vi.resetModules();});+ it("never reads a non-PNG video's bytes looking for PNG colour metadata", async () => {+ const { spawn } = createSpawnSpy([+ {+ kind: "exit",+ code: 0,+ stdout: JSON.stringify({+ streams: [+ {+ codec_type: "video",+ codec_name: "h264",+ width: 16,+ height: 16,+ r_frame_rate: "30/1",+ avg_frame_rate: "30/1",+ },+ ],+ format: { duration: "1" },+ }),+ },+ ]);+ vi.resetModules();+ vi.doMock("child_process", () => ({ spawn }));+ const { extractMediaMetadata: mocked } = await import("./ffprobe.js");+ // Bytes that carry PNG cICP, reported by ffprobe as an untagged H.264 stream.+ const dir = mkdtempSync(`${tmpdir()}/hf-not-png-`);+ const path = `${dir}/clip.mp4`;+ writeFileSync(path, buildMinimalPng());+ const meta = await mocked(path);+ rmSync(dir, { recursive: true, force: true });+ expect(meta.colorSpace).toBeNull();+ });+it("uses nb_frames to bound video duration when stream duration is absent and audio is longer", async () => {const { spawn } = createSpawnSpy([{diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.tsindex 76eb27acff..7659580761 100644--- a/packages/engine/src/utils/ffprobe.ts+++ b/packages/engine/src/utils/ffprobe.ts@@ -5,6 +5,7 @@ import { chunkCrc32 } from "./crc32.js";import { StringDecoder } from "node:string_decoder";import { basename } from "path";import { redactTelemetryString } from "@hyperframes/core";+import { urlInputArgs } from "./urlInput.js";import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";import { ManagedChildProcess } from "./managedChildProcess.js";import { trackChildProcess } from "./processTracker.js";@@ -65,7 +66,15 @@ async function runFfprobe(}const command = getFfprobeBinary();- const proc = spawn(command, ["-v", "error", ...argsWithoutInput, "--", filePath], {+ const args = [+ "-v",+ "error",+ ...urlInputArgs(filePath, { tool: "ffprobe" }),+ ...argsWithoutInput,+ "--",+ filePath,+ ];+ const proc = spawn(command, args, {// Nothing is ever written to the child's stdin; leaving it as a pipe is// what lets a stdin-reading invocation block indefinitely.stdio: ["ignore", "pipe", "pipe"],@@ -769,7 +778,11 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad// ffprobe ran at all, which is exactly the build it exists to cover: one// that reports gbr but does not decode cICP. An HDR PQ PNG then resolved// colorTransfer "" and graded SDR.- const cicp = colorTransfer && colorPrimaries && colorSpaceVal ? null : stillImage()?.colorSpace;+ const isPng = output !== null && isPngImageProbe(output);+ const cicp =+ !isPng || (colorTransfer && colorPrimaries && colorSpaceVal)+ ? null+ : stillImage()?.colorSpace;const merged = {colorTransfer: colorTransfer || cicp?.colorTransfer || "",colorPrimaries: colorPrimaries || cicp?.colorPrimaries || "",diff --git a/packages/engine/src/utils/managedChildProcess.test.ts b/packages/engine/src/utils/managedChildProcess.test.tsindex cf41b3ce87..4c662bf66b 100644--- a/packages/engine/src/utils/managedChildProcess.test.ts+++ b/packages/engine/src/utils/managedChildProcess.test.ts@@ -34,6 +34,57 @@ describe("ManagedChildProcess", () => {});});+ it("keeps no part of a signed URL's query when the tail cut lands inside it", async () => {+ const child = childProcess();+ const managed = new ManagedChildProcess(child, { stderrMaxBytes: 40 });+ child.stderr.emit(+ "data",+ Buffer.from("Error opening https://b.s3.amazonaws.com/v.mp4?X-Amz-Sig"),+ );+ child.stderr.emit(+ "data",+ Buffer.from("nature=supersecret&X-Amz-Credential=AKIDEXAMPLE -t 0.5"),+ );+ child.emit("close", 1, null);++ const { stderr } = await managed.wait();+ expect(stderr).toBe(" -t 0.5");+ for (const leaked of ["secret", "AKID"]) expect(stderr).not.toContain(leaked);+ });++ it.each([+ ["inside the signature value", 66, 64],+ ["right after the query mark", 48, 64],+ ["after a cut severed the URL's head", 66, 30],+ ])("keeps no part of a signature split across chunks %s", async (_where, at, maxBytes) => {+ const child = childProcess();+ const managed = new ManagedChildProcess(child, { stderrMaxBytes: maxBytes });+ const line =+ "Error opening https://b.s3.amazonaws.com/v.mp4?X-Amz-Signature=abcdef1234 -t 0.5\n";+ child.stderr.emit("data", Buffer.from(line.slice(0, at)));+ child.stderr.emit("data", Buffer.from(line.slice(at)));+ child.emit("close", 1, null);++ const { stderr } = await managed.wait();+ expect(stderr).toContain(" -t 0.5");+ for (const leaked of ["abc", "def1234", "1234"]) expect(stderr).not.toContain(leaked);+ });++ it("keeps no signature fragment when the kept tail itself starts inside one", async () => {+ const child = childProcess();+ const managed = new ManagedChildProcess(child, { stderrMaxBytes: 16 });+ const signature = `${"a".repeat(30)}0123456789`;+ child.stderr.emit(+ "data",+ Buffer.from(`x https://b.s3.amazonaws.com/v.mp4?X-Amz-Signature=${signature} -t 0.5\n`),+ );+ child.emit("close", 1, null);++ const { stderr } = await managed.wait();+ expect(stderr).toContain(" -t 0.5");+ for (const leaked of ["aaa", "6789"]) expect(stderr).not.toContain(leaked);+ });+it("escalates abort from SIGTERM to SIGKILL and resolves only after close", async () => {vi.useFakeTimers();const child = childProcess();diff --git a/packages/engine/src/utils/managedChildProcess.ts b/packages/engine/src/utils/managedChildProcess.tsindex 3778e412d6..6963027b9f 100644--- a/packages/engine/src/utils/managedChildProcess.ts+++ b/packages/engine/src/utils/managedChildProcess.ts@@ -1,4 +1,6 @@import type { ChildProcess } from "node:child_process";+import { StringDecoder } from "node:string_decoder";+import { redactUrlQueryStrings } from "@hyperframes/core";export type ManagedProcessTerminationReason =| "exit"@@ -28,6 +30,7 @@ export interface ManagedChildProcessOptions {const DEFAULT_TERMINATION_GRACE_MS = 2_000;const DEFAULT_STDERR_MAX_BYTES = 64 * 1024;+const RAW_STDERR_SLACK_FACTOR = 2;/** Owns cancellation, escalation, stderr and reaping for one child process. */export class ManagedChildProcess {@@ -37,7 +40,9 @@ export class ManagedChildProcess {private resolveOutcome!: (outcome: ManagedChildProcessOutcome) => void;private requestedReason: Exclude<ManagedProcessTerminationReason, "exit" | "spawn_error"> | null =null;- private stderrTail = Buffer.alloc(0);+ private rawStderrTail = "";+ private rawStderrTailCut = false;+ private readonly stderrDecoder = new StringDecoder("utf8");private spawned = false;private settled = false;private deadlineTimer: NodeJS.Timeout | null = null;@@ -83,14 +88,23 @@ export class ManagedChildProcess {private readonly onStderr = (data: Buffer | string): void => {const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);- const maxBytes = this.options.stderrMaxBytes ?? DEFAULT_STDERR_MAX_BYTES;- this.stderrTail = Buffer.concat([this.stderrTail, chunk]);- if (this.stderrTail.byteLength > maxBytes) {- this.stderrTail = this.stderrTail.subarray(this.stderrTail.byteLength - maxBytes);- }+ const joined = this.rawStderrTail + this.stderrDecoder.write(chunk);+ this.rawStderrTail = lastBytes(joined, RAW_STDERR_SLACK_FACTOR * this.maxBytes);+ if (this.rawStderrTail !== joined) this.rawStderrTailCut = true;this.options.onStderr?.(chunk.toString());};+ private get maxBytes(): number {+ return this.options.stderrMaxBytes ?? DEFAULT_STDERR_MAX_BYTES;+ }++ private redactedStderrTail(): string {+ const raw = this.rawStderrTailCut+ ? withoutTokenSeveredByCut(this.rawStderrTail)+ : this.rawStderrTail;+ return lastBytes(redactUrlQueryStrings(raw), this.maxBytes);+ }+private readonly onSpawn = (): void => {this.spawned = true;};@@ -100,7 +114,7 @@ export class ManagedChildProcess {reason: this.requestedReason ?? "exit",exitCode,signal,- stderr: this.stderrTail.toString(),+ stderr: this.redactedStderrTail(),durationMs: this.now() - this.startedAtMs,});};@@ -111,7 +125,7 @@ export class ManagedChildProcess {reason: "spawn_error",exitCode: null,signal: null,- stderr: this.stderrTail.length > 0 ? this.stderrTail.toString() : error.message,+ stderr: this.rawStderrTail.length > 0 ? this.redactedStderrTail() : error.message,durationMs: this.now() - this.startedAtMs,error,});@@ -184,3 +198,14 @@ export class ManagedChildProcess {this.escalationTimer = null;}}++function lastBytes(text: string, maxBytes: number): string {+ return Buffer.byteLength(text) > maxBytes+ ? Buffer.from(text).subarray(-maxBytes).toString()+ : text;+}++function withoutTokenSeveredByCut(text: string): string {+ const firstSpace = text.search(/\s/);+ return firstSpace < 0 ? "" : text.slice(firstSpace);+}diff --git a/packages/engine/src/utils/urlInput.test.ts b/packages/engine/src/utils/urlInput.test.tsnew file mode 100644index 0000000000..c51711420b--- /dev/null+++ b/packages/engine/src/utils/urlInput.test.ts@@ -0,0 +1,62 @@+import { describe, expect, it } from "vitest";+import { VideoSourceExtractionError } from "../services/videoFrameExtractor.js";+import { redactUrlQueriesInError, urlInputArgs } from "./urlInput.js";++const signed =+ "https://bucket.s3.us-east-2.amazonaws.com/a/b?X-Amz-Credential=AKID&X-Amz-Signature=deadbeef";++describe("urlInputArgs", () => {+ const caBundles = ["/nonexistent.crt", import.meta.filename];+ const verifyAndReconnect = [+ "-tls_verify",+ "1",+ "-ca_file",+ import.meta.filename,+ "-rw_timeout",+ "30000000",+ "-reconnect",+ "1",+ "-reconnect_delay_max",+ "2",+ ];++ it("makes ffmpeg verify the certificate of an https input and reconnect on network errors", () => {+ expect(urlInputArgs(signed, { caBundles })).toEqual([+ ...verifyAndReconnect,+ "-reconnect_on_network_error",+ "1",+ ]);+ });++ it("gives ffprobe only the options its 4.0 build accepts", () => {+ expect(urlInputArgs(signed, { tool: "ffprobe", caBundles })).toEqual(verifyAndReconnect);+ });++ it("adds nothing for local paths and plain http", () => {+ expect(urlInputArgs("/plan/compiled/bg.mp4", { caBundles: [] })).toEqual([]);+ expect(urlInputArgs("http://127.0.0.1:8080/bg.mp4", { caBundles: [] })).toEqual([]);+ });++ it("refuses an https input when no CA bundle exists", () => {+ expect(() => urlInputArgs(signed, { caBundles: ["/nonexistent.crt"] })).toThrow(/no CA bundle/);+ });+});++describe("redactUrlQueriesInError", () => {+ it("strips a signed URL's query from the message, diagnostic and stack", () => {+ const error = redactUrlQueriesInError(+ new VideoSourceExtractionError("ffmpeg_failed", false, `read ${signed}`, `${signed}: 403`),+ );+ for (const text of [error.message, error.diagnostic, error.stack]) {+ expect(text).toContain("https://bucket.s3.us-east-2.amazonaws.com/a/b?");+ expect(text).not.toContain("deadbeef");+ }+ });++ it("also strips it from a wrapped cause", () => {+ const error = redactUrlQueriesInError(+ new Error("render failed", { cause: new Error(`read ${signed}`) }),+ );+ expect(String((error.cause as Error).message)).not.toContain("deadbeef");+ });+});diff --git a/packages/engine/src/utils/urlInput.ts b/packages/engine/src/utils/urlInput.tsnew file mode 100644index 0000000000..32cff701da--- /dev/null+++ b/packages/engine/src/utils/urlInput.ts@@ -0,0 +1,42 @@+import { existsSync } from "node:fs";+import { redactUrlQueryStrings } from "@hyperframes/core";++// The Lambda nodejs image (Amazon Linux 2023) ships the first; Debian/Ubuntu the second; macOS the third.+const CA_BUNDLES = [+ "/etc/pki/tls/certs/ca-bundle.crt",+ "/etc/ssl/certs/ca-certificates.crt",+ "/etc/ssl/cert.pem",+];++// Lambda's ffprobe 4.0 accepts these but not -reconnect_on_network_error; a stalled read fails after 30 s.+const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnect_delay_max", "2"];++/** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified. */+export function urlInputArgs(+ input: string,+ {+ tool = "ffmpeg",+ caBundles = CA_BUNDLES,+ }: { tool?: "ffmpeg" | "ffprobe"; caBundles?: readonly string[] } = {},+): string[] {+ if (!/^https:\/\//i.test(input)) return [];+ const caFile = caBundles.find((path) => existsSync(path));+ if (!caFile) {+ throw new Error(+ `[FFmpeg] cannot verify TLS for an https input: no CA bundle at ${caBundles.join(", ")}`,+ );+ }+ const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : [];+ return ["-tls_verify", "1", "-ca_file", caFile, ...RECONNECT_ARGS, ...networkRetry];+}++/** A signed URL grants access until it expires: strip query strings from every string field of an error and its causes. */+export function redactUrlQueriesInError<T>(error: T): T {+ if (!(error instanceof Error)) return error;+ const fields = error as unknown as Record<string, unknown>;+ for (const key of ["message", "stack", ...Object.keys(error)]) {+ if (typeof fields[key] === "string") fields[key] = redactUrlQueryStrings(fields[key]);+ }+ if (error.cause !== error) redactUrlQueriesInError(error.cause);+ return error;+}diff --git a/packages/gcp-cloud-run/src/gcsTransport.ts b/packages/gcp-cloud-run/src/gcsTransport.tsindex 4147ad09ae..32055ca568 100644--- a/packages/gcp-cloud-run/src/gcsTransport.ts+++ b/packages/gcp-cloud-run/src/gcsTransport.ts@@ -241,12 +241,20 @@ function isGcsPreconditionFailed(error: unknown): boolean {* package (pure JS over `node:zlib`) rather than spawning a system tar* binary so the archive format is independent of the container's userland.*/-export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {+export async function tarDirectory(+ sourceDir: string,+ destTarball: string,+ files?: string[],+): Promise<void> {if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);}mkdirSync(dirname(destTarball), { recursive: true });- await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);+ // `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.+ await tar.create(+ { gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },+ files ?? ["."],+ );}/**diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts b/packages/gcp-cloud-run/src/sdk/deploySite.test.tsindex 6c631ef6c9..b7156b421e 100644--- a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts+++ b/packages/gcp-cloud-run/src/sdk/deploySite.test.ts@@ -4,9 +4,10 @@*/import { afterEach, describe, expect, it } from "bun:test";-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import * as tar from "tar";import { asStorage, FakeGcs } from "../__fixtures__/fakeGcs.js";import { deploySite } from "./deploySite.js";@@ -83,4 +84,24 @@ describe("deploySite", () => {expect(handle.siteId).toBe("my-git-sha");expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz");});++ it("packs a symlinked project and its symlinked files as content", async () => {+ const gcs = new FakeGcs();+ const scratch = mkdtempSync(join(tmpdir(), "hf-site-scratch-"));+ tmpDirs.push(scratch);+ const real = mkProject("<html>linked</html>");+ writeFileSync(join(scratch, "logo.txt"), "logo");+ symlinkSync(join(scratch, "logo.txt"), join(real, "logo.txt"));+ symlinkSync(real, join(scratch, "fx"));+ const handle = await deploySite({+ projectDir: join(scratch, "fx"),+ bucketName: "b",+ storage: asStorage(gcs),+ });+ const file = join(scratch, "uploaded.tar.gz");+ writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? "");+ const entries: string[] = [];+ await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });+ expect(entries).toEqual(["File index.html", "File logo.txt"]);+ });});diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.ts b/packages/gcp-cloud-run/src/sdk/deploySite.tsindex dfd9f63ab6..75e0aab3f3 100644--- a/packages/gcp-cloud-run/src/sdk/deploySite.ts+++ b/packages/gcp-cloud-run/src/sdk/deploySite.ts@@ -17,7 +17,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { Storage } from "@google-cloud/storage";-import { hashProjectDir } from "@hyperframes/producer/distributed";+import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";import { formatGcsUri, tarDirectory, uploadFileToGcs } from "../gcsTransport.js";/** Options for {@link deploySite}. */@@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);}- const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);+ const files = listSiteFiles(opts.projectDir);+ const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);const key = `sites/${siteId}/project.tar.gz`;const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key });const storage = opts.storage ?? new Storage();@@ -89,7 +90,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));try {const tarball = join(workdir, "project.tar.gz");- await tarDirectory(opts.projectDir, tarball);+ await tarDirectory(opts.projectDir, tarball, files);const size = statSync(tarball).size;await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");return {diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjsindex 3bf6f6b093..cf3937ba88 100644--- a/packages/producer/scripts/test-classification.mjs+++ b/packages/producer/scripts/test-classification.mjs@@ -21,6 +21,7 @@ const INTEGRATION_TEST_FILES = new Set(["src/services/distributed/plan.masterBus.test.ts","src/services/distributed/plan.test.ts","src/services/distributed/planSizeCap.test.ts",+ "src/services/distributed/planV2RemoteFiles.test.ts","src/services/distributed/renderChunk.test.ts","src/services/fileServer.test.ts","src/services/fileServer.file-race.test.ts",diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.tsindex 2abb295a93..98b5f79530 100644--- a/packages/producer/src/distributed.ts+++ b/packages/producer/src/distributed.ts@@ -41,6 +41,7 @@ export {// Typestype DistributedRenderConfig,type PlanResult,+ type RemoteProjectFile,// ConstantsDEFAULT_CHUNK_SIZE,DEFAULT_MAX_PARALLEL_CHUNKS,@@ -61,6 +62,7 @@ export {createPlanV2FromExecutionPlan,createPlanV2FromV1,getPlanV2ExecutionPlanHash,+ isPlanV2DeferredSourceArtifact,listPlanV2ArtifactsForTarget,materializePlanV2Target,planV2,@@ -75,8 +77,10 @@ export {type PlanV2Artifact,type PlanV2Limitations,type PlanV2Manifest,+ type PlanV2MaterializeOptions,type PlanV2MaterializationResult,type PlanV2MaterializationTarget,+ type PlanV2RemoteSources,type PlanV2Result,type PlanV2WithPublisherOptions,} from "./services/distributed/planV2.js";@@ -128,7 +132,7 @@ export {validateDistributedRenderConfig,validateVariablesPayload,} from "./services/distributed/renderConfigValidation.js";-export { hashProjectDir } from "./services/distributed/projectHash.js";+export { hashProjectDir, listSiteFiles } from "./services/distributed/projectHash.js";// ── Plan protocol compatibility ────────────────────────────────────────────// Workers validate this descriptor before consuming layout-specificdiff --git a/packages/producer/src/plan-parity-analysis.ts b/packages/producer/src/plan-parity-analysis.tsindex bc15285ca8..b8061bbb82 100644--- a/packages/producer/src/plan-parity-analysis.ts+++ b/packages/producer/src/plan-parity-analysis.ts@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";import { createReadStream, readdirSync, statSync, type Dirent } from "node:fs";import { relative, resolve } from "node:path";import { spawn, spawnSync } from "node:child_process";+import { ManagedChildProcess } from "@hyperframes/engine";import type {PlanParityDriverResult,PlanParityMediaMeasurement,@@ -199,60 +200,45 @@ async function hashPath(path: string): Promise<{ sha256: string; bytes: number }async function canonicalPcmAudio(outputPath: string,): Promise<PlanParityMediaMeasurement["pcmAudio"]> {- return new Promise((resolvePromise, reject) => {- const child = spawn(- "ffmpeg",- [- "-hide_banner",- "-loglevel",- "error",- "-i",- outputPath,- "-map",- "0:a:0",- "-vn",- "-ac",- "2",- "-ar",- "48000",- "-f",- "s16le",- "-",- ],- { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },- );- const hash = createHash("sha256");- const stderr: Buffer[] = [];- let bytes = 0;- child.stdout.on("data", (chunk: Buffer) => {- hash.update(chunk);- bytes += chunk.length;- });- child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk));- child.on("error", reject);- child.on("close", (code) => {- if (code !== 0) {- const message = Buffer.concat(stderr).toString("utf-8");- // A video-only fixture is a valid parity input. ffmpeg uses this- // wording when the optional audio map has no match.- if (/matches no streams|does not contain any stream/iu.test(message)) {- resolvePromise(null);- return;- }- reject(new Error(`ffmpeg PCM decode exited ${code}: ${message.trim()}`));- return;- }- if (bytes % 4 !== 0) {- reject(new Error(`canonical stereo s16le byte count ${bytes} is not divisible by 4`));- return;- }- resolvePromise({- sha256: hash.digest("hex"),- sampleCount: bytes / 4,- bytes,- });- });+ const child = spawn(+ "ffmpeg",+ [+ "-hide_banner",+ "-loglevel",+ "error",+ "-i",+ outputPath,+ "-map",+ "0:a:0",+ "-vn",+ "-ac",+ "2",+ "-ar",+ "48000",+ "-f",+ "s16le",+ "-",+ ],+ { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },+ );+ const hash = createHash("sha256");+ let bytes = 0;+ child.stdout.on("data", (chunk: Buffer) => {+ hash.update(chunk);+ bytes += chunk.length;});+ const outcome = await new ManagedChildProcess(child).wait();+ if (outcome.reason === "spawn_error") throw outcome.error;+ if (outcome.exitCode !== 0) {+ // A video-only fixture is a valid parity input. ffmpeg uses this+ // wording when the optional audio map has no match.+ if (/matches no streams|does not contain any stream/iu.test(outcome.stderr)) return null;+ throw new Error(`ffmpeg PCM decode exited ${outcome.exitCode}: ${outcome.stderr.trim()}`);+ }+ if (bytes % 4 !== 0) {+ throw new Error(`canonical stereo s16le byte count ${bytes} is not divisible by 4`);+ }+ return { sha256: hash.digest("hex"), sampleCount: bytes / 4, bytes };}export async function analyzePlanParityDriverResult(diff --git a/packages/producer/src/regression-harness-lambda-local.ts b/packages/producer/src/regression-harness-lambda-local.tsindex 84dd83e553..c475fc704e 100644--- a/packages/producer/src/regression-harness-lambda-local.ts+++ b/packages/producer/src/regression-harness-lambda-local.ts@@ -46,6 +46,7 @@ import type {RenderChunkLambdaResult,SerializableDistributedRenderConfig,} from "@hyperframes/aws-lambda";+import { listSiteFiles } from "./services/distributed/projectHash.js";export type {LambdaLocalRenderResult,@@ -79,7 +80,7 @@ export async function runLambdaLocalRender(const projectKey = `sites/harness/${Date.now()}/project.tar.gz`;const projectS3Path = join(s3Root, projectKey);mkdirSync(dirname(projectS3Path), { recursive: true });- await tarDirectory(input.projectDir, projectS3Path);+ await tarDirectory(input.projectDir, projectS3Path, listSiteFiles(input.projectDir));const fakeS3 = new FilesystemBackedFakeS3(s3Root);const deps: HandlerDeps = {diff --git a/packages/producer/src/services/audioExtractor.ts b/packages/producer/src/services/audioExtractor.tsdeleted file mode 100644index 42208f1a11..0000000000--- a/packages/producer/src/services/audioExtractor.ts+++ /dev/null@@ -1,324 +0,0 @@-// fallow-ignore-file unused-file code-duplication complexity-/**- * Audio Extractor Service- *- * Extracts audio from media elements in the composition HTML,- * applies timeline positioning, and mixes into a single audio track.- */--import { spawn } from "node:child_process";-import { existsSync, mkdirSync, rmSync, readFileSync } from "node:fs";-import { join, dirname } from "node:path";-import { getFfmpegBinary, trackChildProcess } from "@hyperframes/engine";--export interface AudioElement {- id: string;- src: string;- start: number;- duration: number;- mediaStart: number;- volume: number;- tagName: "audio" | "video";-}--export interface AudioTrack {- id: string;- srcPath: string;- start: number;- duration: number;- mediaStart: number;- volume: number;-}--/**- * Parse audio/video elements from HTML to find media with audio.- */-export function parseAudioElements(html: string): AudioElement[] {- const elements: AudioElement[] = [];-- // Match <audio> and <video> elements with data-start- const mediaRegex = /<(audio|video)\s[^>]*data-start=["']([^"']+)["'][^>]*>/gi;- let match;-- while ((match = mediaRegex.exec(html)) !== null) {- const fullTag = match[0];- const tagName = (match[1] ?? "").toLowerCase() as "audio" | "video";- const start = parseFloat(match[2] ?? "");-- // `(?<![\w-])` keeps the plain-id pattern off `data-hf-render-id="…"` (and- // `data-hf-id`), which would otherwise match first and report the wrong id.- const idMatch = fullTag.match(/(?<![\w-])id=["']([^"']+)["']/);- const renderIdMatch = fullTag.match(/data-hf-render-id=["']([^"']+)["']/);- const srcMatch = fullTag.match(/src=["']([^"']+)["']/);- if (!srcMatch) continue;-- const durationMatch = fullTag.match(/data-duration=["']([^"']+)["']/);- const endMatch = fullTag.match(/data-end=["']([^"']+)["']/);- const mediaStartMatch = fullTag.match(/data-media-start=["']([^"']+)["']/);- const volumeMatch = fullTag.match(/data-volume=["']([^"']+)["']/);- const parsedVolume = volumeMatch ? parseFloat(volumeMatch[1] ?? "") : 1.0;- const safeVolume = Number.isFinite(parsedVolume) ? parsedVolume : 1.0;- const durationFromDuration = durationMatch ? parseFloat(durationMatch[1] ?? "") : NaN;- const end = endMatch ? parseFloat(endMatch[1] ?? "") : NaN;- const duration =- !isNaN(durationFromDuration) && durationFromDuration > 0- ? durationFromDuration- : !isNaN(end) && end > start- ? end - start- : 0;-- elements.push({- // The stamped render id is document-unique; the authored id is only- // unique within one composition file. See core's mediaRenderIds.ts.- id: renderIdMatch?.[1] || idMatch?.[1] || `media-${elements.length}`,- src: srcMatch[1] ?? "",- start: isNaN(start) ? 0 : start,- duration,- mediaStart: mediaStartMatch ? parseFloat(mediaStartMatch[1] ?? "") : 0,- volume: safeVolume,- tagName,- });- }-- return elements;-}--/**- * Run an FFmpeg command and return a promise.- */-function runFFmpeg(args: string[]): Promise<void> {- return new Promise((resolve, reject) => {- // See runFfmpeg.ts: keeps a console window off the user's desktop on Windows.- const ffmpeg = spawn(getFfmpegBinary(), args, { windowsHide: true });- trackChildProcess(ffmpeg);- let stderr = "";-- ffmpeg.stderr.on("data", (data) => {- stderr += data.toString();- });-- ffmpeg.on("close", (code) => {- if (code === 0) {- resolve();- } else {- reject(new Error(`FFmpeg failed (code ${code}): ${stderr.slice(-500)}`));- }- });-- ffmpeg.on("error", (err) => {- reject(err);- });- });-}--/**- * Extract audio from a single media file.- */-async function extractAudioTrack(- srcPath: string,- outputPath: string,- playbackStart: number,- duration: number,-): Promise<boolean> {- const outputDir = dirname(outputPath);- if (!existsSync(outputDir)) {- mkdirSync(outputDir, { recursive: true });- }-- const args: string[] = [];-- if (playbackStart > 0) {- args.push("-ss", String(playbackStart));- }-- if (duration > 0) {- args.push("-t", String(duration));- }-- args.push(- "-i",- srcPath,- "-vn",- "-acodec",- "pcm_s16le",- "-ar",- "48000",- "-ac",- "2",- "-y",- outputPath,- );-- try {- await runFFmpeg(args);- return true;- } catch (err) {- console.warn(- `[AudioExtractor] Failed to extract audio from ${srcPath}: ${- err instanceof Error ? err.message : err- }`,- );- return false;- }-}--/**- * Generate a silence audio file.- */-async function generateSilence(outputPath: string, duration: number): Promise<void> {- const outputDir = dirname(outputPath);- if (!existsSync(outputDir)) {- mkdirSync(outputDir, { recursive: true });- }-- await runFFmpeg([- "-f",- "lavfi",- "-i",- "anullsrc=r=48000:cl=stereo",- "-t",- String(duration),- "-acodec",- "pcm_s16le",- "-y",- outputPath,- ]);-}--/**- * Mix multiple audio tracks with timeline positioning.- */-async function mixTracks(- tracks: AudioTrack[],- outputPath: string,- totalDuration: number,-): Promise<void> {- if (tracks.length === 0) {- await generateSilence(outputPath, totalDuration);- return;- }-- const outputDir = dirname(outputPath);- if (!existsSync(outputDir)) {- mkdirSync(outputDir, { recursive: true });- }-- const inputs: string[] = [];- const filterParts: string[] = [];-- tracks.forEach((track, i) => {- inputs.push("-i", track.srcPath);-- const delayMs = Math.round(track.start * 1000);- const trimDuration = track.duration > 0 ? track.duration : totalDuration;-- // See audioMixer.ts for why asetpts sits between apad and atrim.- filterParts.push(- `[${i}:a]atrim=0:${trimDuration},volume=${track.volume},adelay=${delayMs}|${delayMs},apad,asetpts=N/SR/TB,atrim=0:${totalDuration}[a${i}]`,- );- });-- const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");- // amix divides by track count by default (normalize=true). Compensate with- // a volume gain to preserve per-track levels across all FFmpeg versions.- const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest[mixed]`;- const postMixGain = `[mixed]volume=${tracks.length}[out]`;- const fullFilter = [...filterParts, mixFilter, postMixGain].join(";");-- const args = [- ...inputs,- "-filter_complex",- fullFilter,- "-map",- "[out]",- "-acodec",- "aac",- "-b:a",- "192k",- "-t",- String(totalDuration),- "-y",- outputPath,- ];-- await runFFmpeg(args);-}--/**- * Process all audio for a composition.- *- * @param htmlPath - Path to the composition HTML (for parsing media elements)- * @param projectDir - Base directory for resolving relative media paths- * @param workDir - Temporary working directory for intermediate files- * @param outputPath - Final mixed audio output path- * @param totalDuration - Total composition duration in seconds- * @returns true if audio was produced, false if no audio elements found- */-export async function processAudio(- htmlPath: string,- projectDir: string,- workDir: string,- outputPath: string,- totalDuration: number,-): Promise<boolean> {- const html = readFileSync(htmlPath, "utf-8");- const elements = parseAudioElements(html);-- if (elements.length === 0) {- console.log("[AudioExtractor] No audio elements found");- return false;- }-- console.log(`[AudioExtractor] Processing ${elements.length} audio element(s)...`);-- if (!existsSync(workDir)) {- mkdirSync(workDir, { recursive: true });- }-- const tracks: AudioTrack[] = [];-- for (const element of elements) {- // Resolve source path relative to project directory- let srcPath = element.src;- if (!srcPath.startsWith("/") && !srcPath.startsWith("http")) {- srcPath = join(projectDir, srcPath);- }-- if (!existsSync(srcPath)) {- console.warn(`[AudioExtractor] Source not found: ${srcPath}`);- continue;- }-- const extractedPath = join(workDir, `${element.id}-extracted.wav`);- const success = await extractAudioTrack(- srcPath,- extractedPath,- element.mediaStart,- element.duration,- );-- if (success) {- tracks.push({- id: element.id,- srcPath: extractedPath,- start: element.start,- duration: element.duration,- mediaStart: element.mediaStart,- volume: element.volume,- });- }- }-- console.log(`[AudioExtractor] Mixing ${tracks.length} track(s) into final audio...`);- await mixTracks(tracks, outputPath, totalDuration);-- // Clean up work directory- try {- rmSync(workDir, { recursive: true, force: true });- } catch {- // Ignore cleanup errors- }-- console.log("[AudioExtractor] Audio processing complete");- return true;-}diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.tsindex 440f164f90..d5347a21c2 100644--- a/packages/producer/src/services/distributed/chunkFrames.ts+++ b/packages/producer/src/services/distributed/chunkFrames.ts@@ -18,6 +18,8 @@ import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewpoimport { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";import type { LockedRenderConfig } from "../render/stages/freezePlan.js";import { resolveRenderFpsConfig } from "../fileServer.js";+import { defaultLogger } from "../../logger.js";+import type { PlanV2RemoteSources } from "./planV2.js";import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";/** Distinct codes so adapters route retries per cause: a missing artifact may heal, a hash mismatch never. */@@ -136,6 +138,8 @@ export function buildChunkCaptureOptions(input: {encoder: Pick<LockedRenderConfig, "quality" | "deviceScaleFactor" | "variables">;videoCount: number;html: string;+ skipReadinessVideoIds?: string[];+ videoMetadataHints?: CaptureOptions["videoMetadataHints"];}): CaptureOptions {const { dimensions, encoder } = input;const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(input.videoCount);@@ -149,6 +153,8 @@ export function buildChunkCaptureOptions(input: {// The controller's snapshot, so the first capture sees the in-process renderer's `__hfVariables`.variables: encoder.variables,subTimelineWaitMemo: {},+ skipReadinessVideoIds: input.skipReadinessVideoIds,+ videoMetadataHints: input.videoMetadataHints,...(videoCaptureBeyondViewport !== undefined? { captureBeyondViewport: videoCaptureBeyondViewport }: {}),@@ -166,8 +172,23 @@ export async function extractDeferredVideoFramesForChunk(input: {fps: { num: number; den: number };outputRoot: string;cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;+ remoteSources?: PlanV2RemoteSources;}): Promise<ExtractedFrames[]> {const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);+ const remoteSources = input.remoteSources ?? {};+ for (const path of Object.keys(remoteSources)) {+ if (!deferred.some((video) => video.deferredRange!.sourcePath === path)) {+ throw new RenderChunkValidationError(+ MISSING_PLAN_ARTIFACT,+ `[renderChunk] ${path} is read by URL but no deferred video uses it`,+ );+ }+ // The query carries the URL's signature; keep it out of logs.+ defaultLogger.info("[renderChunk] reading video by URL", {+ path,+ url: remoteSources[path]!.split("?")[0],+ });+ }if (deferred.length === 0) return [];const table = createFrameLookupTable(input.planVideos.videos,@@ -195,8 +216,9 @@ export async function extractDeferredVideoFramesForChunk(input: {return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };}const range = video.deferredRange!;- const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));- if (!existsSync(sourcePath)) {+ const remoteUrl = remoteSources[range.sourcePath];+ const sourcePath = remoteUrl ?? join(input.planDir, ...range.sourcePath.split("/"));+ if (remoteUrl === undefined && !existsSync(sourcePath)) {throw new RenderChunkValidationError(MISSING_PLAN_ARTIFACT,`[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 87fb1603b1..ed124b04cb 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -74,6 +74,7 @@ import {type CompositionMetadataJson,freezePlan,type LockedRenderConfig,+ writePlanRemoteFiles,type PlanFileDigests,prehashPlanFiles,} from "../render/stages/freezePlan.js";@@ -309,17 +310,22 @@ export interface BuildLocalExecutionPlanOptions {* chunks write those frames into their own work directory.*/readonly deferVideoExtraction?: boolean;+ /** Project files absent locally: probe their URL and publish their existing digest. */+ readonly remoteFiles?: Readonly<Record<string, RemoteProjectFile>>;/** Shared with the publisher so each plan file is hashed once. */readonly digests?: PlanFileDigests;/** Runs while the audio mixes, once `compiled/` and the video trees are final. */readonly whileAudioMixes?: () => Promise<void>;- /**- * mp4 chunks carry their own slice of the plan audio (renderChunk muxes it). Ignored for other- * formats and with `cfr`, which re-encodes.- */+ /** mp4 chunks carry their plan-audio slice unless cfr re-encodes. */readonly chunksCarryAudio?: boolean;}+export interface RemoteProjectFile {+ readonly url: string;+ readonly sha256: string;+ readonly sizeBytes: number;+}+export function logPlanStage(log: ProducerLogger, stage: string, startedAt: number): void {log.info("[plan] stage", { stage, ms: Date.now() - startedAt });}@@ -1034,6 +1040,7 @@ function createVideoSourceShipper(input: {projectDir: string;workCompiledDir: string;workVideoFramesDir: string;+ isRemote: (srcPath: string) => boolean;}): (srcPath: string) => string {const shipped = new Map<string, string>();return (srcPath) => {@@ -1045,7 +1052,10 @@ function createVideoSourceShipper(input: {let result: string | null = null;for (const root of stagedRel === null ? [input.workCompiledDir, input.projectDir] : []) {const rel = pathInside(root, srcPath);- if (rel !== null && existsSync(join(input.planDir, "compiled", rel))) {+ if (+ rel !== null &&+ (existsSync(join(input.planDir, "compiled", rel)) || input.isRemote(srcPath))+ ) {result = ["compiled", ...rel.split(sep)].join("/");break;}@@ -1175,8 +1185,19 @@ export async function buildLocalExecutionPlan(const needsAlpha = outputNeedsAlpha(config.format);// ── Compile ──+ const remoteFiles = Object.entries(options.remoteFiles ?? {});+ const remoteLocalPath = (rel: string) => join(projectDir, ...rel.split("/"));+ // Keyed by the path each file would have had, which is what every media resolver computes.+ const urlByAbsentLocalPath = new Map(+ remoteFiles.map(([rel, file]) => [remoteLocalPath(rel), file.url]),+ );+ const localPathByUrl = new Map(+ remoteFiles.map(([rel, file]) => [file.url, remoteLocalPath(rel)]),+ );+stageStarted = Date.now();const compileResult = await runCompileStage({+ urlByAbsentLocalPath,projectDir,workDir,htmlPath,@@ -1299,6 +1320,7 @@ export async function buildLocalExecutionPlan(assertNotAborted,materializeSymlinks: true,deferRangeExtraction: options.deferVideoExtraction,+ urlByAbsentLocalPath,});logPlanStage(log, "extract", stageStarted);if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;@@ -1345,6 +1367,7 @@ export async function buildLocalExecutionPlan(projectDir,workCompiledDir: compiledDir,workVideoFramesDir: stagedVideoFrames,+ isRemote: (srcPath) => urlByAbsentLocalPath.has(srcPath),});// Staged frames live inside compiledDir, so they must match first.const sourceRoots: Array<[string, string]> = [@@ -1355,19 +1378,30 @@ export async function buildLocalExecutionPlan(const planVideosJson: PlanVideosJson = buildPlanVideosJson({videos: composition.videos,compositionEnd: job.duration ?? Number.NaN,- extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => ({- videoId: ext.videoId,- srcPath: stableSourcePath(sourceRoots, ext.srcPath),- framePattern: ext.framePattern,- fps: ext.fps,- totalFrames: ext.totalFrames,- metadata: ext.metadata,- ...(ext.deferredRange- ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(ext.srcPath) } }- : {}),- })),+ extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => {+ const sourcePath = localPathByUrl.get(ext.srcPath) ?? ext.srcPath;+ return {+ videoId: ext.videoId,+ srcPath: stableSourcePath(sourceRoots, sourcePath),+ framePattern: ext.framePattern,+ fps: ext.fps,+ totalFrames: ext.totalFrames,+ metadata: ext.metadata,+ ...(ext.deferredRange+ ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(sourcePath) } }+ : {}),+ };+ }),});mkdirSync(join(planDir, "meta"), { recursive: true });+ writePlanRemoteFiles(+ planDir,+ remoteFiles.map(([rel, { sha256, sizeBytes }]) => ({+ path: `compiled/${rel}`,+ sha256,+ sizeBytes,+ })),+ );writeFileSync(join(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),JSON.stringify(planVideosJson, null, 2),@@ -1388,6 +1422,7 @@ export async function buildLocalExecutionPlan(projectDir,workDir,compiledDir: finalCompiledDir,+ urlByAbsentLocalPath,duration: audioSeconds,ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,audioGain: cfg.audioGain,diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.tsindex 114b130be8..305d184cde 100644--- a/packages/producer/src/services/distributed/planV2.test.ts+++ b/packages/producer/src/services/distributed/planV2.test.ts@@ -30,8 +30,10 @@ import {publishPlanV2FromExecutionPlan,planV2WithPublisher,readPlanV2Manifest,+ readValidatedPlanV2Materialization,validatePlanV2MaterializedTarget,} from "./planV2.js";+import { planV2BlobPath } from "./planV2Layout.js";import { LocalPlanV2ArtifactPublisher, type PlanV2ArtifactPublisher } from "./planV2Publisher.js";import { buildPlanVideosJson, isPlanAudioArtifactPath, type PlanVideosJson } from "./shared.js";@@ -470,20 +472,60 @@ describe("Plan v2 manifest", () => {);});- it("keeps a deferred source the page itself loads in every chunk", () => {+ it("ships each compiled/ video source only to the chunks that show it", () => {const root = tempPath("hf-plan-v2-deferred-compiled-source-");const v1 = createV1Plan(root, {video: true,videoStart: 1 / 30,deferredSource: "compiled/hero.mp4",});+ // A second clip shown in both chunks, also shipped from compiled/.+ const videosPath = join(v1, "meta", "videos.json");+ const videos = JSON.parse(readFileSync(videosPath, "utf-8")) as PlanVideosJson;+ const hero = videos.extracted[0]!;+ videos.videos.push({ ...videos.videos[0]!, id: "long", src: "long.mp4", start: 0 });+ videos.extracted.push({+ ...hero,+ videoId: "long",+ deferredRange: { ...hero.deferredRange!, sourcePath: "compiled/long.mp4" },+ });+ writeFileSync(videosPath, JSON.stringify(videos));+ writeFileSync(join(v1, "compiled", "long.mp4"), "long source video");+ refreshV1PlanHash(v1);const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);+ const chunksOf = (path: string) =>+ manifest.artifacts.find((artifact) => artifact.path === path)?.chunks;expect(manifest.limitations.videoDependencyMode).toBe("source-extract");- for (const chunkIndex of [0, 1]) {- const artifacts = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex });- expect(artifacts.some((artifact) => artifact.path === "compiled/hero.mp4")).toBe(true);- }+ expect(chunksOf("compiled/hero.mp4")).toEqual([1]);+ expect(chunksOf("compiled/long.mp4")).toEqual([0, 1]);+ expect(chunksOf("compiled/index.html")).toBe("all");+ expect(chunksOf("compiled/asset.txt")).toBe("all");++ const chunk0Dir = join(root, "chunk-0");+ materializePlanV2Target(join(root, "v2"), { role: "chunk", chunkIndex: 0 }, chunk0Dir);+ expect(existsSync(join(chunk0Dir, "compiled", "hero.mp4"))).toBe(false);+ expect(existsSync(join(chunk0Dir, "compiled", "long.mp4"))).toBe(true);+ expect(validatePlanV2MaterializedTarget(chunk0Dir, { role: "chunk", chunkIndex: 0 })).not.toBe(+ null,+ );++ // A source read by URL is neither verified locally nor copied, and is recorded for the chunk.+ const longDigest = manifest.artifacts.find((a) => a.path === "compiled/long.mp4")!.sha256;+ rmSync(planV2BlobPath(join(root, "v2"), longDigest));+ const remoteSources = { "compiled/long.mp4": "https://example.test/long.mp4" };+ const byUrlDir = join(root, "chunk-0-by-url");+ const chunk0 = { role: "chunk", chunkIndex: 0 } as const;+ materializePlanV2Target(join(root, "v2"), chunk0, byUrlDir, { remoteSources });+ expect(existsSync(join(byUrlDir, "compiled", "long.mp4"))).toBe(false);+ expect(readValidatedPlanV2Materialization(byUrlDir, chunk0)?.remoteSources).toEqual(+ remoteSources,+ );+ expect(() =>+ materializePlanV2Target(join(root, "v2"), chunk0, join(root, "chunk-0-stray"), {+ remoteSources: { "compiled/index.html": "https://example.test/index.html" },+ }),+ ).toThrow(/not a selected deferred video/);});it("rejects a deferred video source outside the plan directory", () => {diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex 2f83170bfd..319082e610 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -29,6 +29,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";import { DISTRIBUTED_FPS, type DistributedFps, isDistributedFps } from "@hyperframes/core";import { createFrameLookupTable, type ExtractedFrames } from "@hyperframes/engine";import {+ readPlanRemoteFiles,planFileSha256,recomputePlanHashFromPlanDir,type ChunkSliceJson,@@ -36,7 +37,12 @@ import {} from "../render/stages/freezePlan.js";import { canonicalJsonStringify, sha256File, sha256Hex } from "../render/stages/planHash.js";import { defaultLogger, type ProducerLogger } from "../../logger.js";-import { buildLocalExecutionPlan, type DistributedRenderConfig, logPlanStage } from "./plan.js";+import {+ type BuildLocalExecutionPlanOptions,+ buildLocalExecutionPlan,+ type DistributedRenderConfig,+ logPlanStage,+} from "./plan.js";import {PLAN_PROTOCOL_V2,readPlanProtocolV1,@@ -140,6 +146,8 @@ export interface PlanV2WithPublisherOptions {* can avoid a second set of data blocks.*/readonly stagingParentDir?: string;+ /** See {@link BuildLocalExecutionPlanOptions.remoteFiles}; needs a publisher with `referenceStoredBlob`. */+ readonly remoteFiles?: BuildLocalExecutionPlanOptions["remoteFiles"];readonly chunksCarryAudio?: boolean;}@@ -159,6 +167,13 @@ export interface PlanV2MaterializationResult {readonly audioPath: string | null;}+export type PlanV2RemoteSources = Readonly<Record<string, string>>;++export interface PlanV2MaterializeOptions {+ /** Deferred video sources the chunk reads by URL; the caller vouches for their bytes. */+ readonly remoteSources?: PlanV2RemoteSources;+}+function isRecord(value: unknown): value is Record<string, unknown> {return value !== null && typeof value === "object" && !Array.isArray(value);}@@ -276,26 +291,35 @@ function artifactTargets(if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {return { chunks: "all", assembler: true };}- // Each chunk's page loads every <video> src at init (a missing one fails the- // chunk), so deferred sources under compiled/ still go to every chunk.+ // A compiled/ file in the map is a deferred video source. Chunks inject its+ // frames and skip its <video> readiness, so only chunks that show it get it.if (- path === "meta/composition.json" ||- path === "meta/videos.json" ||- path.startsWith("compiled/")+ path.startsWith("video-frames/") ||+ path.startsWith("video-sources/") ||+ (path.startsWith("compiled/") && videoDependencies?.has(path))) {- return { chunks: "all", assembler: false };- }- if (path.startsWith("video-frames/") || path.startsWith("video-sources/")) {return {chunks: videoDependencies === null ? "all" : (videoDependencies.get(path) ?? []),assembler: false,};}+ if (+ path === "meta/composition.json" ||+ path === "meta/videos.json" ||+ path.startsWith("compiled/")+ ) {+ return { chunks: "all", assembler: false };+ }// Unknown future execution files go to both roles. Over-including is safe;// silently omitting a new execution dependency is not.return { chunks: "all", assembler: true };}+/** A chunk-scoped compiled/ file is a deferred video source (see `artifactTargets`). */+export function isPlanV2DeferredSourceArtifact(artifact: Readonly<PlanV2Artifact>): boolean {+ return artifact.path.startsWith("compiled/") && artifact.chunks !== "all";+}+function listVideoFramePaths(executionPlanDir: string, videos: PlanVideosJson): ExtractedFrames[] {return videos.extracted.map((video) => {const outputDir = resolveExtractedVideoOutputDir(executionPlanDir, video.videoId);@@ -492,14 +516,18 @@ async function putBlobs(publisher: PlanV2ArtifactPublisher,blobs: readonly Readonly<PlanV2PublishBlob>[],): Promise<void> {+ await putTasks(blobs.map((blob) => () => publisher.putBlob(blob)));+}++async function putTasks(tasks: readonly (() => Promise<void>)[]): Promise<void> {const uploadSlots = 32;let next = 0;let failure: { reason: unknown } | undefined;const upload = async () => {- while (!failure && next < blobs.length) {- const blob = blobs[next++]!;+ while (!failure && next < tasks.length) {+ const task = tasks[next++]!;try {- await publisher.putBlob(blob);+ await task();} catch (reason) {failure ??= { reason };}@@ -512,6 +540,8 @@ async function putBlobs(interface PlanV2Publication {readonly manifest: PlanV2Manifest;readonly blobs: readonly Readonly<PlanV2PublishBlob>[];+ /** Remote project files: referenced by digest, never read by the planner. */+ readonly storedBlobs: readonly Readonly<Omit<PlanV2PublishBlob, "sourcePath">>[];}// Artifact classification intentionally keeps every fail-safe branch together.@@ -574,6 +604,15 @@ function buildPlanV2Publication(...targets,});}+ const storedBlobs = new Map<string, Omit<PlanV2PublishBlob, "sourcePath">>();+ for (const remote of readPlanRemoteFiles(executionPlanDir)) {+ const { sha256, sizeBytes } = remote;+ if (!blobs.has(sha256)) storedBlobs.set(sha256, { sha256, sizeBytes });+ artifacts.push({+ ...remote,+ ...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunksCarryAudio),+ });+ }const base: Omit<PlanV2Manifest, "planHash"> = {protocol: PLAN_PROTOCOL_V2,@@ -597,6 +636,7 @@ function buildPlanV2Publication(planHash: computeManifestHash(base),},blobs: [...blobs.values()],+ storedBlobs: [...storedBlobs.values()],};}@@ -609,6 +649,9 @@ export function createPlanV2FromExecutionPlan(throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);}const publication = buildPlanV2Publication(executionPlanDir);+ if (publication.storedBlobs.length > 0) {+ throw new PlanV2IntegrityError("a local plan v2 directory cannot reference remote files");+ }mkdirSync(dirname(planV2Dir), { recursive: true });const tempDir = mkdtempSync(join(dirname(planV2Dir), ".plan-v2-build-"));try {@@ -654,8 +697,14 @@ async function publishExecutionPlan(let stageStarted = Date.now();const publication = buildPlanV2Publication(executionPlanDir, digests);logPlanStage(log, "publication", stageStarted);+ if (publication.storedBlobs.length > 0 && !publisher.referenceStoredBlob) {+ throw new PlanV2IntegrityError("this publisher cannot reference remote files");+ }stageStarted = Date.now();- await putBlobs(publisher, publication.blobs);+ await putTasks([+ ...publication.blobs.map((blob) => () => publisher.putBlob(blob)),+ ...publication.storedBlobs.map((blob) => () => publisher.referenceStoredBlob!(blob)),+ ]);logPlanStage(log, "upload", stageStarted);stageStarted = Date.now();await publisher.commitManifest(canonicalJsonStringify(publication.manifest));@@ -701,6 +750,7 @@ export async function planV2WithPublisher(await buildLocalExecutionPlan(projectDir, config, stagingRoot, {executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,deferVideoExtraction: true,+ remoteFiles: options.remoteFiles,digests,// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.whileAudioMixes: () =>@@ -970,17 +1020,22 @@ export function materializePlanV2Target(planV2Dir: string,target: PlanV2MaterializationTarget,destinationDir: string,+ options: Readonly<PlanV2MaterializeOptions> = {},): PlanV2MaterializationResult {if (existsSync(destinationDir)) {throw new PlanV2IntegrityError(`materialization destination already exists: ${destinationDir}`);}const manifest = readPlanV2Manifest(planV2Dir);const artifacts = listPlanV2ArtifactsForTarget(manifest, target);+ const remoteSources = options.remoteSources ?? {};+ assertRemoteSources(remoteSources, artifacts);const audioArtifact = artifacts.find((artifact) => isPlanAudioArtifactPath(artifact.path));- const verified = artifacts.map((artifact) => ({- artifact,- sourcePath: verifyBlob(planV2Dir, artifact),- }));+ const verified = artifacts+ .filter((artifact) => !Object.hasOwn(remoteSources, artifact.path))+ .map((artifact) => ({+ artifact,+ sourcePath: verifyBlob(planV2Dir, artifact),+ }));mkdirSync(dirname(destinationDir), { recursive: true });const tempDir = mkdtempSync(join(dirname(destinationDir), ".plan-v2-materialize-"));try {@@ -997,7 +1052,11 @@ export function materializePlanV2Target(if (target.role === "chunk") materializeExtractedVideoDirectories(tempDir);writeFileSync(join(tempDir, PLAN_V2_MATERIALIZATION_MARKER),- canonicalJsonStringify({ manifest, target }),+ canonicalJsonStringify({+ manifest,+ target,+ ...(Object.keys(remoteSources).length > 0 ? { remoteSources } : {}),+ }),"utf-8",);renameSync(tempDir, destinationDir);@@ -1024,11 +1083,34 @@ export function materializePlanV2Target(};}+function assertRemoteSources(+ remoteSources: unknown,+ artifacts: readonly Readonly<PlanV2Artifact>[],+): asserts remoteSources is PlanV2RemoteSources {+ if (!isRecord(remoteSources)) throw new PlanV2IntegrityError("remoteSources must be an object");+ for (const [path, url] of Object.entries(remoteSources)) {+ const artifact = artifacts.find((candidate) => candidate.path === path);+ if (!artifact || !isPlanV2DeferredSourceArtifact(artifact)) {+ throw new PlanV2IntegrityError(`remote source is not a selected deferred video: ${path}`);+ }+ if (typeof url !== "string" || !/^https?:\/\//.test(url)) {+ throw new PlanV2IntegrityError(`remote source for ${path} must be an http(s) URL`);+ }+ }+}+/** Revalidate a materialized subset immediately before execution. */export function validatePlanV2MaterializedTarget(planDir: string,expectedTarget: PlanV2MaterializationTarget,): Readonly<PlanV2Manifest> | null {+ return readValidatedPlanV2Materialization(planDir, expectedTarget)?.manifest ?? null;+}++export function readValidatedPlanV2Materialization(+ planDir: string,+ expectedTarget: PlanV2MaterializationTarget,+): { manifest: Readonly<PlanV2Manifest>; remoteSources: PlanV2RemoteSources } | null {const markerPath = join(planDir, PLAN_V2_MATERIALIZATION_MARKER);if (!existsSync(markerPath)) return null;const marker = readJsonFile(markerPath, "materialization marker");@@ -1046,7 +1128,10 @@ export function validatePlanV2MaterializedTarget(}const manifest = parsePlanV2Manifest(marker.manifest);const required = listPlanV2ArtifactsForTarget(manifest, expectedTarget);+ const remoteSources = marker.remoteSources ?? {};+ assertRemoteSources(remoteSources, required);for (const artifact of required) {+ if (Object.hasOwn(remoteSources, artifact.path)) continue;const path = join(planDir, ...artifact.path.split("/"));if (!existsSync(path) || statSync(path).size !== artifact.sizeBytes) {throw new PlanV2IntegrityError(@@ -1057,5 +1142,5 @@ export function validatePlanV2MaterializedTarget(throw new PlanV2IntegrityError(`materialized artifact hash mismatch: ${artifact.path}`);}}- return manifest;+ return { manifest, remoteSources };}diff --git a/packages/producer/src/services/distributed/planV2Publisher.ts b/packages/producer/src/services/distributed/planV2Publisher.tsindex f6edb8522d..d66813182c 100644--- a/packages/producer/src/services/distributed/planV2Publisher.ts+++ b/packages/producer/src/services/distributed/planV2Publisher.ts@@ -26,6 +26,7 @@ export interface PlanV2ArtifactPublisher {* adapters must finish reading/uploading it before the promise resolves.*/putBlob(blob: Readonly<PlanV2PublishBlob>): Promise<void>;+ referenceStoredBlob?(blob: Readonly<Omit<PlanV2PublishBlob, "sourcePath">>): Promise<void>;/** Commit the manifest only after every referenced blob is durable. */commitManifest(manifestBytes: string): Promise<void>;/** Idempotent best-effort cleanup for an unpublished or partial plan. */diff --git a/packages/producer/src/services/distributed/planV2RemoteFiles.test.ts b/packages/producer/src/services/distributed/planV2RemoteFiles.test.tsnew file mode 100644index 0000000000..84b7b1894a--- /dev/null+++ b/packages/producer/src/services/distributed/planV2RemoteFiles.test.ts@@ -0,0 +1,103 @@+import { afterAll, describe, expect, it } from "bun:test";+import { spawnSync } from "node:child_process";+import { createHash } from "node:crypto";+import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { pathToFileURL } from "node:url";+import { planV2WithPublisher } from "./planV2.js";+import type { PlanV2ArtifactPublisher, PlanV2PublishBlob } from "./planV2Publisher.js";++const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;+const root = mkdtempSync(join(tmpdir(), "hf-plan-remote-"));+afterAll(() => rmSync(root, { recursive: true, force: true }));++class RecordingPublisher implements PlanV2ArtifactPublisher {+ readonly put = new Map<string, Buffer>();+ readonly referenced: string[] = [];+ manifest:+ | { sourcePlanV1Hash: string; artifacts: { path: string; sha256: string; sizeBytes: number }[] }+ | undefined;++ async putBlob(blob: Readonly<PlanV2PublishBlob>): Promise<void> {+ this.put.set(blob.sha256, readFileSync(blob.sourcePath));+ }+ async referenceStoredBlob(blob: Readonly<Omit<PlanV2PublishBlob, "sourcePath">>): Promise<void> {+ this.referenced.push(blob.sha256);+ }+ async commitManifest(manifestBytes: string): Promise<void> {+ this.manifest = JSON.parse(manifestBytes);+ }+ async abort(): Promise<void> {}+}++describe("plan v2 with a remote project file", () => {+ it.skipIf(!HAS_FFMPEG)(+ "probes the video by URL and publishes it by digest without reading it into the plan",+ async () => {+ const clip = join(root, "store", "clip.mp4");+ mkdirSync(join(root, "store"), { recursive: true });+ const ffmpeg = spawnSync("ffmpeg", [+ "-nostdin",+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "testsrc2=size=320x240:rate=30:duration=3",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-g",+ "30",+ clip,+ ]);+ expect(ffmpeg.status).toBe(0);+ const sha256 = createHash("sha256").update(readFileSync(clip)).digest("hex");+ const sizeBytes = statSync(clip).size;++ const projectDir = join(root, "project");+ mkdirSync(projectDir, { recursive: true });+ writeFileSync(+ join(projectDir, "index.html"),+ `<!doctype html>+<html><head><meta charset="utf-8"></head><body>+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="3">+ <video id="bg" src="assets/clip.mp4" muted data-start="0" data-duration="3" data-track-index="0"></video>+ </div>+</body></html>`,+ );+ const planWithDigest = async (digest: string) => {+ const publisher = new RecordingPublisher();+ const url = pathToFileURL(clip).href;+ await planV2WithPublisher(+ projectDir,+ { fps: 30, width: 320, height: 240, format: "mp4" },+ publisher,+ { remoteFiles: { "assets/clip.mp4": { url, sha256: digest, sizeBytes } } },+ );+ return publisher;+ };+ const publisher = await planWithDigest(sha256);++ expect(publisher.referenced).toEqual([sha256]);+ expect(publisher.put.has(sha256)).toBe(false);+ expect(publisher.manifest?.artifacts).toContainEqual(+ expect.objectContaining({ path: "compiled/assets/clip.mp4", sha256, sizeBytes }),+ );+ const videosArtifact = publisher.manifest?.artifacts.find(+ (a) => a.path === "meta/videos.json",+ );+ const videos = publisher.put.get(videosArtifact!.sha256)!.toString();+ expect(JSON.parse(videos).extracted[0].deferredRange.sourcePath).toBe(+ "compiled/assets/clip.mp4",+ );+ expect(videos).not.toContain("file:");+ // The remote file's digest is part of the plan's own content hash.+ const other = await planWithDigest("0".repeat(64));+ expect(other.manifest?.sourcePlanV1Hash).not.toBe(publisher.manifest?.sourcePlanV1Hash);+ },+ 60_000,+ );+});diff --git a/packages/producer/src/services/distributed/projectHash.test.ts b/packages/producer/src/services/distributed/projectHash.test.tsnew file mode 100644index 0000000000..7833f98aaa--- /dev/null+++ b/packages/producer/src/services/distributed/projectHash.test.ts@@ -0,0 +1,74 @@+import { afterEach, describe, expect, it } from "bun:test";+import { createHash } from "node:crypto";+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { hashProjectDir, listSiteFiles } from "./projectHash.js";++const dirs: string[] = [];+function tempDir(): string {+ const dir = mkdtempSync(join(tmpdir(), "hf-project-hash-"));+ dirs.push(dir);+ return dir;+}+function project(): string {+ const dir = tempDir();+ writeFileSync(join(dir, "index.html"), "<html>hi</html>");+ return dir;+}+afterEach(() => {+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });+});++describe("listSiteFiles / hashProjectDir", () => {+ it("follows a symlinked project root", () => {+ const real = project();+ const link = join(tempDir(), "fx");+ symlinkSync(real, link);+ expect(listSiteFiles(link)).toEqual(["index.html"]);+ expect(hashProjectDir(link)).toBe(hashProjectDir(real));+ });++ it("lists symlinked files and folders, and hashes their targets' bytes", () => {+ const dir = project();+ const outside = tempDir();+ mkdirSync(join(outside, "media"));+ writeFileSync(join(outside, "shared.css"), "a{}");+ writeFileSync(join(outside, "media", "clip.txt"), "v1");+ symlinkSync(join(outside, "shared.css"), join(dir, "shared.css"));+ symlinkSync(join(outside, "media"), join(dir, "media"));+ expect(listSiteFiles(dir)).toEqual(["index.html", "media/clip.txt", "shared.css"]);++ const before = hashProjectDir(dir);+ writeFileSync(join(outside, "media", "clip.txt"), "v2");+ expect(hashProjectDir(dir)).not.toBe(before);+ });++ it("skips top-level skip-list folders", () => {+ const dir = project();+ mkdirSync(join(dir, "node_modules"));+ writeFileSync(join(dir, "node_modules", "junk.js"), "x");+ expect(listSiteFiles(dir)).toEqual(["index.html"]);+ });++ it("does not loop on a folder that links back to an ancestor", () => {+ const dir = project();+ symlinkSync(dir, join(dir, "self"));+ expect(listSiteFiles(dir)).toEqual(["index.html"]);+ });++ it("throws on a dangling symlink, naming it", () => {+ const dir = project();+ symlinkSync(join(dir, "missing.png"), join(dir, "broken.png"));+ expect(() => listSiteFiles(dir)).toThrow(/cannot read broken\.png/);+ });++ it("never reuses an id from the unversioned scheme", () => {+ const dir = project();+ const unversioned = createHash("sha256")+ .update("index.html\0<html>hi</html>")+ .digest("hex")+ .slice(0, 16);+ expect(hashProjectDir(dir)).not.toBe(unversioned);+ });+});diff --git a/packages/producer/src/services/distributed/projectHash.ts b/packages/producer/src/services/distributed/projectHash.tsindex bc64c03157..cc514f0a04 100644--- a/packages/producer/src/services/distributed/projectHash.ts+++ b/packages/producer/src/services/distributed/projectHash.ts@@ -2,51 +2,80 @@* Content-addressing for a project directory, shared by the distributed-render* adapters' `deploySite` verbs.*- * Each adapter uploads a project tarball to `…/sites/<siteId>/project.tar.gz`- * and short-circuits the upload when the object already exists. `siteId` is a+ * Each adapter uploads a project under `…/sites/<siteId>/` (AWS: a file list,+ * Cloud Run: a tarball) and skips content already uploaded. `siteId` is a* SHA-256 over the project's files so identical content always maps to the* same key — that contract has to be byte-identical across adapters, which is* exactly why it lives here rather than being copy-pasted per adapter.*/-import { readdirSync, readFileSync } from "node:fs";+import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";import { createHash } from "node:crypto";-import { join, relative } from "node:path";+import { join } from "node:path";import { PLAN_PROJECT_DIR_SKIP_SEGMENTS } from "./plan.js";+// Bump when the hashed set changes so uploads cached under an older scheme are never reused.+const SITE_ID_SCHEME = "site-v2\0";+/**- * SHA-256 over every regular file under `projectDir` (sorted by relative- * path) → 16-character hex prefix. The prefix is the `siteId`.+ * The files a site upload ships, as `/`-separated paths under `projectDir`. Symlinks are followed+ * (root included), top-level {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage+ * does, and a dangling symlink throws.+ */+export function listSiteFiles(projectDir: string): string[] {+ const files: string[] = [];+ const ancestors = new Set([realpathSync(projectDir)]);+ function walk(dir: string, rel: string): void {+ for (const name of readdirSync(dir).sort()) {+ if (rel === "" && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(name)) continue;+ const full = join(dir, name);+ const relPath = rel === "" ? name : `${rel}/${name}`;+ let stat;+ try {+ stat = statSync(full);+ } catch (err) {+ throw new Error(`[listSiteFiles] cannot read ${relPath} (dangling symlink?)`, {+ cause: err,+ });+ }+ if (stat.isFile()) {+ files.push(relPath);+ } else if (stat.isDirectory()) {+ const real = realpathSync(full);+ // A folder linking back to an ancestor is skipped rather than walked forever.+ if (ancestors.has(real)) continue;+ ancestors.add(real);+ walk(full, relPath);+ ancestors.delete(real);+ }+ }+ }+ walk(projectDir, "");+ return files;+}++/**+ * SHA-256 over a scheme tag plus each of {@link listSiteFiles}'s paths and+ * bytes → 16-character hex prefix. The prefix is the `siteId`.** The hash includes the relative path plus every byte of each file, so a* same-bytes rename still yields a fresh id. We trim to 16 chars because the- * full 64 isn't useful in an object key for legibility. Top-level segments in- * {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} (e.g. `node_modules`) are skipped to- * match what the plan stage copies.+ * full 64 isn't useful in an object key for legibility. Pass `files` when the+ * caller already listed them, so the hashed set is the shipped set.** Reads are synchronous: project trees are typically tens of MB at most* (HTML/CSS/JS plus a few composition assets), so the simpler shape wins over* a streaming pipeline.*/-export function hashProjectDir(projectDir: string): string {- const hash = createHash("sha256");- const files: string[] = [];- function walk(dir: string, isRoot: boolean): void {- for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) =>- a.name < b.name ? -1 : a.name > b.name ? 1 : 0,- )) {- if (isRoot && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(entry.name)) continue;- const full = join(dir, entry.name);- if (entry.isDirectory()) walk(full, false);- else if (entry.isFile()) files.push(full);- }- }- walk(projectDir, true);- for (const file of files) {- const rel = relative(projectDir, file).replaceAll("\\", "/");+export function hashProjectDir(+ projectDir: string,+ files: readonly string[] = listSiteFiles(projectDir),+): string {+ const hash = createHash("sha256").update(SITE_ID_SCHEME);+ for (const rel of files) {hash.update(rel);hash.update("\0");- hash.update(readFileSync(file));+ hash.update(readFileSync(join(projectDir, rel)));}return hash.digest("hex").slice(0, 16);}diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.tsindex 69beb54408..ee8b663d75 100644--- a/packages/producer/src/services/distributed/renderChunk.test.ts+++ b/packages/producer/src/services/distributed/renderChunk.test.ts@@ -20,13 +20,29 @@import { afterAll, beforeAll, describe, expect, it } from "bun:test";import { spawnSync } from "node:child_process";-import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";+import {+ copyFileSync,+ cpSync,+ createReadStream,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ readdirSync,+ rmSync,+ statSync,+ writeFileSync,+} from "node:fs";+import { createServer } from "node:http";+import type { AddressInfo } from "node:net";import { tmpdir } from "node:os";import { join } from "node:path";import { type CaptureOptions, extractVideoFramesRange } from "@hyperframes/engine";import { HOST_CHROME_FAILURE_PATTERNS } from "./__test_utils__/hostChromeFailures.js";+import { defaultLogger } from "../../logger.js";import { RenderQualityError } from "../renderOrchestrator.js";import { plan } from "./plan.js";+import { materializePlanV2Target, planV2, readPlanV2Manifest } from "./planV2.js";import {CHUNK_INDEX_OUT_OF_RANGE,distributedCaptureSessionDependencies,@@ -53,6 +69,30 @@ const FIXTURE_HTML = `<!doctype html></body></html>`;+/** Serves one file with Range support and counts the bytes it sent. */+async function serveFileByRange(path: string) {+ const size = statSync(path).size;+ const served = { bytes: 0, requests: 0 };+ const server = createServer((req, res) => {+ served.requests++;+ const match = /bytes=(\d+)-(\d*)/.exec(req.headers.range ?? "");+ const start = match ? Number(match[1]) : 0;+ const end = match?.[2] ? Number(match[2]) : size - 1;+ res.writeHead(match ? 206 : 200, {+ "Accept-Ranges": "bytes",+ "Content-Length": end - start + 1,+ ...(match ? { "Content-Range": `bytes ${start}-${end}/${size}` } : {}),+ });+ const body = createReadStream(path, { start, end, highWaterMark: 16 * 1024 });+ body.on("data", (chunk) => (served.bytes += chunk.length));+ body.pipe(res);+ res.on("close", () => body.destroy());+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/video.mp4`;+ return { url, size, served, close: () => server.close() };+}+let runRoot: string;let projectDir: string;let planDir: string;@@ -530,6 +570,113 @@ describe("renderChunk() — script failure", () => {}, 60_000);});+describe("renderChunk() — a video the chunk does not show", () => {+ it("renders the same without the video's file, with no media warning", async () => {+ if (!hasChrome) {+ console.warn(+ "[renderChunk.test] skipping unshown-video test — chrome-headless-shell not available on this host",+ );+ return;+ }++ const videoDir = join(runRoot, "project-two-videos");+ mkdirSync(videoDir, { recursive: true });+ for (const [name, seconds] of [+ ["short.mp4", 2],+ ["long.mp4", 3],+ ] as const) {+ const synth = spawnSync("ffmpeg", [+ "-v",+ "error",+ "-y",+ "-f",+ "lavfi",+ "-i",+ `testsrc2=s=64x36:r=30:d=${seconds}`,+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ join(videoDir, name),+ ]);+ if (synth.status !== 0) throw new Error(synth.stderr.toString());+ }+ writeFileSync(+ join(videoDir, "index.html"),+ // Clips over a second long, so the planner defers them to the chunks.+ FIXTURE_HTML.replace('data-duration="0.16667"', 'data-duration="2.4"').replace(+ "</div>",+ '<video id="short" src="short.mp4" muted data-start="0" data-duration="1.2" style="position:absolute;left:0;top:0;width:64px"></video>' ++ '<video id="long" src="long.mp4" muted data-start="0" data-duration="2.4" style="position:absolute;right:0;bottom:0;width:64px"></video></div>',+ ),+ "utf-8",+ );+ const v2Dir = join(runRoot, "plan-two-videos");+ await planV2(+ videoDir,+ { fps: 30, width: 160, height: 120, format: "png-sequence", chunkSize: 36 },+ v2Dir,+ );+ const manifest = readPlanV2Manifest(v2Dir);+ expect(manifest.artifacts.find((a) => a.path === "compiled/short.mp4")?.chunks).toEqual([0]);++ const without = join(runRoot, "chunk1-without-short");+ const withShort = join(runRoot, "chunk1-with-short");+ materializePlanV2Target(v2Dir, { role: "chunk", chunkIndex: 1 }, without);+ cpSync(without, withShort, { recursive: true });+ copyFileSync(join(videoDir, "short.mp4"), join(withShort, "compiled", "short.mp4"));++ const warnings: unknown[] = [];+ const skipped: (readonly string[] | undefined)[] = [];+ const originalWarn = defaultLogger.warn;+ const originalCreate = distributedCaptureSessionDependencies.createCaptureSession;+ defaultLogger.warn = (message, meta) => {+ warnings.push(meta);+ originalWarn(message, meta);+ };+ distributedCaptureSessionDependencies.createCaptureSession = ((...args) => {+ skipped.push(args[2].skipReadinessVideoIds);+ return originalCreate(...args);+ }) as typeof originalCreate;+ let a, b;+ try {+ a = await renderChunk(without, 1, join(runRoot, "chunk1-out-without"));+ b = await renderChunk(withShort, 1, join(runRoot, "chunk1-out-with"));+ } catch (err) {+ const message = err instanceof Error ? err.message : String(err);+ if (!HOST_CHROME_FAILURE_PATTERNS.test(message)) throw err;+ console.warn(+ "[renderChunk.test] skipping unshown-video test — host Chrome stack can't render. Diagnostic:",+ message.slice(0, 240),+ );+ return;+ } finally {+ defaultLogger.warn = originalWarn;+ distributedCaptureSessionDependencies.createCaptureSession = originalCreate;+ }++ expect(JSON.stringify(warnings)).not.toContain("media_load_failed");+ // Only the unshown video is skipped; the shown one keeps today's readiness wait.+ expect(new Set(skipped.map(String))).toEqual(new Set(["short"]));+ expect(a.sha256).toBe(b.sha256);++ // The shown video read by URL instead of from the plan renders the same chunk.+ const server = await serveFileByRange(join(videoDir, "long.mp4"));+ const viaUrl = join(runRoot, "chunk1-long-by-url");+ try {+ materializePlanV2Target(v2Dir, { role: "chunk", chunkIndex: 1 }, viaUrl, {+ remoteSources: { "compiled/long.mp4": server.url },+ });+ expect(existsSync(join(viaUrl, "compiled", "long.mp4"))).toBe(false);+ const byUrl = await renderChunk(viaUrl, 1, join(runRoot, "chunk1-out-by-url"));+ expect(server.served.requests).toBeGreaterThan(0);+ expect(byUrl.sha256).toBe(a.sha256);+ } finally {+ server.close();+ }+ }, 120_000);+});+describe("resolvePresetForLockedEncoder", () => {// Tiny fast tests for the codec-override helper. No Chrome, no ffmpeg —// exists so a refactor that moves the override (e.g. into@@ -671,6 +818,118 @@ describe("extractDeferredVideoFramesForChunk", () => {}});+ // Lossless keeps the file large next to what one seek reads.+ const big = join(root, "big.mp4");+ const synthBig = () => {+ if (existsSync(big)) return;+ const synth = spawnSync("ffmpeg", [+ ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=640x360:r=30:d=120"],+ ...["-c:v", "libx264", "-preset", "ultrafast", "-qp", "0", "-g", "30", big],+ ]);+ if (synth.status !== 0) throw new Error(synth.stderr.toString());+ };+ const bigVideos = () => {+ const videos = planVideos(false);+ const bigRange = {+ ...range,+ sourcePath: "compiled/big.mp4",+ startTime: 0,+ durationSeconds: 120,+ };+ videos.videos[0] = { ...videos.videos[0]!, end: 120, mediaStart: 0 };+ videos.extracted[0] = { ...videos.extracted[0]!, totalFrames: 3600, deferredRange: bigRange };+ return videos;+ };+ const extract = (outputRoot: string, remoteSources?: Record<string, string>) =>+ extractDeferredVideoFramesForChunk({+ planDir: root,+ planVideos: bigVideos(),+ frames: frameSpan(1800, 1815),+ fps: { num: 30, den: 1 },+ outputRoot,+ cfg: {},+ remoteSources,+ });++ it("reads a source by URL in byte ranges and extracts the same frames", async () => {+ synthBig();+ mkdirSync(join(root, "compiled"), { recursive: true });+ copyFileSync(big, join(root, "compiled", "big.mp4"));+ const server = await serveFileByRange(big);+ const logged: unknown[] = [];+ const originalInfo = defaultLogger.info;+ defaultLogger.info = (message, meta) => void logged.push({ message, ...meta });+ try {+ const [local] = await extract(join(root, "big-local"));+ rmSync(join(root, "compiled", "big.mp4"));+ const [remote] = await extract(join(root, "big-remote"), {+ "compiled/big.mp4": `${server.url}?X-Amz-Signature=secret`,+ });+ expect(logged).toEqual([+ {+ message: "[renderChunk] reading video by URL",+ path: "compiled/big.mp4",+ url: server.url,+ },+ ]);+ expect([...remote!.framePaths.keys()]).toEqual([...local!.framePaths.keys()]);+ for (const [index, path] of remote!.framePaths) {+ expect(readFileSync(path).equals(readFileSync(local!.framePaths.get(index)!))).toBe(true);+ }+ // What it sends past the seek is socket buffering (a few MB), not the file.+ expect(server.served.bytes).toBeLessThan(server.size / 3);+ } finally {+ defaultLogger.info = originalInfo;+ server.close();+ }+ }, 60_000);++ it("never surfaces a signed URL's query when ffmpeg fails", async () => {+ synthBig();+ // Echoes its arguments (long enough that the diagnostic's tail cut can land inside the URL),+ // then ends on the URL the way a real read failure does.+ const failingFfmpeg = join(root, "failing-ffmpeg");+ writeFileSync(+ failingFfmpeg,+ '#!/bin/sh\necho "cannot read: $*" >&2\n' ++ 'for a in "$@"; do case $a in http*) echo "$a: Server returned 403 Forbidden" >&2;; esac; done\n' ++ "exit 1\n",+ { mode: 0o755 },+ );+ const server = await serveFileByRange(big);+ const originalFfmpeg = process.env.HYPERFRAMES_FFMPEG_PATH;+ process.env.HYPERFRAMES_FFMPEG_PATH = failingFfmpeg;+ try {+ const url = `${server.url}?X-Amz-Signature=supersecret`;+ const error = await extract(join(root, "big-failed"), { "compiled/big.mp4": url }).catch(+ (caught: unknown) => caught,+ );+ expect(error).toBeInstanceOf(Error);+ const { message, stack } = error as Error;+ const text = JSON.stringify({ ...(error as object), message, stack });+ expect(text).toContain("/video.mp4?…");+ expect(text).not.toContain("supersecret");+ } finally {+ if (originalFfmpeg === undefined) delete process.env.HYPERFRAMES_FFMPEG_PATH;+ else process.env.HYPERFRAMES_FFMPEG_PATH = originalFfmpeg;+ server.close();+ }+ }, 60_000);++ it("refuses a URL for a file no deferred video reads", async () => {+ await expect(+ extractDeferredVideoFramesForChunk({+ planDir,+ planVideos: planVideos(false),+ frames: frameSpan(0, 5),+ fps: { num: 30, den: 1 },+ outputRoot: join(root, "chunk-stray-url"),+ cfg: {},+ remoteSources: { "compiled/other.mp4": "http://127.0.0.1:1/other.mp4" },+ }),+ ).rejects.toMatchObject({ code: MISSING_PLAN_ARTIFACT });+ });+it("extracts nothing for a chunk past the clip, and holds no frames", async () => {const [chunk] = await extractDeferredVideoFramesForChunk({planDir,diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 0abc78de07..789fe83323 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -58,7 +58,7 @@ import {resolveConfig,} from "@hyperframes/engine";import { defaultLogger } from "../../logger.js";-import { applyRenderWarningPolicy } from "../renderOrchestrator.js";+import { applyRenderWarningPolicy, collectVideoMetadataHints } from "../renderOrchestrator.js";import { runEncodeStage } from "../render/stages/encodeStage.js";import { runCaptureStage } from "../render/stages/captureStage.js";import { createCapturePlan } from "../render/capturePlan.js";@@ -88,7 +88,7 @@ import {} from "./shared.js";import { muxChunkWithAudio } from "./chunkAudio.js";import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";-import { validatePlanV2MaterializedTarget } from "./planV2.js";+import { readValidatedPlanV2Materialization } from "./planV2.js";import {buildChunkCaptureOptions,createChunkVideoFrameInjectorFactory,@@ -416,10 +416,12 @@ export async function renderChunk(const plan = JSON.parse(readFileSync(planJsonPath, "utf-8")) as PlanJson;readPlanProtocolV1(plan, DISTRIBUTED_RENDER_CAPABILITIES.roles.chunk);const planHashStarted = Date.now();- const v2Manifest = validatePlanV2MaterializedTarget(planDir, {+ const v2Materialization = readValidatedPlanV2Materialization(planDir, {role: "chunk",chunkIndex,});+ const v2Manifest = v2Materialization?.manifest ?? null;+ const remoteSources = v2Materialization?.remoteSources ?? {};for (const required of [encoderJsonPath, chunksJsonPath]) {if (!existsSync(required)) {throw new RenderChunkValidationError(@@ -575,6 +577,30 @@ export async function renderChunk(// its own injector hook because the hook remembers the last injected frame// per video; sharing that state across a fresh retry page can suppress the// first injection and produce a blank/stale frame.+ const deferredVideoFrames = planVideos+ ? await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos,+ frames: Array.from(+ { length: slice.endFrame - slice.startFrame },+ (_, i) => slice.startFrame + i,+ ),+ fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },+ outputRoot: videoFramesDir,+ cfg,+ remoteSources,+ })+ : [];+ // The browser cannot load a source read by URL; size its <video> from the plan instead.+ const remoteVideoIds = new Set(+ planVideos?.extracted+ .filter(+ (video) =>+ video.deferredRange && Object.hasOwn(remoteSources, video.deferredRange.sourcePath),+ )+ .map((video) => video.videoId),+ );+ const remoteVideos = deferredVideoFrames.filter((video) => remoteVideoIds.has(video.videoId));const videoFrameLookup =planVideos && planVideos.extracted.length > 0? createFrameLookupTable(@@ -585,17 +611,7 @@ export async function renderChunk(planVideos.extracted.filter((video) => !video.deferredRange),v2Manifest === null ? "dense-v1" : "sparse-v2",),- ...(await extractDeferredVideoFramesForChunk({- planDir,- planVideos,- frames: Array.from(- { length: slice.endFrame - slice.startFrame },- (_, i) => slice.startFrame + i,- ),- fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },- outputRoot: videoFramesDir,- cfg,- })),+ ...deferredVideoFrames,],resolveRenderFpsConfig(job.config.fps).value,)@@ -620,6 +636,11 @@ export async function renderChunk(encoder,videoCount: planVideos?.videos.length ?? 0,html: readFileSync(join(compiledDir, "index.html"), "utf-8"),+ // The plan ships a deferred source only to chunks that show it; don't wait on the rest.+ skipReadinessVideoIds: deferredVideoFrames+ .filter((video) => video.framePaths.size === 0 || remoteVideoIds.has(video.videoId))+ .map((video) => video.videoId),+ videoMetadataHints: collectVideoMetadataHints(remoteVideos),});// Resolve worker count up-front. Sequential capture reuses the initializeddiff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.tsindex f25739a5fc..5e08891826 100644--- a/packages/producer/src/services/htmlCompiler.ts+++ b/packages/producer/src/services/htmlCompiler.ts@@ -473,6 +473,7 @@ async function resolveMediaDuration(tagName: string,elementIdentity: string,log?: ProducerLogger,+ urlByAbsentLocalPath?: ReadonlyMap<string, string>,): Promise<{ duration: number | null; resolvedPath: string }> {let filePath = src;@@ -490,9 +491,11 @@ async function resolveMediaDuration(} else {filePath = decodedUrlPath(src);if (!filePath.startsWith("/")) filePath = join(baseDir, filePath);+ // ffprobe reads only the header of a remote source, so it is probed in place.+ filePath = urlByAbsentLocalPath?.get(filePath) ?? filePath;}- if (!existsSync(filePath)) {+ if (!isHttpUrl(filePath) && !existsSync(filePath)) {return { duration: null, resolvedPath: filePath };}@@ -607,6 +610,7 @@ async function compileHtmlFile(baseDir: string,downloadDir: string,log?: ProducerLogger,+ urlByAbsentLocalPath?: ReadonlyMap<string, string>,): Promise<{ html: string; unresolvedCompositions: UnresolvedElement[] }> {const { html: staticCompiled, unresolved } = compileTimingAttrs(html);@@ -628,6 +632,7 @@ async function compileHtmlFile(el.tagName,el.id,log,+ urlByAbsentLocalPath,).then(({ duration }) => ({ id: el.id, duration })),),);@@ -722,6 +727,7 @@ async function parseSubCompositions(projectDir: string,downloadDir: string,visited: Set<string> = new Set(),+ urlByAbsentLocalPath?: ReadonlyMap<string, string>,): Promise<{ subCompositions: Map<string, string> }> {const subCompositions = new Map<string, string>();@@ -766,6 +772,8 @@ async function parseSubCompositions(item.rawSubHtml,dirname(item.filePath),downloadDir,+ undefined,+ urlByAbsentLocalPath,);const nested = await parseSubCompositions(@@ -773,6 +781,7 @@ async function parseSubCompositions(projectDir,downloadDir,item.nestedVisited,+ urlByAbsentLocalPath,);return {@@ -1884,6 +1893,8 @@ export interface CompileForRenderOptions {* mismatch warning). Optional so non-render callers can omit it.*/log?: ProducerLogger;+ /** Absolute project paths not on disk, readable at these URLs instead. */+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;/*** Threaded through to {@link injectDeterministicFontFaces}. When `true`,* deterministic font resolution and exhausted transient fetch failures@@ -1998,10 +2009,17 @@ export async function compileForRender(projectDir,downloadDir,options.log,+ options.urlByAbsentLocalPath,);// Compile each referenced sub-composition so the inliner can hoist it.- const { subCompositions } = await parseSubCompositions(compiledHtml, projectDir, downloadDir);+ const { subCompositions } = await parseSubCompositions(+ compiledHtml,+ projectDir,+ downloadDir,+ undefined,+ options.urlByAbsentLocalPath,+ );// Ensure the HTML is a full document before inlining sub-compositions.// When index.html is a fragment (no <html>/<head>/<body>), linkedom.parseHTML()diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.tsindex 0cbf2bea1f..9523d7d89a 100644--- a/packages/producer/src/services/render/stages/audioStage.ts+++ b/packages/producer/src/services/render/stages/audioStage.ts@@ -42,6 +42,7 @@ export interface AudioStageInput {/** Read-only view of `composition.audios`. */audios: CompositionMetadata["audios"];masterFxChain?: string;+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;abortSignal: AbortSignal | undefined;assertNotAborted: () => void;/** Where a per-track failure's detail goes. Optional so tests need not pass one. */@@ -82,6 +83,7 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageRaudioGain,audios,masterFxChain,+ urlByAbsentLocalPath,abortSignal,assertNotAborted,log,@@ -109,7 +111,7 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageRaudioOutputPath,duration,abortSignal,- { ffmpegProcessTimeout, audioGain, masterFxChain },+ { ffmpegProcessTimeout, audioGain, masterFxChain, urlByAbsentLocalPath },compiledDir,);} catch (err) {diff --git a/packages/producer/src/services/render/stages/compileStage.ts b/packages/producer/src/services/render/stages/compileStage.tsindex 51bf5b7f20..ca8684cb9a 100644--- a/packages/producer/src/services/render/stages/compileStage.ts+++ b/packages/producer/src/services/render/stages/compileStage.ts@@ -89,6 +89,8 @@ export interface CompileStageInput {allowSystemFontCapture?: boolean;/** Render-time variable overrides (`--variables`); see CompileForRenderOptions.variables. */variables?: Record<string, unknown>;+ /** Absolute project paths not on disk, readable at these URLs instead. */+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;}export interface CompileStageResult {@@ -171,6 +173,7 @@ export async function runCompileStage(input: CompileStageInput): Promise<CompileallowSystemFontCapture,abortSignal,variables: input.variables,+ urlByAbsentLocalPath: input.urlByAbsentLocalPath,animatedGifCacheDir: cfg.extractCacheDir? join(cfg.extractCacheDir, "animated-gif"): undefined,diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.tsindex 5208f5fac5..ece85cb34e 100644--- a/packages/producer/src/services/render/stages/extractVideosStage.ts+++ b/packages/producer/src/services/render/stages/extractVideosStage.ts@@ -86,6 +86,7 @@ export interface ExtractVideosStageInput {*/materializeSymlinks?: boolean;deferRangeExtraction?: boolean;+ urlByAbsentLocalPath?: ReadonlyMap<string, string>;}export interface ExtractVideosStageResult {@@ -386,6 +387,7 @@ export async function runExtractVideosStage(assertNotAborted,materializeSymlinks,deferRangeExtraction,+ urlByAbsentLocalPath,} = input;const stage2Start = Date.now();@@ -510,6 +512,7 @@ export async function runExtractVideosStage(collectProbeFailures: extractionPolicy.failureMode === "enforce",deferRangeExtraction,contentKeyedDirs: [join(compiledDir, REMOTE_MEDIA_SUBDIR)],+ urlByAbsentLocalPath,},abortSignal,cfg,diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.tsindex f69d017a82..ba054823d2 100644--- a/packages/producer/src/services/render/stages/freezePlan.ts+++ b/packages/producer/src/services/render/stages/freezePlan.ts@@ -273,6 +273,26 @@ function collectPlanAssetShas(return { compositionHtml, assets };}+/** Plan file left in remote storage: hashed and published by digest, never copied into the plan. */+export interface PlanRemoteFile {+ readonly path: string;+ readonly sha256: string;+ readonly sizeBytes: number;+}++const PLAN_REMOTE_FILES_RELATIVE_PATH = "meta/remote-files.json";++export function writePlanRemoteFiles(planDir: string, files: readonly PlanRemoteFile[]): void {+ if (files.length === 0) return;+ const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : 1));+ writeFileSync(join(planDir, PLAN_REMOTE_FILES_RELATIVE_PATH), JSON.stringify(sorted), "utf-8");+}++export function readPlanRemoteFiles(planDir: string): PlanRemoteFile[] {+ const path = join(planDir, PLAN_REMOTE_FILES_RELATIVE_PATH);+ return existsSync(path) ? (JSON.parse(readFileSync(path, "utf-8")) as PlanRemoteFile[]) : [];+}+/*** Read a frozen plan directory back from disk and recompute its* content-addressed `planHash` over the actual on-disk bytes — includingdiff --git a/packages/producer/src/utils/ffprobeArgvContract.test.ts b/packages/producer/src/utils/ffprobeArgvContract.test.tsindex a4817f9168..6c45daf69b 100644--- a/packages/producer/src/utils/ffprobeArgvContract.test.ts+++ b/packages/producer/src/utils/ffprobeArgvContract.test.ts@@ -318,3 +318,26 @@ describe("ffprobe argv contract", () => {}});});++describe("ffmpeg/ffprobe stderr has one owner", () => {+ const SPAWNS_MEDIA_TOOL =+ /\bspawn\(\s*(?:getFfmpegBinary\(\)|getFfprobeBinary\(\)|command\b|["'`]ff(?:mpeg|probe)["'`])/;+ const runtimeSources = (dir: string): string[] =>+ readdirSync(dir).flatMap((entry) => {+ const abs = join(dir, entry);+ if (statSync(abs).isDirectory()) return runtimeSources(abs);+ return /\.ts$/.test(entry) && !/\.(?:test|d)\.ts$/.test(entry) ? [abs] : [];+ });++ it("every spawn hands its stderr to ManagedChildProcess, which redacts signed URLs", () => {+ // Its stderr is what errors and logs quote; any other reader would need its own redaction.+ const offenders = ["engine", "producer", "aws-lambda"]+ .flatMap((pkg) => runtimeSources(join(REPO_ROOT, "packages", pkg, "src")))+ .filter((abs) => {+ const code = stripComments(readFileSync(abs, "utf8"));+ return SPAWNS_MEDIA_TOOL.test(code) && !code.includes("new ManagedChildProcess(");+ })+ .map((abs) => relative(REPO_ROOT, abs));+ expect(offenders).toEqual([]);+ });+});diff --git a/scripts/check-no-main-deletions.mjs b/scripts/check-no-main-deletions.mjsindex c9087087c3..bd7fcd04e9 100644--- a/scripts/check-no-main-deletions.mjs+++ b/scripts/check-no-main-deletions.mjs@@ -1182,6 +1182,10 @@ export const ALLOWED_DELETIONS = new Map(["packages/engine/src/services/frameCapture-wslFullChrome.test.ts","tests the WSL full-Chrome GPU fallback, replaced by the probe without WAYLAND_DISPLAY (frameCapture-wslWayland.test.ts)",],+ [+ "packages/producer/src/services/audioExtractor.ts",+ "imported by nothing; its own ffmpeg spawn quoted stderr unredacted, outside ManagedChildProcess",+ ],["scripts/prune-fallow-audit-caches.mjs", FALLOW_PRUNE_REASON],["scripts/prune-fallow-audit-caches.test.mjs", FALLOW_PRUNE_REASON],]);diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.jsonindex 8c23d51a74..d0af3d6dda 100644--- a/scripts/package-size-budgets.json+++ b/scripts/package-size-budgets.json@@ -1,5 +1,5 @@{- "@hyperframes/aws-lambda": { "packed": 71152, "unpacked": 256920 },+ "@hyperframes/aws-lambda": { "packed": 60823, "unpacked": 209843 },"@hyperframes/cli": { "packed": 7517641, "unpacked": 22401400 },"@hyperframes/core": { "packed": 1636442, "unpacked": 3903211 },"@hyperframes/engine": { "packed": 428347, "unpacked": 1436293 },@@ -8,7 +8,7 @@"@hyperframes/lint": { "packed": 168745, "unpacked": 654607 },"@hyperframes/parsers": { "packed": 190591, "unpacked": 831243 },"@hyperframes/player": { "packed": 157145, "unpacked": 541601 },- "@hyperframes/producer": { "packed": 729380, "unpacked": 2672493 },+ "@hyperframes/producer": { "packed": 768210, "unpacked": 2807938 },"@hyperframes/sdk": { "packed": 90662, "unpacked": 325069 },"@hyperframes/shader-transitions": { "packed": 212864, "unpacked": 825937 },"@hyperframes/studio": { "packed": 5998328, "unpacked": 24360208 },diff --git a/scripts/test-reachability.json b/scripts/test-reachability.jsonindex 621c2140fb..46d3918c86 100644--- a/scripts/test-reachability.json+++ b/scripts/test-reachability.json@@ -61,7 +61,7 @@],"sources": {"packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",- "packages/producer/scripts/test-classification.mjs": "18ec188edb061f2a729d261a64fb22682064e4f438413fdeb4d502e709cc8684",+ "packages/producer/scripts/test-classification.mjs": "1f3380ad44971fbf728239b89c35a92d25ccf8ee416f1d77458bddfb7f222965",".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864","packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1","packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",