Skip to content

Write HTML. Render video. Built for agents.

fix(producer): export places a scene's media where preview plays it (#5454)

Miguel Ángel2905162d925f
diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts
index 1711954b43..a810dc4d85 100644
--- a/packages/producer/src/services/htmlCompiler.ts
+++ b/packages/producer/src/services/htmlCompiler.ts
@@ -111,6 +111,7 @@ export interface CompiledComposition {
videos: VideoElement[];
audios: AudioElement[];
images: ImageElement[];
+ droppedMediaIds: Set<string>;
masterFxChain?: string;
unresolvedCompositions: UnresolvedElement[];
/** Assets that resolve outside projectDir. Keys are the path used in HTML, values are absolute filesystem paths. */
@@ -2147,7 +2148,7 @@ export async function compileForRender(
// within one composition file: two scenes declaring `<video id="clip">` — or
// two bare `<video>`s, both auto-numbered `hf-video-0` — collapsed into one
// entry and injected frames onto whichever element came first. See #3340.
- const { videos, audios, images } = collectRenderMedia(html);
+ const { videos, audios, images, droppedMediaIds } = collectRenderMedia(html);
const masterFxChain = parseMasterFxChain(html);
// Advisory video checks (sparse keyframes, VFR). Fire-and-forget — these spawn
@@ -2201,6 +2202,7 @@ export async function compileForRender(
videos,
audios,
images,
+ droppedMediaIds,
...(masterFxChain ? { masterFxChain } : {}),
unresolvedCompositions,
externalAssets,
diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts
index 2528d82635..9d2cf6f7d8 100644
--- a/packages/producer/src/services/render/stages/probeStage.test.ts
+++ b/packages/producer/src/services/render/stages/probeStage.test.ts
@@ -271,6 +271,7 @@ function makeProbeInput(overrides: {
videos: [],
audios: [],
images: [],
+ droppedMediaIds: new Set<string>(),
unresolvedCompositions: [],
externalAssets: new Map(),
width: 1920,
@@ -411,6 +412,106 @@ describe("runProbeStage — forceScreenshot threading", () => {
expect(mediaPreflightComposition).toBe(input.composition);
});
+ // Scene in-point 5 and clip trim 1: the collector starts picture and sound at source 6.
+ it.each([
+ { pageStart: 1, expected: 6 },
+ { pageStart: 3, expected: 8 },
+ ])(
+ "keeps a trimmed clip's picture and sound together inside a trimmed scene (page start $pageStart)",
+ async ({ pageStart, expected }) => {
+ resetRetryMocks();
+ browserMediaResults = [
+ {
+ id: "a",
+ tagName: "video",
+ src: "a.mp4",
+ start: 0,
+ end: 18,
+ duration: 18,
+ mediaStart: pageStart,
+ loop: false,
+ hasAudio: true,
+ volume: 1,
+ muted: false,
+ },
+ ];
+ const { runProbeStage } = await import("./probeStage.js");
+ const input = makeProbeInput({});
+ const window = { src: "a.mp4", start: 0, end: 6, mediaStart: 6 };
+ input.composition.videos.push({ id: "a", ...window, loop: false, hasAudio: true });
+ input.composition.audios.push({ id: "a-audio", ...window, layer: 0, type: "video" });
+ input.compiled.html = `<div data-composition-id="scene" data-start="0" data-duration="6" data-playback-start="5">
+ <video id="a" src="a.mp4" data-start="0" data-duration="18" data-playback-start="1" data-has-audio="true"></video>
+ </div>`;
+
+ await runProbeStage(input);
+
+ expect(input.composition.videos[0]?.mediaStart).toBe(expected);
+ expect(input.composition.audios[0]?.mediaStart).toBe(expected);
+ },
+ );
+
+ it("does not add back clips the collector dropped", async () => {
+ resetRetryMocks();
+ const common = { start: 1, end: 4, duration: 3, mediaStart: 0, volume: 1, muted: false };
+ browserMediaResults = [
+ { ...common, id: "early", tagName: "audio", src: "early.m4a" },
+ { ...common, id: "early__hf2", tagName: "audio", src: "early.m4a" },
+ { ...common, id: "clip__hf2", tagName: "video", src: "clip.mp4" },
+ { ...common, id: "pic__hf2", tagName: "image", src: "pic.png" },
+ ];
+ const { runProbeStage } = await import("./probeStage.js");
+ const input = makeProbeInput({});
+ input.composition.audios.push({
+ id: "early",
+ src: "early.m4a",
+ start: 1,
+ end: 4,
+ mediaStart: 0,
+ layer: 0,
+ type: "audio",
+ });
+ input.compiled.droppedMediaIds = new Set(["early__hf2", "clip__hf2", "pic__hf2"]);
+
+ await runProbeStage(input);
+
+ expect(input.composition.audios.map((audio) => audio.id)).toEqual(["early"]);
+ expect(input.composition.videos).toEqual([]);
+ expect(input.composition.images).toEqual([]);
+ });
+
+ it("moves an audio's start without moving a video track that shares its id", async () => {
+ resetRetryMocks();
+ browserMediaResults = [
+ {
+ id: "v-audio",
+ tagName: "audio",
+ src: "a.m4a",
+ start: 0,
+ end: 6,
+ duration: 6,
+ mediaStart: 4,
+ loop: false,
+ hasAudio: false,
+ volume: 1,
+ muted: false,
+ },
+ ];
+ const { runProbeStage } = await import("./probeStage.js");
+ const input = makeProbeInput({});
+ input.composition.audios.push(
+ { id: "v-audio", src: "a.m4a", start: 0, end: 6, mediaStart: 2, layer: 0, type: "audio" },
+ { id: "v-audio", src: "v.mp4", start: 0, end: 6, mediaStart: 0, layer: 0, type: "video" },
+ );
+ input.compiled.html =
+ `<video id="v" src="v.mp4" data-start="0" data-duration="6" data-has-audio="true"></video>` +
+ `<audio id="v-audio" src="a.m4a" data-start="0" data-duration="6" data-playback-start="2"></audio>`;
+
+ await runProbeStage(input);
+
+ expect(input.composition.audios.map((audio) => audio.mediaStart)).toEqual([4, 0]);
+ });
+
it("reconciles a nested source's selected runtime URL before media-type preflight", async () => {
resetRetryMocks();
browserMediaResults = [
diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts
index 1a0977e310..0203e8f963 100644
--- a/packages/producer/src/services/render/stages/probeStage.ts
+++ b/packages/producer/src/services/render/stages/probeStage.ts
@@ -44,9 +44,9 @@ import {
isTransientBrowserError,
probeBeginFrameLiveness,
} from "@hyperframes/engine";
-import { fpsToNumber } from "@hyperframes/core";
+import { MEDIA_RENDER_ID_ATTR, fpsToNumber, readMediaStart } from "@hyperframes/core";
import { extractMediaSrcMutations } from "@hyperframes/parsers";
-import type { CompiledComposition } from "../../htmlCompiler.js";
+import type { BrowserMediaElement, CompiledComposition } from "../../htmlCompiler.js";
import {
discoverMediaFromBrowser,
discoverAudioVolumeAutomationFromTimeline,
@@ -181,6 +181,40 @@ export function hasVariableBoundMedia(
});
}
+/** Each clip's own start point as the compiled markup gives it, keyed like the browser probe. */
+function authoredMediaStarts(html: string): Map<string, number> {
+ const { document } = parseHTML(html);
+ const starts = new Map<string, number>();
+ for (const media of document.querySelectorAll("video, audio")) {
+ const id = media.getAttribute(MEDIA_RENDER_ID_ATTR) || media.getAttribute("id");
+ if (id && !starts.has(id)) starts.set(id, readMediaStart(media));
+ }
+ return starts;
+}
+
+/**
+ * Take a start point the page changed by moving the collected one by the same amount, so the
+ * in-point its scene added survives, and a video's sound track moves with its picture.
+ */
+function adoptRuntimeMediaStart(
+ composition: Pick<CompositionMetadata, "videos" | "audios">,
+ el: BrowserMediaElement,
+ authored: number,
+): void {
+ const shift = el.mediaStart - authored;
+ if (el.mediaStart <= 0 || Math.abs(shift) <= BROWSER_MEDIA_EPSILON) return;
+ const entries =
+ el.tagName === "video"
+ ? [
+ ...composition.videos.filter((video) => video.id === el.id),
+ ...composition.audios.filter(
+ (audio) => audio.type === "video" && audio.id === `${el.id}-audio`,
+ ),
+ ]
+ : composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id);
+ for (const entry of entries) entry.mediaStart += shift;
+}
+
function reconcileBrowserMediaEnd(
existingEnd: number,
projectedEnd: number,
@@ -524,6 +558,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
const existingVideoIds = new Set(composition.videos.map((v) => v.id));
const existingAudioIds = new Set(composition.audios.map((a) => a.id));
const existingImageIds = new Set(composition.images.map((image) => image.id));
+ const authoredStarts = authoredMediaStarts(compiled.html);
pruneMutedBrowserMedia(composition, browserMedia, existingAudioIds);
@@ -558,13 +593,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
sourceChanged,
el.durationInferred,
);
- if (
- el.mediaStart > 0 &&
- (existing.mediaStart <= 0 ||
- Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
- ) {
- existing.mediaStart = el.mediaStart;
- }
+ adoptRuntimeMediaStart(
+ composition,
+ el,
+ authoredStarts.get(el.id) ?? existing.mediaStart,
+ );
if (el.hasAudio && !el.muted && !existing.hasAudio) {
existing.hasAudio = true;
}
@@ -572,7 +605,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
existing.loop = true;
}
}
- } else {
+ } else if (!compiled.droppedMediaIds.has(el.id)) {
// New video discovered from browser
composition.videos.push({
id: el.id,
@@ -605,13 +638,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
sourceChanged,
el.durationInferred,
);
- if (
- el.mediaStart > 0 &&
- (existing.mediaStart <= 0 ||
- Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
- ) {
- existing.mediaStart = el.mediaStart;
- }
+ adoptRuntimeMediaStart(
+ composition,
+ el,
+ authoredStarts.get(el.id) ?? existing.mediaStart,
+ );
if (
el.volume > 0 &&
Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
@@ -619,7 +650,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
existing.volume = el.volume;
}
}
- } else {
+ } else if (!compiled.droppedMediaIds.has(el.id)) {
composition.audios.push({
id: el.id,
src,
@@ -650,7 +681,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
existing.end = projectedEnd;
}
}
- } else {
+ } else if (!compiled.droppedMediaIds.has(el.id)) {
composition.images.push({
id: el.id,
src,
diff --git a/packages/producer/src/services/renderMediaCollector.test.ts b/packages/producer/src/services/renderMediaCollector.test.ts
index 1e2af71195..d5f80ee93a 100644
--- a/packages/producer/src/services/renderMediaCollector.test.ts
+++ b/packages/producer/src/services/renderMediaCollector.test.ts
@@ -1,5 +1,10 @@
import { describe, expect, it } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import { MEDIA_RENDER_ID_ATTR } from "@hyperframes/core";
+import { resolveTimelineExtractionWindow } from "@hyperframes/engine";
+import { compileForRender } from "./htmlCompiler.js";
import { collectRenderMedia } from "./renderMediaCollector.js";
describe("collectRenderMedia host windows", () => {
@@ -96,3 +101,200 @@ describe("collectRenderMedia host windows", () => {
expect(audios.find((audio) => audio.id === "global-audio")).toMatchObject({ start: 2, end: 8 });
});
});
+
+async function compileProject(files: Record<string, string>) {
+ const dir = mkdtempSync(join(tmpdir(), "hf-media-"));
+ try {
+ for (const [name, html] of Object.entries(files)) writeFileSync(join(dir, name), html);
+ return await compileForRender(dir, join(dir, "index.html"), dir);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+const inlineScene = (sceneTiming: string, extra = "") =>
+ `<!doctype html><html><body>` +
+ `<div id="root" data-composition-id="main" data-start="0" data-duration="12" data-width="1280" data-height="720">` +
+ `<div id="scene" data-composition-id="scene" class="clip" ${sceneTiming}>` +
+ `<video id="clip" class="clip" src="clip.mp4" data-start="0" data-duration="19" muted></video>` +
+ `<audio id="vo" src="vo.m4a" data-start="0" data-duration="19"></audio>` +
+ extra +
+ `</div></div></body></html>`;
+
+const media = (attrs: string) =>
+ `<video ${MEDIA_RENDER_ID_ATTR}="clip" id="clip" src="clip.mp4" ${attrs}></video>` +
+ `<audio ${MEDIA_RENDER_ID_ATTR}="vo" id="vo" src="vo.m4a" ${attrs}></audio>`;
+
+describe("collectRenderMedia in-point hosts", () => {
+ it("plays a head-trimmed scene's media from its in-point", () => {
+ const html =
+ `<div data-composition-id="scene" data-start="0" data-duration="12" data-playback-start="7">` +
+ media(`data-start="0" data-duration="19" data-end="19"`) +
+ `</div>`;
+
+ const { videos, audios } = collectRenderMedia(html);
+ const video = videos.find((v) => v.id === "clip")!;
+ expect(video).toMatchObject({ start: 0, end: 12, mediaStart: 7 });
+ expect(resolveTimelineExtractionWindow(video, video.end - video.start, 12)).toMatchObject({
+ compositionStart: 0,
+ mediaStart: 7,
+ durationSeconds: 12,
+ });
+ expect(audios.find((a) => a.id === "vo")).toMatchObject({ start: 0, end: 12, mediaStart: 7 });
+ });
+
+ it("finds an inline in-point host in a compiled document with no sub-composition files", async () => {
+ const { videos, audios } = await compileProject({
+ "index.html": inlineScene(`data-start="0" data-duration="12" data-playback-start="7"`),
+ });
+ expect(videos.find((v) => v.id === "clip")).toMatchObject({ start: 0, end: 12, mediaStart: 7 });
+ expect(audios.find((a) => a.id === "vo")).toMatchObject({ start: 0, end: 12, mediaStart: 7 });
+ });
+
+ it("starts an inline scene's media at the scene's start in a compiled document", async () => {
+ const legacy = `<video id="legacy" src="l.mp4" data-start="6" data-duration="3" data-hf-media-start-basis="global" muted></video>`;
+ const { videos, audios } = await compileProject({
+ "index.html": inlineScene(`data-start="6" data-duration="6"`, legacy),
+ });
+ expect(videos.find((v) => v.id === "clip")).toMatchObject({ start: 6, end: 12, mediaStart: 0 });
+ expect(audios.find((a) => a.id === "vo")).toMatchObject({ start: 6, end: 12, mediaStart: 0 });
+ // Already root time: the scene's start is not added again.
+ expect(videos.find((v) => v.id === "legacy")).toMatchObject({ start: 6, end: 9 });
+ });
+
+ it("leaves media running past the root composition's end uncut", () => {
+ const html =
+ `<div data-composition-id="main" data-start="0" data-duration="12">` +
+ `<audio id="music" src="m.mp3" data-start="0" data-duration="19" data-end="19"></audio>` +
+ `</div>`;
+
+ expect(collectRenderMedia(html).audios[0]).toMatchObject({ start: 0, end: 19 });
+ });
+
+ it("places an in-point scene inside a later inline scene", () => {
+ const html =
+ `<div data-composition-id="outer" data-start="10" data-duration="10">` +
+ `<div data-composition-id="inner" data-start="0" data-duration="5" data-playback-start="3">` +
+ media(`data-start="0" data-duration="19" data-end="19"`) +
+ `</div></div>`;
+
+ const { videos, audios } = collectRenderMedia(html);
+ expect(videos.find((v) => v.id === "clip")).toMatchObject({
+ start: 10,
+ end: 15,
+ mediaStart: 3,
+ });
+ expect(audios.find((a) => a.id === "vo")).toMatchObject({ start: 10, end: 15, mediaStart: 3 });
+ });
+
+ it("keeps a looping video inside its in-point scene's slot", () => {
+ const html =
+ `<div data-composition-id="scene" data-start="2" data-duration="5" data-playback-start="3">` +
+ `<video id="bg" src="bg.mp4" loop data-start="0" data-duration="10"></video>` +
+ `</div>`;
+
+ expect(collectRenderMedia(html).videos.find((v) => v.id === "bg")).toMatchObject({
+ start: 2,
+ end: 7,
+ mediaStart: 0,
+ });
+ });
+
+ it("never lets an author id take a render id's window", () => {
+ const html =
+ `<div data-composition-file="a.html" data-composition-id="a" data-start="5" data-duration="6">` +
+ `<video ${MEDIA_RENDER_ID_ATTR}="clip" id="x" src="a.mp4" data-start="0" data-duration="12"></video>` +
+ `</div>` +
+ `<video id="clip" src="b.mp4" data-start="0" data-duration="3"></video>`;
+
+ expect(collectRenderMedia(html).videos[0]).toMatchObject({ src: "a.mp4", start: 5, end: 11 });
+ });
+
+ it("skips an element the parsers do not report when matching author ids", () => {
+ const html =
+ `<div data-composition-id="main"><img id="clip">` +
+ `<div data-composition-id="scene" data-start="5" data-duration="6">` +
+ `<video id="clip" src="a.mp4" data-start="0" data-duration="12"></video>` +
+ `</div></div>`;
+
+ expect(collectRenderMedia(html).videos[0]).toMatchObject({ start: 5, end: 11 });
+ });
+
+ it("keeps a scene clip's window when a root element without a src shares its id", async () => {
+ const { videos, images } = await compileProject({
+ "index.html":
+ `<!doctype html><html><body>` +
+ `<div id="root" data-composition-id="main" data-start="0" data-duration="12" data-width="1280" data-height="720">` +
+ `<div data-composition-src="a.html" data-start="5" data-duration="6"></div>` +
+ `<img id="clip" data-start="0" data-duration="3"><img id="pic">` +
+ `</div></body></html>`,
+ "a.html":
+ `<template><div data-composition-id="a" data-width="1280" data-height="720">` +
+ `<video id="clip" src="clip.mp4" data-start="0" data-duration="12" muted></video>` +
+ `<img id="pic" src="p.png" data-start="1" data-duration="2">` +
+ `</div></template>`,
+ });
+ // The windows main exports for this project.
+ expect(videos.find((v) => v.id === "clip")).toMatchObject({ start: 5, end: 11 });
+ expect(images.find((i) => i.id === "pic")).toMatchObject({ start: 6, end: 8 });
+ });
+
+ it("cuts a split scene's media to the second half's slot", () => {
+ const html =
+ `<div data-composition-file="scene.html" data-composition-id="scene" data-start="5" data-duration="7" data-playback-start="5">` +
+ media(`data-start="0" data-duration="12" data-end="12" data-playback-start="1"`) +
+ `<audio ${MEDIA_RENDER_ID_ATTR}="early" id="early" src="early.m4a" data-start="1" data-duration="3" data-end="4"></audio>` +
+ `<video ${MEDIA_RENDER_ID_ATTR}="early-clip" id="early-clip" src="early.mp4" data-start="1" data-duration="3"></video>` +
+ `<audio ${MEDIA_RENDER_ID_ATTR}="fast" id="fast" src="fast.m4a" data-start="3" data-duration="20" data-end="23" data-playback-rate="2"></audio>` +
+ `</div>`;
+
+ const { videos, audios } = collectRenderMedia(html);
+ expect(videos.find((v) => v.id === "clip")).toMatchObject({ start: 5, end: 12, mediaStart: 6 });
+ expect(audios.find((a) => a.id === "vo")).toMatchObject({ start: 5, end: 12, mediaStart: 6 });
+ // Wholly before the in-point: never played.
+ expect(audios.find((a) => a.id === "early")).toBeUndefined();
+ expect(videos.find((v) => v.id === "early-clip")).toBeUndefined();
+ // Cut at the host's end, its 2 s head consumed at 2x.
+ expect(audios.find((a) => a.id === "fast")).toMatchObject({ start: 5, end: 12, mediaStart: 4 });
+ });
+
+ it("reports the clips a split scene's in-point cuts away", async () => {
+ const host = (timing: string) =>
+ `<div data-composition-id="scene" data-composition-src="scene.html" ${timing}></div>`;
+ const compiled = await compileProject({
+ "index.html":
+ `<!doctype html><html><body>` +
+ `<div id="root" data-composition-id="main" data-start="0" data-duration="12" data-width="1280" data-height="720">` +
+ host(`data-start="0" data-duration="5"`) +
+ host(`data-start="5" data-duration="7" data-playback-start="5"`) +
+ `</div></body></html>`,
+ "scene.html":
+ `<template><div data-composition-id="scene" data-width="1280" data-height="720">` +
+ `<audio id="early" src="early.m4a" data-start="1" data-duration="3"></audio>` +
+ `</div></template>`,
+ });
+
+ expect(compiled.audios.map(({ id, start, end }) => ({ id, start, end }))).toEqual([
+ { id: "early", start: 1, end: 4 },
+ ]);
+ expect([...compiled.droppedMediaIds]).toEqual(["early__hf2"]);
+ });
+
+ it("changes nothing for a host without an in-point", () => {
+ const scene = (inPoint: string) =>
+ `<div data-composition-id="inline" data-start="3" data-duration="4"${inPoint}>` +
+ media(`data-start="1" data-duration="9" data-end="10" data-playback-start="2"`) +
+ `</div>` +
+ `<div data-composition-file="scene.html" data-composition-id="scene" data-start="2" data-duration="6"${inPoint}>` +
+ `<video ${MEDIA_RENDER_ID_ATTR}="nested" id="nested" src="n.mp4" data-start="1" data-duration="9" data-has-audio="true"></video>` +
+ `</div>`;
+
+ const absent = collectRenderMedia(scene(""));
+ expect(collectRenderMedia(scene(` data-playback-start="0"`))).toEqual(absent);
+ expect(absent.audios.find((a) => a.id === "nested-audio")).toMatchObject({
+ start: 3,
+ end: 8,
+ mediaStart: 0,
+ });
+ });
+});
diff --git a/packages/producer/src/services/renderMediaCollector.ts b/packages/producer/src/services/renderMediaCollector.ts
index c497280ce2..6414acec40 100644
--- a/packages/producer/src/services/renderMediaCollector.ts
+++ b/packages/producer/src/services/renderMediaCollector.ts
@@ -1,23 +1,25 @@
/**
* Collect the render pipeline's media list from the fully inlined document.
*
- * Sub-composition media used to be gathered from each composition FILE before
- * inlining, then merged with the main document's media and deduplicated by
- * element id. That merge is unsound: ids are unique per file, not per render
- * document, so two scenes that both declare `<video id="clip">` — or that both
- * declare a bare `<video>` and get the per-file auto-id `hf-video-0` — collapse
- * into a single entry. See mediaRenderIds.ts for the full failure.
- *
- * Reading the inlined document instead makes the render document the single
- * source of truth for what media exists: every element is present exactly once,
- * `assignMediaRenderIds` has already given it a document-unique key, and the
- * timeline offsets are recoverable from the composition hosts it sits inside.
+ * Ids are unique per composition file, not per render document (see mediaRenderIds.ts), so media is
+ * read from the inlined document: each element once, keyed by its render id or else its author id,
+ * and placed by the composition hosts it sits inside.
*/
import { parseHTML } from "linkedom";
-import { MEDIA_RENDER_ID_ATTR, resolveAuthoredTimingWindow } from "@hyperframes/core";
+import {
+ MEDIA_RENDER_ID_ATTR,
+ normalizeRateSpec,
+ resolveAuthoredTimingWindow,
+ shiftRateLane,
+ sourceTimeAt,
+} from "@hyperframes/core";
import {
MEDIA_START_BASIS_ATTR,
+ compositionOriginSeconds,
+ cutToHostSlots,
+ hostInPointSeconds,
+ type HostSlot,
readMediaStartBasis,
resolveAbsoluteMediaStartSeconds,
type MediaStartBasis,
@@ -34,13 +36,6 @@ import {
type AudioElement,
} from "@hyperframes/engine";
-/**
- * Marks a host element that `inlineSubCompositions` hoisted a composition into.
- * Set unconditionally on every inlined host, which makes it the reliable signal
- * for "this ancestor shifts its children along the timeline".
- */
-const COMPOSITION_HOST_ATTR = "data-composition-file";
-
/**
* Where a composition host closes in its parent's time, or null when unbounded.
* Delegates to `resolveAuthoredTimingWindow`, the same window the runtime uses to hide descendants.
@@ -62,9 +57,26 @@ interface HostWindow {
limit: number;
/** Whether authored media time is composition-local or legacy root-global. */
basis: MediaStartBasis;
+ /** Root-time slots of the enclosing hosts with an in-point. */
+ slots: HostSlot[];
}
-const ROOT_WINDOW: HostWindow = { offset: 0, limit: Infinity, basis: "local" };
+const ROOT_WINDOW: HostWindow = { offset: 0, limit: Infinity, basis: "local", slots: [] };
+
+// Every composition host shifts what it holds, inline or inlined from a file, as in core's start resolver.
+const COMPOSITION_HOST_SELECTOR =
+ "[data-composition-id], [data-composition-src], [data-composition-file]";
+
+/** Composition hosts that place `element`, root first. */
+function enclosingHosts(element: Element): Element[] {
+ const hosts: Element[] = [];
+ for (let ancestor = element.parentElement; ancestor; ancestor = ancestor.parentElement) {
+ if (ancestor.matches(COMPOSITION_HOST_SELECTOR)) {
+ hosts.unshift(ancestor);
+ }
+ }
+ return hosts;
+}
/**
* Fold a media element's chain of composition hosts into one window.
@@ -82,48 +94,53 @@ function resolveHostWindow(
startCache: Map<RefResolverEl, number>,
visiting: Set<RefResolverEl>,
): HostWindow {
- const hosts: Element[] = [];
- for (let ancestor = element.parentElement; ancestor; ancestor = ancestor.parentElement) {
- if (ancestor.hasAttribute(COMPOSITION_HOST_ATTR)) hosts.push(ancestor);
- }
+ const hosts = enclosingHosts(element);
if (hosts.length === 0) return ROOT_WINDOW;
let offset = 0;
let limit = Infinity;
- // parentElement walks leaf → root; the offsets accumulate root → leaf.
- for (const host of hosts.reverse()) {
+ const slots: HostSlot[] = [];
+ for (const [index, host] of hosts.entries()) {
const hostStart = resolveReferencedStart(document, host, startCache, visiting);
const hostEnd = resolveHostEnd(host, hostStart);
- if (hostEnd != null) limit = Math.min(limit, offset + hostEnd);
- offset += hostStart;
+ if (hostEnd != null && !isRootComposition(host, index)) {
+ limit = Math.min(limit, offset + hostEnd);
+ }
+ if (hostInPointSeconds(host) > 0) {
+ slots.push({ start: offset + hostStart, end: offset + (hostEnd ?? Infinity) });
+ }
+ offset += compositionOriginSeconds(hostStart, host);
}
+ return { offset, limit, basis: mediaStartBasis(element), slots };
+}
+
+// The root composition's end is the render's length, not a cut: a cut would move clip fade-outs.
+const isRootComposition = (host: Element, depth: number): boolean =>
+ depth === 0 && !host.matches("[data-composition-src], [data-composition-file]");
+
+function mediaStartBasis(element: Element): MediaStartBasis {
const tag = element.tagName.toLowerCase();
- const basis =
- tag === "video" || tag === "audio"
- ? readMediaStartBasis(element.getAttribute(MEDIA_START_BASIS_ATTR))
- : "local";
- return {
- offset,
- limit,
- basis,
- };
+ return tag === "video" || tag === "audio"
+ ? readMediaStartBasis(element.getAttribute(MEDIA_START_BASIS_ATTR))
+ : "local";
}
/**
- * Map each render id to the window of the composition hosts it is nested in.
- * Keyed on the render id rather than document position so the caller never has
- * to assume two separate parses walk the document in the same order.
+ * Map each media id the parsers report (render id, else author id: render ids are only stamped
+ * when files are inlined) to its hosts' window, keyed on id so two parses need not walk in step.
*/
function collectHostWindows(html: string): Map<string, HostWindow> {
const { document } = parseHTML(html);
const windows = new Map<string, HostWindow>();
const startCache = new Map<RefResolverEl, number>();
const visiting = new Set<RefResolverEl>();
- for (const element of document.querySelectorAll(`[${MEDIA_RENDER_ID_ATTR}]`)) {
+ // What the parsers report; an author id never displaces a render id, or another author id.
+ for (const element of document.querySelectorAll("video, audio, img[src]")) {
const renderId = element.getAttribute(MEDIA_RENDER_ID_ATTR);
- if (!renderId) continue;
+ const id = renderId || element.getAttribute("id");
+ if (!id || (!renderId && windows.has(id))) continue;
windows.set(
- renderId,
+ id,
resolveHostWindow(element as unknown as Element, document, startCache, visiting),
);
}
@@ -131,15 +148,15 @@ function collectHostWindows(html: string): Map<string, HostWindow> {
}
/**
- * Shift a scene-relative window onto the root timeline.
- * Returns null when the clip starts after its host has already ended, matching
- * the `start < absoluteEnd` drop the file-tree walk applied.
+ * Shift a scene-relative window onto the root timeline; null when the clip starts after its host
+ * ends (the file-tree walk's `start < absoluteEnd` drop) or its in-point hosts cut all of it.
+ * `start` stays the clip's origin; `played` is the part the in-point hosts' slots keep.
*/
function toAbsoluteWindow(
start: number,
end: number,
window: HostWindow,
-): { start: number; end: number } | null {
+): { start: number; end: number; played: { start: number; end: number } } | null {
const absoluteStart = resolveAbsoluteMediaStartSeconds({
authoredStart: start,
hostStart: window.offset,
@@ -151,13 +168,37 @@ function toAbsoluteWindow(
hostStart: window.offset,
basis: window.basis,
});
- return { start: absoluteStart, end: Math.min(absoluteEnd, window.limit) };
+ const absolute = { start: absoluteStart, end: Math.min(absoluteEnd, window.limit) };
+ const played = cutToHostSlots(absolute, window.slots);
+ if (played.end === played.start && absolute.end > absolute.start) return null;
+ return { start: absolute.start, end: played.end, played };
+}
+
+/** Start a clip where its in-point hosts let it play, that much further into its source. */
+function startAtPlayed<T extends AudioElement | VideoElement>(
+ clip: T,
+ played: { start: number },
+): T {
+ const head = played.start - clip.start;
+ if (head <= 0) return clip;
+ // ponytail: a loop runs from mediaStart, so it only moves to its slot and keeps main's phase error;
+ // an exact cut needs a visible start separate from the loop origin in the engine.
+ if ("loop" in clip && clip.loop) return { ...clip, start: played.start };
+ const rate = normalizeRateSpec(clip.playbackRate);
+ return {
+ ...clip,
+ start: played.start,
+ mediaStart: clip.mediaStart + sourceTimeAt(rate, head),
+ playbackRate: shiftRateLane(rate, head),
+ };
}
export interface RenderMedia {
videos: VideoElement[];
audios: AudioElement[];
images: ImageElement[];
+ /** Ids of clips in the document that never play; the browser probe must not add them back. */
+ droppedMediaIds: Set<string>;
}
/**
@@ -171,17 +212,25 @@ export interface RenderMedia {
export function collectRenderMedia(html: string): RenderMedia {
const windows = collectHostWindows(html);
const windowFor = (id: string): HostWindow => windows.get(id) ?? ROOT_WINDOW;
+ const droppedMediaIds = new Set<string>();
const videos: VideoElement[] = [];
for (const video of parseVideoElements(html)) {
const absolute = toAbsoluteWindow(video.start, video.end, windowFor(video.id));
- if (absolute) videos.push({ ...video, ...absolute });
+ if (!absolute) {
+ droppedMediaIds.add(video.id);
+ continue;
+ }
+ videos.push(
+ startAtPlayed({ ...video, start: absolute.start, end: absolute.end }, absolute.played),
+ );
}
const images: ImageElement[] = [];
for (const image of parseImageElements(html)) {
const absolute = toAbsoluteWindow(image.start, image.end, windowFor(image.id));
- if (absolute) images.push({ ...image, ...absolute });
+ if (absolute) images.push({ ...image, start: absolute.start, end: absolute.end });
+ else droppedMediaIds.add(image.id);
}
// A <video data-has-audio> track is reported as "<renderId>-audio"; strip the
@@ -189,17 +238,20 @@ export function collectRenderMedia(html: string): RenderMedia {
const audios: AudioElement[] = [];
for (const audio of parseAudioElements(html)) {
const elementId = audio.type === "video" ? audio.id.replace(/-audio$/, "") : audio.id;
- // The mixer reads end === 0 as "run to the natural media length", so an
- // unbounded track must stay unbounded rather than collapse onto its start.
- const authoredEnd = audio.end > 0 ? audio.end : Infinity;
- const absolute = toAbsoluteWindow(audio.start, authoredEnd, windowFor(elementId));
- if (!absolute) continue;
- audios.push({
- ...audio,
- start: absolute.start,
- end: Number.isFinite(absolute.end) ? absolute.end : 0,
- });
+ const placed = placeAudio(audio, windowFor(elementId));
+ if (placed) audios.push(placed);
+ else droppedMediaIds.add(elementId);
}
- return { videos, audios, images };
+ return { videos, audios, images, droppedMediaIds };
+}
+
+function placeAudio(audio: AudioElement, window: HostWindow): AudioElement | null {
+ // The mixer reads end === 0 as "run to the natural media length", so an
+ // unbounded track must stay unbounded rather than collapse onto its start.
+ const authoredEnd = audio.end > 0 ? audio.end : Infinity;
+ const absolute = toAbsoluteWindow(audio.start, authoredEnd, window);
+ if (!absolute) return null;
+ const end = Number.isFinite(absolute.end) ? absolute.end : 0;
+ return startAtPlayed({ ...audio, start: absolute.start, end }, absolute.played);
}