Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): a split sub-composition plays its videos from its in-point in preview and timeline (#5413)
Miguel Ángel480807ec4be4
diff --git a/packages/cli/src/timeline/describeProject.inPoint.test.ts b/packages/cli/src/timeline/describeProject.inPoint.test.tsnew file mode 100644index 0000000000..2fde62feee--- /dev/null+++ b/packages/cli/src/timeline/describeProject.inPoint.test.ts@@ -0,0 +1,61 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { beforeAll, describe, expect, it, onTestFinished } from "vitest";+import { ensureDOMParser } from "../utils/dom.js";+import { describeProject } from "./describeProject.js";++// What Studio's split writes when a 12 s scene clip is cut at 5 s and then at 9 s.+const SPLIT_INDEX = `<!DOCTYPE html><html><head></head><body><div data-composition-id="main" data-duration="12">+<div data-playback-start="0" id="scene" class="clip" data-composition-id="scene" data-composition-src="compositions/scene.html" data-start="0" data-duration="5" data-track-index="0"></div><div data-playback-start="5" id="scene-split" class="clip" data-composition-id="scene-split" data-composition-src="compositions/scene.html" data-start="5" data-duration="4" data-track-index="0"></div><div data-playback-start="9" id="scene-split-split" class="clip" data-composition-id="scene-split-split" data-composition-src="compositions/scene.html" data-start="9" data-duration="3" data-track-index="0"></div>+</div></body></html>`;+const SCENE = `<template id="scene-template"><div data-composition-id="scene" data-duration="12">+<video id="v1" class="clip" src="../sample.mp4" muted data-start="0" data-duration="4" data-media-start="1" data-track-index="0"></video>+<video id="v2" class="clip" src="../sample.mp4" muted data-start="4" data-duration="4" data-media-start="5" data-track-index="0"></video>+<video id="v3" class="clip" src="../sample.mp4" muted data-start="8" data-duration="4" data-media-start="2" data-track-index="0"></video>+</div></template>`;++beforeAll(ensureDOMParser);++describe("a sub-composition's in-point", () => {+ it("shifts each later split half's videos by its in-point and cuts them to that half", async () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-timeline-inpoint-"));+ onTestFinished(() => rmSync(dir, { recursive: true, force: true }));+ mkdirSync(join(dir, "compositions"));+ writeFileSync(join(dir, "index.html"), SPLIT_INDEX);+ writeFileSync(join(dir, "compositions", "scene.html"), SCENE);++ const timeline = await describeProject(join(dir, "index.html"));+ const videos = timeline.tracks.find((t) => t.kind === "video")!.rows;++ // The first half's in-point is 0, so it shows its children's authored extent, as before.+ expect(videos.map((r) => [r.host, r.id, r.absStart, r.absEnd])).toEqual([+ ["scene", "v1", 0, 4],+ ["scene", "v2", 4, 8],+ ["scene-split", "v2", 5, 8],+ ["scene", "v3", 8, 12],+ ["scene-split", "v3", 8, 9],+ ["scene-split-split", "v3", 9, 12],+ ]);+ });++ it("cuts nothing for a host whose in-point is 0, as Studio stamps on every inserted scene", async () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-timeline-inpoint0-"));+ onTestFinished(() => rmSync(dir, { recursive: true, force: true }));+ mkdirSync(join(dir, "compositions"));+ writeFileSync(+ join(dir, "index.html"),+ `<div data-composition-id="main" data-duration="12"><div id="scene" data-composition-src="compositions/scene.html" data-start="1" data-duration="3" data-playback-start="0"></div></div>`,+ );+ writeFileSync(join(dir, "compositions", "scene.html"), SCENE);++ const timeline = await describeProject(join(dir, "index.html"));+ const videos = timeline.tracks.find((t) => t.kind === "video")!.rows;++ expect(videos.map((r) => [r.id, r.absStart, r.absEnd])).toEqual([+ ["v1", 1, 5],+ ["v2", 5, 9],+ ["v3", 9, 13],+ ]);+ });+});diff --git a/packages/cli/src/timeline/describeProject.ts b/packages/cli/src/timeline/describeProject.tsindex 1475a6e7b9..af41810915 100644--- a/packages/cli/src/timeline/describeProject.ts+++ b/packages/cli/src/timeline/describeProject.ts@@ -24,7 +24,13 @@ import {type StructureNode,type TrackKind,} from "@hyperframes/parsers";-import { resolveMediaStartSeconds } from "@hyperframes/core/media-timing";+import {+ compositionOriginSeconds,+ cutToHostSlots,+ hostInPointSeconds,+ resolveMediaStartSeconds,+ type HostSlot,+} from "@hyperframes/core/media-timing";import {extractAudioMetadata,extractMediaMetadata,@@ -47,7 +53,8 @@ export interface TimelineRow extends ClipFact {pendingReason: string | null;/** Why `data-automation` / `data-fx-chain` could not be read; `null` when fine or absent. */laneError: string | null;- /** Start and end on the main timeline, in seconds. `start`/`end` are local to the owning file's composition. */+ /** Main-timeline window in seconds: the played part under an in-point host, while `duration` stays+ * the authored length. `start`/`end` are local to the owning file's composition. */absStart: number;absEnd: number;/** Project-relative path of the file that declares this clip. */@@ -133,7 +140,7 @@ interface DocScope {startCache: Map<Element, number>;/** Sub-composition files must stay inside the project. */projectDir: string;- /** Main-timeline start of this document's root: 0 for index.html, the host's start for a sub-composition. */+ /** Main-timeline time of this document's local t=0: 0 for index.html, see `compositionOriginSeconds`. */origin: number;/** Project-relative path of this document, with `/` separators. */file: string;@@ -277,7 +284,12 @@ async function resolveRowTiming(scope: DocScope, node: DomNode, depth: number):const authored = resolveReferencedDuration(scope.doc, el, scope.startCache, new Set());const host = el.getAttribute("data-composition-src");const absStart = mainTimelineStart(scope, el, start);- const children = host && depth === 0 ? await readSubComposition(host, scope, absStart) : [];+ const slots: HostSlot[] =+ hostInPointSeconds(el) > 0 ? [{ start: absStart, end: absStart + (authored ?? Infinity) }] : [];+ const children =+ host && depth === 0+ ? await readSubComposition(host, scope, compositionOriginSeconds(absStart, el), slots)+ : [];const kind = el.tagName.toLowerCase();const duration = MEDIA_TAG.test(kind)? await resolveMediaRowDuration(scope, el, kind as MediaTag, authored)@@ -334,6 +346,7 @@ async function readSubComposition(src: string,parent: DocScope,origin: number,+ slots: readonly HostSlot[],): Promise<ClipDraft[]> {const authored = resolve(parent.dir, src);const file = realFileInside(parent.projectDir, authored);@@ -359,7 +372,13 @@ async function readSubComposition(const rows = await Promise.all(topLevelElements(toNode(root)).map((node) => describeRow(scope, node, 1)),);- return rows.sort(byStart);+ return rows.flatMap((row) => cutRow(row, slots)).sort(byStart);+}++function cutRow(row: ClipDraft, slots: readonly HostSlot[]): ClipDraft[] {+ const kept = cutToHostSlots({ start: row.absStart, end: row.absEnd }, slots);+ if (kept.end === kept.start && row.absEnd > row.absStart) return [];+ return [{ ...row, absStart: roundMs(kept.start), absEnd: roundMs(kept.end) }];}/** The file's real path when it is a regular file inside the project (symlinks resolved), else null. */diff --git a/packages/core/src/mediaTiming.ts b/packages/core/src/mediaTiming.tsindex db129fe170..9061828632 100644--- a/packages/core/src/mediaTiming.ts+++ b/packages/core/src/mediaTiming.ts@@ -1,3 +1,5 @@+import { readMediaOffsetSeconds } from "@hyperframes/parsers/media-duration";+export const MEDIA_START_BASIS_ATTR = "data-hf-media-start-basis";export type MediaStartBasis = "local" | "global";@@ -20,8 +22,8 @@ export function resolveAbsoluteMediaStartSeconds(input: {}/** The one rule for a media element's root-timeline start; the runtime and the CLI both call it.- * With no literal start, an auto-injected start, or a host at t<=0 there is nothing for the basis- * to disambiguate, so the ordinary start resolution applies. */+ * `hostStart` is the composition's origin. With no literal start, an auto-injected start, or an+ * origin at 0 there is nothing for the basis to disambiguate, so ordinary resolution applies. */export function resolveMediaStartSeconds(input: MediaStartInput & { ordinaryStart: () => number },): number {@@ -48,5 +50,34 @@ export interface MediaStartInput {function hasLiteralNestedStart(input: MediaStartInput,): input is MediaStartInput & { authoredStart: number } {- return !input.hasAutoStart && input.authoredStart != null && input.hostStart > 0;+ return !input.hasAutoStart && input.authoredStart != null && input.hostStart !== 0;+}++/** A composition host's in-point; only a host with one shifts or cuts what it holds. */+export function hostInPointSeconds(host: Pick<Element, "getAttribute">): number {+ return readMediaOffsetSeconds((name) => host.getAttribute(name));+}++/** Main-timeline time of a sub-composition's local t=0: its host's start minus its in-point. */+export function compositionOriginSeconds(+ hostStart: number,+ host: Pick<Element, "getAttribute">,+): number {+ return hostStart - hostInPointSeconds(host);+}++export interface HostSlot {+ start: number;+ end: number;+}++/** Each in-point host cuts the span to its own window. */+export function cutToHostSlots(+ span: { start: number; end: number },+ hosts: readonly HostSlot[],+): { start: number; end: number } {+ return hosts.reduce((kept, host) => {+ const start = Math.max(kept.start, host.start);+ return { start, end: Math.max(start, Math.min(kept.end, host.end)) };+ }, span);}diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.tsindex ec3d5b5d60..6eeae7d7d2 100644--- a/packages/core/src/runtime/audioClockSource.test.ts+++ b/packages/core/src/runtime/audioClockSource.test.ts@@ -141,6 +141,22 @@ afterEach(() => {});describe("the audio the playhead follows", () => {+ it("does not hold the playhead on a clip an in-point host has not started yet", async () => {+ mount(+ `<div id="half" class="clip" data-composition-id="half" data-composition-file="compositions/scene.html"` ++ ` data-start="5" data-duration="5" data-playback-start="5">` ++ `<div><audio id="a" data-start="2" data-duration="8" src="/assets/a.mp3"></audio></div></div>`,+ );+ const audio = document.getElementById("a") as HTMLAudioElement;+ Object.defineProperty(audio, "readyState", { value: 0, writable: true, configurable: true });+ initSandboxRuntimeModular();+ await flush();+ window.__player?.play();+ await flush();+ stepFrames(240);+ expect(window.__player!.getTime()).toBeGreaterThan(3.5);+ });+it("keeps following a playing voice when a clip earlier in the page starts late", async () => {mount(`<audio id="sfx" data-start="1" data-duration="2" src="/assets/sfx.mp3"></audio>` +diff --git a/packages/core/src/runtime/init.noInPointParity.test.ts b/packages/core/src/runtime/init.noInPointParity.test.tsnew file mode 100644index 0000000000..31786bb18e--- /dev/null+++ b/packages/core/src/runtime/init.noInPointParity.test.ts@@ -0,0 +1,475 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { initSandboxRuntimeModular } from "./init";+import { WebAudioTransport } from "./webAudioTransport";+import type { RuntimeTimelineLike } from "./types";+import {+ createMockTimeline,+ resetRuntimeFixtureDom,+ stubDuration,+} from "./runtimeSeekFixture.test-helpers";++// Guard: a project with no non-zero in-point plays exactly as before in-points existed. Every+// expected value below was recorded by running these fixtures against the pre-in-point runtime.++const ROOT_12 = `data-composition-id="main" data-root="true" data-duration="12"`;+const host = (attrs: string, inner: string) =>+ `<div class="clip" data-composition-file="compositions/scene.html" ${attrs}>${inner}</div>`;++interface Fixture {+ name: string;+ html: string;+ timelines?: Record<string, number>;+ sources?: Record<string, number>;+}++const FIXTURES: Fixture[] = [+ {+ name: "one level",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="2" data-duration="6"`, `<div><video id="v" data-start="1" data-duration="3" data-media-start="0.5"></video></div>`)}</div>`,+ },+ {+ name: "two levels",+ html: `<div ${ROOT_12}>${host(`data-composition-id="o" data-start="1" data-duration="8"`, host(`data-composition-id="i" data-start="1" data-duration="5"`, `<video id="v" data-start="0.5" data-duration="6"></video>`))}</div>`,+ },+ {+ name: "three levels",+ html: `<div ${ROOT_12}>${host(`data-composition-id="a" data-start="1" data-duration="10"`, host(`data-composition-id="b" data-start="1" data-duration="5"`, host(`data-composition-id="c" data-start="1"`, `<video id="v" data-start="0" data-duration="9"></video>`)))}</div>`,+ },+ {+ name: "root length from its own timeline",+ html: `<div data-composition-id="main" data-root="true">${host(`data-composition-id="s" data-start="0" data-duration="10"`, `<video id="v" data-start="0" data-duration="10"></video>`)}</div>`,+ timelines: { main: 2 },+ },+ {+ name: "host with data-end",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="1" data-end="4"`, `<video id="v" data-start="0" data-duration="6"></video>`)}</div>`,+ },+ {+ name: "relative start",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="2" data-duration="8"`, `<div id="x" data-start="0" data-duration="2"></div><video id="v" data-start="x + 1" data-duration="3"></video>`)}</div>`,+ },+ {+ name: "anonymous host",+ html: `<div ${ROOT_12}>${host(`data-start="1" data-duration="5" data-playback-start="0"`, `<div data-composition-id="scene" data-duration="12"><video id="v" data-start="4" data-duration="4"></video></div>`)}</div>`,+ },+ {+ name: "host under a different id",+ html: `<div ${ROOT_12}>${host(`id="montage" data-composition-id="montage" data-start="3" data-duration="5"`, `<div data-composition-id="scene-10"><video id="v" data-start="1" data-duration="6"></video></div>`)}</div>`,+ },+ {+ name: "root-time media",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="3" data-duration="6"`, `<video id="v" data-start="4" data-duration="3" data-hf-media-start-basis="global"></video>`)}</div>`,+ },+ {+ name: "looping clip",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="1" data-duration="10"`, `<video id="v" loop data-start="1" data-duration="8"></video>`)}</div>`,+ sources: { v: 3 },+ },+ {+ name: "faded clip",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="1" data-duration="10"`, `<video id="v" data-start="1" data-duration="6" data-fade-in="1" data-fade-out="2"></video>`)}</div>`,+ },+ {+ name: "audio, same and cross origin",+ html: `<div ${ROOT_12}>${host(`data-composition-id="s" data-start="2" data-duration="4"`, `<audio id="same" src="a.mp3" data-start="1" data-duration="5"></audio><audio id="cross" src="https://cdn.example.com/a.mp3" data-start="0.5"></audio>`)}</div>`,+ },+ {+ name: "scene trimmed shorter than its file",+ html: `<div ${ROOT_12}>${host(`id="scene" data-composition-id="scene" data-start="2" data-duration="3" data-playback-start="0"`, `<div data-composition-id="scene" data-duration="6"><video id="v" data-start="0"></video><audio id="cross" src="https://cdn.example.com/a.mp3" data-start="0"></audio></div>`)}</div>`,+ timelines: { scene: 6 },+ },+];++const SAMPLES = [0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5, 10.5, 11.5];+const round = (value: number | undefined) =>+ value != null && Number.isFinite(value) ? Math.round(value * 1e6) / 1e6 : String(value);++function mount(fixture: Fixture) {+ document.body.innerHTML = fixture.html;+ for (const media of document.querySelectorAll<HTMLMediaElement>("video, audio")) {+ stubDuration(media, fixture.sources?.[media.id] ?? 30);+ Object.defineProperty(media, "currentTime", { value: -1, writable: true, configurable: true });+ media.load = () => {};+ media.play = vi.fn(() => Promise.resolve());+ media.pause = vi.fn();+ }+ window.__timelines = Object.fromEntries(+ Object.entries({ main: 12, ...fixture.timelines }).map(([id, seconds]) => [+ id,+ createMockTimeline(seconds),+ ]),+ ) as Record<string, RuntimeTimelineLike>;+}++/** Start, then source time and volume at each sample, then what play() hands the audio transport. */+async function record(fixture: Fixture) {+ mount(fixture);+ vi.spyOn(console, "info").mockImplementation(() => {});+ vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue({} as AudioBuffer);+ const capture = vi+ .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")+ .mockResolvedValue(null);+ const decoded = vi.spyOn(WebAudioTransport.prototype, "schedulePlayback").mockResolvedValue(null);+ initSandboxRuntimeModular();+ // The audio transport comes up on a microtask, as in startPlayback (init.test.ts).+ await Promise.resolve();+ const media = [...document.querySelectorAll<HTMLMediaElement>("video, audio")];+ const trace: Record<string, unknown[]> = {};+ for (const el of media) trace[el.id] = [round(window.__hfResolveMediaStartSeconds?.(el))];+ for (const t of SAMPLES) {+ window.__player?.renderSeek(t);+ for (const el of media) trace[el.id]!.push([round(el.currentTime), round(el.volume)]);+ }+ window.__player?.renderSeek(0);+ window.__player?.play();+ const audio = media.filter((el) => el.tagName === "AUDIO");+ if (audio.length > 0) await vi.waitFor(() => expect(decoded).toHaveBeenCalledTimes(audio.length));+ return {+ trace,+ capture: capture.mock.calls.map(([el, start, mediaStart]) => [el.id, round(start), mediaStart]),+ decoded: decoded.mock.calls.map(([el, , start, mediaStart, , , , rate, clipDuration]) => [+ el.id,+ round(start),+ mediaStart,+ rate,+ round(clipDuration),+ ]),+ };+}++const EXPECTED: Record<string, unknown> = {+ "one level": {+ trace: {+ v: [+ 3,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [1, 1],+ [2, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "two levels": {+ trace: {+ v: [+ 2.5,+ [-1, 1],+ [-1, 1],+ [0, 1],+ [1, 1],+ [2, 1],+ [3, 1],+ [4, 1],+ [4, 1],+ [4, 1],+ [4, 1],+ [4, 1],+ [4, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "three levels": {+ trace: {+ v: [+ 3,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 1],+ [5.5, 1],+ [6.5, 1],+ [7.5, 1],+ [8.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "root length from its own timeline": {+ trace: {+ v: [+ 0,+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 1],+ [5.5, 1],+ [6.5, 1],+ [7.5, 1],+ [8.5, 1],+ [9.5, 1],+ [10, 1],+ [10, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "host with data-end": {+ trace: {+ v: [+ 1,+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "relative start": {+ trace: {+ v: [+ 5,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "anonymous host": {+ trace: {+ v: [+ 5,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [3.5, 1],+ [3.5, 1],+ [3.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "host under a different id": {+ trace: {+ v: [+ 4,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "root-time media": {+ trace: {+ v: [+ 4,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "looping clip": {+ trace: {+ v: [+ 2,+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [0.5, 1],+ [1.5, 1],+ [1.5, 1],+ [1.5, 1],+ ],+ },+ capture: [],+ decoded: [],+ },+ "faded clip": {+ trace: {+ v: [+ 2,+ [-1, 1],+ [-1, 1],+ [0.5, 0.5],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 0.75],+ [5.5, 0.25],+ [5.5, 0.25],+ [5.5, 0.25],+ [5.5, 0.25],+ [5.5, 0.25],+ ],+ },+ capture: [],+ decoded: [],+ },+ "audio, same and cross origin": {+ trace: {+ same: [+ 3,+ [-1, 1],+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ [2.5, 1],+ ],+ cross: [+ 2.5,+ [-1, 1],+ [-1, 1],+ [0, 1],+ [1, 1],+ [2, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ [3, 1],+ ],+ },+ capture: [["same", 3, 0]],+ decoded: [+ ["same", 3, 0, 1, 3],+ ["cross", 2.5, 0, 1, 3.5],+ ],+ },+ "scene trimmed shorter than its file": {+ trace: {+ v: [+ 2,+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ ],+ cross: [+ 2,+ [-1, 1],+ [-1, 1],+ [0.5, 1],+ [1.5, 1],+ [2.5, 1],+ [3.5, 1],+ [4.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ [5.5, 1],+ ],+ },+ capture: [],+ decoded: [["cross", 2, 0, 1, 6]],+ },+};++describe("a project with no in-point", () => {+ const originalAudioContext = (globalThis as Record<string, unknown>).AudioContext;+ beforeEach(() => {+ resetRuntimeFixtureDom();+ (globalThis as Record<string, unknown>).AudioContext = class {+ state = "running";+ destination = {};+ currentTime = 0;+ resume = () => Promise.resolve();+ suspend = () => Promise.resolve();+ createGain = () => ({ gain: { value: 1 }, connect() {}, disconnect() {} });+ createMediaElementSource = () => ({ connect() {}, disconnect() {} });+ };+ });+ afterEach(() => {+ window.__hfRuntimeTeardown?.();+ vi.restoreAllMocks();+ (globalThis as Record<string, unknown>).AudioContext = originalAudioContext;+ delete window.__player;+ delete window.__playerReady;+ delete window.__hf;+ });++ it.each(FIXTURES)("plays $name as before in-points", async (fixture) => {+ expect(await record(fixture)).toEqual(EXPECTED[fixture.name]);+ });+});diff --git a/packages/core/src/runtime/init.subCompositionPlacement.test.ts b/packages/core/src/runtime/init.subCompositionPlacement.test.tsindex a5fc0bac14..c1d0d05f88 100644--- a/packages/core/src/runtime/init.subCompositionPlacement.test.ts+++ b/packages/core/src/runtime/init.subCompositionPlacement.test.ts@@ -3,7 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { initSandboxRuntimeModular } from "./init";import type { RuntimeTimelineLike } from "./types";import { STUDIO_MANUAL_EDIT_GESTURE_ATTR } from "../editing/draftMarkers";-import { resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";+import { sourceTimeAt } from "../speedRamp";+import { readElementRateSpec } from "./playbackRate";+import {+ createMockTimeline,+ resetRuntimeFixtureDom,+ stubDuration,+} from "./runtimeSeekFixture.test-helpers";// Pins where a running runtime places a sub-composition's timeline, against what a fresh load does.describe("runtime sub-composition placement", () => {@@ -225,4 +231,171 @@ describe("runtime sub-composition placement", () => {expect(remove).not.toHaveBeenCalledWith(scene);});++ /** What Studio's split writes for a 12 s scene cut at 5 s and 9 s, each half mounting its own copy. */+ function loadSplitScene() {+ const videos = [+ ["v1", 0, 1],+ ["v2", 4, 5],+ ["v3", 8, 2],+ ]+ .map(+ ([name, start, mediaStart]) =>+ `<video class="${name}" data-start="${start}" data-duration="4" data-media-start="${mediaStart}"></video>`,+ )+ .join("");+ const hosts = [+ ["scene", 0, 5, 0],+ ["scene-split", 5, 4, 5],+ ["scene-split-split", 9, 3, 9],+ ]+ .map(+ ([id, start, duration, inPoint]) =>+ `<div id="${id}" class="clip" data-composition-id="${id}" data-composition-file="compositions/scene.html" ` ++ `data-start="${start}" data-duration="${duration}" data-playback-start="${inPoint}"><div>${videos}</div></div>`,+ )+ .join("");+ mountInRoot(hosts);+ }++ /** A 12 s root around `markup`; every video's source runs `sourceSeconds` and has not been seeked (-1). */+ function mountInRoot(markup: string, sourceSeconds = 30) {+ document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-duration="12">${markup}</div>`;+ for (const video of document.querySelectorAll("video")) {+ stubDuration(video, sourceSeconds);+ Object.defineProperty(video, "currentTime", {+ value: -1,+ writable: true,+ configurable: true,+ });+ video.play = vi.fn(() => Promise.resolve());+ video.pause = vi.fn();+ }+ window.__timelines = { main: createMockTimeline(12) };+ initSandboxRuntimeModular();+ }++ /** The half after a split at 5 s: it starts at 5 with in-point 5, so scene time t plays at t. */+ function loadSecondHalf(inner: string, sourceSeconds?: number) {+ mountInRoot(+ `<div class="clip" data-composition-id="half" data-composition-file="compositions/scene.html" ` ++ `data-start="5" data-duration="7" data-playback-start="5"><div>${inner}</div></div>`,+ sourceSeconds,+ );+ }++ const video = (id: string) => document.getElementById(id) as HTMLVideoElement;++ const sourceTimeOf = (host: string, video: string) =>+ document.querySelector<HTMLVideoElement>(`#${host} .${video}`)!.currentTime;++ it("shifts a split scene's videos by each half's in-point and cuts them to that half", () => {+ loadSplitScene();+ window.__player?.renderSeek(4.5);+ expect(sourceTimeOf("scene", "v2")).toBe(5.5);+ // The second half's v2 lands at 4-8 but is cut until that half starts at 5.+ expect(sourceTimeOf("scene-split", "v2")).toBe(-1);++ window.__player?.renderSeek(5);+ expect(sourceTimeOf("scene-split", "v2")).toBe(6);+ window.__player?.renderSeek(8.5);+ expect(sourceTimeOf("scene-split", "v3")).toBe(2.5);+ window.__player?.renderSeek(10);+ expect(sourceTimeOf("scene-split-split", "v3")).toBe(4);+ });++ it("lets a scene's video play past a shorter root animation when the root authors no length", () => {+ document.body.innerHTML =+ `<div data-composition-id="main" data-root="true">` ++ `<div class="clip" data-composition-id="scene" data-composition-file="compositions/scene.html" data-start="0" data-duration="10">` ++ `<video id="long" data-start="0" data-duration="10"></video></div></div>`;+ stubDuration(video("long"), 10);+ Object.defineProperty(video("long"), "currentTime", { value: -1, writable: true });+ video("long").play = vi.fn(() => Promise.resolve());+ video("long").pause = vi.fn();+ // The root's own timeline is a 2 s title fade.+ window.__timelines = { main: createMockTimeline(2) };+ initSandboxRuntimeModular();++ const at = (t: number) => (window.__player?.renderSeek(t), video("long").currentTime);+ expect([at(1), at(5), at(8)]).toEqual([1, 5, 8]);+ });++ it("keeps a scene trimmed to 3 s driving its 6 s file past the trim, as before in-points", () => {+ mountInRoot(+ `<div id="scene" class="clip" data-composition-id="scene" data-composition-file="compositions/scene.html" ` ++ `data-start="2" data-duration="3" data-playback-start="0">` ++ `<div data-composition-id="scene" data-duration="6"><video id="body" data-start="0"></video></div></div>`,+ );+ window.__player?.renderSeek(6);+ expect(video("body").currentTime).toBe(4);+ });++ it("cuts a split half that has no id at its own end, not its file's", () => {+ // Studio's split of a host without an id: the half is found by its composition attributes.+ mountInRoot(+ `<div class="clip" data-composition-file="compositions/scene.html" data-start="5" data-duration="4" ` ++ `data-playback-start="5"><div data-composition-id="scene" data-duration="12">` ++ `<video id="late" data-start="8" data-duration="4"></video></div></div>`,+ );+ window.__player?.renderSeek(8.5);+ expect(video("late").currentTime).toBe(0.5);+ window.__player?.renderSeek(10);+ expect(video("late").currentTime).toBe(0.5);+ });++ it("keeps a video in a composition nested inside the half silent until the half starts", () => {+ loadSecondHalf(+ `<div data-composition-id="inner" data-composition-file="compositions/inner.html" data-start="0" data-duration="12">` ++ `<video id="deep" data-start="0" data-duration="12"></video></div>`,+ );+ window.__player?.renderSeek(1);+ expect(video("deep").currentTime).toBe(-1);+ window.__player?.renderSeek(6);+ expect(video("deep").currentTime).toBe(6);+ });++ it("loops a straddling clip over its whole file, not from the cut", () => {+ loadSecondHalf(+ `<video id="loop" loop data-start="2" data-duration="10" data-media-start="0"></video>`,+ 4,+ );+ const at = (t: number) => (window.__player?.renderSeek(t), video("loop").currentTime);+ expect([at(5), at(6), at(7.5)]).toEqual([3, 0, 1.5]);+ });++ it("follows a straddling clip's speed ramp from the clip's own start", () => {+ const ramp = JSON.stringify({+ version: 1,+ lanes: [+ {+ target: "rate",+ points: [+ { t: 0, v: 1 },+ { t: 8, v: 2 },+ ],+ },+ ],+ });+ loadSecondHalf(+ `<video id="ramp" data-start="2" data-duration="8" data-automation='${ramp}'></video>`,+ );+ window.__player?.renderSeek(7);+ expect(video("ramp").currentTime).toBeCloseTo(+ sourceTimeAt(readElementRateSpec(video("ramp")), 5),+ 6,+ );+ });++ it("times a straddling clip's fade-out from the clip's own end", () => {+ loadSecondHalf(`<video id="fade" data-start="4" data-duration="4" data-fade-out="2"></video>`);+ window.__player?.renderSeek(7);+ expect(video("fade").volume).toBeCloseTo(0.5, 6);+ });++ it("times a straddling clip's fade-in from the clip's own start", () => {+ loadSecondHalf(`<video id="fade" data-start="4" data-duration="4" data-fade-in="2"></video>`);+ window.__player?.renderSeek(5);+ expect(video("fade").volume).toBeCloseTo(0.5, 6);+ });});diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.tsindex 7d4fef3648..5cf42e2fb3 100644--- a/packages/core/src/runtime/init.test.ts+++ b/packages/core/src/runtime/init.test.ts@@ -5347,6 +5347,92 @@ describe("initSandboxRuntimeModular", () => {expect(decodeSpy).toHaveBeenCalledWith(audio);});+ it("holds a decoded clip a split half cut until that half starts, timed from the clip's own start", async () => {+ const audio = mountAudio("https://cdn.example.com/track.mp3", {+ "data-start": "2",+ "data-duration": "4",+ });+ // A half at 2 s with in-point 3 puts the clip's own t=0 at 1 s; the half cuts it to 2-5 s.+ const half = document.createElement("div");+ for (const [name, value] of Object.entries({+ "data-composition-id": "half",+ "data-composition-file": "half.html",+ "data-start": "2",+ "data-duration": "4",+ "data-playback-start": "3",+ }))+ half.setAttribute(name, value);+ audio.replaceWith(half);+ half.appendChild(audio);+ vi.spyOn(console, "info").mockImplementation(() => {});+ const buffer = {} as AudioBuffer;+ vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(buffer);+ const schedule = vi+ .spyOn(WebAudioTransport.prototype, "schedulePlayback")+ .mockResolvedValue(null);++ await startPlayback();++ await vi.waitFor(() => expect(schedule).toHaveBeenCalled());+ const [el, decoded, clipStart, mediaStart, , , , , clipDuration, windowStart] =+ schedule.mock.calls[0]!;+ expect([el, decoded, clipStart, mediaStart, clipDuration, windowStart]).toEqual([+ audio,+ buffer,+ 1,+ 0,+ 4,+ 2,+ ]);+ });++ it("stops a decoded clip at the end of a scene whose length comes from its animation", async () => {+ const audio = mountAudio("https://cdn.example.com/track.mp3", { "data-start": "1" });+ audio.removeAttribute("data-duration");+ const scene = document.createElement("div");+ scene.setAttribute("data-composition-id", "scene");+ scene.setAttribute("data-composition-file", "scene.html");+ scene.setAttribute("data-start", "0");+ audio.replaceWith(scene);+ scene.appendChild(audio);+ window.__timelines = { main: createMockTimeline(10), scene: createMockTimeline(3) };+ vi.spyOn(console, "info").mockImplementation(() => {});+ vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(+ {} as AudioBuffer,+ );+ const schedule = vi+ .spyOn(WebAudioTransport.prototype, "schedulePlayback")+ .mockResolvedValue(null);++ await startPlayback();++ await vi.waitFor(() => expect(schedule).toHaveBeenCalled());+ expect(schedule.mock.calls[0]![8]).toBe(2);+ });++ it("never routes a clip that an in-point host cuts away entirely", async () => {+ const audio = mountAudio("/assets/early.mp3", { "data-start": "0", "data-duration": "3" });+ // A half at 5 s with in-point 5: the clip's own 0-3 s plays at 0-3 s, before the half starts.+ const half = document.createElement("div");+ for (const [name, value] of Object.entries({+ "data-composition-id": "half",+ "data-composition-file": "half.html",+ "data-start": "5",+ "data-duration": "5",+ "data-playback-start": "5",+ }))+ half.setAttribute(name, value);+ audio.replaceWith(half);+ half.appendChild(audio);+ const capture = vi+ .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")+ .mockResolvedValue(null);++ await startPlayback();++ expect(capture).not.toHaveBeenCalled();+ });+it("leaves the element audible on native output when decode also fails", async () => {const audio = mountAudio("https://cdn.example.com/track.mp3");vi.spyOn(console, "info").mockImplementation(() => {});diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 76217e6d12..62d4894495 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -138,7 +138,7 @@ import {} from "./timelineRebindPolicy";import { installStudioCustomEase } from "./customEase";import { parseStrictFiniteTimingNumber, resolveMediaElementDurationSeconds } from "./playbackRate";-import { MEDIA_START_BASIS_ATTR } from "../mediaTiming";+import { cutToHostSlots, MEDIA_START_BASIS_ATTR } from "../mediaTiming";import { settleFirstFrameCompositionReadiness } from "../compositionReadiness";import { AUTHORED_DURATION_ATTR, AUTHORED_END_ATTR } from "./authoredTiming";import {@@ -979,6 +979,16 @@ export function initSandboxRuntimeModular(): void {return { compositionRoot, inheritedStart, inheritedDuration };};+ const resolveMediaHostSlots = (element: Element) =>+ timingResolverFor(true).resolveHostSlotsForElement(element);++ /** A clip's own window from `start` and `data-duration`, cut by its in-point hosts. */+ const playedMediaWindow = (element: HTMLMediaElement, start: number) => {+ const duration = parseStrictFiniteTimingNumber(element.dataset.duration);+ const end = duration != null && duration > 0 ? start + duration : Infinity;+ return cutToHostSlots({ start, end }, resolveMediaHostSlots(element));+ };+// Single owner: `createRuntimeStartTimeResolver` (startResolver.ts). The clip// manifest resolves media starts through the same method, so what the studio// draws and what the transport plays cannot drift apart.@@ -2936,6 +2946,7 @@ export function initSandboxRuntimeModular(): void {explicitDuration,});},+ resolveHostSlots: resolveMediaHostSlots,});const resolveMediaClipIndex = (): MediaClipIndex => {@@ -4574,9 +4585,9 @@ export function initSandboxRuntimeModular(): void {if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue;if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue;const start = resolveAbsoluteMediaStartSeconds(el);- const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);- const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;- if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, start, end)) continue;+ const kept = playedMediaWindow(el, start);+ if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, kept.start, kept.end))+ continue;if (el === followed) return { el, start };const runsUntil = start + (resolveMediaElementDurationSeconds(el) ?? Infinity);if (!longest || runsUntil > longest.runsUntil) longest = { el, start, runsUntil };@@ -4790,9 +4801,8 @@ export function initSandboxRuntimeModular(): void {if (!el.hasAttribute("data-start")) continue;const start = resolveAbsoluteMediaStartSeconds(el);if (!Number.isFinite(start)) continue;- const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);- const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;- if (!isInClipWindow(timeSeconds, start, end)) continue;+ const kept = playedMediaWindow(el, start);+ if (!isInClipWindow(timeSeconds, kept.start, kept.end)) continue;const mediaStart = readElementPlaybackStart(el);const relTime = timeSeconds - start + mediaStart;if (relTime >= 0) {@@ -4824,18 +4834,18 @@ export function initSandboxRuntimeModular(): void {const durationAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);let clipDuration =durationAttr != null && durationAttr > 0 ? durationAttr : Number.POSITIVE_INFINITY;- const compositionRoot = rawEl.closest("[data-composition-id]");- if (compositionRoot) {- const inheritedStart = resolveStartForElement(compositionRoot, 0);- const inheritedDuration = resolveDurationForElement(compositionRoot, {- includeAuthoredTimingAttrs: true,- });- if (inheritedDuration != null && inheritedDuration > 0) {- clipDuration = Math.min(- clipDuration,- Math.max(0, inheritedStart + inheritedDuration - compStart),- );- }+ const { inheritedStart, inheritedDuration } = resolveMediaCompositionContext(rawEl);+ if (inheritedStart != null && inheritedDuration != null && inheritedDuration > 0) {+ clipDuration = Math.min(+ clipDuration,+ Math.max(0, inheritedStart + inheritedDuration - compStart),+ );+ }+ const slots = resolveMediaHostSlots(rawEl);+ const kept = cutToHostSlots({ start: compStart, end: compStart + clipDuration }, slots);+ if (slots.length > 0) {+ if (kept.end <= kept.start) continue;+ clipDuration = kept.end - compStart;}// Decided BEFORE the transport is asked, because the two verdicts want// two different fallback chains — and only one of them is the chain@@ -4899,6 +4909,7 @@ export function initSandboxRuntimeModular(): void {gen,state.playbackRate,clipDuration,+ kept.start,);});});diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.tsindex 6b2e81a80b..34dedbfa66 100644--- a/packages/core/src/runtime/media.test.ts+++ b/packages/core/src/runtime/media.test.ts@@ -911,6 +911,26 @@ describe("syncRuntimeMedia", () => {expect(clip.el.play).not.toHaveBeenCalled();});+ it("judges a head-cut clip's cue-ahead by its rate at the cut, not at its own start", () => {+ // The clip's own t=0 is 2 and an in-point cuts it to start at 5, where the ramp has reached 3x.+ const rate: HfAutomationLane = {+ target: "rate",+ points: [+ { t: 0, v: 1 },+ { t: 3, v: 3 },+ ],+ };+ const clip = createMockClip({ start: 5, origin: 2, end: 6, rate }, "audio");+ syncRuntimeMedia({+ clips: [clip],+ timeSeconds: 4.98,+ playing: true,+ playbackRate: 1,+ cueAheadSeconds: 0.034,+ });+ expect(clip.el.play).not.toHaveBeenCalled();+ });+it("keeps a clip it started early playing when the next tick is shorter", () => {const clip = createMockClip({ start: 5, end: 6 }, "audio");syncRuntimeMedia({diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.tsindex 7ad1cbabec..b8490149c0 100644--- a/packages/core/src/runtime/media.ts+++ b/packages/core/src/runtime/media.ts@@ -10,6 +10,7 @@ import { clampAudioGain } from "../audioGain.js";import { isMemberGroupHidden } from "../audioGroups.js";import { findInjectedRenderFrame } from "./renderFrameSibling.js";import { registerSeekCompletion } from "./adapters/seek-dispatch.js";+import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";export {readElementPlaybackRate,readElementRateSpec,@@ -77,6 +78,8 @@ export function resolveRuntimeMediaClipDuration(params: {export type RuntimeMediaClip = {el: HTMLVideoElement | HTMLAudioElement;start: number;+ /** The clip's own t=0 when a host's in-point cut its head; source time, lanes and fades run from it. */+ origin?: number;mediaStart: number;duration: number;end: number;@@ -101,6 +104,7 @@ export type RuntimeMediaClip = {export function refreshRuntimeMediaCache(params?: {resolveStartSeconds?: (element: Element) => number;resolveDurationSeconds?: (element: HTMLVideoElement | HTMLAudioElement) => number | null;+ resolveHostSlots?: (element: HTMLVideoElement | HTMLAudioElement) => HostSlot[];shouldIncludeElement?: (element: HTMLVideoElement | HTMLAudioElement) => boolean;/*** Build clips for exactly these elements instead of scanning the document for@@ -131,7 +135,7 @@ export function refreshRuntimeMediaCache(params?: {const videoClips: RuntimeMediaClip[] = [];let maxMediaEnd = 0;for (const el of timedMediaEls) {- const start = params?.resolveStartSeconds+ let start = params?.resolveStartSeconds? params.resolveStartSeconds(el): Number.parseFloat(el.dataset.start ?? "0");if (!Number.isFinite(start)) continue;@@ -148,13 +152,20 @@ export function refreshRuntimeMediaCache(params?: {duration = Math.max(0, timeAtSourceTime(rate, sourceDuration - mediaStart));}const hasKnownDuration = Number.isFinite(duration) && duration >= 0;- const end = hasKnownDuration ? start + duration : Number.POSITIVE_INFINITY;+ let end = hasKnownDuration ? start + duration : Number.POSITIVE_INFINITY;+ const origin = start;+ const kept = cutToHostSlots({ start, end }, params?.resolveHostSlots?.(el) ?? []);+ if (kept.start !== start || kept.end !== end) {+ ({ start, end } = kept);+ duration = end - start;+ }const volumeRaw = Number.parseFloat(el.dataset.volume ?? "");const clip: RuntimeMediaClip = {el,start,+ origin,mediaStart,- duration: hasKnownDuration ? duration : Number.POSITIVE_INFINITY,+ duration: Number.isFinite(end) ? duration : Number.POSITIVE_INFINITY,end,volume: Number.isFinite(volumeRaw) ? volumeRaw : null,playbackRate,@@ -326,6 +337,7 @@ export function syncRuntimeMedia(params: {const { el } = clip;if (!el.isConnected) continue;const clipRate = clip.rate ?? clip.playbackRate;+ const origin = clip.origin ?? clip.start;const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop;const inWindow = isInClipWindow(params.timeSeconds, clip.start, clip.end);const dueIn = clip.start - params.timeSeconds;@@ -333,7 +345,7 @@ export function syncRuntimeMedia(params: {el.tagName === "AUDIO" &&!inWindow &&dueIn > 0 &&- dueIn * Math.max(1, rateAt(clipRate, 0)) <=+ dueIn * Math.max(1, rateAt(clipRate, clip.start - origin)) <=Math.max(params.cueAheadSeconds ?? 0,startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0,@@ -348,7 +360,7 @@ export function syncRuntimeMedia(params: {(params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) &&isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration());let relTime =- sourceTimeAt(clipRate, Math.max(0, Math.min(params.timeSeconds, clip.end) - clip.start)) ++ sourceTimeAt(clipRate, Math.max(0, Math.min(params.timeSeconds, clip.end) - origin)) +clip.mediaStart;const isHeldVideoTail =isTerminalVideo ||@@ -402,7 +414,7 @@ export function syncRuntimeMedia(params: {// different position than it renders, or ran it off the end entirely on a// trimmed clip. The FX lanes on this same feature use clip-local elapsed;// there is one time base, and this is it.- const laneGain = elementVolumeLaneGain(el, params.timeSeconds - clip.start);+ const laneGain = elementVolumeLaneGain(el, params.timeSeconds - origin);if (laneGain !== null) {authorVolume = clampAudioGain(laneGain);} else if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) {@@ -413,7 +425,7 @@ export function syncRuntimeMedia(params: {// `relTime` is a position inside the media SOURCE — it carries `mediaStart`// and the playback rate — so it only coincides with the envelope's time base// for an untrimmed clip playing at 1x from t=0.- const elapsedInClip = params.timeSeconds - clip.start;+ const elapsedInClip = params.timeSeconds - origin;authorVolume = clampAudioGain(interpolateVolumeGain(clip.volumeKeyframes, elapsedInClip));} else if (params.isWebAudioRouted?.(el)) {authorVolume = fallbackAuthorVolume;@@ -447,7 +459,11 @@ export function syncRuntimeMedia(params: {// Clip-local fade on top of the resolved level, matching render's afade-after-volume.const fades = clip.fades ?? NO_FADES;if (fades.fadeIn > 0 || fades.fadeOut > 0) {- authorVolume *= fadeGain(params.timeSeconds - clip.start, clip.duration, fades);+ authorVolume *= fadeGain(+ params.timeSeconds - origin,+ clip.duration + clip.start - origin,+ fades,+ );}// A data-hidden ancestor is silent in the export (audioMixer.ts drops@@ -475,7 +491,7 @@ export function syncRuntimeMedia(params: {// was overridden after init.ts set it.if (el.preload !== "auto") el.preload = "auto";// Per-element rate × global transport rate- const baseRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;+ const baseRate = rateAt(clipRate, params.timeSeconds - origin) * params.playbackRate;// Too slow for the browser to play: kept paused and stepped onto the playhead by seeks.const playing = params.playing && baseRate >= MIN_NATIVE_PLAYBACK_RATE;// Drift correction — three tiers:diff --git a/packages/core/src/runtime/startResolver.test.ts b/packages/core/src/runtime/startResolver.test.tsindex 8b42c9b390..b523d618c8 100644--- a/packages/core/src/runtime/startResolver.test.ts+++ b/packages/core/src/runtime/startResolver.test.ts@@ -452,3 +452,46 @@ describe("documentRef", () => {expect(resolver.resolveStartForElement(dependent)).toBe(2);});});++describe("a host's in-point", () => {+ afterEach(() => {+ document.body.innerHTML = "";+ });++ it("shifts what a host holds by the host's in-point, through nested hosts", () => {+ document.body.innerHTML =+ `<div data-composition-id="main">` ++ `<div id="half" data-composition-id="scene" data-start="5" data-duration="4" data-playback-start="5">` ++ `<video id="v2" data-start="4"></video><video id="v3" data-start="8"></video></div>` ++ `<div data-composition-id="early" data-start="2" data-duration="4" data-playback-start="5">` ++ `<video id="legacy" data-start="3" data-hf-media-start-basis="global"></video></div>` ++ `<div data-composition-id="outer" data-start="5" data-duration="10" data-playback-start="1">` ++ `<div id="inner" data-composition-id="inner" data-start="2" data-duration="5">` ++ `<video id="deep" data-start="0"></video></div></div></div>`;+ const resolver = createRuntimeStartTimeResolver({});+ const media = (id: string) =>+ resolver.resolveMediaStartForElement(document.getElementById(id)!);++ // The half starts at 5 with in-point 5: local t plays at 5 + t - 5.+ expect([media("v2"), media("v3")]).toEqual([4, 8]);+ expect(resolver.resolveHostStartForElement(document.getElementById("v2")!)).toBe(0);+ // Root-time media keeps its authored time, even where the in-point puts local 0 before 0.+ expect(media("legacy")).toBe(3);+ // Outer at 5 with in-point 1 puts the inner host at 5 - 1 + 2 = 6.+ expect(resolver.resolveStartForElement(document.getElementById("inner")!)).toBe(6);+ expect(media("deep")).toBe(6);+ });++ it("places a relative start where the same local time written as a number lands", () => {+ // A host at 0 with in-point 5 puts local 0 at -5.+ document.body.innerHTML =+ `<div data-composition-id="main"><div data-composition-id="half" data-start="0" data-duration="7" data-playback-start="5">` ++ `<div id="x" data-start="0" data-duration="1"></div>` ++ `<video id="relative" data-start="x + 3"></video><video id="numeric" data-start="4"></video></div></div>`;+ const resolver = createRuntimeStartTimeResolver({});+ const media = (id: string) =>+ resolver.resolveMediaStartForElement(document.getElementById(id)!);++ expect([media("relative"), media("numeric")]).toEqual([-1, -1]);+ });+});diff --git a/packages/core/src/runtime/startResolver.ts b/packages/core/src/runtime/startResolver.tsindex 878b4377ea..51f78dc912 100644--- a/packages/core/src/runtime/startResolver.ts+++ b/packages/core/src/runtime/startResolver.ts@@ -12,6 +12,9 @@ import {import { isMediaElement } from "./domRealm";import { parseStartExpression } from "./startExpression";import {+ compositionOriginSeconds,+ hostInPointSeconds,+ type HostSlot,isRootGlobalMediaStart,MEDIA_START_BASIS_ATTR,resolveMediaStartSeconds,@@ -33,6 +36,7 @@ export function createRuntimeStartTimeResolver(params: {resolveDurationForElement: (element: Element) => number | null;resolveMediaStartForElement: (element: Element) => number;resolveHostStartForElement: (element: Element) => number;+ resolveHostSlotsForElement: (element: Element) => HostSlot[];isRootGlobalMediaStartForElement: (element: Element) => boolean;} {const timelineRegistry = params.timelineRegistry ?? {};@@ -50,61 +54,92 @@ export function createRuntimeStartTimeResolver(params: {);};+ const authoredAttr = (element: Element, name: string) =>+ includeAuthoredTimingAttrs ? element.getAttribute(name) : null;++ const resolveAuthoredDuration = (element: Element): number | null =>+ resolveAuthoredTimingWindow({+ start: 0,+ duration: element.getAttribute("data-duration"),+ authoredDuration: authoredAttr(element, "data-hf-authored-duration"),+ })?.duration ??+ resolveAuthoredTimingWindow({+ start: resolveStartForElementInternal(element, 0),+ end: element.getAttribute("data-end"),+ authoredEnd: authoredAttr(element, "data-hf-authored-end"),+ })?.duration ??+ null;++ const positive = (value: number | null | undefined): number | null =>+ value != null && Number.isFinite(value) && value > 0 ? value : null;++ const resolveRegisteredTimelineDuration = (element: Element): number | null => {+ const compositionId = element.getAttribute("data-composition-id");+ return compositionId+ ? positive(readTimelineDurationSeconds(timelineRegistry[compositionId]))+ : null;+ };+const resolveDurationForElement = (element: Element): number | null => {const cached = durationCache.get(element);if (cached !== undefined) return cached;- let resolved: number | null = null;- const durationTiming = resolveAuthoredTimingWindow({- start: 0,- duration: element.getAttribute("data-duration"),- authoredDuration: includeAuthoredTimingAttrs- ? element.getAttribute("data-hf-authored-duration")- : null,- });- if (durationTiming?.duration != null && durationTiming.duration > 0) {- resolved = durationTiming.duration;- }- if (resolved == null || resolved <= 0) {- const start = resolveStartForElementInternal(element, 0);- const endTiming = resolveAuthoredTimingWindow({- start,- end: element.getAttribute("data-end"),- authoredEnd: includeAuthoredTimingAttrs- ? element.getAttribute("data-hf-authored-end")+ const resolved =+ resolveAuthoredDuration(element) ??+ positive(+ isMediaElement(element)+ ? resolveNaturalMediaTimelineDuration(element, element.duration): null,- });- if (endTiming?.duration != null && endTiming.duration > 0) {- resolved = endTiming.duration;- }- }- if ((resolved == null || resolved <= 0) && isMediaElement(element)) {- resolved = resolveNaturalMediaTimelineDuration(element, element.duration);- }- if (resolved == null || resolved <= 0) resolved = resolveTimedImageDurationSeconds(element);- if (resolved == null || resolved <= 0) {- const compositionId = element.getAttribute("data-composition-id");- if (compositionId) {- const timelineDuration = readTimelineDurationSeconds(timelineRegistry[compositionId]);- if (timelineDuration != null && timelineDuration > 0) resolved = timelineDuration;- }- }- if (resolved != null && Number.isFinite(resolved) && resolved > 0) {- durationCache.set(element, resolved);- return resolved;- }- durationCache.set(element, null);- return null;+ ) ??+ positive(resolveTimedImageDurationSeconds(element)) ??+ resolveRegisteredTimelineDuration(element);+ durationCache.set(element, resolved);+ return resolved;};+ const isCompositionHost = (element: Element): boolean =>+ element.hasAttribute("data-composition-src") ||+ element.hasAttribute("data-composition-id") ||+ element.hasAttribute("data-composition-file");++ // A mounted composition root without its own `data-start` takes its timing from the host it was+ // loaded into: the host may use a different id than the file, or none (an anonymous host).+ const inheritedTimingHost = (element: Element): Element | null => {+ const parent = element.parentElement;+ if (!parent || !element.hasAttribute("data-composition-id")) return null;+ if (parseStartExpression(element.getAttribute("data-start"))) return null;+ return isCompositionHost(parent) ? parent : null;+ };++ const resolveCompositionOrigin = (compositionRoot: Element, fallback: number): number =>+ compositionOriginSeconds(+ resolveStartForElementInternal(compositionRoot, fallback),+ inheritedTimingHost(compositionRoot) ?? compositionRoot,+ );+const resolveHostOffsetForElement = (element: Element, fallback: number): number => {- if (element.hasAttribute("data-composition-id")) {- const parentComposition = element.parentElement?.closest("[data-composition-id]");- if (!parentComposition) return 0;- return resolveStartForElementInternal(parentComposition, fallback);+ const compositionRoot = element.hasAttribute("data-composition-id")+ ? element.parentElement?.closest("[data-composition-id]")+ : element.closest("[data-composition-id]");+ return compositionRoot ? resolveCompositionOrigin(compositionRoot, fallback) : 0;+ };++ const computeStart = (element: Element, fallback: number): number => {+ const expression = parseStartExpression(element.getAttribute("data-start"));+ if (!expression) {+ const host = inheritedTimingHost(element);+ return host ? resolveStartForElementInternal(host, fallback) : fallback;}- const compositionRoot = element.closest("[data-composition-id]");- if (!compositionRoot) return 0;- return resolveStartForElementInternal(compositionRoot, fallback);+ if (expression.kind === "absolute") {+ // Negative when a host's in-point is past its start; the host's slot cuts that part.+ return resolveHostOffsetForElement(element, fallback) + Math.max(0, expression.value);+ }+ const target = findReferenceTarget(expression.refId);+ if (!target) return fallback;+ const targetStart = resolveStartForElementInternal(target, 0);+ const targetDuration = resolveDurationForElement(target) ?? 0;+ // Clamped at local 0, as a numeric start is, once an in-point puts local 0 before 0.+ const floor = Math.min(0, resolveHostOffsetForElement(element, 0));+ return Math.max(floor, targetStart + targetDuration + expression.offset);};const resolveStartForElementInternal = (element: Element, fallback: number): number => {@@ -117,53 +152,7 @@ export function createRuntimeStartTimeResolver(params: {}visiting.add(element);try {- const expression = parseStartExpression(element.getAttribute("data-start"));- if (!expression) {- // If this element is a loaded composition inner root (has data-composition-id- // but no data-start), walk up to the host parent which carries the actual- // timing. This happens when the host uses a different data-composition-id- // than the loaded file — e.g. host="montage" but file has "scene-10", or- // when the host itself has no data-composition-id at all (an "anonymous"- // host) and the composition's own id was restored onto the inlined wrapper.- // Check data-composition-src (runtime, not yet inlined), data-composition-id- // (bundled/compiled host with its own id), and data-composition-file (the- // marker every inlined host gets, compiled or bundled, once- // data-composition-src is stripped — covers the anonymous-host case).- if (element.hasAttribute("data-composition-id")) {- const parent = element.parentElement;- if (- parent &&- (parent.hasAttribute("data-composition-src") ||- parent.hasAttribute("data-composition-id") ||- parent.hasAttribute("data-composition-file"))- ) {- const parentStart = resolveStartForElementInternal(parent, fallback);- startCache.set(element, parentStart);- return parentStart;- }- }- startCache.set(element, fallback);- return fallback;- }- if (expression.kind === "absolute") {- const absolute = Math.max(0, expression.value);- const resolved = Math.max(0, resolveHostOffsetForElement(element, fallback) + absolute);- startCache.set(element, resolved);- return resolved;- }- const target = findReferenceTarget(expression.refId);- if (!target) {- startCache.set(element, fallback);- return fallback;- }- const targetStart = resolveStartForElementInternal(target, 0);- const targetDuration = resolveDurationForElement(target);- if (targetDuration == null || targetDuration <= 0) {- const unresolved = Math.max(0, targetStart + expression.offset);- startCache.set(element, unresolved);- return unresolved;- }- const resolved = Math.max(0, targetStart + targetDuration + expression.offset);+ const resolved = computeStart(element, fallback);startCache.set(element, resolved);return resolved;} finally {@@ -186,7 +175,7 @@ export function createRuntimeStartTimeResolver(params: {const compositionRoot = element.closest("[data-composition-id]");return {authoredStart: parseStrictFiniteTimingNumber(element.getAttribute("data-start")),- hostStart: compositionRoot ? resolveStartForElementInternal(compositionRoot, 0) : 0,+ hostStart: compositionRoot ? resolveCompositionOrigin(compositionRoot, 0) : 0,hasAutoStart: element.hasAttribute("data-hf-auto-start"),basis: element.getAttribute(MEDIA_START_BASIS_ATTR),};@@ -200,6 +189,19 @@ export function createRuntimeStartTimeResolver(params: {});};+ // Hosts with an in-point, found by their composition attributes so an anonymous one counts too.+ const resolveHostSlotsForElement = (element: Element): HostSlot[] => {+ const slots: HostSlot[] = [];+ for (let node = element.parentElement; node; node = node.parentElement) {+ if (!isCompositionHost(node) || inheritedTimingHost(node)) continue;+ if (hostInPointSeconds(node) <= 0) continue;+ const start = resolveStartForElementInternal(node, 0);+ const duration = resolveAuthoredDuration(node);+ slots.push({ start, end: duration ? start + duration : Infinity });+ }+ return slots;+ };+const isRootGlobalMediaStartForElement = (element: Element): boolean =>isMediaElement(element) && isRootGlobalMediaStart(mediaStartInput(element));@@ -209,6 +211,7 @@ export function createRuntimeStartTimeResolver(params: {resolveDurationForElement: (element: Element) => resolveDurationForElement(element),resolveMediaStartForElement,resolveHostStartForElement: (element: Element) => resolveHostOffsetForElement(element, 0),+ resolveHostSlotsForElement,isRootGlobalMediaStartForElement,};}diff --git a/packages/core/src/runtime/timeline.inPoint.test.ts b/packages/core/src/runtime/timeline.inPoint.test.tsnew file mode 100644index 0000000000..a02d30df22--- /dev/null+++ b/packages/core/src/runtime/timeline.inPoint.test.ts@@ -0,0 +1,46 @@+import { afterEach, describe, expect, it } from "vitest";+import { collectRuntimeTimelinePayload } from "./timeline";++describe("Studio's clip list under a host with an in-point", () => {+ afterEach(() => {+ document.body.innerHTML = "";+ });++ it("lists only the part each clip plays, as `hyperframes timeline` does", () => {+ // Local 0-4 and 1-3 fall before the in-point; local 8-12 plays at 3-7.+ document.body.innerHTML =+ `<div data-composition-id="main" data-duration="10">` ++ `<div id="half" data-composition-id="half" data-composition-file="scene.html" data-start="0" data-duration="7" data-playback-start="5">` ++ `<video id="v1" data-start="0" data-duration="4"></video>` ++ `<div id="t" data-start="1" data-duration="2"></div>` ++ `<video id="v3" data-start="8" data-duration="4"></video></div></div>`;++ const { clips } = collectRuntimeTimelinePayload({ canonicalFps: 30 });++ expect(clips.map((c) => [c.id, c.start, c.duration])).toEqual([+ ["half", 0, 7],+ ["v3", 3, 4],+ ]);+ });++ it("keeps a nested host it cuts away so what that host holds stays nested in Studio", () => {+ // `lower` plays at local 1-3 of `half`, before the in-point; `bed` outlives it into 5-7.+ document.body.innerHTML =+ `<div data-composition-id="main" data-duration="20">` ++ `<div id="half" data-composition-id="half" data-start="5" data-duration="5" data-playback-start="5">` ++ `<div id="lower" data-composition-id="lower" data-composition-file="lower.html" data-start="1" data-duration="2">` ++ `<audio id="bed" data-start="0" data-duration="6"></audio></div></div></div>`;++ const { clips } = collectRuntimeTimelinePayload({ canonicalFps: 30 });++ expect(clips.map((c) => [c.id, c.start, c.duration])).toEqual([+ ["half", 5, 5],+ ["lower", 5, 0],+ ["bed", 5, 2],+ ]);+ // Studio's top-level rule: no parent, or a parent missing from the list.+ const ids = new Set(clips.map((c) => c.compositionId));+ const topLevel = clips.filter((c) => !c.parentCompositionId || !ids.has(c.parentCompositionId));+ expect(topLevel.map((c) => c.id)).toEqual(["half"]);+ });+});diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.tsindex 911e897c00..459959432f 100644--- a/packages/core/src/runtime/timeline.ts+++ b/packages/core/src/runtime/timeline.ts@@ -29,6 +29,7 @@ import { COMPOSITION_CONTRACT_VERSION } from "../compositionContract.js";import { runtimeProtocolMetadata } from "./protocol.js";import { isElementNode, isMediaElement } from "./domRealm";import { LOOP_INFLATED_TIMELINE_SECONDS } from "./compositionLength";+import { cutToHostSlots } from "../mediaTiming";export function isRuntimeElementVisibleAt(rawNode: HTMLElement,@@ -422,13 +423,14 @@ export function collectRuntimeTimelinePayload(params: {continue;const compositionContext = resolveNearestCompositionContext(node, root);const tag = node.tagName.toLowerCase();- const start =+ let start =tag === "video" || tag === "audio"? startResolver.resolveMediaStartForElement(node): startResolver.resolveStartForElement(node, compositionContext.inheritedStart ?? 0);const nodeCompositionId = node.getAttribute("data-composition-id");+ const isNestedHost = !!nodeCompositionId && nodeCompositionId !== rootCompositionId;let duration = parseElementDurationAttr(node);- if (duration == null && nodeCompositionId && nodeCompositionId !== rootCompositionId) {+ if (duration == null && isNestedHost) {duration = resolveTimelineDurationSeconds(nodeCompositionId);}if (duration == null && isMediaElement(node)) {@@ -448,6 +450,14 @@ export function collectRuntimeTimelinePayload(params: {if (duration == null || duration <= 0) continue;duration = clampDurationToRootWindow(start, duration);if (duration <= 0) continue;+ const slots = startResolver.resolveHostSlotsForElement(node);+ if (slots.length > 0) {+ const played = cutToHostSlots({ start, end: start + duration }, slots);+ const keepZeroLengthSoStudioNestsChildren = isNestedHost;+ if (played.end <= played.start && !keepZeroLengthSoStudioNestsChildren) continue;+ start = played.start;+ duration = played.end - played.start;+ }const end = start + duration;maxEnd = Math.max(maxEnd, end);const kind: RuntimeTimelineClip["kind"] =diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.tsindex 2c01d95f9f..dcb384c6c6 100644--- a/packages/core/src/runtime/webAudioTransport.test.ts+++ b/packages/core/src/runtime/webAudioTransport.test.ts@@ -500,6 +500,16 @@ describe("WebAudioTransport", () => {expect(mock.startFn).toHaveBeenCalledWith(108, 1.5, 4);});+ it("holds a clip whose head an in-point cut until its window opens, at that point's source", async () => {+ const { transport, mock, gen } = setupTransport(100);+ // The clip's own t=0 is 1, its window opens at 2 and ends at 5: from comp time 0, wait 2 s, then+ // play source 1 to 4; from comp time 1.5, the same.+ await transport.schedulePlayback(mockEl, mockBuffer, 1, 0, () => 0, 1, gen, 1, 4, 2);+ expect(mock.startFn).toHaveBeenLastCalledWith(102, 1, 3);+ await transport.schedulePlayback(mockEl, mockBuffer, 1, 0, () => 1.5, 1, gen, 1, 4, 2);+ expect(mock.startFn).toHaveBeenLastCalledWith(100.5, 1, 3);+ });+it("does not schedule a clip whose window has already elapsed", async () => {const { transport, mock, gen } = setupTransport(100);// elapsed=15 > clipDuration=10 → nothing to playdiff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.tsindex 52766f4a7e..d02770764f 100644--- a/packages/core/src/runtime/webAudioTransport.ts+++ b/packages/core/src/runtime/webAudioTransport.ts@@ -69,22 +69,17 @@ function startBoundedSource(globalRate: number;mediaRate: number;clipDuration: number;+ headCut: number;},): boolean {- const { elapsed, mediaStart, scheduledAt, globalRate, mediaRate, clipDuration } = opts;+ const { elapsed, mediaStart, scheduledAt, globalRate, mediaRate, clipDuration, headCut } = opts;const hasBound = Number.isFinite(clipDuration) && clipDuration > 0;- const clipSourceLen = clipDuration * mediaRate;- if (elapsed >= 0) {- const sourceElapsed = elapsed * mediaRate;- const remaining = clipSourceLen - sourceElapsed;- if (hasBound && remaining <= 0) return false;- if (hasBound) node.start(0, sourceElapsed + mediaStart, remaining);- else node.start(0, sourceElapsed + mediaStart);- return true;- }- const delay = -elapsed / globalRate;- if (hasBound) node.start(scheduledAt + delay, mediaStart, clipSourceLen);- else node.start(scheduledAt + delay, mediaStart);+ const from = Math.max(elapsed, headCut, 0);+ const remaining = (clipDuration - from) * mediaRate;+ if (hasBound && remaining <= 0) return false;+ const when = from > elapsed ? scheduledAt + (from - elapsed) / globalRate : 0;+ if (hasBound) node.start(when, from * mediaRate + mediaStart, remaining);+ else node.start(when, from * mediaRate + mediaStart);return true;}@@ -667,6 +662,7 @@ export class WebAudioTransport {generation: number,rate = 1,clipDuration = Number.POSITIVE_INFINITY,+ windowStart = compositionStart,): Promise<ScheduledSource | null> {if (!this._ctx || !this._masterGain) return null;if (generation !== this._playGeneration) return null;@@ -715,6 +711,7 @@ export class WebAudioTransport {globalRate: safeRate,mediaRate,clipDuration,+ headCut: windowStart - compositionStart,})) {// Playhead already past the clip end — discard the nodes we built.