Skip to content

Write HTML. Render video. Built for agents.

feat(core): audible videos in preview web audio, audio groups, meters and carve (2/8) (#4814)

Vance Ingalls3c299acc7f6f
diff --git a/docs/reference/audio-effects.mdx b/docs/reference/audio-effects.mdx
index df00a9c5fa..428895e63f 100644
--- a/docs/reference/audio-effects.mdx
+++ b/docs/reference/audio-effects.mdx
@@ -20,10 +20,10 @@ nothing to load beside the markup.
| `data-fx-chain` | The effects, in signal order | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes |
| `data-automation` | Envelopes on volume or an effect parameter | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes |
| `data-fx-carve` | A voiceover carve's own settings | JSON | `<audio>`, `<video>` | No — see [below](#data-fx-carve) |
-| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>` only | Yes |
+| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>`, audible `<video>` | Yes |
-`data-audio-group` is the odd one: a bare string, and **ignored on `<video>`**.
-The other three are JSON. Of those, only `data-fx-chain` and `data-automation`
+`data-audio-group` is the odd one: a bare string, and **ignored on a muted
+`<video>`** (`muted`, or `data-has-audio="false"`). The other three are JSON. Of those, only `data-fx-chain` and `data-automation`
can also sit on an `<hf-audio-group>`, where they apply to the group's whole bus;
`data-fx-carve` belongs to a clip.
@@ -170,8 +170,9 @@ dangles.
- `<hf-audio-group>` is optional metadata: `data-label`, `data-volume`,
`data-hidden`, and its own `data-fx-chain` / `data-automation`. A group with
members but no element still resolves, using its id as the label.
-- **Audio only.** `data-audio-group` on a `<video>` is ignored, and groups do
- not nest.
+- **Audio and audible video.** A `<video>` with sound (`data-has-audio="true"`,
+ no `muted`) joins its group in preview and render; on a muted video
+ `data-audio-group` is ignored. Groups do not nest.
- `data-audio-group=""` is no group at all.
- The element must be inert. The runtime injects
`hf-audio-group{display:none!important}` so an unknown custom element cannot
diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json
index 75c95a1ba2..f2626dd280 100644
--- a/packages/core/package-subpaths.json
+++ b/packages/core/package-subpaths.json
@@ -218,6 +218,12 @@
"types": "./dist/clipFacts.d.ts",
"environments": ["browser", "bun", "node"]
},
+ "./audible-video": {
+ "source": "./src/audibleVideo.ts",
+ "runtime": "./dist/audibleVideo.js",
+ "types": "./dist/audibleVideo.d.ts",
+ "environments": ["browser", "bun", "node"]
+ },
"./audio-gain": {
"source": "./src/audioGain.ts",
"runtime": "./dist/audioGain.js",
diff --git a/packages/core/package.json b/packages/core/package.json
index 83a64b567f..836954ea54 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -232,6 +232,12 @@
"import": "./src/clipFacts.ts",
"types": "./src/clipFacts.ts"
},
+ "./audible-video": {
+ "bun": "./src/audibleVideo.ts",
+ "node": "./dist/audibleVideo.js",
+ "import": "./src/audibleVideo.ts",
+ "types": "./src/audibleVideo.ts"
+ },
"./audio-gain": {
"bun": "./src/audioGain.ts",
"node": "./dist/audioGain.js",
@@ -614,6 +620,10 @@
"import": "./dist/clipFacts.js",
"types": "./dist/clipFacts.d.ts"
},
+ "./audible-video": {
+ "import": "./dist/audibleVideo.js",
+ "types": "./dist/audibleVideo.d.ts"
+ },
"./audio-gain": {
"import": "./dist/audioGain.js",
"types": "./dist/audioGain.d.ts"
diff --git a/packages/core/src/audibleVideo.test.ts b/packages/core/src/audibleVideo.test.ts
new file mode 100644
index 0000000000..1d0ea261a5
--- /dev/null
+++ b/packages/core/src/audibleVideo.test.ts
@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+ AUDIBLE_MEDIA_SELECTOR,
+ audibleVideoNeedsWebAudio,
+ isAudibleVideoElement,
+} from "./audibleVideo";
+
+const el = (tagName: string, attrs: Record<string, string>) => ({
+ tagName,
+ hasAttribute: (n: string) => n in attrs,
+ getAttribute: (n: string) => (n in attrs ? (attrs[n] ?? null) : null),
+});
+
+describe("isAudibleVideoElement", () => {
+ it("unmuted video with data-has-audio=true or no attribute is audible", () => {
+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "true" }))).toBe(true);
+ expect(isAudibleVideoElement(el("video", {}))).toBe(true);
+ });
+ it("muted, false or empty is not audible", () => {
+ expect(isAudibleVideoElement(el("VIDEO", { muted: "" }))).toBe(false);
+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "false" }))).toBe(false);
+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "" }))).toBe(false);
+ });
+ it("only the exact value true counts, so TRUE is not audible", () => {
+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "TRUE" }))).toBe(false);
+ });
+ it("never true for non-video", () => {
+ expect(isAudibleVideoElement(el("AUDIO", {}))).toBe(false);
+ });
+});
+
+describe("AUDIBLE_MEDIA_SELECTOR", () => {
+ it("matches audio plus exactly the videos the predicate calls audible", () => {
+ document.body.innerHTML = `
+ <audio id="a" data-start="0"></audio>
+ <video id="v-true" data-start="0" data-has-audio="true"></video>
+ <video id="v-absent" data-start="0"></video>
+ <video id="v-muted" data-start="0" data-has-audio="true" muted></video>
+ <video id="v-false" data-start="0" data-has-audio="false"></video>
+ <video id="v-empty" data-start="0" data-has-audio=""></video>`;
+ const matched = Array.from(document.querySelectorAll(AUDIBLE_MEDIA_SELECTOR), (n) => n.id);
+ expect(matched).toEqual(["a", "v-true", "v-absent"]);
+ const audibleVideos = Array.from(document.querySelectorAll("video"))
+ .filter(isAudibleVideoElement)
+ .map((n) => n.id);
+ expect(audibleVideos).toEqual(matched.filter((id) => id.startsWith("v-")));
+ });
+});
+
+describe("audibleVideoNeedsWebAudio", () => {
+ it("keeps a unity, unprocessed video on native output", () => {
+ expect(audibleVideoNeedsWebAudio({})).toBe(false);
+ expect(audibleVideoNeedsWebAudio({ volume: 1, fxChain: "", automation: null })).toBe(false);
+ expect(audibleVideoNeedsWebAudio({ volume: Number.NaN })).toBe(false);
+ });
+
+ it.each([{ volume: 1.5 }, { fxChain: "[]" }, { automation: "{}" }, { audioGroup: "music" }])(
+ "routes a video carrying %o",
+ (fields) => {
+ expect(audibleVideoNeedsWebAudio(fields)).toBe(true);
+ },
+ );
+});
diff --git a/packages/core/src/audibleVideo.ts b/packages/core/src/audibleVideo.ts
new file mode 100644
index 0000000000..30c9aa5fd2
--- /dev/null
+++ b/packages/core/src/audibleVideo.ts
@@ -0,0 +1,25 @@
+export const AUDIBLE_VIDEO_QUALIFIER =
+ ':not([muted]):is(:not([data-has-audio]),[data-has-audio="true"])';
+
+export const AUDIBLE_MEDIA_SELECTOR = `audio[data-start], video[data-start]${AUDIBLE_VIDEO_QUALIFIER}`;
+
+export function isAudibleVideoElement(el: {
+ tagName: string;
+ hasAttribute(name: string): boolean;
+ getAttribute(name: string): string | null;
+}): boolean {
+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;
+ const declared = el.getAttribute("data-has-audio");
+ return declared === null || declared === "true";
+}
+
+export function audibleVideoNeedsWebAudio(fields: {
+ volume?: number | null;
+ fxChain?: string | null;
+ automation?: string | null;
+ audioGroup?: string | null;
+}): boolean {
+ return (
+ (fields.volume ?? 1) > 1 || Boolean(fields.fxChain || fields.automation || fields.audioGroup)
+ );
+}
diff --git a/packages/core/src/audioGroups.test.ts b/packages/core/src/audioGroups.test.ts
index 606112d2de..722da0a225 100644
--- a/packages/core/src/audioGroups.test.ts
+++ b/packages/core/src/audioGroups.test.ts
@@ -67,9 +67,12 @@ describe("resolveAudioGroups", () => {
expect(resolveAudioGroups(document)[0].memberIds).toEqual(["vo-1"]);
});
- it("ignores a data-audio-group on a video element (audio only in v1)", () => {
- document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;
- expect(resolveAudioGroups(document)).toEqual([]);
+ it("groups an audible video and ignores a muted or silent one", () => {
+ document.body.innerHTML = `
+ <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>
+ <video id="v-2" muted data-audio-group="voiceover"></video>
+ <video id="v-3" data-has-audio="false" data-audio-group="voiceover"></video>`;
+ expect(resolveAudioGroups(document).map((g) => g.memberIds)).toEqual([["v-1"]]);
});
it("reads the group element's fx chain, automation, volume and hidden", () => {
@@ -115,11 +118,12 @@ describe("audioGroupOf", () => {
expect(audioGroupOf(document.getElementById("vo-1") as Element)).toBeNull();
});
- it("ignores video membership so preview matches the audio-only render", () => {
- document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;
- const el = document.getElementById("v-1") as Element;
- expect(audioGroupOf(el)).toBeNull();
- expect(resolveAudioGroups(document)).toEqual([]);
+ it("reads an audible video's membership and ignores a muted video's", () => {
+ document.body.innerHTML = `
+ <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>
+ <video id="v-2" muted data-audio-group="voiceover"></video>`;
+ expect(audioGroupOf(document.getElementById("v-1") as Element)).toBe("voiceover");
+ expect(audioGroupOf(document.getElementById("v-2") as Element)).toBeNull();
});
it("normalizes an empty membership attribute to null", () => {
diff --git a/packages/core/src/audioGroups.ts b/packages/core/src/audioGroups.ts
index 6b2a6b263c..b0cc736d11 100644
--- a/packages/core/src/audioGroups.ts
+++ b/packages/core/src/audioGroups.ts
@@ -12,6 +12,7 @@
import { HF_AUDIO_FX_ATTR } from "./audioFx.js";
import { AUDIO_GROUP_RENDER_ID_ATTR, MEDIA_RENDER_ID_ATTR } from "./compiler/mediaRenderIds.js";
import { HF_AUDIO_AUTOMATION_ATTR } from "./audioAutomation.js";
+import { isAudibleVideoElement } from "./audibleVideo.js";
export const HF_AUDIO_GROUP_TAG = "hf-audio-group";
export const HF_AUDIO_GROUP_ATTR = "data-audio-group";
@@ -129,7 +130,11 @@ export function isMemberGroupHidden(
export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] {
const membersByGroup = new Map<string, string[]>();
- for (const member of root.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)) {
+ const candidates = root.querySelectorAll(
+ `audio[${HF_AUDIO_GROUP_ATTR}], video[${HF_AUDIO_GROUP_ATTR}]`,
+ );
+ for (const member of candidates) {
+ if (!isGroupableMember(member)) continue;
// The render-stamped instance key when the compiler has been through
// (`assignMediaRenderIds`), else the author id. An author id is unique only
// per composition FILE, so a sub-composition declaring a bus AND its members
@@ -195,8 +200,8 @@ export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): st
return out;
}
-/** The group an audio member belongs to, or null. Membership is audio-only in
- * v1, matching `resolveAudioGroups` and the render mixer; video and group-bus
+/** The group an audio or audible-video member belongs to, or null, matching
+ * `resolveAudioGroups` and the render mixer; muted video and group-bus
* attributes are inert.
*
* Tolerant of objects that only partially implement `Element` (test doubles
@@ -204,11 +209,16 @@ export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): st
* `getAttribute` simply has no group, mirroring `readChain`'s style in
* `runtime/audioFx.ts`. */
export function audioGroupOf(el: Element): string | null {
- if (typeof el.tagName !== "string" || el.tagName.toLowerCase() !== "audio") return null;
- if (typeof el.getAttribute !== "function") return null;
+ if (typeof el.tagName !== "string" || typeof el.getAttribute !== "function") return null;
+ if (!isGroupableMember(el)) return null;
return el.getAttribute(HF_AUDIO_GROUP_ATTR) || null;
}
+function isGroupableMember(el: Element): boolean {
+ if (el.tagName.toLowerCase() === "audio") return true;
+ return typeof el.hasAttribute === "function" && isAudibleVideoElement(el);
+}
+
/**
* Make `<hf-audio-group>` inert, once per document.
*
diff --git a/packages/core/src/compiler/mediaRenderIds.test.ts b/packages/core/src/compiler/mediaRenderIds.test.ts
index 4e6bfab329..f0411a1f39 100644
--- a/packages/core/src/compiler/mediaRenderIds.test.ts
+++ b/packages/core/src/compiler/mediaRenderIds.test.ts
@@ -166,6 +166,16 @@ describe("audio group render ids", () => {
expect(d.querySelector("audio")?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");
});
+ it("stamps an audible video member and skips a muted one", () => {
+ const d = doc(`<hf-audio-group id="vo"></hf-audio-group>
+ <video id="v" src="a.mp4" data-has-audio="true" data-audio-group="vo"></video>
+ <video id="m" src="b.mp4" muted data-audio-group="vo"></video>`);
+ assignMediaRenderIds(d);
+ const [audible, muted] = [...d.querySelectorAll("video")];
+ expect(audible?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");
+ expect(muted?.hasAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe(false);
+ });
+
it("leaves a member alone when no bus element declares its group", () => {
const d = doc(`<audio id="a" src="a.wav" data-audio-group="ghost"></audio>`);
assignMediaRenderIds(d);
diff --git a/packages/core/src/compiler/mediaRenderIds.ts b/packages/core/src/compiler/mediaRenderIds.ts
index 9e2a545e7a..ad9227a563 100644
--- a/packages/core/src/compiler/mediaRenderIds.ts
+++ b/packages/core/src/compiler/mediaRenderIds.ts
@@ -25,6 +25,8 @@
* fix scene footage.
*/
+import { AUDIBLE_VIDEO_QUALIFIER } from "../audibleVideo.js";
+
export const MEDIA_RENDER_ID_ATTR = "data-hf-render-id";
/**
@@ -141,7 +143,7 @@ function assignAudioGroupRenderIds(document: DocumentLike, taken: Set<string>):
if (busesById.size === 0) return;
for (const member of document.querySelectorAll(
- "audio[data-audio-group]",
+ `audio[data-audio-group], video[data-audio-group]${AUDIBLE_VIDEO_QUALIFIER}`,
) as Iterable<ScopedElementLike>) {
const groupId = member.getAttribute("data-audio-group");
const buses = groupId ? busesById.get(groupId) : undefined;
diff --git a/packages/core/src/editing/affordances.test.ts b/packages/core/src/editing/affordances.test.ts
index 81574e3deb..86db7d912c 100644
--- a/packages/core/src/editing/affordances.test.ts
+++ b/packages/core/src/editing/affordances.test.ts
@@ -140,6 +140,16 @@ describe("resolveEditingAffordances — sections", () => {
expect(s).toMatchObject({ media: true, colorGrading: true });
});
+ it("audible video: audioFx; silent video: no audioFx", () => {
+ expect(
+ resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: true })).sections.audioFx,
+ ).toBe(true);
+ expect(
+ resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: false })).sections.audioFx,
+ ).toBe(false);
+ expect(resolveEditingAffordances(baseFacts({ tag: "video" })).sections.audioFx).toBe(false);
+ });
+
it("audio: media but not colorGrading", () => {
const s = resolveEditingAffordances(baseFacts({ tag: "audio" })).sections;
expect(s).toMatchObject({ media: true, colorGrading: false });
diff --git a/packages/core/src/editing/affordances.ts b/packages/core/src/editing/affordances.ts
index 682494094f..238a17a6b2 100644
--- a/packages/core/src/editing/affordances.ts
+++ b/packages/core/src/editing/affordances.ts
@@ -69,6 +69,7 @@ export interface EditableElementFacts {
hasTimingStart: boolean;
/** count of GSAP tweens targeting this element */
animationCount: number;
+ hasAudio?: boolean;
}
/**
@@ -218,7 +219,8 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSect
return {
text: facts.hasEditableText && !facts.isCompositionHost && !facts.isInsideLockedComposition,
media: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",
- audioFx: facts.tag === "audio" || isAudioBus,
+ audioFx:
+ facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),
colorGrading: facts.tag === "video" || facts.tag === "img",
// A bus has no clip range at all — no `data-start`, no duration, and its
// automation clock is composition time — so Start/Duration/End would be
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 31a4898f3e..c1f1725a0c 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -183,6 +183,11 @@ export {
} from "./compiler/subCompositionValidity";
export { RUNTIME_BOOTSTRAP_ATTR, stripEmbeddedRuntimeScripts } from "./compiler/htmlDocument";
export { queryByAttr } from "./utils/cssSelector";
+export {
+ AUDIBLE_MEDIA_SELECTOR,
+ audibleVideoNeedsWebAudio,
+ isAudibleVideoElement,
+} from "./audibleVideo";
export { decodeUrlPathVariants } from "./utils/urlPath";
export {
clearGifFramesBeforeNext,
diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts
index 4a60c5155b..76a95cbb2d 100644
--- a/packages/core/src/runtime/init.test.ts
+++ b/packages/core/src/runtime/init.test.ts
@@ -5279,6 +5279,175 @@ describe("initSandboxRuntimeModular", () => {
expect(seeks.filter((t) => t > 0.1)).toEqual([]);
});
});
+ describe("an audible <video> routed through Web Audio", () => {
+ const ctx = useMockAudioContext();
+
+ function mountMedia(tag: "audio" | "video", attrs: Record<string, string> = {}) {
+ let root = document.querySelector<HTMLElement>("[data-root]");
+ if (!root) {
+ root = document.createElement("div");
+ root.setAttribute("data-composition-id", "main");
+ root.setAttribute("data-root", "true");
+ root.setAttribute("data-start", "0");
+ root.setAttribute("data-duration", "10");
+ root.setAttribute("data-width", "1920");
+ root.setAttribute("data-height", "1080");
+ document.body.appendChild(root);
+ }
+ const el = document.createElement(tag);
+ el.setAttribute("data-start", "0");
+ el.setAttribute("data-duration", "10");
+ el.setAttribute("src", "/assets/talk.mp4");
+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);
+ el.load = () => {};
+ el.play = vi.fn(() => Promise.resolve());
+ root.appendChild(el);
+ window.__timelines = { main: createMockTimeline(10) };
+ return el;
+ }
+
+ function spyCapture() {
+ return vi
+ .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")
+ .mockResolvedValue(null);
+ }
+
+ it("schedules it through the media-element transport with its above-unity gain", async () => {
+ const video = mountMedia("video", { "data-has-audio": "true", "data-volume": "2" });
+ const captureSpy = spyCapture();
+
+ await startPlayback();
+
+ expect(captureSpy).toHaveBeenCalledTimes(1);
+ expect(captureSpy.mock.calls[0]?.[0]).toBe(video);
+ expect(captureSpy.mock.calls[0]?.[4]).toBe(2);
+ });
+
+ it("leaves a plain audible video at unity on native output", async () => {
+ mountMedia("video", { "data-has-audio": "true" });
+
+ await startPlayback();
+ await Promise.resolve();
+
+ expect(ctx.mediaElementSources).toBe(0);
+ });
+
+ it.each([
+ ["data-fx-chain", "[]"],
+ ["data-automation", "[]"],
+ ["data-audio-group", "music"],
+ ])("acquires a media element source for an audible video carrying %s", async (name, value) => {
+ mountMedia("video", { "data-has-audio": "true", [name]: value });
+
+ await startPlayback();
+ await Promise.resolve();
+
+ expect(ctx.mediaElementSources).toBe(1);
+ });
+
+ it("never schedules a muted or data-has-audio=false video", async () => {
+ mountMedia("video", { "data-has-audio": "true", muted: "" });
+ mountMedia("video", { "data-has-audio": "false" });
+ const captureSpy = spyCapture();
+
+ await startPlayback();
+
+ expect(captureSpy).not.toHaveBeenCalled();
+ });
+
+ it("routes only the <audio> of a legacy split (muted video + audio on the same file)", async () => {
+ mountMedia("video", { muted: "" });
+ const audio = mountMedia("audio");
+ const captureSpy = spyCapture();
+
+ await startPlayback();
+
+ expect(captureSpy).toHaveBeenCalledTimes(1);
+ expect(captureSpy.mock.calls[0]?.[0]).toBe(audio);
+ });
+
+ it("never whole-file decodes a video whose capture failed, and leaves it unmuted", async () => {
+ const video = mountMedia("video", {
+ "data-has-audio": "true",
+ "data-fx-chain": "[]",
+ "data-playback-rate": "2",
+ });
+ const plainVideo = mountMedia("video", { "data-has-audio": "true" });
+ const audio = mountMedia("audio");
+ spyCapture();
+ const decodeSpy = vi
+ .spyOn(WebAudioTransport.prototype, "decodeAudioElement")
+ .mockResolvedValue(null);
+
+ await startPlayback();
+ await Promise.resolve();
+
+ expect(decodeSpy).toHaveBeenCalledWith(audio);
+ expect(decodeSpy).not.toHaveBeenCalledWith(video);
+ expect(decodeSpy).not.toHaveBeenCalledWith(plainVideo);
+ expect(video.muted).toBe(false);
+ });
+
+ const rateRamp = JSON.stringify({
+ version: 1,
+ lanes: [
+ {
+ target: "rate",
+ points: [
+ { t: 0, v: 1 },
+ { t: 5, v: 2 },
+ ],
+ },
+ ],
+ });
+
+ it("captures a ramped audible video in a group, so the group bus carries it", async () => {
+ const video = mountMedia("video", {
+ "data-has-audio": "true",
+ "data-audio-group": "music",
+ "data-automation": rateRamp,
+ });
+ const captureSpy = spyCapture();
+
+ await startPlayback();
+
+ expect(captureSpy.mock.calls.map((call) => call[0])).toContain(video);
+ });
+
+ it("never decodes a ramped audio clip whose capture failed", async () => {
+ const audio = mountMedia("audio", { "data-automation": rateRamp });
+ spyCapture();
+ const decodeSpy = vi
+ .spyOn(WebAudioTransport.prototype, "decodeAudioElement")
+ .mockResolvedValue(null);
+
+ await startPlayback();
+ await Promise.resolve();
+
+ expect(decodeSpy).not.toHaveBeenCalledWith(audio);
+ expect(audio.muted).toBe(false);
+ });
+
+ it("adds exactly one reschedule when a routed video's data-hidden toggles mid-playback", async () => {
+ const video = mountMedia("video", {
+ "data-has-audio": "true",
+ "data-fx-chain": "[]",
+ "data-hidden": "",
+ });
+ await startPlayback();
+ const captureSpy = spyCapture();
+ const generationSpy = vi.spyOn(WebAudioTransport.prototype, "startGeneration");
+
+ window.__player?.seek(1, { keepPlaying: true });
+ const seekOnly = generationSpy.mock.calls.length;
+ generationSpy.mockClear();
+ video.removeAttribute("data-hidden");
+ window.__player?.seek(2, { keepPlaying: true });
+
+ expect(generationSpy.mock.calls.length).toBe(seekOnly + 1);
+ expect(captureSpy.mock.calls.at(-1)?.[0]).toBe(video);
+ });
+ });
});
/**
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 012ec73c62..e5b60f33d5 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -136,6 +136,7 @@ import {
isMediaElement,
isVideoElement,
} from "./domRealm";
+import { audibleVideoNeedsWebAudio, isAudibleVideoElement } from "../audibleVideo";
/**
* A `window.__timelines` entry is authored content and may be a PARTIAL
@@ -331,7 +332,15 @@ function pageAnimationsForOnePass(): () => Animation[] {
// A `<video>` joins only for a gain `el.volume` cannot express: capture is a one-way door.
const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
- isAudioElement(el) || (isVideoElement(el) && Number.parseFloat(el.dataset.volume ?? "") > 1);
+ isAudioElement(el) ||
+ (isVideoElement(el) &&
+ isAudibleVideoElement(el) &&
+ audibleVideoNeedsWebAudio({
+ volume: Number.parseFloat(el.dataset.volume ?? ""),
+ fxChain: el.getAttribute("data-fx-chain"),
+ automation: el.getAttribute("data-automation"),
+ audioGroup: el.getAttribute("data-audio-group"),
+ }));
const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>
Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);
@@ -2589,8 +2598,8 @@ export function initSandboxRuntimeModular(): void {
// sync with a `data-hidden` toggle made mid-playback.
const groupHiddenLast = new WeakMap<Element, boolean>();
const groupHasUncapturedMember = (groupId: string, currentTime: number): boolean => {
- for (const el of document.querySelectorAll("audio[data-start]")) {
- if (!isMediaElement(el) || audioGroupOf(el) !== groupId) continue;
+ for (const el of webAudioMediaIn(document)) {
+ if (audioGroupOf(el) !== groupId) continue;
if (webAudio.routesElement(el) || isSilencedByHidden(el)) continue;
const start = resolveAbsoluteMediaStartSeconds(el);
const duration = parseStrictFiniteTimingNumber(el.dataset.duration);
@@ -4801,8 +4810,6 @@ export function initSandboxRuntimeModular(): void {
// that existed before (#3458).
const route = classifyWebAudioMediaRoute(rawEl);
reportWebAudioMediaRoute(rawEl, route);
- // Decoded buffers cannot follow a rate curve without shifting pitch; the media element can.
- if (typeof readElementRateSpec(rawEl) !== "number") continue;
// The cross-origin verdict's BEST outcome is decode, since a CDN that
// sends `Access-Control-Allow-Origin` (the author just never wrote the
// `crossorigin` attribute) decodes fine and keeps the whole FX graph.
@@ -4823,6 +4830,8 @@ export function initSandboxRuntimeModular(): void {
// A video's picture must keep playing from the element, so it has no decode fallback.
if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)
return;
+ // Decoded buffers cannot follow a rate curve without shifting pitch; the media element can.
+ if (typeof readElementRateSpec(rawEl) !== "number") return;
const effectiveRate = state.playbackRate * readElementPlaybackRate(rawEl);
// Deliberately the FX/automation pair and NOT
// `nativeUnexpressibleProcessing()`, which this route's diagnostic uses.
diff --git a/packages/core/src/runtime/webAudioRoute.ts b/packages/core/src/runtime/webAudioRoute.ts
index 3459a4f82b..f759f6a323 100644
--- a/packages/core/src/runtime/webAudioRoute.ts
+++ b/packages/core/src/runtime/webAudioRoute.ts
@@ -5,50 +5,15 @@ import { postRuntimeMessage } from "./bridge";
import type { RuntimeJson } from "./types";
/**
- * Which transport may claim an `<audio>` element's output.
+ * Which transport may claim an `<audio>` or audible `<video>` element's output.
*
- * `createMediaElementSource()` is the runtime's PRIMARY audio path, and it is
- * a one-way door: the node permanently reroutes the element away from its
- * native output and is cached for the element's lifetime. That matters because
- * of a spec behaviour that looks nothing like a failure — per the Web Audio
- * spec's MediaElementAudioSourceNode security section, a node built over a
- * resource that fails the CORS-cross-origin check outputs SILENCE. It does not
- * throw, so the `try/catch` around the call in `webAudioTransport.ts` never
- * fires, nothing reaches `swallow()`, and the composition plays perfectly —
- * timeline advancing, visuals animating — with no audio at all (#3458).
- *
- * The only defence is to decide BEFORE the call, which is what this module is:
- * a pure classifier, so the same verdict can be reached at media-discovery time
- * (to emit a diagnostic) and at schedule time (to actually withhold the node)
- * without those two ever drifting apart.
- *
- * That guarantee holds for every caller that routes through this classifier —
- * it is NOT a runtime-wide interception of `createMediaElementSource`. The
- * timeline transport (`webAudioTransport.ts`, via `init.ts`) always goes
- * through it; a UI surface that builds its own throwaway `AudioContext` for
- * an unrelated purpose (e.g. the asset sidebar's preview player,
- * `AudioRow.tsx`) has to call it too, and is expected to. Known gap: an
- * element playing a `MediaStream` via `srcObject` instead of `src`/`<source>`
- * has no origin for this module to judge — `routeCandidates` only reads
- * `src`-shaped attributes, so a `srcObject` element always reads as
- * `web-audio` here, correctly or not. Nothing in this codebase feeds
- * `createMediaElementSource` from a `srcObject` element today, so this is
- * recorded as a boundary rather than fixed.
- *
- * Second known gap, same shape: `corsSilenceReason` judges the RAW url string —
- * the same-origin URL the author wrote, or whatever the browser resolved into
- * `currentSrc` — not wherever a server-side redirect chain actually lands.
- * A same-origin URL that 302s to a cross-origin CDN reads as `web-audio` here
- * and gets a real `createMediaElementSource` node; whether that node is
- * silent then depends on the redirect target's CORS headers, which this
- * classifier never sees (following the chain to inspect the final response
- * would turn a pure, synchronous verdict — needed on every schedule call —
- * into an async fetch). A cross-origin URL that redirects back to same-origin
- * has the opposite miss: classified `decode-only` and sent down the fetch
- * fallback when Web Audio capture would have worked fine either way. Not
- * fixed for the same reason as `srcObject` — no caller in this codebase
- * routes media through a redirecting URL today — but worth knowing before
- * trusting this classifier's verdict for one that does.
+ * `createMediaElementSource()` is a one-way door, and over a resource that fails
+ * the CORS-cross-origin check it outputs SILENCE without throwing (#3458). So the
+ * verdict is reached BEFORE the call by this pure classifier, shared by media
+ * discovery (diagnostic) and scheduling (withholding the node). Callers that build
+ * their own `AudioContext` (e.g. `AudioRow.tsx`) must route through it too.
+ * Known gaps: `srcObject` elements always read `web-audio`, and the RAW url is
+ * judged, not a redirect chain's final target.
*/
export type WebAudioMediaRoute =
/** Same-origin, CORS-opted-in, or a scheme the check doesn't apply to. */
@@ -259,8 +224,8 @@ export function nativeUnexpressibleProcessing(el: HTMLMediaElement): string[] {
* element is not the audio source there in the first place.
*
* Same signal `mediaProxy.ts`'s `isRenderMode` gates on. The `<video>` half of
- * that check (the injected render-frame sibling) is deliberately not mirrored:
- * only `<audio>` ever reaches this module.
+ * that check (the injected render-frame sibling) is not mirrored: the
+ * export-seek config alone gates reporting.
*/
function isRenderMode(): boolean {
// Read through an inline cast rather than the ambient `Window` augmentation
diff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.ts
index a3f0ab87fc..b8741ad644 100644
--- a/packages/engine/src/services/audioMixer.grouping.test.ts
+++ b/packages/engine/src/services/audioMixer.grouping.test.ts
@@ -544,3 +544,44 @@ describe("duplicate bus instances", () => {
]);
});
});
+
+describe("audible video members", () => {
+ const withBus = (busAttrs: string, videoAttrs: string) =>
+ `<div id="root" data-composition-id="main" data-start="0" data-duration="4">
+ <hf-audio-group id="vo" data-volume="0.5" ${busAttrs}></hf-audio-group>
+ <video id="talk" src="talk.mp4" data-start="0" data-duration="4"
+ data-has-audio="true" data-audio-group="vo" ${videoAttrs}></video>
+ </div>`;
+
+ it("puts an audible video's track in its group sub-mix", () => {
+ const tracks = parseAudioElements(withBus("", ""));
+ expect(tracks.map((t) => [t.type, t.groupId, t.groupVolume])).toEqual([["video", "vo", 0.5]]);
+ });
+
+ it("drops an audible video from the mix when its group is hidden", () => {
+ expect(parseAudioElements(withBus("data-hidden", ""))).toEqual([]);
+ });
+
+ it("keeps a ramped audible video in its group sub-mix with its rate lane", () => {
+ const lane = JSON.stringify({
+ version: 1,
+ lanes: [
+ {
+ target: "rate",
+ points: [
+ { t: 0, v: 1 },
+ { t: 2, v: 2 },
+ ],
+ },
+ ],
+ });
+ const [track] = parseAudioElements(withBus("", `data-automation='${lane}'`));
+ expect(track?.groupId).toBe("vo");
+ expect(typeof track?.playbackRate).toBe("object");
+ });
+
+ it("ignores a muted video's group membership", () => {
+ const tracks = parseAudioElements(withBus("data-hidden", "muted"));
+ expect(tracks.every((t) => t.groupId === undefined)).toBe(true);
+ });
+});
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 598ad3030e..af39c210e2 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -1619,6 +1619,11 @@ describe("parseAudioElements — hidden tracks", () => {
expect(parseAudioElements(html).map((track) => track.id)).toEqual(["master"]);
});
+
+ it("does not mix a muted video that still declares data-has-audio", () => {
+ const html = `<video id="v" src="v.mp4" muted data-has-audio="true" data-start="0" data-duration="3"></video>`;
+ expect(parseAudioElements(html)).toEqual([]);
+ });
});
describe("parseAudioElements data-fx-chain", () => {
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 6f58cf75ad..a529255658 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -5,7 +5,7 @@
* Processes and mixes audio tracks using FFmpeg.
*/
-import { isSelfOrAncestorHidden } from "./mediaHidden.js";
+import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
import { join, dirname, isAbsolute, relative } from "path";
import { parseHTML } from "linkedom";
@@ -44,6 +44,7 @@ import {
} from "@hyperframes/core/audio-automation";
import { chainTailSeconds } from "@hyperframes/core/audio-fx-tail";
import {
+ isAudibleVideoElement,
MEDIA_RENDER_ID_ATTR,
normalizePlaybackRate,
normalizeRateSpec,
@@ -54,8 +55,7 @@ import {
timeAtSourceTime,
type RateSpec,
} from "@hyperframes/core";
-import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups";
-import { AUDIO_GROUP_RENDER_ID_ATTR } from "@hyperframes/core";
+import { resolveAudioGroups } from "@hyperframes/core/audio-groups";
import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js";
import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
@@ -76,21 +76,6 @@ export type { AudioElement, MixResult } from "./audioMixer.types.js";
*/
export const MIXED_AUDIO_FILENAME = "audio.m4a";
-/**
- * The bus key a member belongs to, as `resolveAudioGroups` keys them.
- *
- * The compiler's `data-hf-group-render-id` names one INSTANCE of a bus; the
- * author's `data-audio-group` names it only within its own composition file. A
- * sub-composition declaring a bus and its members, used twice, therefore had
- * both instances' members under one key: one sub-mix for two independent buses,
- * one instance's fader and chain over the other's audio, and — with only the
- * second muted — BOTH instances dropped from the export. Uncompiled documents
- * (the live preview) carry no stamp and read exactly as before.
- */
-function memberGroupKey(el: RefResolverEl): string | null {
- return el.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR) ?? el.getAttribute(HF_AUDIO_GROUP_ATTR);
-}
-
function clampVolume(volume: number): number {
return clampAudioGain(volume);
}
@@ -597,10 +582,7 @@ export function parseAudioElements(html: string): AudioElement[] {
const groupsById = new Map(
resolveAudioGroups(document).map((group) => [group.id, group] as const),
);
- const memberGroupHidden = (el: AudioMediaElement): boolean => {
- const groupId = memberGroupKey(el);
- return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false;
- };
+ const memberGroupHidden = (el: AudioMediaElement): boolean => isMemberGroupHidden(groupsById, el);
// <audio> and <video data-has-audio> tracks differ only in the emitted id
@@ -610,15 +592,14 @@ export function parseAudioElements(html: string): AudioElement[] {
id: string,
src: string,
type: AudioElement["type"],
+ joinsGroup: boolean,
): AudioElement => {
const layerAttr = el.getAttribute("data-layer");
const volumeAttr = el.getAttribute("data-volume");
const fades = readElementFades(el);
const fxChain = el.getAttribute(HF_AUDIO_FX_ATTR);
const automation = el.getAttribute(HF_AUDIO_AUTOMATION_ATTR);
- // Audio only in v1 (matches resolveAudioGroups, which only scans
- // `audio[data-audio-group]`) — a stray attribute on a <video> is inert.
- const groupId = type === "audio" ? memberGroupKey(el) : null;
+ const groupId = joinsGroup ? memberGroupKey(el) : null;
const group = groupId ? groupsById.get(groupId) : undefined;
return {
id,
@@ -658,15 +639,16 @@ export function parseAudioElements(html: string): AudioElement[] {
// member from the mix, the same way `isHidden` drops one track.
if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;
if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
- elements.push(build(el, id, src, "audio"));
+ elements.push(build(el, id, src, "audio", true));
}
for (const el of document.querySelectorAll('video[id][data-has-audio="true"]')) {
+ if (!isAudibleVideoElement(el)) continue;
const id = trackId(el);
const src = resolveMediaElementSrc(el);
- if (!id || !src || isHidden(el)) continue;
+ if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;
if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
- elements.push(build(el, `${id}-audio`, src, "video"));
+ elements.push(build(el, `${id}-audio`, src, "video", true));
}
return elements;
diff --git a/packages/engine/src/services/mediaHidden.ts b/packages/engine/src/services/mediaHidden.ts
index 0fe24a38e3..61c5fb21b4 100644
--- a/packages/engine/src/services/mediaHidden.ts
+++ b/packages/engine/src/services/mediaHidden.ts
@@ -1,3 +1,6 @@
+import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups";
+import { AUDIO_GROUP_RENDER_ID_ATTR } from "@hyperframes/core";
+
interface HiddenCheckEl {
hasAttribute(name: string): boolean;
parentElement: HiddenCheckEl | null;
@@ -9,3 +12,24 @@ export function isSelfOrAncestorHidden(el: HiddenCheckEl): boolean {
}
return false;
}
+
+interface GroupKeyEl {
+ getAttribute(name: string): string | null;
+}
+
+export function memberGroupKey(el: GroupKeyEl): string | null {
+ return el.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR) ?? el.getAttribute(HF_AUDIO_GROUP_ATTR);
+}
+
+export type AudioGroupsById = ReadonlyMap<string, { hidden?: boolean }>;
+
+export function audioGroupsById(
+ document: Parameters<typeof resolveAudioGroups>[0],
+): AudioGroupsById {
+ return new Map(resolveAudioGroups(document).map((group) => [group.id, group] as const));
+}
+
+export function isMemberGroupHidden(groupsById: AudioGroupsById, el: GroupKeyEl): boolean {
+ const groupId = memberGroupKey(el);
+ return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false;
+}
diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts
index 925da5e65e..e643be22b9 100644
--- a/packages/engine/src/services/videoFrameExtractor.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.test.ts
@@ -3714,4 +3714,16 @@ describe("parseVideoElements hidden flag", () => {
["c", undefined],
]);
});
+
+ it("marks only an audible video hidden when its audio group is hidden", () => {
+ const videos = parseVideoElements(
+ `<hf-audio-group id="g" data-hidden></hf-audio-group>` +
+ `<video id="a" src="a.mp4" data-has-audio="true" data-audio-group="g"></video>` +
+ `<video id="b" src="b.mp4" muted data-audio-group="g"></video>`,
+ );
+ expect(videos.map((v) => [v.id, v.hidden])).toEqual([
+ ["a", true],
+ ["b", undefined],
+ ]);
+ });
});
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index 56d077dd4e..3b8879f87d 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -6,13 +6,14 @@
* Videos are replaced with <img> elements during capture.
*/
-import { isSelfOrAncestorHidden } from "./mediaHidden.js";
+import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";
import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";
import { join } from "path";
import { parseHTML } from "linkedom";
import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";
import {
MEDIA_RENDER_ID_ATTR,
+ isAudibleVideoElement,
fpsToFfmpegArg,
fpsToNumber,
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
@@ -667,6 +668,7 @@ export function parseVideoElements(html: string): VideoElement[] {
const startCache = new Map<RefResolverEl, number>();
const visiting = new Set<RefResolverEl>();
+ const groupsById = audioGroupsById(document);
const videoEls = document.querySelectorAll("video");
let autoIdCounter = 0;
for (const el of videoEls) {
@@ -722,7 +724,10 @@ export function parseVideoElements(html: string): VideoElement[] {
playbackRate: readElementRateSpec(el),
loop: el.hasAttribute("loop"),
hasAudio: hasAudioAttr === "true",
- ...(isSelfOrAncestorHidden(el) ? { hidden: true } : {}),
+ ...(isSelfOrAncestorHidden(el) ||
+ (isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el))
+ ? { hidden: true }
+ : {}),
});
}
diff --git a/packages/lint/src/rules/media.test.ts b/packages/lint/src/rules/media.test.ts
index 0f55557ad5..ab940d8d95 100644
--- a/packages/lint/src/rules/media.test.ts
+++ b/packages/lint/src/rules/media.test.ts
@@ -808,9 +808,21 @@ describe("audio_group_no_members", () => {
expect(finding?.message).not.toContain('"music"');
});
- it("does not count video as group membership", async () => {
+ it("counts an audible video as group membership", async () => {
const res = await lintHyperframeHtml(
- doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" data-audio-group="voiceover"></video>
+ doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" data-has-audio="true" data-audio-group="voiceover"></video>
+ <audio id="s-1" src="s.wav" data-start="0" data-duration="2" data-audio-group="sfx"></audio>`),
+ );
+ expect(
+ res.findings.some(
+ (finding) => finding.code === "audio_group_no_members" && finding.elementId === "voiceover",
+ ),
+ ).toBe(false);
+ });
+
+ it("does not count a muted video as group membership", async () => {
+ const res = await lintHyperframeHtml(
+ doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" muted data-audio-group="voiceover"></video>
<audio id="s-1" src="s.wav" data-start="0" data-duration="2" data-audio-group="sfx"></audio>`),
);
expect(
diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.ts
index 6d492c7c14..695705b63d 100644
--- a/packages/lint/src/rules/media.ts
+++ b/packages/lint/src/rules/media.ts
@@ -993,7 +993,7 @@ const AUDIO_GROUP_TIMING_ATTRS = ["data-start", "data-duration", "data-track-ind
/**
* A bus nobody joined does nothing, silently.
*
- * `resolveAudioGroups` builds groups from the MEMBERS (`audio[data-audio-group]`)
+ * `resolveAudioGroups` builds groups from the MEMBERS (audio or audible video)
* and only then looks for a matching `<hf-audio-group>` element, so a bus whose
* id no clip names is dropped entirely — its fader, FX chain and automation
* never reach preview or render, and nothing says so. One typo is enough:
@@ -1004,7 +1004,7 @@ const AUDIO_GROUP_TIMING_ATTRS = ["data-start", "data-duration", "data-track-ind
function findAudioGroupNoMembersFindings(ctx: LintContext): HyperframeLintFinding[] {
const memberGroupIds = new Set(
ctx.tags
- .filter((tag) => tag.name === "audio")
+ .filter((tag) => tag.name === "audio" || (tag.name === "video" && isAudibleVideoTag(tag.raw)))
.map((tag) => readAttr(tag.raw, "data-audio-group"))
.filter((id): id is string => Boolean(id)),
);
diff --git a/packages/lint/src/utils.test.ts b/packages/lint/src/utils.test.ts
index b6d201ec6c..cc78ecfdbf 100644
--- a/packages/lint/src/utils.test.ts
+++ b/packages/lint/src/utils.test.ts
@@ -1,5 +1,6 @@
import { describe, it, expect } from "vitest";
import {
+ isAudibleVideoTag,
parseHtmlStructure,
stripCssComments,
stripJsComments,
@@ -232,3 +233,16 @@ describe("stripCssComments", () => {
}
});
});
+
+describe("isAudibleVideoTag", () => {
+ it("treats an absent marker and the exact value true as audible", () => {
+ expect(isAudibleVideoTag('<video src="a.mp4">')).toBe(true);
+ expect(isAudibleVideoTag('<video data-has-audio="true">')).toBe(true);
+ });
+ it("treats muted, false, empty and non-exact values as silent", () => {
+ expect(isAudibleVideoTag('<video muted data-has-audio="true">')).toBe(false);
+ expect(isAudibleVideoTag('<video data-has-audio="false">')).toBe(false);
+ expect(isAudibleVideoTag('<video data-has-audio="">')).toBe(false);
+ expect(isAudibleVideoTag('<video data-has-audio="TRUE">')).toBe(false);
+ });
+});
diff --git a/packages/lint/src/utils.ts b/packages/lint/src/utils.ts
index 265bc9110c..57f0fc0bc4 100644
--- a/packages/lint/src/utils.ts
+++ b/packages/lint/src/utils.ts
@@ -239,12 +239,12 @@ export function hasAttrName(tagSource: string, attr: string): boolean {
return new RegExp(`(?:^|\\s)${escaped}(?:\\s*=|\\s|/?>)`, "i").test(attrs);
}
-// An explicit data-has-audio is authoritative for the compiler; "false" and empty mean silent.
+// An explicit data-has-audio is authoritative for the compiler; only the exact value "true" means audible.
export function isAudibleVideoTag(tagSource: string): boolean {
if (hasAttrName(tagSource, "muted")) return false;
if (!hasAttrName(tagSource, "data-has-audio")) return true;
const declared = readAttr(tagSource, "data-has-audio");
- return declared !== null && declared.trim().toLowerCase() !== "false";
+ return declared === "true";
}
export function mediaTimeWindow(tagSource: string): { start: number; end: number } | null {
diff --git a/packages/sdk/src/editing/affordances.test.ts b/packages/sdk/src/editing/affordances.test.ts
index 8d2da06aa6..7e8146d06e 100644
--- a/packages/sdk/src/editing/affordances.test.ts
+++ b/packages/sdk/src/editing/affordances.test.ts
@@ -26,6 +26,22 @@ describe("resolveElementAffordances (live DOM)", () => {
expect(a.capabilities.canSelect).toBe(true);
});
+ it("audible video => audioFx; muted or data-has-audio=false video => no audioFx", () => {
+ const model = { text: null, animationIds: [], start: null };
+ expect(resolveElementAffordances(el(`<video></video>`), model).sections.audioFx).toBe(true);
+ expect(
+ resolveElementAffordances(el(`<video data-has-audio="true"></video>`), model).sections
+ .audioFx,
+ ).toBe(true);
+ expect(resolveElementAffordances(el(`<video muted></video>`), model).sections.audioFx).toBe(
+ false,
+ );
+ expect(
+ resolveElementAffordances(el(`<video data-has-audio="false"></video>`), model).sections
+ .audioFx,
+ ).toBe(false);
+ });
+
it("absolutely-positioned div with inline left/top => canMove", () => {
const d = el(`<div style="position:absolute;left:5px;top:5px"></div>`);
const a = resolveElementAffordances(d, { text: null, animationIds: [], start: null });
diff --git a/packages/sdk/src/editing/affordances.ts b/packages/sdk/src/editing/affordances.ts
index efb5e62f99..33f58b00da 100644
--- a/packages/sdk/src/editing/affordances.ts
+++ b/packages/sdk/src/editing/affordances.ts
@@ -58,5 +58,12 @@ export function resolveElementAffordances(
hasEditableText: modelEl?.text != null,
hasTimingStart: modelEl ? modelEl.start != null : liveEl.hasAttribute("data-start"),
animationCount: modelEl?.animationIds.length ?? 0,
+ hasAudio: isAudibleVideo(liveEl),
});
}
+
+function isAudibleVideo(el: HTMLElement): boolean {
+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;
+ const hasAudio = el.getAttribute("data-has-audio");
+ return hasAudio === null || hasAudio === "true";
+}
diff --git a/packages/studio/src/components/editor/domEditingLayers.test.ts b/packages/studio/src/components/editor/domEditingLayers.test.ts
index a67b998af4..629275a827 100644
--- a/packages/studio/src/components/editor/domEditingLayers.test.ts
+++ b/packages/studio/src/components/editor/domEditingLayers.test.ts
@@ -7,6 +7,7 @@ import {
buildTextFieldChildLocator,
readHfId,
liveLayerElement,
+ domEditSelectionToFacts,
} from "./domEditingLayers";
import type { DomEditTextField } from "./domEditingTypes";
@@ -332,3 +333,23 @@ describe("liveLayerElement", () => {
).toBe("strip");
});
});
+
+describe("domEditSelectionToFacts hasAudio", () => {
+ async function factsFor(html: string) {
+ document.body.innerHTML = html;
+ const node = document.body.firstElementChild;
+ if (!(node instanceof HTMLElement)) throw new Error("expected element");
+ const selection = await resolveDomEditSelection(node, opts);
+ document.body.innerHTML = "";
+ if (!selection) throw new Error("expected selection");
+ return domEditSelectionToFacts(selection);
+ }
+
+ it("is true for an unmuted video, false for muted or data-has-audio=false", async () => {
+ expect((await factsFor(`<video id="v1"></video>`)).hasAudio).toBe(true);
+ expect((await factsFor(`<video id="v2" data-has-audio="true"></video>`)).hasAudio).toBe(true);
+ expect((await factsFor(`<video id="v3" muted></video>`)).hasAudio).toBe(false);
+ expect((await factsFor(`<video id="v4" data-has-audio="false"></video>`)).hasAudio).toBe(false);
+ expect((await factsFor(`<div id="d1"></div>`)).hasAudio).toBe(false);
+ });
+});
diff --git a/packages/studio/src/components/editor/domEditingLayers.ts b/packages/studio/src/components/editor/domEditingLayers.ts
index df08fc8770..acc24b7c38 100644
--- a/packages/studio/src/components/editor/domEditingLayers.ts
+++ b/packages/studio/src/components/editor/domEditingLayers.ts
@@ -1,4 +1,5 @@
import { probeSourceElement } from "./probeSourceElement";
+import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers";
import type { PatchOperation } from "../../utils/sourcePatcher";
import {
resolveEditingAffordances,
@@ -248,6 +249,7 @@ export function domEditSelectionToFacts(
hasEditableText: selection.textFields.length > 0,
hasTimingStart: selection.dataAttributes.start != null,
animationCount,
+ hasAudio: isAudibleVideoNode(selection.element),
};
}
diff --git a/packages/studio/src/components/editor/mediaAudioEdits.test.ts b/packages/studio/src/components/editor/mediaAudioEdits.test.ts
index df089f55e7..9f2e0203b1 100644
--- a/packages/studio/src/components/editor/mediaAudioEdits.test.ts
+++ b/packages/studio/src/components/editor/mediaAudioEdits.test.ts
@@ -209,4 +209,63 @@ describe("buildKeepSoundCutoutEdit", () => {
const doc = parse(applyEdit(taken, edit));
expect(doc.getElementById("clip-audio-2")?.getAttribute("src")).toBe("assets/talk.mp4");
});
+
+ it("reads the authored src from the source, not the live element", () => {
+ const sub = source.replace('src="assets/talk.mp4"', 'src="../assets/a.mp4"');
+ const edit = buildKeepSoundCutoutEdit({
+ video: liveVideo(),
+ videoId: "clip",
+ target: { id: "clip" },
+ cutoutSrc: "assets/cut.webm",
+ });
+ const doc = parse(applyEdit(sub, edit));
+ expect(doc.getElementById("clip-audio")?.getAttribute("src")).toBe("../assets/a.mp4");
+ expect(doc.getElementById("clip")?.getAttribute("src")).toBe("assets/cut.webm");
+ });
+
+ it("takes the first <source> child when the video has no src attribute", () => {
+ const withSource = source.replace(
+ /<video id="clip"([^>]*?) src="assets\/talk.mp4"([^>]*)><\/video>/,
+ '<video id="clip"$1$2><source src="assets/s.mp4" type="video/mp4"></video>',
+ );
+ const edit = buildKeepSoundCutoutEdit({
+ video: liveVideo(),
+ videoId: "clip",
+ target: { id: "clip" },
+ cutoutSrc: "assets/cut.webm",
+ });
+ const doc = parse(applyEdit(withSource, edit));
+ expect(doc.getElementById("clip-audio")?.getAttribute("src")).toBe("assets/s.mp4");
+ });
+
+ it("reports when the audio insert did not land", () => {
+ const edit = buildKeepSoundCutoutEdit({
+ video: liveVideo(),
+ videoId: "clip",
+ target: { id: "missing" },
+ cutoutSrc: "assets/cut.webm",
+ });
+ edit.prepareContent(source);
+ expect(edit.audioInserted()).toBe(false);
+ const ok = buildKeepSoundCutoutEdit({
+ video: liveVideo(),
+ videoId: "clip",
+ target: { id: "clip" },
+ cutoutSrc: "assets/cut.webm",
+ });
+ ok.prepareContent(source);
+ expect(ok.audioInserted()).toBe(true);
+ });
+
+ it("copies loop onto the linked audio", () => {
+ const video = makeVideo({ id: "clip", src: "assets/talk.mp4", loop: "" });
+ const edit = buildKeepSoundCutoutEdit({
+ video,
+ videoId: "clip",
+ target: { id: "clip" },
+ cutoutSrc: "assets/cut.webm",
+ });
+ const doc = parse(applyEdit(source, edit));
+ expect(doc.getElementById("clip-audio")?.hasAttribute("loop")).toBe(true);
+ });
});
diff --git a/packages/studio/src/components/editor/mediaAudioEdits.ts b/packages/studio/src/components/editor/mediaAudioEdits.ts
index 3ab2317fa0..48f9371633 100644
--- a/packages/studio/src/components/editor/mediaAudioEdits.ts
+++ b/packages/studio/src/components/editor/mediaAudioEdits.ts
@@ -12,7 +12,9 @@ import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
import {
escapeHtmlAttribute,
findTagByTarget,
+ applyPatchByTarget,
readAttributeByTarget,
+ unescapeHtmlAttribute,
type PatchOperation,
type PatchTarget,
} from "../../utils/sourcePatcher";
@@ -24,6 +26,9 @@ import { generateId } from "../../utils/generateId";
const MEDIA_LINK_ATTR = "data-link";
const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";
+const SOUND_NOT_KEPT_MESSAGE =
+ "Background removed, but the sound could not be kept on a linked audio track. Undo to restore the original clip.";
+
const MOVED_SOUND_ATTRS = [
"data-volume",
HF_AUDIO_FADE_IN_ATTR,
@@ -131,6 +136,15 @@ function formatAttrs(attrs: Array<[string, string]>): string {
return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");
}
+function readAuthoredSrc(source: string, target: PatchTarget): string {
+ const tag = findTagByTarget(source, target);
+ if (!tag) return "";
+ const own = /\bsrc=(["'])([^"']*)\1/.exec(tag.tag);
+ if (own?.[2]) return unescapeHtmlAttribute(own[2]);
+ const firstSource = /^\s*<source\b[^>]*?\bsrc=(["'])([^"']*)\1/i.exec(source.slice(tag.end + 1));
+ return firstSource?.[2] ? unescapeHtmlAttribute(firstSource[2]) : "";
+}
+
function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
const match = findTagByTarget(source, target);
if (!match) return source;
@@ -145,6 +159,7 @@ function insertBeforeTarget(source: string, target: PatchTarget, markup: string)
export interface KeepSoundCutoutEdit {
ops: PatchOperation[];
prepareContent: (source: string) => string;
+ audioInserted: () => boolean;
}
export function buildKeepSoundCutoutEdit(input: {
@@ -155,10 +170,11 @@ export function buildKeepSoundCutoutEdit(input: {
}): KeepSoundCutoutEdit {
const { video, target } = input;
const linkId = mintLinkId(video.ownerDocument);
- const originalSrc = video.getAttribute("src") ?? "";
+ let audioInserted = false;
const movedSound: Array<[string, string]> = [];
- const ops: PatchOperation[] = [...cutoutOps({ isVideo: true, cutoutSrc: input.cutoutSrc })];
+ const cutoutSrcOp = htmlAttr("src", input.cutoutSrc);
+ const ops: PatchOperation[] = [htmlAttr("muted", "true"), dataAttr("has-audio", null)];
for (const name of MOVED_SOUND_ATTRS) {
const value = video.getAttribute(name);
if (value === null) continue;
@@ -174,6 +190,7 @@ export function buildKeepSoundCutoutEdit(input: {
ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));
const prepareContent = (source: string): string => {
+ const originalSrc = readAuthoredSrc(source, target);
const audioId = firstFreeName(
`${input.videoId || "video"}-audio`,
new Set(collectHtmlIds(source)),
@@ -193,10 +210,14 @@ export function buildKeepSoundCutoutEdit(input: {
[MEDIA_LINK_ATTR, linkId],
...movedSound,
];
- return insertBeforeTarget(source, target, `<audio ${formatAttrs(attrs)}></audio>`);
+ if (video.hasAttribute("loop")) attrs.push(["loop", ""]);
+ const cut = applyPatchByTarget(source, target, cutoutSrcOp);
+ const inserted = insertBeforeTarget(cut, target, `<audio ${formatAttrs(attrs)}></audio>`);
+ audioInserted = inserted !== cut;
+ return inserted;
};
- return { ops, prepareContent };
+ return { ops, prepareContent, audioInserted: () => audioInserted };
}
export interface MediaEditContext {
@@ -231,23 +252,31 @@ export async function commitCutout(
const isVideo = isVideoSelection(ctx.element);
const label = "Remove background";
const keep = isVideo && keepSound;
- const landed = keep
- ? await commitKeepSoundCutout(ctx, cutoutSrc, label)
- : await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+ if (!keep) {
+ const applied = await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+ if (!applied) throw new Error("Couldn't apply the cutout");
+ return "Applied cutout";
+ }
+ const { landed, soundKept } = await commitKeepSoundCutout(ctx, cutoutSrc, label);
if (!landed) throw new Error("Couldn't apply the cutout");
- return keep ? SOUND_KEPT_ON_LINKED_AUDIO_STAGE : "Applied cutout";
+ if (!soundKept) throw new Error(SOUND_NOT_KEPT_MESSAGE);
+ return SOUND_KEPT_ON_LINKED_AUDIO_STAGE;
}
-function commitKeepSoundCutout(
+async function commitKeepSoundCutout(
ctx: MediaEditContext,
cutoutSrc: string,
label: string,
-): Promise<boolean> {
+): Promise<{ landed: boolean; soundKept: boolean }> {
const edit = buildKeepSoundCutoutEdit({
video: ctx.element.element,
videoId: ctx.element.id ?? null,
target: buildDomEditPatchTarget(ctx.element),
cutoutSrc,
});
- return ctx.commit(ctx.element, edit.ops, { label, prepareContent: edit.prepareContent });
+ const landed = await ctx.commit(ctx.element, edit.ops, {
+ label,
+ prepareContent: edit.prepareContent,
+ });
+ return { landed, soundKept: landed && edit.audioInserted() };
}
diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
index 7991bfcd4f..5c44f617c5 100644
--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
@@ -103,11 +103,11 @@ function mount(dataAttributes: Record<string, string>, alone = false, voices = 2
return { host, onSetAttributeQuiet, onSetAttributeLive };
}
-function mountGroup(memberStart: number) {
+function mountGroup(memberStart: number, memberTag: "audio" | "video" = "audio") {
const bus = document.createElement("hf-audio-group");
bus.id = "voiceover";
document.body.append(bus);
- const member = document.createElement("audio");
+ const member = document.createElement(memberTag);
member.id = "vo-1";
member.setAttribute("data-audio-group", "voiceover");
member.setAttribute("data-start", String(memberStart));
@@ -649,6 +649,16 @@ describe("AudioFxGroup dynamic carve", () => {
leaveShelf(host);
});
+ it("seeks a group audition to an audible video member's span", () => {
+ act(() =>
+ usePlayerStore.setState({ isPlaying: false, currentTime: 2, requestedSeekTime: null }),
+ );
+ const host = mountGroup(7, "video");
+ hoverPreset(host);
+ expect(store().requestedSeekTime).toBe(7);
+ leaveShelf(host);
+ });
+
it("leaves a transport the author started alone", () => {
// Stopping their playback because they passed over a preset would be the
// panel taking a decision nobody offered it.
@@ -1621,7 +1631,14 @@ describe("AudioFxGroup carve by default", () => {
describe("AudioFxGroup carve source list", () => {
/** Mount a bed alongside tracks named however the test needs. */
const mountWith = (
- tracks: { id: string; src?: string; start?: string; duration?: string }[],
+ tracks: {
+ id: string;
+ src?: string;
+ start?: string;
+ duration?: string;
+ tag?: "audio" | "video";
+ muted?: boolean;
+ }[],
bedAttrs: Record<string, string> = {},
) => {
const bed = document.createElement("audio");
@@ -1629,8 +1646,9 @@ describe("AudioFxGroup carve source list", () => {
for (const [k, v] of Object.entries(bedAttrs)) bed.setAttribute(`data-${k}`, v);
document.body.append(bed);
for (const t of tracks) {
- const el = document.createElement("audio");
+ const el = document.createElement(t.tag ?? "audio");
el.id = t.id;
+ if (t.muted) el.setAttribute("muted", "");
if (t.src) el.setAttribute("src", t.src);
if (t.start !== undefined) el.setAttribute("data-start", t.start);
if (t.duration !== undefined) el.setAttribute("data-duration", t.duration);
@@ -1660,6 +1678,16 @@ describe("AudioFxGroup carve source list", () => {
return { host, options, boxes, onSetAttributeQuiet };
};
+ it("offers an audible video as a voice and never a muted one", () => {
+ const { options } = mountWith([
+ { id: "a-roll", tag: "video" },
+ { id: "b-roll", tag: "video", muted: true },
+ { id: "narration" },
+ ]);
+ expect(options).toContain("a-roll");
+ expect(options).not.toContain("b-roll");
+ });
+
it("reads the one voice out instead of offering a picker with one entry", () => {
// A question with one answer is not a question. It is also the common case: a
// narration and a bed, with a couple of stings that cannot be the voice.
diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx
index df0a0d9361..93439868d1 100644
--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx
+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx
@@ -54,6 +54,7 @@ import {
resolveAudioGroups,
} from "@hyperframes/core/audio-groups";
import { useFxLevelling } from "./useFxLevelling.js";
+import { isCarveVoiceElement } from "./useFxCarveGrouping.js";
function auditionSpan(startRaw: string | undefined, durationRaw: string | undefined) {
const start = Number.parseFloat(startRaw ?? "");
@@ -70,8 +71,9 @@ function auditionSpansFor(element: DomEditSelection): AuditionSpan[] {
if (element.tagName?.toLowerCase() !== HF_AUDIO_GROUP_TAG || !element.id) return [];
const doc = element.element?.ownerDocument;
if (!doc) return [];
- return [...doc.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)]
+ return [...doc.querySelectorAll(`[${HF_AUDIO_GROUP_ATTR}]`)]
.filter((member) => member.getAttribute(HF_AUDIO_GROUP_ATTR) === element.id)
+ .filter(isCarveVoiceElement)
.flatMap((member) => {
const span = auditionSpan(
member.getAttribute("data-start") ?? undefined,
diff --git a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx
index 4186ae24b0..8507215654 100644
--- a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx
+++ b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx
@@ -652,7 +652,12 @@ function makeHandlers() {
ops: PatchOperation[],
options: BatchOptions,
) => Promise<boolean>
- >(async () => true),
+ >(async (_selection, _ops, options) => {
+ options.prepareContent?.(
+ '<div data-composition-id="main"><video id="s1-bg" src="assets/intro-loop.mp4" data-start="2"></video></div>',
+ );
+ return true;
+ }),
};
}
@@ -871,14 +876,13 @@ describe.each<PanelKind>(["flat", "design"])("%s panel — cutout keeps the soun
const [selection, ops, options] = onSetAttributeBatch.mock.calls[0] ?? [];
expect(selection).toBe(element);
expect(ops).toEqual([
- htmlOp("src", "assets/cut.webm"),
htmlOp("muted", "true"),
dataOp("has-audio", null),
dataOp("volume", null),
dataOp("link", "link"),
]);
const source =
- '<div data-composition-id="main"><video id="s1-bg" src="assets/cut.webm" muted data-start="2" data-duration="3" data-track-index="0" data-link="link"></video></div>';
+ '<div data-composition-id="main"><video id="s1-bg" src="assets/intro-loop.mp4" muted data-start="2" data-duration="3" data-track-index="0" data-link="link"></video></div>';
const prepared = options?.prepareContent?.(source) ?? "";
const audio = new DOMParser()
.parseFromString(prepared, "text/html")
diff --git a/packages/studio/src/components/editor/useFxCarve.ts b/packages/studio/src/components/editor/useFxCarve.ts
index 27e9ae7b1b..44959f0826 100644
--- a/packages/studio/src/components/editor/useFxCarve.ts
+++ b/packages/studio/src/components/editor/useFxCarve.ts
@@ -25,6 +25,7 @@ import {
carverAgainst,
collectCarveCandidates,
CARVE_ABORTED,
+ isCarveVoiceElement,
isPromiseLike,
resolveNextCarveSettings,
} from "./useFxCarveGrouping.js";
@@ -101,7 +102,7 @@ function resolveCarveVoices(doc: Document, sources: readonly string[]): CarveCli
// By tag name, not `instanceof HTMLAudioElement`: these elements belong to
// the composition's iframe document, so the constructor they were made
// from is not this realm's and the instanceof is false for every one.
- if (el?.tagName !== "AUDIO") continue;
+ if (!isCarveVoiceElement(el)) continue;
const src = el.getAttribute("src");
if (!src) continue;
const clock = readClipClock((name) => el.getAttribute(name));
@@ -179,7 +180,7 @@ export function useFxCarve(
} => {
const doc = element.element?.ownerDocument;
if (!doc || !couldBeBed) return { sourceOptions: [], autoSourceIds: [] };
- const others = Array.from(doc.querySelectorAll<HTMLAudioElement>("audio[id]")).filter(
+ const others = Array.from(doc.querySelectorAll("audio[id], video[id]")).filter(
(a) => a.id !== element.id,
);
// Only tracks that are actually playing while this bed is. A voice somewhere
diff --git a/packages/studio/src/components/editor/useFxCarveGrouping.test.ts b/packages/studio/src/components/editor/useFxCarveGrouping.test.ts
index 5dfc2d2b0d..cbe4e1f589 100644
--- a/packages/studio/src/components/editor/useFxCarveGrouping.test.ts
+++ b/packages/studio/src/components/editor/useFxCarveGrouping.test.ts
@@ -54,6 +54,19 @@ describe("collectCarveCandidates", () => {
});
});
+describe("collectCarveCandidates with video", () => {
+ it("offers an ungrouped audible video, not a muted one", () => {
+ const doc = previewDoc(`
+ <video id="a-roll"></video>
+ <video id="b-roll" muted></video>
+ <audio id="bed"></audio>
+ `);
+ const others = Array.from(doc.querySelectorAll("video[id]"));
+ const ids = collectCarveCandidates(doc, others, () => true, "bed").map((c) => c.id);
+ expect(ids).toEqual(["a-roll"]);
+ });
+});
+
describe("carverAgainst", () => {
// The far-end guard has to see through a GROUP source. A plural carve names a
// group — that is what the lint rule pushes authors toward — so matching raw
diff --git a/packages/studio/src/components/editor/useFxCarveGrouping.ts b/packages/studio/src/components/editor/useFxCarveGrouping.ts
index 5f79cf1308..5fa3a6a8c4 100644
--- a/packages/studio/src/components/editor/useFxCarveGrouping.ts
+++ b/packages/studio/src/components/editor/useFxCarveGrouping.ts
@@ -20,6 +20,11 @@ import {
resolveAudioGroups,
resolveCarveSourceIds,
} from "@hyperframes/core/audio-groups";
+import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers";
+
+export function isCarveVoiceElement(el: Element | null): el is Element {
+ return el !== null && (el.tagName === "AUDIO" || isAudibleVideoNode(el));
+}
/**
* An id for a new voiceover group, de-duped against every id already in the
@@ -130,7 +135,7 @@ function excludedFor(groupByMember: Map<string, { id: string }>, bedId?: string)
export function collectCarveCandidates(
doc: Document,
- others: readonly HTMLAudioElement[],
+ others: readonly Element[],
overlapsBed: (a: Element) => boolean,
/** The bed's own id, so neither it nor the group it belongs to is offered. */
bedId?: string,
@@ -141,7 +146,7 @@ export function collectCarveCandidates(
const excluded = excludedFor(groupByMember, bedId);
const offeredGroupIds = new Set<string>();
const described: CarveCandidate[] = [];
- for (const a of others) {
+ for (const a of others.filter(isCarveVoiceElement)) {
const group = groupByMember.get(a.id);
if (excluded.has(a.id) || (group && excluded.has(group.id))) continue;
if (!group) {
@@ -161,7 +166,7 @@ export function collectCarveCandidates(
// iframe document, so the constructor is a different realm's and the
// instanceof is false for every one (mirrors resolveCarveVoices in
// useFxCarve.ts).
- .filter((el): el is HTMLElement => el?.tagName === "AUDIO");
+ .filter((el): el is HTMLElement => isCarveVoiceElement(el));
if (!members.some(overlapsBed)) continue;
offeredGroupIds.add(group.id);
described.push({
diff --git a/packages/studio/src/player/components/TimelineFxButton.test.tsx b/packages/studio/src/player/components/TimelineFxButton.test.tsx
index 0ceac9cd44..ef865a7198 100644
--- a/packages/studio/src/player/components/TimelineFxButton.test.tsx
+++ b/packages/studio/src/player/components/TimelineFxButton.test.tsx
@@ -181,13 +181,13 @@ describe("TimelineFxButton", () => {
<TimelineFxButton
variant="group-pointer"
clipCount={2}
- refusal="Video audio can't be grouped yet — only audio clips can join a group."
+ refusal="This track can't be grouped."
onGroupClips={onGroupClips}
/>,
);
act(() => byTextButton(host, "FX")?.click());
const dialog = document.querySelector('[role="dialog"]');
- expect(dialog?.textContent).toContain("Video audio can't be grouped yet");
+ expect(dialog?.textContent).toContain("This track can't be grouped");
expect(document.querySelector('input[aria-label="Group name"]')).toBeNull();
expect(
Array.from(document.body.querySelectorAll("button")).some((b) => b.textContent === "Group"),
diff --git a/packages/studio/src/player/components/TimelineTrackHeader.test.tsx b/packages/studio/src/player/components/TimelineTrackHeader.test.tsx
index 8246bf8785..1d36898aa8 100644
--- a/packages/studio/src/player/components/TimelineTrackHeader.test.tsx
+++ b/packages/studio/src/player/components/TimelineTrackHeader.test.tsx
@@ -869,6 +869,43 @@ describe("TimelineTrackHeader", () => {
act(() => view.root.unmount());
});
+ it("offers grouping on a track of audible videos, not on muted b-roll", () => {
+ const aRoll: TimelineElement = {
+ ...VOICE,
+ id: "a-roll",
+ domId: "a-roll",
+ tag: "video",
+ hasAudio: true,
+ };
+ const aRoll2: TimelineElement = { ...aRoll, id: "a-roll-2", domId: "a-roll-2" };
+ const bRoll: TimelineElement = {
+ ...aRoll,
+ id: "b-roll",
+ domId: "b-roll",
+ hasAudio: undefined,
+ };
+ const bRoll2: TimelineElement = { ...bRoll, id: "b-roll-2", domId: "b-roll-2" };
+ const opts = {
+ keyframeClip: aRoll,
+ trackElements: [aRoll, aRoll2],
+ clipCount: 2,
+ animations: [],
+ expanded: false,
+ };
+ const pointer = (host: HTMLElement) =>
+ host.querySelector<HTMLButtonElement>(
+ 'button[aria-label="Effects — group these clips first"]',
+ );
+ const view = renderHeader(opts);
+ const button = pointer(view.host);
+ expect(button).not.toBeNull();
+ act(() => button?.click());
+ expect(document.body.textContent).not.toContain("can't be grouped");
+ view.rerender({ ...opts, keyframeClip: bRoll, trackElements: [bRoll, bRoll2] });
+ expect(pointer(view.host)).toBeNull();
+ act(() => view.root.unmount());
+ });
+
// The header is a 48px column of exactly TWO lines — what the row is, then
// what you can do to it. The group pointer used to render as a sibling of
// both, making a third: 17 + 24 + 24 + gaps in a 48px box, which
diff --git a/packages/studio/src/player/components/TimelineTrackHeader.tsx b/packages/studio/src/player/components/TimelineTrackHeader.tsx
index 25e27f60b2..5b0da20c94 100644
--- a/packages/studio/src/player/components/TimelineTrackHeader.tsx
+++ b/packages/studio/src/player/components/TimelineTrackHeader.tsx
@@ -226,11 +226,9 @@ export function TimelineTrackHeader({
const singleAudioClip =
isAudioTrack && clipCount === 1 && trackElements.length > 0 ? trackElements[0] : null;
const isTrackGrouped = trackElements.some((el) => el.audioGroup);
- // A video track carries sound the render mixes but preview never routes
- // through Web Audio, which is why §1.4 keeps groups audio-only. It still
- // needs to be TOLD that, so it earns the button and a refusal.
- const isVideoWithAudioTrack =
- !isAudioTrack && trackElements.some((el) => el.tag.toLowerCase() === "video");
+ const audioBearingClips = isAudioTrack
+ ? trackElements
+ : trackElements.filter((el) => el.hasAudio === true);
const writeClipFxChain = (
clip: TimelineElement,
next: HfAudioFxChain,
@@ -256,7 +254,7 @@ export function TimelineTrackHeader({
// UI showed the track as grouped. The button is withheld instead of acting on
// a subset, which is also why the carve path's loud guard cannot catch this:
// the unresolvable ids were filtered out before the call.
- const groupableClipIds = trackElements.map(runtimeAudioId);
+ const groupableClipIds = audioBearingClips.map(runtimeAudioId);
const allGroupableClipIds = groupableClipIds.every((id): id is string => id !== null)
? groupableClipIds
: null;
@@ -342,25 +340,14 @@ export function TimelineTrackHeader({
onOpenRack={() => openClipFxRack(singleAudioClip)}
/>
)}
- {showAudioEffects &&
- clipCount > 1 &&
- !isTrackGrouped &&
- (isAudioTrack ? canGroupWholeTrack : isVideoWithAudioTrack) && (
- <TimelineFxButton
- variant="group-pointer"
- clipCount={trackElements.length}
- defaultLabel={trackLabel}
- // Groups are audio-only in v1 (§1.4). A video track showing no
- // button at all is the silent limit §5 forbids, so it gets the
- // button and a reason instead.
- refusal={
- isAudioTrack
- ? undefined
- : "Video audio can't be grouped yet — only audio clips can join a group."
- }
- onGroupClips={groupUngroupedClips}
- />
- )}
+ {showAudioEffects && clipCount > 1 && !isTrackGrouped && canGroupWholeTrack && (
+ <TimelineFxButton
+ variant="group-pointer"
+ clipCount={audioBearingClips.length}
+ defaultLabel={trackLabel}
+ onGroupClips={groupUngroupedClips}
+ />
+ )}
{/* The lane disclosure, on the row's own layout rather than by
swapping it for a keyframe-layer row. */}
{disclosable && (
diff --git a/packages/studio/src/player/lib/timelineElementHelpers.ts b/packages/studio/src/player/lib/timelineElementHelpers.ts
index c5e3b944de..2b2c13d945 100644
--- a/packages/studio/src/player/lib/timelineElementHelpers.ts
+++ b/packages/studio/src/player/lib/timelineElementHelpers.ts
@@ -231,6 +231,15 @@ export function isVideoAudible(opts: {
return opts.tag.toLowerCase() === "video" && !opts.muted;
}
+export function isAudibleVideoNode(el: Element): boolean {
+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;
+ return isVideoAudible({
+ tag: "video",
+ hasAudioAttr: el.getAttribute("data-has-audio"),
+ muted: false,
+ });
+}
+
/** What the mixer gets: the compiler's `data-has-audio` rule, muted and volume. */
function applyAudioMetadataFromElement(entry: TimelineElement, el: Element): void {
const media = resolveMediaElement(el) ?? el;
diff --git a/packages/studio/src/utils/audioMeterMath.test.ts b/packages/studio/src/utils/audioMeterMath.test.ts
index 95884a0581..b5fbfb2518 100644
--- a/packages/studio/src/utils/audioMeterMath.test.ts
+++ b/packages/studio/src/utils/audioMeterMath.test.ts
@@ -1,5 +1,7 @@
import { describe, expect, it } from "vitest";
+import type { TimelineElement } from "../player";
import {
+ hasProjectAudio,
levelToFraction,
markFraction,
SILENT_CHANNEL,
@@ -42,3 +44,38 @@ describe("stepPair", () => {
expect(stepPair(rest, { l: 1, r: 0 }, 32, 16)).not.toBe(rest);
});
});
+
+describe("hasProjectAudio", () => {
+ const clip = (overrides: Partial<TimelineElement>): TimelineElement => ({
+ id: "c",
+ tag: "div",
+ start: 0,
+ duration: 1,
+ track: 0,
+ ...overrides,
+ });
+
+ it("ignores an audible video left on native output at unity gain", () => {
+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true })])).toBe(false);
+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, volume: 1 })])).toBe(false);
+ });
+
+ it.each<[string, Partial<TimelineElement>]>([
+ ["an above-unity gain", { volume: 2 }],
+ ["an fx chain", { fxChain: "[]" }],
+ ["automation", { automation: "{}" }],
+ ["a group", { audioGroup: "music" }],
+ ])("counts an audible video routed through Web Audio by %s", (_, fields) => {
+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, ...fields })])).toBe(true);
+ });
+
+ it("ignores a muted video even when it carries processing", () => {
+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, muted: true, volume: 2 })])).toBe(
+ false,
+ );
+ });
+
+ it("ignores a silent video", () => {
+ expect(hasProjectAudio([clip({ tag: "video" })])).toBe(false);
+ });
+});
diff --git a/packages/studio/src/utils/audioMeterMath.ts b/packages/studio/src/utils/audioMeterMath.ts
index 0cd84ff1e8..10395f9fc5 100644
--- a/packages/studio/src/utils/audioMeterMath.ts
+++ b/packages/studio/src/utils/audioMeterMath.ts
@@ -1,5 +1,6 @@
import { useMemo } from "react";
import { audioDbToGain, audioGainToDb } from "@hyperframes/core/audio-gain";
+import { audibleVideoNeedsWebAudio } from "@hyperframes/core/audible-video";
import { usePlayerStore, type TimelineElement } from "../player";
import { isAudioTimelineElement } from "./timelineInspector";
@@ -67,9 +68,14 @@ export function stepPair(
return [stepChannel(l, raw?.l ?? 0, now, dtMs), stepChannel(r, raw?.r ?? 0, now, dtMs)];
}
-/** The meters appear only for a project with at least one audio clip or group. */
-function hasProjectAudio(elements: readonly TimelineElement[]): boolean {
- return elements.some((el) => isAudioTimelineElement(el) || el.audioGroup !== undefined);
+/** The meters appear only for a project with an audio clip, a group, or a video Web Audio carries. */
+export function hasProjectAudio(elements: readonly TimelineElement[]): boolean {
+ return elements.some(
+ (el) =>
+ isAudioTimelineElement(el) ||
+ el.audioGroup !== undefined ||
+ (el.hasAudio === true && !el.muted && audibleVideoNeedsWebAudio(el)),
+ );
}
/** Scans once per `elements` change, not on every store update (the playhead ticks each frame). */
diff --git a/packages/studio/src/utils/sourcePatcher.ts b/packages/studio/src/utils/sourcePatcher.ts
index 47ef37cc70..57a250e859 100644
--- a/packages/studio/src/utils/sourcePatcher.ts
+++ b/packages/studio/src/utils/sourcePatcher.ts
@@ -21,7 +21,7 @@ export function escapeHtmlAttribute(value: string): string {
}
/** Reverse escapeHtmlAttribute so callers get the original value. */
-function unescapeHtmlAttribute(value: string): string {
+export function unescapeHtmlAttribute(value: string): string {
return value
.replace(/&quot;/g, '"')
.replace(/&lt;/g, "<")