Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(core): audible videos in preview web audio, audio groups, meters and carve (2/8) (#4814)
Vance Ingalls3c299acc7f6f
diff --git a/docs/reference/audio-effects.mdx b/docs/reference/audio-effects.mdxindex df00a9c5fa..428895e63f 100644--- a/docs/reference/audio-effects.mdx+++ b/docs/reference/audio-effects.mdx@@ -20,10 +20,10 @@ nothing to load beside the markup.| `data-fx-chain` | The effects, in signal order | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes || `data-automation` | Envelopes on volume or an effect parameter | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes || `data-fx-carve` | A voiceover carve's own settings | JSON | `<audio>`, `<video>` | No — see [below](#data-fx-carve) |-| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>` only | Yes |+| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>`, audible `<video>` | Yes |-`data-audio-group` is the odd one: a bare string, and **ignored on `<video>`**.-The other three are JSON. Of those, only `data-fx-chain` and `data-automation`+`data-audio-group` is the odd one: a bare string, and **ignored on a muted+`<video>`** (`muted`, or `data-has-audio="false"`). The other three are JSON. Of those, only `data-fx-chain` and `data-automation`can also sit on an `<hf-audio-group>`, where they apply to the group's whole bus;`data-fx-carve` belongs to a clip.@@ -170,8 +170,9 @@ dangles.- `<hf-audio-group>` is optional metadata: `data-label`, `data-volume`,`data-hidden`, and its own `data-fx-chain` / `data-automation`. A group withmembers but no element still resolves, using its id as the label.-- **Audio only.** `data-audio-group` on a `<video>` is ignored, and groups do- not nest.+- **Audio and audible video.** A `<video>` with sound (`data-has-audio="true"`,+ no `muted`) joins its group in preview and render; on a muted video+ `data-audio-group` is ignored. Groups do not nest.- `data-audio-group=""` is no group at all.- The element must be inert. The runtime injects`hf-audio-group{display:none!important}` so an unknown custom element cannotdiff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.jsonindex 75c95a1ba2..f2626dd280 100644--- a/packages/core/package-subpaths.json+++ b/packages/core/package-subpaths.json@@ -218,6 +218,12 @@"types": "./dist/clipFacts.d.ts","environments": ["browser", "bun", "node"]},+ "./audible-video": {+ "source": "./src/audibleVideo.ts",+ "runtime": "./dist/audibleVideo.js",+ "types": "./dist/audibleVideo.d.ts",+ "environments": ["browser", "bun", "node"]+ },"./audio-gain": {"source": "./src/audioGain.ts","runtime": "./dist/audioGain.js",diff --git a/packages/core/package.json b/packages/core/package.jsonindex 83a64b567f..836954ea54 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -232,6 +232,12 @@"import": "./src/clipFacts.ts","types": "./src/clipFacts.ts"},+ "./audible-video": {+ "bun": "./src/audibleVideo.ts",+ "node": "./dist/audibleVideo.js",+ "import": "./src/audibleVideo.ts",+ "types": "./src/audibleVideo.ts"+ },"./audio-gain": {"bun": "./src/audioGain.ts","node": "./dist/audioGain.js",@@ -614,6 +620,10 @@"import": "./dist/clipFacts.js","types": "./dist/clipFacts.d.ts"},+ "./audible-video": {+ "import": "./dist/audibleVideo.js",+ "types": "./dist/audibleVideo.d.ts"+ },"./audio-gain": {"import": "./dist/audioGain.js","types": "./dist/audioGain.d.ts"diff --git a/packages/core/src/audibleVideo.test.ts b/packages/core/src/audibleVideo.test.tsnew file mode 100644index 0000000000..1d0ea261a5--- /dev/null+++ b/packages/core/src/audibleVideo.test.ts@@ -0,0 +1,63 @@+import { describe, expect, it } from "vitest";+import {+ AUDIBLE_MEDIA_SELECTOR,+ audibleVideoNeedsWebAudio,+ isAudibleVideoElement,+} from "./audibleVideo";++const el = (tagName: string, attrs: Record<string, string>) => ({+ tagName,+ hasAttribute: (n: string) => n in attrs,+ getAttribute: (n: string) => (n in attrs ? (attrs[n] ?? null) : null),+});++describe("isAudibleVideoElement", () => {+ it("unmuted video with data-has-audio=true or no attribute is audible", () => {+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "true" }))).toBe(true);+ expect(isAudibleVideoElement(el("video", {}))).toBe(true);+ });+ it("muted, false or empty is not audible", () => {+ expect(isAudibleVideoElement(el("VIDEO", { muted: "" }))).toBe(false);+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "false" }))).toBe(false);+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "" }))).toBe(false);+ });+ it("only the exact value true counts, so TRUE is not audible", () => {+ expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "TRUE" }))).toBe(false);+ });+ it("never true for non-video", () => {+ expect(isAudibleVideoElement(el("AUDIO", {}))).toBe(false);+ });+});++describe("AUDIBLE_MEDIA_SELECTOR", () => {+ it("matches audio plus exactly the videos the predicate calls audible", () => {+ document.body.innerHTML = `+ <audio id="a" data-start="0"></audio>+ <video id="v-true" data-start="0" data-has-audio="true"></video>+ <video id="v-absent" data-start="0"></video>+ <video id="v-muted" data-start="0" data-has-audio="true" muted></video>+ <video id="v-false" data-start="0" data-has-audio="false"></video>+ <video id="v-empty" data-start="0" data-has-audio=""></video>`;+ const matched = Array.from(document.querySelectorAll(AUDIBLE_MEDIA_SELECTOR), (n) => n.id);+ expect(matched).toEqual(["a", "v-true", "v-absent"]);+ const audibleVideos = Array.from(document.querySelectorAll("video"))+ .filter(isAudibleVideoElement)+ .map((n) => n.id);+ expect(audibleVideos).toEqual(matched.filter((id) => id.startsWith("v-")));+ });+});++describe("audibleVideoNeedsWebAudio", () => {+ it("keeps a unity, unprocessed video on native output", () => {+ expect(audibleVideoNeedsWebAudio({})).toBe(false);+ expect(audibleVideoNeedsWebAudio({ volume: 1, fxChain: "", automation: null })).toBe(false);+ expect(audibleVideoNeedsWebAudio({ volume: Number.NaN })).toBe(false);+ });++ it.each([{ volume: 1.5 }, { fxChain: "[]" }, { automation: "{}" }, { audioGroup: "music" }])(+ "routes a video carrying %o",+ (fields) => {+ expect(audibleVideoNeedsWebAudio(fields)).toBe(true);+ },+ );+});diff --git a/packages/core/src/audibleVideo.ts b/packages/core/src/audibleVideo.tsnew file mode 100644index 0000000000..30c9aa5fd2--- /dev/null+++ b/packages/core/src/audibleVideo.ts@@ -0,0 +1,25 @@+export const AUDIBLE_VIDEO_QUALIFIER =+ ':not([muted]):is(:not([data-has-audio]),[data-has-audio="true"])';++export const AUDIBLE_MEDIA_SELECTOR = `audio[data-start], video[data-start]${AUDIBLE_VIDEO_QUALIFIER}`;++export function isAudibleVideoElement(el: {+ tagName: string;+ hasAttribute(name: string): boolean;+ getAttribute(name: string): string | null;+}): boolean {+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;+ const declared = el.getAttribute("data-has-audio");+ return declared === null || declared === "true";+}++export function audibleVideoNeedsWebAudio(fields: {+ volume?: number | null;+ fxChain?: string | null;+ automation?: string | null;+ audioGroup?: string | null;+}): boolean {+ return (+ (fields.volume ?? 1) > 1 || Boolean(fields.fxChain || fields.automation || fields.audioGroup)+ );+}diff --git a/packages/core/src/audioGroups.test.ts b/packages/core/src/audioGroups.test.tsindex 606112d2de..722da0a225 100644--- a/packages/core/src/audioGroups.test.ts+++ b/packages/core/src/audioGroups.test.ts@@ -67,9 +67,12 @@ describe("resolveAudioGroups", () => {expect(resolveAudioGroups(document)[0].memberIds).toEqual(["vo-1"]);});- it("ignores a data-audio-group on a video element (audio only in v1)", () => {- document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;- expect(resolveAudioGroups(document)).toEqual([]);+ it("groups an audible video and ignores a muted or silent one", () => {+ document.body.innerHTML = `+ <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>+ <video id="v-2" muted data-audio-group="voiceover"></video>+ <video id="v-3" data-has-audio="false" data-audio-group="voiceover"></video>`;+ expect(resolveAudioGroups(document).map((g) => g.memberIds)).toEqual([["v-1"]]);});it("reads the group element's fx chain, automation, volume and hidden", () => {@@ -115,11 +118,12 @@ describe("audioGroupOf", () => {expect(audioGroupOf(document.getElementById("vo-1") as Element)).toBeNull();});- it("ignores video membership so preview matches the audio-only render", () => {- document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;- const el = document.getElementById("v-1") as Element;- expect(audioGroupOf(el)).toBeNull();- expect(resolveAudioGroups(document)).toEqual([]);+ it("reads an audible video's membership and ignores a muted video's", () => {+ document.body.innerHTML = `+ <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>+ <video id="v-2" muted data-audio-group="voiceover"></video>`;+ expect(audioGroupOf(document.getElementById("v-1") as Element)).toBe("voiceover");+ expect(audioGroupOf(document.getElementById("v-2") as Element)).toBeNull();});it("normalizes an empty membership attribute to null", () => {diff --git a/packages/core/src/audioGroups.ts b/packages/core/src/audioGroups.tsindex 6b2a6b263c..b0cc736d11 100644--- a/packages/core/src/audioGroups.ts+++ b/packages/core/src/audioGroups.ts@@ -12,6 +12,7 @@import { HF_AUDIO_FX_ATTR } from "./audioFx.js";import { AUDIO_GROUP_RENDER_ID_ATTR, MEDIA_RENDER_ID_ATTR } from "./compiler/mediaRenderIds.js";import { HF_AUDIO_AUTOMATION_ATTR } from "./audioAutomation.js";+import { isAudibleVideoElement } from "./audibleVideo.js";export const HF_AUDIO_GROUP_TAG = "hf-audio-group";export const HF_AUDIO_GROUP_ATTR = "data-audio-group";@@ -129,7 +130,11 @@ export function isMemberGroupHidden(export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] {const membersByGroup = new Map<string, string[]>();- for (const member of root.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)) {+ const candidates = root.querySelectorAll(+ `audio[${HF_AUDIO_GROUP_ATTR}], video[${HF_AUDIO_GROUP_ATTR}]`,+ );+ for (const member of candidates) {+ if (!isGroupableMember(member)) continue;// The render-stamped instance key when the compiler has been through// (`assignMediaRenderIds`), else the author id. An author id is unique only// per composition FILE, so a sub-composition declaring a bus AND its members@@ -195,8 +200,8 @@ export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): streturn out;}-/** The group an audio member belongs to, or null. Membership is audio-only in- * v1, matching `resolveAudioGroups` and the render mixer; video and group-bus+/** The group an audio or audible-video member belongs to, or null, matching+ * `resolveAudioGroups` and the render mixer; muted video and group-bus* attributes are inert.** Tolerant of objects that only partially implement `Element` (test doubles@@ -204,11 +209,16 @@ export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): st* `getAttribute` simply has no group, mirroring `readChain`'s style in* `runtime/audioFx.ts`. */export function audioGroupOf(el: Element): string | null {- if (typeof el.tagName !== "string" || el.tagName.toLowerCase() !== "audio") return null;- if (typeof el.getAttribute !== "function") return null;+ if (typeof el.tagName !== "string" || typeof el.getAttribute !== "function") return null;+ if (!isGroupableMember(el)) return null;return el.getAttribute(HF_AUDIO_GROUP_ATTR) || null;}+function isGroupableMember(el: Element): boolean {+ if (el.tagName.toLowerCase() === "audio") return true;+ return typeof el.hasAttribute === "function" && isAudibleVideoElement(el);+}+/*** Make `<hf-audio-group>` inert, once per document.*diff --git a/packages/core/src/compiler/mediaRenderIds.test.ts b/packages/core/src/compiler/mediaRenderIds.test.tsindex 4e6bfab329..f0411a1f39 100644--- a/packages/core/src/compiler/mediaRenderIds.test.ts+++ b/packages/core/src/compiler/mediaRenderIds.test.ts@@ -166,6 +166,16 @@ describe("audio group render ids", () => {expect(d.querySelector("audio")?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");});+ it("stamps an audible video member and skips a muted one", () => {+ const d = doc(`<hf-audio-group id="vo"></hf-audio-group>+ <video id="v" src="a.mp4" data-has-audio="true" data-audio-group="vo"></video>+ <video id="m" src="b.mp4" muted data-audio-group="vo"></video>`);+ assignMediaRenderIds(d);+ const [audible, muted] = [...d.querySelectorAll("video")];+ expect(audible?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");+ expect(muted?.hasAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe(false);+ });+it("leaves a member alone when no bus element declares its group", () => {const d = doc(`<audio id="a" src="a.wav" data-audio-group="ghost"></audio>`);assignMediaRenderIds(d);diff --git a/packages/core/src/compiler/mediaRenderIds.ts b/packages/core/src/compiler/mediaRenderIds.tsindex 9e2a545e7a..ad9227a563 100644--- a/packages/core/src/compiler/mediaRenderIds.ts+++ b/packages/core/src/compiler/mediaRenderIds.ts@@ -25,6 +25,8 @@* fix scene footage.*/+import { AUDIBLE_VIDEO_QUALIFIER } from "../audibleVideo.js";+export const MEDIA_RENDER_ID_ATTR = "data-hf-render-id";/**@@ -141,7 +143,7 @@ function assignAudioGroupRenderIds(document: DocumentLike, taken: Set<string>):if (busesById.size === 0) return;for (const member of document.querySelectorAll(- "audio[data-audio-group]",+ `audio[data-audio-group], video[data-audio-group]${AUDIBLE_VIDEO_QUALIFIER}`,) as Iterable<ScopedElementLike>) {const groupId = member.getAttribute("data-audio-group");const buses = groupId ? busesById.get(groupId) : undefined;diff --git a/packages/core/src/editing/affordances.test.ts b/packages/core/src/editing/affordances.test.tsindex 81574e3deb..86db7d912c 100644--- a/packages/core/src/editing/affordances.test.ts+++ b/packages/core/src/editing/affordances.test.ts@@ -140,6 +140,16 @@ describe("resolveEditingAffordances — sections", () => {expect(s).toMatchObject({ media: true, colorGrading: true });});+ it("audible video: audioFx; silent video: no audioFx", () => {+ expect(+ resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: true })).sections.audioFx,+ ).toBe(true);+ expect(+ resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: false })).sections.audioFx,+ ).toBe(false);+ expect(resolveEditingAffordances(baseFacts({ tag: "video" })).sections.audioFx).toBe(false);+ });+it("audio: media but not colorGrading", () => {const s = resolveEditingAffordances(baseFacts({ tag: "audio" })).sections;expect(s).toMatchObject({ media: true, colorGrading: false });diff --git a/packages/core/src/editing/affordances.ts b/packages/core/src/editing/affordances.tsindex 682494094f..238a17a6b2 100644--- a/packages/core/src/editing/affordances.ts+++ b/packages/core/src/editing/affordances.ts@@ -69,6 +69,7 @@ export interface EditableElementFacts {hasTimingStart: boolean;/** count of GSAP tweens targeting this element */animationCount: number;+ hasAudio?: boolean;}/**@@ -218,7 +219,8 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSectreturn {text: facts.hasEditableText && !facts.isCompositionHost && !facts.isInsideLockedComposition,media: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",- audioFx: facts.tag === "audio" || isAudioBus,+ audioFx:+ facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),colorGrading: facts.tag === "video" || facts.tag === "img",// A bus has no clip range at all — no `data-start`, no duration, and its// automation clock is composition time — so Start/Duration/End would bediff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 31a4898f3e..c1f1725a0c 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -183,6 +183,11 @@ export {} from "./compiler/subCompositionValidity";export { RUNTIME_BOOTSTRAP_ATTR, stripEmbeddedRuntimeScripts } from "./compiler/htmlDocument";export { queryByAttr } from "./utils/cssSelector";+export {+ AUDIBLE_MEDIA_SELECTOR,+ audibleVideoNeedsWebAudio,+ isAudibleVideoElement,+} from "./audibleVideo";export { decodeUrlPathVariants } from "./utils/urlPath";export {clearGifFramesBeforeNext,diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.tsindex 4a60c5155b..76a95cbb2d 100644--- a/packages/core/src/runtime/init.test.ts+++ b/packages/core/src/runtime/init.test.ts@@ -5279,6 +5279,175 @@ describe("initSandboxRuntimeModular", () => {expect(seeks.filter((t) => t > 0.1)).toEqual([]);});});+ describe("an audible <video> routed through Web Audio", () => {+ const ctx = useMockAudioContext();++ function mountMedia(tag: "audio" | "video", attrs: Record<string, string> = {}) {+ let root = document.querySelector<HTMLElement>("[data-root]");+ if (!root) {+ root = document.createElement("div");+ root.setAttribute("data-composition-id", "main");+ root.setAttribute("data-root", "true");+ root.setAttribute("data-start", "0");+ root.setAttribute("data-duration", "10");+ root.setAttribute("data-width", "1920");+ root.setAttribute("data-height", "1080");+ document.body.appendChild(root);+ }+ const el = document.createElement(tag);+ el.setAttribute("data-start", "0");+ el.setAttribute("data-duration", "10");+ el.setAttribute("src", "/assets/talk.mp4");+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);+ el.load = () => {};+ el.play = vi.fn(() => Promise.resolve());+ root.appendChild(el);+ window.__timelines = { main: createMockTimeline(10) };+ return el;+ }++ function spyCapture() {+ return vi+ .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")+ .mockResolvedValue(null);+ }++ it("schedules it through the media-element transport with its above-unity gain", async () => {+ const video = mountMedia("video", { "data-has-audio": "true", "data-volume": "2" });+ const captureSpy = spyCapture();++ await startPlayback();++ expect(captureSpy).toHaveBeenCalledTimes(1);+ expect(captureSpy.mock.calls[0]?.[0]).toBe(video);+ expect(captureSpy.mock.calls[0]?.[4]).toBe(2);+ });++ it("leaves a plain audible video at unity on native output", async () => {+ mountMedia("video", { "data-has-audio": "true" });++ await startPlayback();+ await Promise.resolve();++ expect(ctx.mediaElementSources).toBe(0);+ });++ it.each([+ ["data-fx-chain", "[]"],+ ["data-automation", "[]"],+ ["data-audio-group", "music"],+ ])("acquires a media element source for an audible video carrying %s", async (name, value) => {+ mountMedia("video", { "data-has-audio": "true", [name]: value });++ await startPlayback();+ await Promise.resolve();++ expect(ctx.mediaElementSources).toBe(1);+ });++ it("never schedules a muted or data-has-audio=false video", async () => {+ mountMedia("video", { "data-has-audio": "true", muted: "" });+ mountMedia("video", { "data-has-audio": "false" });+ const captureSpy = spyCapture();++ await startPlayback();++ expect(captureSpy).not.toHaveBeenCalled();+ });++ it("routes only the <audio> of a legacy split (muted video + audio on the same file)", async () => {+ mountMedia("video", { muted: "" });+ const audio = mountMedia("audio");+ const captureSpy = spyCapture();++ await startPlayback();++ expect(captureSpy).toHaveBeenCalledTimes(1);+ expect(captureSpy.mock.calls[0]?.[0]).toBe(audio);+ });++ it("never whole-file decodes a video whose capture failed, and leaves it unmuted", async () => {+ const video = mountMedia("video", {+ "data-has-audio": "true",+ "data-fx-chain": "[]",+ "data-playback-rate": "2",+ });+ const plainVideo = mountMedia("video", { "data-has-audio": "true" });+ const audio = mountMedia("audio");+ spyCapture();+ const decodeSpy = vi+ .spyOn(WebAudioTransport.prototype, "decodeAudioElement")+ .mockResolvedValue(null);++ await startPlayback();+ await Promise.resolve();++ expect(decodeSpy).toHaveBeenCalledWith(audio);+ expect(decodeSpy).not.toHaveBeenCalledWith(video);+ expect(decodeSpy).not.toHaveBeenCalledWith(plainVideo);+ expect(video.muted).toBe(false);+ });++ const rateRamp = JSON.stringify({+ version: 1,+ lanes: [+ {+ target: "rate",+ points: [+ { t: 0, v: 1 },+ { t: 5, v: 2 },+ ],+ },+ ],+ });++ it("captures a ramped audible video in a group, so the group bus carries it", async () => {+ const video = mountMedia("video", {+ "data-has-audio": "true",+ "data-audio-group": "music",+ "data-automation": rateRamp,+ });+ const captureSpy = spyCapture();++ await startPlayback();++ expect(captureSpy.mock.calls.map((call) => call[0])).toContain(video);+ });++ it("never decodes a ramped audio clip whose capture failed", async () => {+ const audio = mountMedia("audio", { "data-automation": rateRamp });+ spyCapture();+ const decodeSpy = vi+ .spyOn(WebAudioTransport.prototype, "decodeAudioElement")+ .mockResolvedValue(null);++ await startPlayback();+ await Promise.resolve();++ expect(decodeSpy).not.toHaveBeenCalledWith(audio);+ expect(audio.muted).toBe(false);+ });++ it("adds exactly one reschedule when a routed video's data-hidden toggles mid-playback", async () => {+ const video = mountMedia("video", {+ "data-has-audio": "true",+ "data-fx-chain": "[]",+ "data-hidden": "",+ });+ await startPlayback();+ const captureSpy = spyCapture();+ const generationSpy = vi.spyOn(WebAudioTransport.prototype, "startGeneration");++ window.__player?.seek(1, { keepPlaying: true });+ const seekOnly = generationSpy.mock.calls.length;+ generationSpy.mockClear();+ video.removeAttribute("data-hidden");+ window.__player?.seek(2, { keepPlaying: true });++ expect(generationSpy.mock.calls.length).toBe(seekOnly + 1);+ expect(captureSpy.mock.calls.at(-1)?.[0]).toBe(video);+ });+ });});/**diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 012ec73c62..e5b60f33d5 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -136,6 +136,7 @@ import {isMediaElement,isVideoElement,} from "./domRealm";+import { audibleVideoNeedsWebAudio, isAudibleVideoElement } from "../audibleVideo";/*** A `window.__timelines` entry is authored content and may be a PARTIAL@@ -331,7 +332,15 @@ function pageAnimationsForOnePass(): () => Animation[] {// A `<video>` joins only for a gain `el.volume` cannot express: capture is a one-way door.const joinsWebAudio = (el: Element): el is HTMLMediaElement =>- isAudioElement(el) || (isVideoElement(el) && Number.parseFloat(el.dataset.volume ?? "") > 1);+ isAudioElement(el) ||+ (isVideoElement(el) &&+ isAudibleVideoElement(el) &&+ audibleVideoNeedsWebAudio({+ volume: Number.parseFloat(el.dataset.volume ?? ""),+ fxChain: el.getAttribute("data-fx-chain"),+ automation: el.getAttribute("data-automation"),+ audioGroup: el.getAttribute("data-audio-group"),+ }));const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);@@ -2589,8 +2598,8 @@ export function initSandboxRuntimeModular(): void {// sync with a `data-hidden` toggle made mid-playback.const groupHiddenLast = new WeakMap<Element, boolean>();const groupHasUncapturedMember = (groupId: string, currentTime: number): boolean => {- for (const el of document.querySelectorAll("audio[data-start]")) {- if (!isMediaElement(el) || audioGroupOf(el) !== groupId) continue;+ for (const el of webAudioMediaIn(document)) {+ if (audioGroupOf(el) !== groupId) continue;if (webAudio.routesElement(el) || isSilencedByHidden(el)) continue;const start = resolveAbsoluteMediaStartSeconds(el);const duration = parseStrictFiniteTimingNumber(el.dataset.duration);@@ -4801,8 +4810,6 @@ export function initSandboxRuntimeModular(): void {// that existed before (#3458).const route = classifyWebAudioMediaRoute(rawEl);reportWebAudioMediaRoute(rawEl, route);- // Decoded buffers cannot follow a rate curve without shifting pitch; the media element can.- if (typeof readElementRateSpec(rawEl) !== "number") continue;// The cross-origin verdict's BEST outcome is decode, since a CDN that// sends `Access-Control-Allow-Origin` (the author just never wrote the// `crossorigin` attribute) decodes fine and keeps the whole FX graph.@@ -4823,6 +4830,8 @@ export function initSandboxRuntimeModular(): void {// A video's picture must keep playing from the element, so it has no decode fallback.if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)return;+ // Decoded buffers cannot follow a rate curve without shifting pitch; the media element can.+ if (typeof readElementRateSpec(rawEl) !== "number") return;const effectiveRate = state.playbackRate * readElementPlaybackRate(rawEl);// Deliberately the FX/automation pair and NOT// `nativeUnexpressibleProcessing()`, which this route's diagnostic uses.diff --git a/packages/core/src/runtime/webAudioRoute.ts b/packages/core/src/runtime/webAudioRoute.tsindex 3459a4f82b..f759f6a323 100644--- a/packages/core/src/runtime/webAudioRoute.ts+++ b/packages/core/src/runtime/webAudioRoute.ts@@ -5,50 +5,15 @@ import { postRuntimeMessage } from "./bridge";import type { RuntimeJson } from "./types";/**- * Which transport may claim an `<audio>` element's output.+ * Which transport may claim an `<audio>` or audible `<video>` element's output.*- * `createMediaElementSource()` is the runtime's PRIMARY audio path, and it is- * a one-way door: the node permanently reroutes the element away from its- * native output and is cached for the element's lifetime. That matters because- * of a spec behaviour that looks nothing like a failure — per the Web Audio- * spec's MediaElementAudioSourceNode security section, a node built over a- * resource that fails the CORS-cross-origin check outputs SILENCE. It does not- * throw, so the `try/catch` around the call in `webAudioTransport.ts` never- * fires, nothing reaches `swallow()`, and the composition plays perfectly —- * timeline advancing, visuals animating — with no audio at all (#3458).- *- * The only defence is to decide BEFORE the call, which is what this module is:- * a pure classifier, so the same verdict can be reached at media-discovery time- * (to emit a diagnostic) and at schedule time (to actually withhold the node)- * without those two ever drifting apart.- *- * That guarantee holds for every caller that routes through this classifier —- * it is NOT a runtime-wide interception of `createMediaElementSource`. The- * timeline transport (`webAudioTransport.ts`, via `init.ts`) always goes- * through it; a UI surface that builds its own throwaway `AudioContext` for- * an unrelated purpose (e.g. the asset sidebar's preview player,- * `AudioRow.tsx`) has to call it too, and is expected to. Known gap: an- * element playing a `MediaStream` via `srcObject` instead of `src`/`<source>`- * has no origin for this module to judge — `routeCandidates` only reads- * `src`-shaped attributes, so a `srcObject` element always reads as- * `web-audio` here, correctly or not. Nothing in this codebase feeds- * `createMediaElementSource` from a `srcObject` element today, so this is- * recorded as a boundary rather than fixed.- *- * Second known gap, same shape: `corsSilenceReason` judges the RAW url string —- * the same-origin URL the author wrote, or whatever the browser resolved into- * `currentSrc` — not wherever a server-side redirect chain actually lands.- * A same-origin URL that 302s to a cross-origin CDN reads as `web-audio` here- * and gets a real `createMediaElementSource` node; whether that node is- * silent then depends on the redirect target's CORS headers, which this- * classifier never sees (following the chain to inspect the final response- * would turn a pure, synchronous verdict — needed on every schedule call —- * into an async fetch). A cross-origin URL that redirects back to same-origin- * has the opposite miss: classified `decode-only` and sent down the fetch- * fallback when Web Audio capture would have worked fine either way. Not- * fixed for the same reason as `srcObject` — no caller in this codebase- * routes media through a redirecting URL today — but worth knowing before- * trusting this classifier's verdict for one that does.+ * `createMediaElementSource()` is a one-way door, and over a resource that fails+ * the CORS-cross-origin check it outputs SILENCE without throwing (#3458). So the+ * verdict is reached BEFORE the call by this pure classifier, shared by media+ * discovery (diagnostic) and scheduling (withholding the node). Callers that build+ * their own `AudioContext` (e.g. `AudioRow.tsx`) must route through it too.+ * Known gaps: `srcObject` elements always read `web-audio`, and the RAW url is+ * judged, not a redirect chain's final target.*/export type WebAudioMediaRoute =/** Same-origin, CORS-opted-in, or a scheme the check doesn't apply to. */@@ -259,8 +224,8 @@ export function nativeUnexpressibleProcessing(el: HTMLMediaElement): string[] {* element is not the audio source there in the first place.** Same signal `mediaProxy.ts`'s `isRenderMode` gates on. The `<video>` half of- * that check (the injected render-frame sibling) is deliberately not mirrored:- * only `<audio>` ever reaches this module.+ * that check (the injected render-frame sibling) is not mirrored: the+ * export-seek config alone gates reporting.*/function isRenderMode(): boolean {// Read through an inline cast rather than the ambient `Window` augmentationdiff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.tsindex a3f0ab87fc..b8741ad644 100644--- a/packages/engine/src/services/audioMixer.grouping.test.ts+++ b/packages/engine/src/services/audioMixer.grouping.test.ts@@ -544,3 +544,44 @@ describe("duplicate bus instances", () => {]);});});++describe("audible video members", () => {+ const withBus = (busAttrs: string, videoAttrs: string) =>+ `<div id="root" data-composition-id="main" data-start="0" data-duration="4">+ <hf-audio-group id="vo" data-volume="0.5" ${busAttrs}></hf-audio-group>+ <video id="talk" src="talk.mp4" data-start="0" data-duration="4"+ data-has-audio="true" data-audio-group="vo" ${videoAttrs}></video>+ </div>`;++ it("puts an audible video's track in its group sub-mix", () => {+ const tracks = parseAudioElements(withBus("", ""));+ expect(tracks.map((t) => [t.type, t.groupId, t.groupVolume])).toEqual([["video", "vo", 0.5]]);+ });++ it("drops an audible video from the mix when its group is hidden", () => {+ expect(parseAudioElements(withBus("data-hidden", ""))).toEqual([]);+ });++ it("keeps a ramped audible video in its group sub-mix with its rate lane", () => {+ const lane = JSON.stringify({+ version: 1,+ lanes: [+ {+ target: "rate",+ points: [+ { t: 0, v: 1 },+ { t: 2, v: 2 },+ ],+ },+ ],+ });+ const [track] = parseAudioElements(withBus("", `data-automation='${lane}'`));+ expect(track?.groupId).toBe("vo");+ expect(typeof track?.playbackRate).toBe("object");+ });++ it("ignores a muted video's group membership", () => {+ const tracks = parseAudioElements(withBus("data-hidden", "muted"));+ expect(tracks.every((t) => t.groupId === undefined)).toBe(true);+ });+});diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex 598ad3030e..af39c210e2 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -1619,6 +1619,11 @@ describe("parseAudioElements — hidden tracks", () => {expect(parseAudioElements(html).map((track) => track.id)).toEqual(["master"]);});++ it("does not mix a muted video that still declares data-has-audio", () => {+ const html = `<video id="v" src="v.mp4" muted data-has-audio="true" data-start="0" data-duration="3"></video>`;+ expect(parseAudioElements(html)).toEqual([]);+ });});describe("parseAudioElements data-fx-chain", () => {diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 6f58cf75ad..a529255658 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -5,7 +5,7 @@* Processes and mixes audio tracks using FFmpeg.*/-import { isSelfOrAncestorHidden } from "./mediaHidden.js";+import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";import { join, dirname, isAbsolute, relative } from "path";import { parseHTML } from "linkedom";@@ -44,6 +44,7 @@ import {} from "@hyperframes/core/audio-automation";import { chainTailSeconds } from "@hyperframes/core/audio-fx-tail";import {+ isAudibleVideoElement,MEDIA_RENDER_ID_ATTR,normalizePlaybackRate,normalizeRateSpec,@@ -54,8 +55,7 @@ import {timeAtSourceTime,type RateSpec,} from "@hyperframes/core";-import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups";-import { AUDIO_GROUP_RENDER_ID_ATTR } from "@hyperframes/core";+import { resolveAudioGroups } from "@hyperframes/core/audio-groups";import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js";import type { AudioVolumeKeyframe } from "./audioMixer.types.js";@@ -76,21 +76,6 @@ export type { AudioElement, MixResult } from "./audioMixer.types.js";*/export const MIXED_AUDIO_FILENAME = "audio.m4a";-/**- * The bus key a member belongs to, as `resolveAudioGroups` keys them.- *- * The compiler's `data-hf-group-render-id` names one INSTANCE of a bus; the- * author's `data-audio-group` names it only within its own composition file. A- * sub-composition declaring a bus and its members, used twice, therefore had- * both instances' members under one key: one sub-mix for two independent buses,- * one instance's fader and chain over the other's audio, and — with only the- * second muted — BOTH instances dropped from the export. Uncompiled documents- * (the live preview) carry no stamp and read exactly as before.- */-function memberGroupKey(el: RefResolverEl): string | null {- return el.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR) ?? el.getAttribute(HF_AUDIO_GROUP_ATTR);-}-function clampVolume(volume: number): number {return clampAudioGain(volume);}@@ -597,10 +582,7 @@ export function parseAudioElements(html: string): AudioElement[] {const groupsById = new Map(resolveAudioGroups(document).map((group) => [group.id, group] as const),);- const memberGroupHidden = (el: AudioMediaElement): boolean => {- const groupId = memberGroupKey(el);- return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false;- };+ const memberGroupHidden = (el: AudioMediaElement): boolean => isMemberGroupHidden(groupsById, el);// <audio> and <video data-has-audio> tracks differ only in the emitted id@@ -610,15 +592,14 @@ export function parseAudioElements(html: string): AudioElement[] {id: string,src: string,type: AudioElement["type"],+ joinsGroup: boolean,): AudioElement => {const layerAttr = el.getAttribute("data-layer");const volumeAttr = el.getAttribute("data-volume");const fades = readElementFades(el);const fxChain = el.getAttribute(HF_AUDIO_FX_ATTR);const automation = el.getAttribute(HF_AUDIO_AUTOMATION_ATTR);- // Audio only in v1 (matches resolveAudioGroups, which only scans- // `audio[data-audio-group]`) — a stray attribute on a <video> is inert.- const groupId = type === "audio" ? memberGroupKey(el) : null;+ const groupId = joinsGroup ? memberGroupKey(el) : null;const group = groupId ? groupsById.get(groupId) : undefined;return {id,@@ -658,15 +639,16 @@ export function parseAudioElements(html: string): AudioElement[] {// member from the mix, the same way `isHidden` drops one track.if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;- elements.push(build(el, id, src, "audio"));+ elements.push(build(el, id, src, "audio", true));}for (const el of document.querySelectorAll('video[id][data-has-audio="true"]')) {+ if (!isAudibleVideoElement(el)) continue;const id = trackId(el);const src = resolveMediaElementSrc(el);- if (!id || !src || isHidden(el)) continue;+ if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;- elements.push(build(el, `${id}-audio`, src, "video"));+ elements.push(build(el, `${id}-audio`, src, "video", true));}return elements;diff --git a/packages/engine/src/services/mediaHidden.ts b/packages/engine/src/services/mediaHidden.tsindex 0fe24a38e3..61c5fb21b4 100644--- a/packages/engine/src/services/mediaHidden.ts+++ b/packages/engine/src/services/mediaHidden.ts@@ -1,3 +1,6 @@+import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups";+import { AUDIO_GROUP_RENDER_ID_ATTR } from "@hyperframes/core";+interface HiddenCheckEl {hasAttribute(name: string): boolean;parentElement: HiddenCheckEl | null;@@ -9,3 +12,24 @@ export function isSelfOrAncestorHidden(el: HiddenCheckEl): boolean {}return false;}++interface GroupKeyEl {+ getAttribute(name: string): string | null;+}++export function memberGroupKey(el: GroupKeyEl): string | null {+ return el.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR) ?? el.getAttribute(HF_AUDIO_GROUP_ATTR);+}++export type AudioGroupsById = ReadonlyMap<string, { hidden?: boolean }>;++export function audioGroupsById(+ document: Parameters<typeof resolveAudioGroups>[0],+): AudioGroupsById {+ return new Map(resolveAudioGroups(document).map((group) => [group.id, group] as const));+}++export function isMemberGroupHidden(groupsById: AudioGroupsById, el: GroupKeyEl): boolean {+ const groupId = memberGroupKey(el);+ return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false;+}diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.tsindex 925da5e65e..e643be22b9 100644--- a/packages/engine/src/services/videoFrameExtractor.test.ts+++ b/packages/engine/src/services/videoFrameExtractor.test.ts@@ -3714,4 +3714,16 @@ describe("parseVideoElements hidden flag", () => {["c", undefined],]);});++ it("marks only an audible video hidden when its audio group is hidden", () => {+ const videos = parseVideoElements(+ `<hf-audio-group id="g" data-hidden></hf-audio-group>` ++ `<video id="a" src="a.mp4" data-has-audio="true" data-audio-group="g"></video>` ++ `<video id="b" src="b.mp4" muted data-audio-group="g"></video>`,+ );+ expect(videos.map((v) => [v.id, v.hidden])).toEqual([+ ["a", true],+ ["b", undefined],+ ]);+ });});diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 56d077dd4e..3b8879f87d 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -6,13 +6,14 @@* Videos are replaced with <img> elements during capture.*/-import { isSelfOrAncestorHidden } from "./mediaHidden.js";+import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";import { join } from "path";import { parseHTML } from "linkedom";import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";import {MEDIA_RENDER_ID_ATTR,+ isAudibleVideoElement,fpsToFfmpegArg,fpsToNumber,MEDIA_DURATION_CLAMP_EPSILON_SECONDS,@@ -667,6 +668,7 @@ export function parseVideoElements(html: string): VideoElement[] {const startCache = new Map<RefResolverEl, number>();const visiting = new Set<RefResolverEl>();+ const groupsById = audioGroupsById(document);const videoEls = document.querySelectorAll("video");let autoIdCounter = 0;for (const el of videoEls) {@@ -722,7 +724,10 @@ export function parseVideoElements(html: string): VideoElement[] {playbackRate: readElementRateSpec(el),loop: el.hasAttribute("loop"),hasAudio: hasAudioAttr === "true",- ...(isSelfOrAncestorHidden(el) ? { hidden: true } : {}),+ ...(isSelfOrAncestorHidden(el) ||+ (isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el))+ ? { hidden: true }+ : {}),});}diff --git a/packages/lint/src/rules/media.test.ts b/packages/lint/src/rules/media.test.tsindex 0f55557ad5..ab940d8d95 100644--- a/packages/lint/src/rules/media.test.ts+++ b/packages/lint/src/rules/media.test.ts@@ -808,9 +808,21 @@ describe("audio_group_no_members", () => {expect(finding?.message).not.toContain('"music"');});- it("does not count video as group membership", async () => {+ it("counts an audible video as group membership", async () => {const res = await lintHyperframeHtml(- doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" data-audio-group="voiceover"></video>+ doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" data-has-audio="true" data-audio-group="voiceover"></video>+ <audio id="s-1" src="s.wav" data-start="0" data-duration="2" data-audio-group="sfx"></audio>`),+ );+ expect(+ res.findings.some(+ (finding) => finding.code === "audio_group_no_members" && finding.elementId === "voiceover",+ ),+ ).toBe(false);+ });++ it("does not count a muted video as group membership", async () => {+ const res = await lintHyperframeHtml(+ doc(`${BUS}<video id="v" src="v.mp4" data-start="0" data-duration="5" muted data-audio-group="voiceover"></video><audio id="s-1" src="s.wav" data-start="0" data-duration="2" data-audio-group="sfx"></audio>`),);expect(diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.tsindex 6d492c7c14..695705b63d 100644--- a/packages/lint/src/rules/media.ts+++ b/packages/lint/src/rules/media.ts@@ -993,7 +993,7 @@ const AUDIO_GROUP_TIMING_ATTRS = ["data-start", "data-duration", "data-track-ind/*** A bus nobody joined does nothing, silently.*- * `resolveAudioGroups` builds groups from the MEMBERS (`audio[data-audio-group]`)+ * `resolveAudioGroups` builds groups from the MEMBERS (audio or audible video)* and only then looks for a matching `<hf-audio-group>` element, so a bus whose* id no clip names is dropped entirely — its fader, FX chain and automation* never reach preview or render, and nothing says so. One typo is enough:@@ -1004,7 +1004,7 @@ const AUDIO_GROUP_TIMING_ATTRS = ["data-start", "data-duration", "data-track-indfunction findAudioGroupNoMembersFindings(ctx: LintContext): HyperframeLintFinding[] {const memberGroupIds = new Set(ctx.tags- .filter((tag) => tag.name === "audio")+ .filter((tag) => tag.name === "audio" || (tag.name === "video" && isAudibleVideoTag(tag.raw))).map((tag) => readAttr(tag.raw, "data-audio-group")).filter((id): id is string => Boolean(id)),);diff --git a/packages/lint/src/utils.test.ts b/packages/lint/src/utils.test.tsindex b6d201ec6c..cc78ecfdbf 100644--- a/packages/lint/src/utils.test.ts+++ b/packages/lint/src/utils.test.ts@@ -1,5 +1,6 @@import { describe, it, expect } from "vitest";import {+ isAudibleVideoTag,parseHtmlStructure,stripCssComments,stripJsComments,@@ -232,3 +233,16 @@ describe("stripCssComments", () => {}});});++describe("isAudibleVideoTag", () => {+ it("treats an absent marker and the exact value true as audible", () => {+ expect(isAudibleVideoTag('<video src="a.mp4">')).toBe(true);+ expect(isAudibleVideoTag('<video data-has-audio="true">')).toBe(true);+ });+ it("treats muted, false, empty and non-exact values as silent", () => {+ expect(isAudibleVideoTag('<video muted data-has-audio="true">')).toBe(false);+ expect(isAudibleVideoTag('<video data-has-audio="false">')).toBe(false);+ expect(isAudibleVideoTag('<video data-has-audio="">')).toBe(false);+ expect(isAudibleVideoTag('<video data-has-audio="TRUE">')).toBe(false);+ });+});diff --git a/packages/lint/src/utils.ts b/packages/lint/src/utils.tsindex 265bc9110c..57f0fc0bc4 100644--- a/packages/lint/src/utils.ts+++ b/packages/lint/src/utils.ts@@ -239,12 +239,12 @@ export function hasAttrName(tagSource: string, attr: string): boolean {return new RegExp(`(?:^|\\s)${escaped}(?:\\s*=|\\s|/?>)`, "i").test(attrs);}-// An explicit data-has-audio is authoritative for the compiler; "false" and empty mean silent.+// An explicit data-has-audio is authoritative for the compiler; only the exact value "true" means audible.export function isAudibleVideoTag(tagSource: string): boolean {if (hasAttrName(tagSource, "muted")) return false;if (!hasAttrName(tagSource, "data-has-audio")) return true;const declared = readAttr(tagSource, "data-has-audio");- return declared !== null && declared.trim().toLowerCase() !== "false";+ return declared === "true";}export function mediaTimeWindow(tagSource: string): { start: number; end: number } | null {diff --git a/packages/sdk/src/editing/affordances.test.ts b/packages/sdk/src/editing/affordances.test.tsindex 8d2da06aa6..7e8146d06e 100644--- a/packages/sdk/src/editing/affordances.test.ts+++ b/packages/sdk/src/editing/affordances.test.ts@@ -26,6 +26,22 @@ describe("resolveElementAffordances (live DOM)", () => {expect(a.capabilities.canSelect).toBe(true);});+ it("audible video => audioFx; muted or data-has-audio=false video => no audioFx", () => {+ const model = { text: null, animationIds: [], start: null };+ expect(resolveElementAffordances(el(`<video></video>`), model).sections.audioFx).toBe(true);+ expect(+ resolveElementAffordances(el(`<video data-has-audio="true"></video>`), model).sections+ .audioFx,+ ).toBe(true);+ expect(resolveElementAffordances(el(`<video muted></video>`), model).sections.audioFx).toBe(+ false,+ );+ expect(+ resolveElementAffordances(el(`<video data-has-audio="false"></video>`), model).sections+ .audioFx,+ ).toBe(false);+ });+it("absolutely-positioned div with inline left/top => canMove", () => {const d = el(`<div style="position:absolute;left:5px;top:5px"></div>`);const a = resolveElementAffordances(d, { text: null, animationIds: [], start: null });diff --git a/packages/sdk/src/editing/affordances.ts b/packages/sdk/src/editing/affordances.tsindex efb5e62f99..33f58b00da 100644--- a/packages/sdk/src/editing/affordances.ts+++ b/packages/sdk/src/editing/affordances.ts@@ -58,5 +58,12 @@ export function resolveElementAffordances(hasEditableText: modelEl?.text != null,hasTimingStart: modelEl ? modelEl.start != null : liveEl.hasAttribute("data-start"),animationCount: modelEl?.animationIds.length ?? 0,+ hasAudio: isAudibleVideo(liveEl),});}++function isAudibleVideo(el: HTMLElement): boolean {+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;+ const hasAudio = el.getAttribute("data-has-audio");+ return hasAudio === null || hasAudio === "true";+}diff --git a/packages/studio/src/components/editor/domEditingLayers.test.ts b/packages/studio/src/components/editor/domEditingLayers.test.tsindex a67b998af4..629275a827 100644--- a/packages/studio/src/components/editor/domEditingLayers.test.ts+++ b/packages/studio/src/components/editor/domEditingLayers.test.ts@@ -7,6 +7,7 @@ import {buildTextFieldChildLocator,readHfId,liveLayerElement,+ domEditSelectionToFacts,} from "./domEditingLayers";import type { DomEditTextField } from "./domEditingTypes";@@ -332,3 +333,23 @@ describe("liveLayerElement", () => {).toBe("strip");});});++describe("domEditSelectionToFacts hasAudio", () => {+ async function factsFor(html: string) {+ document.body.innerHTML = html;+ const node = document.body.firstElementChild;+ if (!(node instanceof HTMLElement)) throw new Error("expected element");+ const selection = await resolveDomEditSelection(node, opts);+ document.body.innerHTML = "";+ if (!selection) throw new Error("expected selection");+ return domEditSelectionToFacts(selection);+ }++ it("is true for an unmuted video, false for muted or data-has-audio=false", async () => {+ expect((await factsFor(`<video id="v1"></video>`)).hasAudio).toBe(true);+ expect((await factsFor(`<video id="v2" data-has-audio="true"></video>`)).hasAudio).toBe(true);+ expect((await factsFor(`<video id="v3" muted></video>`)).hasAudio).toBe(false);+ expect((await factsFor(`<video id="v4" data-has-audio="false"></video>`)).hasAudio).toBe(false);+ expect((await factsFor(`<div id="d1"></div>`)).hasAudio).toBe(false);+ });+});diff --git a/packages/studio/src/components/editor/domEditingLayers.ts b/packages/studio/src/components/editor/domEditingLayers.tsindex df08fc8770..acc24b7c38 100644--- a/packages/studio/src/components/editor/domEditingLayers.ts+++ b/packages/studio/src/components/editor/domEditingLayers.ts@@ -1,4 +1,5 @@import { probeSourceElement } from "./probeSourceElement";+import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers";import type { PatchOperation } from "../../utils/sourcePatcher";import {resolveEditingAffordances,@@ -248,6 +249,7 @@ export function domEditSelectionToFacts(hasEditableText: selection.textFields.length > 0,hasTimingStart: selection.dataAttributes.start != null,animationCount,+ hasAudio: isAudibleVideoNode(selection.element),};}diff --git a/packages/studio/src/components/editor/mediaAudioEdits.test.ts b/packages/studio/src/components/editor/mediaAudioEdits.test.tsindex df089f55e7..9f2e0203b1 100644--- a/packages/studio/src/components/editor/mediaAudioEdits.test.ts+++ b/packages/studio/src/components/editor/mediaAudioEdits.test.ts@@ -209,4 +209,63 @@ describe("buildKeepSoundCutoutEdit", () => {const doc = parse(applyEdit(taken, edit));expect(doc.getElementById("clip-audio-2")?.getAttribute("src")).toBe("assets/talk.mp4");});++ it("reads the authored src from the source, not the live element", () => {+ const sub = source.replace('src="assets/talk.mp4"', 'src="../assets/a.mp4"');+ const edit = buildKeepSoundCutoutEdit({+ video: liveVideo(),+ videoId: "clip",+ target: { id: "clip" },+ cutoutSrc: "assets/cut.webm",+ });+ const doc = parse(applyEdit(sub, edit));+ expect(doc.getElementById("clip-audio")?.getAttribute("src")).toBe("../assets/a.mp4");+ expect(doc.getElementById("clip")?.getAttribute("src")).toBe("assets/cut.webm");+ });++ it("takes the first <source> child when the video has no src attribute", () => {+ const withSource = source.replace(+ /<video id="clip"([^>]*?) src="assets\/talk.mp4"([^>]*)><\/video>/,+ '<video id="clip"$1$2><source src="assets/s.mp4" type="video/mp4"></video>',+ );+ const edit = buildKeepSoundCutoutEdit({+ video: liveVideo(),+ videoId: "clip",+ target: { id: "clip" },+ cutoutSrc: "assets/cut.webm",+ });+ const doc = parse(applyEdit(withSource, edit));+ expect(doc.getElementById("clip-audio")?.getAttribute("src")).toBe("assets/s.mp4");+ });++ it("reports when the audio insert did not land", () => {+ const edit = buildKeepSoundCutoutEdit({+ video: liveVideo(),+ videoId: "clip",+ target: { id: "missing" },+ cutoutSrc: "assets/cut.webm",+ });+ edit.prepareContent(source);+ expect(edit.audioInserted()).toBe(false);+ const ok = buildKeepSoundCutoutEdit({+ video: liveVideo(),+ videoId: "clip",+ target: { id: "clip" },+ cutoutSrc: "assets/cut.webm",+ });+ ok.prepareContent(source);+ expect(ok.audioInserted()).toBe(true);+ });++ it("copies loop onto the linked audio", () => {+ const video = makeVideo({ id: "clip", src: "assets/talk.mp4", loop: "" });+ const edit = buildKeepSoundCutoutEdit({+ video,+ videoId: "clip",+ target: { id: "clip" },+ cutoutSrc: "assets/cut.webm",+ });+ const doc = parse(applyEdit(source, edit));+ expect(doc.getElementById("clip-audio")?.hasAttribute("loop")).toBe(true);+ });});diff --git a/packages/studio/src/components/editor/mediaAudioEdits.ts b/packages/studio/src/components/editor/mediaAudioEdits.tsindex 3ab2317fa0..48f9371633 100644--- a/packages/studio/src/components/editor/mediaAudioEdits.ts+++ b/packages/studio/src/components/editor/mediaAudioEdits.ts@@ -12,7 +12,9 @@ import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";import {escapeHtmlAttribute,findTagByTarget,+ applyPatchByTarget,readAttributeByTarget,+ unescapeHtmlAttribute,type PatchOperation,type PatchTarget,} from "../../utils/sourcePatcher";@@ -24,6 +26,9 @@ import { generateId } from "../../utils/generateId";const MEDIA_LINK_ATTR = "data-link";const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";+const SOUND_NOT_KEPT_MESSAGE =+ "Background removed, but the sound could not be kept on a linked audio track. Undo to restore the original clip.";+const MOVED_SOUND_ATTRS = ["data-volume",HF_AUDIO_FADE_IN_ATTR,@@ -131,6 +136,15 @@ function formatAttrs(attrs: Array<[string, string]>): string {return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");}+function readAuthoredSrc(source: string, target: PatchTarget): string {+ const tag = findTagByTarget(source, target);+ if (!tag) return "";+ const own = /\bsrc=(["'])([^"']*)\1/.exec(tag.tag);+ if (own?.[2]) return unescapeHtmlAttribute(own[2]);+ const firstSource = /^\s*<source\b[^>]*?\bsrc=(["'])([^"']*)\1/i.exec(source.slice(tag.end + 1));+ return firstSource?.[2] ? unescapeHtmlAttribute(firstSource[2]) : "";+}+function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {const match = findTagByTarget(source, target);if (!match) return source;@@ -145,6 +159,7 @@ function insertBeforeTarget(source: string, target: PatchTarget, markup: string)export interface KeepSoundCutoutEdit {ops: PatchOperation[];prepareContent: (source: string) => string;+ audioInserted: () => boolean;}export function buildKeepSoundCutoutEdit(input: {@@ -155,10 +170,11 @@ export function buildKeepSoundCutoutEdit(input: {}): KeepSoundCutoutEdit {const { video, target } = input;const linkId = mintLinkId(video.ownerDocument);- const originalSrc = video.getAttribute("src") ?? "";+ let audioInserted = false;const movedSound: Array<[string, string]> = [];- const ops: PatchOperation[] = [...cutoutOps({ isVideo: true, cutoutSrc: input.cutoutSrc })];+ const cutoutSrcOp = htmlAttr("src", input.cutoutSrc);+ const ops: PatchOperation[] = [htmlAttr("muted", "true"), dataAttr("has-audio", null)];for (const name of MOVED_SOUND_ATTRS) {const value = video.getAttribute(name);if (value === null) continue;@@ -174,6 +190,7 @@ export function buildKeepSoundCutoutEdit(input: {ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));const prepareContent = (source: string): string => {+ const originalSrc = readAuthoredSrc(source, target);const audioId = firstFreeName(`${input.videoId || "video"}-audio`,new Set(collectHtmlIds(source)),@@ -193,10 +210,14 @@ export function buildKeepSoundCutoutEdit(input: {[MEDIA_LINK_ATTR, linkId],...movedSound,];- return insertBeforeTarget(source, target, `<audio ${formatAttrs(attrs)}></audio>`);+ if (video.hasAttribute("loop")) attrs.push(["loop", ""]);+ const cut = applyPatchByTarget(source, target, cutoutSrcOp);+ const inserted = insertBeforeTarget(cut, target, `<audio ${formatAttrs(attrs)}></audio>`);+ audioInserted = inserted !== cut;+ return inserted;};- return { ops, prepareContent };+ return { ops, prepareContent, audioInserted: () => audioInserted };}export interface MediaEditContext {@@ -231,23 +252,31 @@ export async function commitCutout(const isVideo = isVideoSelection(ctx.element);const label = "Remove background";const keep = isVideo && keepSound;- const landed = keep- ? await commitKeepSoundCutout(ctx, cutoutSrc, label)- : await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });+ if (!keep) {+ const applied = await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });+ if (!applied) throw new Error("Couldn't apply the cutout");+ return "Applied cutout";+ }+ const { landed, soundKept } = await commitKeepSoundCutout(ctx, cutoutSrc, label);if (!landed) throw new Error("Couldn't apply the cutout");- return keep ? SOUND_KEPT_ON_LINKED_AUDIO_STAGE : "Applied cutout";+ if (!soundKept) throw new Error(SOUND_NOT_KEPT_MESSAGE);+ return SOUND_KEPT_ON_LINKED_AUDIO_STAGE;}-function commitKeepSoundCutout(+async function commitKeepSoundCutout(ctx: MediaEditContext,cutoutSrc: string,label: string,-): Promise<boolean> {+): Promise<{ landed: boolean; soundKept: boolean }> {const edit = buildKeepSoundCutoutEdit({video: ctx.element.element,videoId: ctx.element.id ?? null,target: buildDomEditPatchTarget(ctx.element),cutoutSrc,});- return ctx.commit(ctx.element, edit.ops, { label, prepareContent: edit.prepareContent });+ const landed = await ctx.commit(ctx.element, edit.ops, {+ label,+ prepareContent: edit.prepareContent,+ });+ return { landed, soundKept: landed && edit.audioInserted() };}diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsxindex 7991bfcd4f..5c44f617c5 100644--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx@@ -103,11 +103,11 @@ function mount(dataAttributes: Record<string, string>, alone = false, voices = 2return { host, onSetAttributeQuiet, onSetAttributeLive };}-function mountGroup(memberStart: number) {+function mountGroup(memberStart: number, memberTag: "audio" | "video" = "audio") {const bus = document.createElement("hf-audio-group");bus.id = "voiceover";document.body.append(bus);- const member = document.createElement("audio");+ const member = document.createElement(memberTag);member.id = "vo-1";member.setAttribute("data-audio-group", "voiceover");member.setAttribute("data-start", String(memberStart));@@ -649,6 +649,16 @@ describe("AudioFxGroup dynamic carve", () => {leaveShelf(host);});+ it("seeks a group audition to an audible video member's span", () => {+ act(() =>+ usePlayerStore.setState({ isPlaying: false, currentTime: 2, requestedSeekTime: null }),+ );+ const host = mountGroup(7, "video");+ hoverPreset(host);+ expect(store().requestedSeekTime).toBe(7);+ leaveShelf(host);+ });+it("leaves a transport the author started alone", () => {// Stopping their playback because they passed over a preset would be the// panel taking a decision nobody offered it.@@ -1621,7 +1631,14 @@ describe("AudioFxGroup carve by default", () => {describe("AudioFxGroup carve source list", () => {/** Mount a bed alongside tracks named however the test needs. */const mountWith = (- tracks: { id: string; src?: string; start?: string; duration?: string }[],+ tracks: {+ id: string;+ src?: string;+ start?: string;+ duration?: string;+ tag?: "audio" | "video";+ muted?: boolean;+ }[],bedAttrs: Record<string, string> = {},) => {const bed = document.createElement("audio");@@ -1629,8 +1646,9 @@ describe("AudioFxGroup carve source list", () => {for (const [k, v] of Object.entries(bedAttrs)) bed.setAttribute(`data-${k}`, v);document.body.append(bed);for (const t of tracks) {- const el = document.createElement("audio");+ const el = document.createElement(t.tag ?? "audio");el.id = t.id;+ if (t.muted) el.setAttribute("muted", "");if (t.src) el.setAttribute("src", t.src);if (t.start !== undefined) el.setAttribute("data-start", t.start);if (t.duration !== undefined) el.setAttribute("data-duration", t.duration);@@ -1660,6 +1678,16 @@ describe("AudioFxGroup carve source list", () => {return { host, options, boxes, onSetAttributeQuiet };};+ it("offers an audible video as a voice and never a muted one", () => {+ const { options } = mountWith([+ { id: "a-roll", tag: "video" },+ { id: "b-roll", tag: "video", muted: true },+ { id: "narration" },+ ]);+ expect(options).toContain("a-roll");+ expect(options).not.toContain("b-roll");+ });+it("reads the one voice out instead of offering a picker with one entry", () => {// A question with one answer is not a question. It is also the common case: a// narration and a bed, with a couple of stings that cannot be the voice.diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsxindex df0a0d9361..93439868d1 100644--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx@@ -54,6 +54,7 @@ import {resolveAudioGroups,} from "@hyperframes/core/audio-groups";import { useFxLevelling } from "./useFxLevelling.js";+import { isCarveVoiceElement } from "./useFxCarveGrouping.js";function auditionSpan(startRaw: string | undefined, durationRaw: string | undefined) {const start = Number.parseFloat(startRaw ?? "");@@ -70,8 +71,9 @@ function auditionSpansFor(element: DomEditSelection): AuditionSpan[] {if (element.tagName?.toLowerCase() !== HF_AUDIO_GROUP_TAG || !element.id) return [];const doc = element.element?.ownerDocument;if (!doc) return [];- return [...doc.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)]+ return [...doc.querySelectorAll(`[${HF_AUDIO_GROUP_ATTR}]`)].filter((member) => member.getAttribute(HF_AUDIO_GROUP_ATTR) === element.id)+ .filter(isCarveVoiceElement).flatMap((member) => {const span = auditionSpan(member.getAttribute("data-start") ?? undefined,diff --git a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsxindex 4186ae24b0..8507215654 100644--- a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx+++ b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx@@ -652,7 +652,12 @@ function makeHandlers() {ops: PatchOperation[],options: BatchOptions,) => Promise<boolean>- >(async () => true),+ >(async (_selection, _ops, options) => {+ options.prepareContent?.(+ '<div data-composition-id="main"><video id="s1-bg" src="assets/intro-loop.mp4" data-start="2"></video></div>',+ );+ return true;+ }),};}@@ -871,14 +876,13 @@ describe.each<PanelKind>(["flat", "design"])("%s panel — cutout keeps the sounconst [selection, ops, options] = onSetAttributeBatch.mock.calls[0] ?? [];expect(selection).toBe(element);expect(ops).toEqual([- htmlOp("src", "assets/cut.webm"),htmlOp("muted", "true"),dataOp("has-audio", null),dataOp("volume", null),dataOp("link", "link"),]);const source =- '<div data-composition-id="main"><video id="s1-bg" src="assets/cut.webm" muted data-start="2" data-duration="3" data-track-index="0" data-link="link"></video></div>';+ '<div data-composition-id="main"><video id="s1-bg" src="assets/intro-loop.mp4" muted data-start="2" data-duration="3" data-track-index="0" data-link="link"></video></div>';const prepared = options?.prepareContent?.(source) ?? "";const audio = new DOMParser().parseFromString(prepared, "text/html")diff --git a/packages/studio/src/components/editor/useFxCarve.ts b/packages/studio/src/components/editor/useFxCarve.tsindex 27e9ae7b1b..44959f0826 100644--- a/packages/studio/src/components/editor/useFxCarve.ts+++ b/packages/studio/src/components/editor/useFxCarve.ts@@ -25,6 +25,7 @@ import {carverAgainst,collectCarveCandidates,CARVE_ABORTED,+ isCarveVoiceElement,isPromiseLike,resolveNextCarveSettings,} from "./useFxCarveGrouping.js";@@ -101,7 +102,7 @@ function resolveCarveVoices(doc: Document, sources: readonly string[]): CarveCli// By tag name, not `instanceof HTMLAudioElement`: these elements belong to// the composition's iframe document, so the constructor they were made// from is not this realm's and the instanceof is false for every one.- if (el?.tagName !== "AUDIO") continue;+ if (!isCarveVoiceElement(el)) continue;const src = el.getAttribute("src");if (!src) continue;const clock = readClipClock((name) => el.getAttribute(name));@@ -179,7 +180,7 @@ export function useFxCarve(} => {const doc = element.element?.ownerDocument;if (!doc || !couldBeBed) return { sourceOptions: [], autoSourceIds: [] };- const others = Array.from(doc.querySelectorAll<HTMLAudioElement>("audio[id]")).filter(+ const others = Array.from(doc.querySelectorAll("audio[id], video[id]")).filter((a) => a.id !== element.id,);// Only tracks that are actually playing while this bed is. A voice somewherediff --git a/packages/studio/src/components/editor/useFxCarveGrouping.test.ts b/packages/studio/src/components/editor/useFxCarveGrouping.test.tsindex 5dfc2d2b0d..cbe4e1f589 100644--- a/packages/studio/src/components/editor/useFxCarveGrouping.test.ts+++ b/packages/studio/src/components/editor/useFxCarveGrouping.test.ts@@ -54,6 +54,19 @@ describe("collectCarveCandidates", () => {});});+describe("collectCarveCandidates with video", () => {+ it("offers an ungrouped audible video, not a muted one", () => {+ const doc = previewDoc(`+ <video id="a-roll"></video>+ <video id="b-roll" muted></video>+ <audio id="bed"></audio>+ `);+ const others = Array.from(doc.querySelectorAll("video[id]"));+ const ids = collectCarveCandidates(doc, others, () => true, "bed").map((c) => c.id);+ expect(ids).toEqual(["a-roll"]);+ });+});+describe("carverAgainst", () => {// The far-end guard has to see through a GROUP source. A plural carve names a// group — that is what the lint rule pushes authors toward — so matching rawdiff --git a/packages/studio/src/components/editor/useFxCarveGrouping.ts b/packages/studio/src/components/editor/useFxCarveGrouping.tsindex 5f79cf1308..5fa3a6a8c4 100644--- a/packages/studio/src/components/editor/useFxCarveGrouping.ts+++ b/packages/studio/src/components/editor/useFxCarveGrouping.ts@@ -20,6 +20,11 @@ import {resolveAudioGroups,resolveCarveSourceIds,} from "@hyperframes/core/audio-groups";+import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers";++export function isCarveVoiceElement(el: Element | null): el is Element {+ return el !== null && (el.tagName === "AUDIO" || isAudibleVideoNode(el));+}/*** An id for a new voiceover group, de-duped against every id already in the@@ -130,7 +135,7 @@ function excludedFor(groupByMember: Map<string, { id: string }>, bedId?: string)export function collectCarveCandidates(doc: Document,- others: readonly HTMLAudioElement[],+ others: readonly Element[],overlapsBed: (a: Element) => boolean,/** The bed's own id, so neither it nor the group it belongs to is offered. */bedId?: string,@@ -141,7 +146,7 @@ export function collectCarveCandidates(const excluded = excludedFor(groupByMember, bedId);const offeredGroupIds = new Set<string>();const described: CarveCandidate[] = [];- for (const a of others) {+ for (const a of others.filter(isCarveVoiceElement)) {const group = groupByMember.get(a.id);if (excluded.has(a.id) || (group && excluded.has(group.id))) continue;if (!group) {@@ -161,7 +166,7 @@ export function collectCarveCandidates(// iframe document, so the constructor is a different realm's and the// instanceof is false for every one (mirrors resolveCarveVoices in// useFxCarve.ts).- .filter((el): el is HTMLElement => el?.tagName === "AUDIO");+ .filter((el): el is HTMLElement => isCarveVoiceElement(el));if (!members.some(overlapsBed)) continue;offeredGroupIds.add(group.id);described.push({diff --git a/packages/studio/src/player/components/TimelineFxButton.test.tsx b/packages/studio/src/player/components/TimelineFxButton.test.tsxindex 0ceac9cd44..ef865a7198 100644--- a/packages/studio/src/player/components/TimelineFxButton.test.tsx+++ b/packages/studio/src/player/components/TimelineFxButton.test.tsx@@ -181,13 +181,13 @@ describe("TimelineFxButton", () => {<TimelineFxButtonvariant="group-pointer"clipCount={2}- refusal="Video audio can't be grouped yet — only audio clips can join a group."+ refusal="This track can't be grouped."onGroupClips={onGroupClips}/>,);act(() => byTextButton(host, "FX")?.click());const dialog = document.querySelector('[role="dialog"]');- expect(dialog?.textContent).toContain("Video audio can't be grouped yet");+ expect(dialog?.textContent).toContain("This track can't be grouped");expect(document.querySelector('input[aria-label="Group name"]')).toBeNull();expect(Array.from(document.body.querySelectorAll("button")).some((b) => b.textContent === "Group"),diff --git a/packages/studio/src/player/components/TimelineTrackHeader.test.tsx b/packages/studio/src/player/components/TimelineTrackHeader.test.tsxindex 8246bf8785..1d36898aa8 100644--- a/packages/studio/src/player/components/TimelineTrackHeader.test.tsx+++ b/packages/studio/src/player/components/TimelineTrackHeader.test.tsx@@ -869,6 +869,43 @@ describe("TimelineTrackHeader", () => {act(() => view.root.unmount());});+ it("offers grouping on a track of audible videos, not on muted b-roll", () => {+ const aRoll: TimelineElement = {+ ...VOICE,+ id: "a-roll",+ domId: "a-roll",+ tag: "video",+ hasAudio: true,+ };+ const aRoll2: TimelineElement = { ...aRoll, id: "a-roll-2", domId: "a-roll-2" };+ const bRoll: TimelineElement = {+ ...aRoll,+ id: "b-roll",+ domId: "b-roll",+ hasAudio: undefined,+ };+ const bRoll2: TimelineElement = { ...bRoll, id: "b-roll-2", domId: "b-roll-2" };+ const opts = {+ keyframeClip: aRoll,+ trackElements: [aRoll, aRoll2],+ clipCount: 2,+ animations: [],+ expanded: false,+ };+ const pointer = (host: HTMLElement) =>+ host.querySelector<HTMLButtonElement>(+ 'button[aria-label="Effects — group these clips first"]',+ );+ const view = renderHeader(opts);+ const button = pointer(view.host);+ expect(button).not.toBeNull();+ act(() => button?.click());+ expect(document.body.textContent).not.toContain("can't be grouped");+ view.rerender({ ...opts, keyframeClip: bRoll, trackElements: [bRoll, bRoll2] });+ expect(pointer(view.host)).toBeNull();+ act(() => view.root.unmount());+ });+// The header is a 48px column of exactly TWO lines — what the row is, then// what you can do to it. The group pointer used to render as a sibling of// both, making a third: 17 + 24 + 24 + gaps in a 48px box, whichdiff --git a/packages/studio/src/player/components/TimelineTrackHeader.tsx b/packages/studio/src/player/components/TimelineTrackHeader.tsxindex 25e27f60b2..5b0da20c94 100644--- a/packages/studio/src/player/components/TimelineTrackHeader.tsx+++ b/packages/studio/src/player/components/TimelineTrackHeader.tsx@@ -226,11 +226,9 @@ export function TimelineTrackHeader({const singleAudioClip =isAudioTrack && clipCount === 1 && trackElements.length > 0 ? trackElements[0] : null;const isTrackGrouped = trackElements.some((el) => el.audioGroup);- // A video track carries sound the render mixes but preview never routes- // through Web Audio, which is why §1.4 keeps groups audio-only. It still- // needs to be TOLD that, so it earns the button and a refusal.- const isVideoWithAudioTrack =- !isAudioTrack && trackElements.some((el) => el.tag.toLowerCase() === "video");+ const audioBearingClips = isAudioTrack+ ? trackElements+ : trackElements.filter((el) => el.hasAudio === true);const writeClipFxChain = (clip: TimelineElement,next: HfAudioFxChain,@@ -256,7 +254,7 @@ export function TimelineTrackHeader({// UI showed the track as grouped. The button is withheld instead of acting on// a subset, which is also why the carve path's loud guard cannot catch this:// the unresolvable ids were filtered out before the call.- const groupableClipIds = trackElements.map(runtimeAudioId);+ const groupableClipIds = audioBearingClips.map(runtimeAudioId);const allGroupableClipIds = groupableClipIds.every((id): id is string => id !== null)? groupableClipIds: null;@@ -342,25 +340,14 @@ export function TimelineTrackHeader({onOpenRack={() => openClipFxRack(singleAudioClip)}/>)}- {showAudioEffects &&- clipCount > 1 &&- !isTrackGrouped &&- (isAudioTrack ? canGroupWholeTrack : isVideoWithAudioTrack) && (- <TimelineFxButton- variant="group-pointer"- clipCount={trackElements.length}- defaultLabel={trackLabel}- // Groups are audio-only in v1 (§1.4). A video track showing no- // button at all is the silent limit §5 forbids, so it gets the- // button and a reason instead.- refusal={- isAudioTrack- ? undefined- : "Video audio can't be grouped yet — only audio clips can join a group."- }- onGroupClips={groupUngroupedClips}- />- )}+ {showAudioEffects && clipCount > 1 && !isTrackGrouped && canGroupWholeTrack && (+ <TimelineFxButton+ variant="group-pointer"+ clipCount={audioBearingClips.length}+ defaultLabel={trackLabel}+ onGroupClips={groupUngroupedClips}+ />+ )}{/* The lane disclosure, on the row's own layout rather than byswapping it for a keyframe-layer row. */}{disclosable && (diff --git a/packages/studio/src/player/lib/timelineElementHelpers.ts b/packages/studio/src/player/lib/timelineElementHelpers.tsindex c5e3b944de..2b2c13d945 100644--- a/packages/studio/src/player/lib/timelineElementHelpers.ts+++ b/packages/studio/src/player/lib/timelineElementHelpers.ts@@ -231,6 +231,15 @@ export function isVideoAudible(opts: {return opts.tag.toLowerCase() === "video" && !opts.muted;}+export function isAudibleVideoNode(el: Element): boolean {+ if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;+ return isVideoAudible({+ tag: "video",+ hasAudioAttr: el.getAttribute("data-has-audio"),+ muted: false,+ });+}+/** What the mixer gets: the compiler's `data-has-audio` rule, muted and volume. */function applyAudioMetadataFromElement(entry: TimelineElement, el: Element): void {const media = resolveMediaElement(el) ?? el;diff --git a/packages/studio/src/utils/audioMeterMath.test.ts b/packages/studio/src/utils/audioMeterMath.test.tsindex 95884a0581..b5fbfb2518 100644--- a/packages/studio/src/utils/audioMeterMath.test.ts+++ b/packages/studio/src/utils/audioMeterMath.test.ts@@ -1,5 +1,7 @@import { describe, expect, it } from "vitest";+import type { TimelineElement } from "../player";import {+ hasProjectAudio,levelToFraction,markFraction,SILENT_CHANNEL,@@ -42,3 +44,38 @@ describe("stepPair", () => {expect(stepPair(rest, { l: 1, r: 0 }, 32, 16)).not.toBe(rest);});});++describe("hasProjectAudio", () => {+ const clip = (overrides: Partial<TimelineElement>): TimelineElement => ({+ id: "c",+ tag: "div",+ start: 0,+ duration: 1,+ track: 0,+ ...overrides,+ });++ it("ignores an audible video left on native output at unity gain", () => {+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true })])).toBe(false);+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, volume: 1 })])).toBe(false);+ });++ it.each<[string, Partial<TimelineElement>]>([+ ["an above-unity gain", { volume: 2 }],+ ["an fx chain", { fxChain: "[]" }],+ ["automation", { automation: "{}" }],+ ["a group", { audioGroup: "music" }],+ ])("counts an audible video routed through Web Audio by %s", (_, fields) => {+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, ...fields })])).toBe(true);+ });++ it("ignores a muted video even when it carries processing", () => {+ expect(hasProjectAudio([clip({ tag: "video", hasAudio: true, muted: true, volume: 2 })])).toBe(+ false,+ );+ });++ it("ignores a silent video", () => {+ expect(hasProjectAudio([clip({ tag: "video" })])).toBe(false);+ });+});diff --git a/packages/studio/src/utils/audioMeterMath.ts b/packages/studio/src/utils/audioMeterMath.tsindex 0cd84ff1e8..10395f9fc5 100644--- a/packages/studio/src/utils/audioMeterMath.ts+++ b/packages/studio/src/utils/audioMeterMath.ts@@ -1,5 +1,6 @@import { useMemo } from "react";import { audioDbToGain, audioGainToDb } from "@hyperframes/core/audio-gain";+import { audibleVideoNeedsWebAudio } from "@hyperframes/core/audible-video";import { usePlayerStore, type TimelineElement } from "../player";import { isAudioTimelineElement } from "./timelineInspector";@@ -67,9 +68,14 @@ export function stepPair(return [stepChannel(l, raw?.l ?? 0, now, dtMs), stepChannel(r, raw?.r ?? 0, now, dtMs)];}-/** The meters appear only for a project with at least one audio clip or group. */-function hasProjectAudio(elements: readonly TimelineElement[]): boolean {- return elements.some((el) => isAudioTimelineElement(el) || el.audioGroup !== undefined);+/** The meters appear only for a project with an audio clip, a group, or a video Web Audio carries. */+export function hasProjectAudio(elements: readonly TimelineElement[]): boolean {+ return elements.some(+ (el) =>+ isAudioTimelineElement(el) ||+ el.audioGroup !== undefined ||+ (el.hasAudio === true && !el.muted && audibleVideoNeedsWebAudio(el)),+ );}/** Scans once per `elements` change, not on every store update (the playhead ticks each frame). */diff --git a/packages/studio/src/utils/sourcePatcher.ts b/packages/studio/src/utils/sourcePatcher.tsindex 47ef37cc70..57a250e859 100644--- a/packages/studio/src/utils/sourcePatcher.ts+++ b/packages/studio/src/utils/sourcePatcher.ts@@ -21,7 +21,7 @@ export function escapeHtmlAttribute(value: string): string {}/** Reverse escapeHtmlAttribute so callers get the original value. */-function unescapeHtmlAttribute(value: string): string {+export function unescapeHtmlAttribute(value: string): string {return value.replace(/"/g, '"').replace(/</g, "<")