Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(studio): waveform strip, clip light, peak marks and video beat source (5/8) (#4817)
Vance Ingalls824e4100c636
diff --git a/packages/cli/src/commands/beats.ts b/packages/cli/src/commands/beats.tsindex 536811479f..9ae8ff1d65 100644--- a/packages/cli/src/commands/beats.ts+++ b/packages/cli/src/commands/beats.ts@@ -23,7 +23,8 @@ function resolveMusicTarget(project: ProjectDir): { rel: string; audioPath: striconst src = findMusicAudioSrc(readFileSync(project.indexPath, "utf-8"));if (!src) {fail(- 'No music track found. Add data-timeline-role="music" to the <audio> element ' ++ 'No music track found. Add data-timeline-role="music" to the <audio> element, or to a ' ++ '<video data-has-audio="true"> whose own sound is the music ' +"(or give it an id like music/bgm/soundtrack).",);}diff --git a/packages/core/src/beats/beatFile.test.ts b/packages/core/src/beats/beatFile.test.tsnew file mode 100644index 0000000000..38a79991bc--- /dev/null+++ b/packages/core/src/beats/beatFile.test.ts@@ -0,0 +1,29 @@+import { describe, expect, it } from "vitest";+import { findMusicAudioSrc } from "./beatFile";++describe("findMusicAudioSrc", () => {+ it("finds a video with sound tagged as music", () => {+ const html = `<video id="clip" src="band.mp4" data-has-audio="true" data-timeline-role="music"></video>`;+ expect(findMusicAudioSrc(html)).toBe("band.mp4");+ });++ it("finds an audible video by a music id", () => {+ expect(findMusicAudioSrc(`<video id="soundtrack" src="s.mp4" data-has-audio="true">`)).toBe(+ "s.mp4",+ );+ });++ it("skips a muted video and one without declared sound", () => {+ const html = [+ `<video id="music" src="muted.mp4" data-has-audio="true" muted>`,+ `<video id="bgm" src="undeclared.mp4">`,+ `<audio id="music-bed" src="bed.mp3">`,+ ].join("");+ expect(findMusicAudioSrc(html)).toBe("bed.mp3");+ });++ it("keeps document order across audio and video", () => {+ const html = `<audio id="music" src="first.mp3"></audio><video id="bgm" src="v.mp4" data-has-audio="true">`;+ expect(findMusicAudioSrc(html)).toBe("first.mp3");+ });+});diff --git a/packages/core/src/beats/beatFile.ts b/packages/core/src/beats/beatFile.tsindex 31f2e0b71b..3235fffb58 100644--- a/packages/core/src/beats/beatFile.ts+++ b/packages/core/src/beats/beatFile.ts@@ -87,18 +87,25 @@ function attr(tag: string, name: string): string | null {return m ? m[1]! : null;}+function isSoundingVideoTag(tag: string): boolean {+ const unquoted = tag.replace(/"[^"]*"|'[^']*'/g, '""');+ return attr(tag, "data-has-audio") === "true" && !/\smuted(?=[\s/>=])/i.test(unquoted);+}+/*** Find the music track's src in composition HTML, applying the SAME rules as the- * Studio's `isMusicTrack` so the CLI and Studio agree on which `<audio>` is music:- * the FIRST `<audio>` (in document order) where data-timeline-role="music", or —+ * Studio's `isMusicTrack` so the CLI and Studio agree on which clip is music:+ * the FIRST `<audio>`, or `<video data-has-audio="true">` without `muted` (in+ * document order) where data-timeline-role="music", or —* when no role is set — whose id matches the music regex. An explicit non-music* role excludes the element. Returns the raw src attribute, or null.*/export function findMusicAudioSrc(html: string): string | null {// `[^>]*` spans newlines (it's a negated class, not `.`), so multi-line opening// tags are handled. HyperFrames authors src as an attribute on <audio>.- const tags = html.match(/<audio\b[^>]*>/gi) ?? [];+ const tags = html.match(/<(?:audio|video)\b[^>]*>/gi) ?? [];for (const tag of tags) {+ if (/^<video/i.test(tag) && !isSoundingVideoTag(tag)) continue;const src = attr(tag, "src");if (!src) continue;const role = attr(tag, "data-timeline-role");diff --git a/packages/studio-server/src/createStudioApi.ts b/packages/studio-server/src/createStudioApi.tsindex d39576c446..3c1e3cde3e 100644--- a/packages/studio-server/src/createStudioApi.ts+++ b/packages/studio-server/src/createStudioApi.ts@@ -10,6 +10,7 @@ import { registerThumbnailRoutes } from "./routes/thumbnail.js";import { registerWaveformRoutes } from "./routes/waveform.js";import { registerFreezeFrameRoutes } from "./routes/freezeFrame.js";import { registerLoudnessRoutes } from "./routes/loudness.js";+import { registerPeakRoutes } from "./routes/peaks.js";import { registerFontRoutes } from "./routes/fonts.js";import { registerRegistryRoutes } from "./routes/registry.js";import { registerSelectionRoutes } from "./routes/selection.js";@@ -65,6 +66,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {registerWaveformRoutes(api, adapter);registerFreezeFrameRoutes(api, adapter);registerLoudnessRoutes(api, adapter);+ registerPeakRoutes(api, adapter);registerFontRoutes(api);registerRegistryRoutes(api, adapter);registerGlobalAssetRoutes(api);diff --git a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.tsnew file mode 100644index 0000000000..a2f24c8c40--- /dev/null+++ b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts@@ -0,0 +1,29 @@+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { describe, expect, it, vi } from "vitest";++// Stand-in binaries: ffprobe reports a stereo stream, ffmpeg writes samples and then fails.+const fakeBin = vi.hoisted(() => ({ dir: "" }));+vi.mock("@hyperframes/parsers/ff-binaries", () => ({+ findFfBinary: (name: string) => join(fakeBin.dir, name),+}));++import { decodePeakMap } from "./peakMap";++describe("decodePeakMap, decode failure", () => {+ it.skipIf(process.platform === "win32")(+ "rejects partial output when ffmpeg exits nonzero",+ async () => {+ fakeBin.dir = mkdtempSync(join(tmpdir(), "hf-fake-ff-"));+ const script = (name: string, body: string) => {+ const file = join(fakeBin.dir, name);+ writeFileSync(file, `#!/bin/sh\n${body}\n`);+ chmodSync(file, 0o755);+ };+ script("ffprobe", "echo 2");+ script("ffmpeg", "head -c 38400 /dev/zero; exit 1");+ await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/);+ },+ );+});diff --git a/packages/studio-server/src/helpers/peakMap.test.ts b/packages/studio-server/src/helpers/peakMap.test.tsnew file mode 100644index 0000000000..6299574e75--- /dev/null+++ b/packages/studio-server/src/helpers/peakMap.test.ts@@ -0,0 +1,149 @@+import { execFileSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { describe, expect, it } from "vitest";+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";+import {+ accumulatePeaks,+ buildPeakMapCacheKey,+ createPeakAccumulator,+ decodePeakMap,+ finishPeaks,+} from "./peakMap.js";++function stereo(frames: Array<[number, number]>): Buffer {+ const buf = Buffer.alloc(frames.length * 8);+ frames.forEach(([l, r], i) => {+ buf.writeFloatLE(l, i * 8);+ buf.writeFloatLE(r, i * 8 + 4);+ });+ return buf;+}++describe("peak accumulation", () => {+ it("keeps the absolute max of either channel per bin, un-normalized", () => {+ const acc = createPeakAccumulator(4, 0.5, 2);+ accumulatePeaks(+ acc,+ stereo([+ [0.1, -0.3],+ [0.2, 0],+ [0, 0.05],+ [-0.04, 0],+ ]),+ );+ expect(finishPeaks(acc)).toEqual([0.3, 0.05]);+ });++ it("carries a frame split across chunks", () => {+ const acc = createPeakAccumulator(2, 1, 2);+ const whole = stereo([+ [0.5, 0],+ [0, -0.9],+ ]);+ accumulatePeaks(acc, whole.subarray(0, 5));+ accumulatePeaks(acc, whole.subarray(5));+ expect(finishPeaks(acc)).toEqual([0.9]);+ expect(acc.frame).toBe(2);+ });+});++describe("peak accumulation, mono", () => {+ it("reads one channel per frame", () => {+ const acc = createPeakAccumulator(2, 1, 1);+ const buf = Buffer.alloc(8);+ buf.writeFloatLE(-1, 0);+ buf.writeFloatLE(0.5, 4);+ accumulatePeaks(acc, buf);+ expect(finishPeaks(acc)).toEqual([1]);+ });+});++describe("buildPeakMapCacheKey", () => {+ it("changes when the file is replaced and never collides with a waveform entry", () => {+ const a = buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 });+ expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 }));+ expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 }));+ expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 }));+ expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/);+ });+});++describe("decodePeakMap", () => {+ const ffmpeg = findFfBinary("ffmpeg");+ it.skipIf(!ffmpeg)(+ "reads a 0 dBFS video track near 1.0 and a -12 dBFS one near 0.25",+ async () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));+ try {+ const make = (name: string, volume: string) => {+ const file = join(dir, name);+ execFileSync(ffmpeg ?? "ffmpeg", [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "color=c=black:s=32x32:d=1",+ "-f",+ "lavfi",+ "-i",+ `sine=frequency=440:duration=1,volume=${volume}`,+ "-shortest",+ "-c:v",+ "libx264",+ "-c:a",+ "pcm_s16le",+ file,+ ]);+ return file;+ };+ const loud = await decodePeakMap(make("loud.mov", "8"));+ const quiet = await decodePeakMap(make("quiet.mov", "2"));+ expect(Math.max(...loud)).toBeGreaterThan(0.95);+ expect(Math.max(...quiet)).toBeLessThan(0.3);+ expect(loud.length).toBeGreaterThanOrEqual(19);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ },+ );++ it.skipIf(!ffmpeg)(+ "decodes the probed first audio stream when a louder stereo stream follows it",+ async () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));+ try {+ const file = join(dir, "multi.mkv");+ execFileSync(ffmpeg ?? "ffmpeg", [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "aevalsrc=0.1*sin(2*PI*440*t):s=48000:d=1",+ "-f",+ "lavfi",+ "-i",+ "aevalsrc=0.9*sin(2*PI*440*t)|0.9*sin(2*PI*440*t):s=48000:d=1",+ "-map",+ "0:a",+ "-map",+ "1:a",+ "-c:a",+ "pcm_f32le",+ "-disposition:a",+ "0",+ file,+ ]);+ const peaks = await decodePeakMap(file);+ expect(peaks.length).toBeGreaterThanOrEqual(19);+ expect(peaks.length).toBeLessThanOrEqual(21);+ expect(Math.max(...peaks)).toBeCloseTo(0.1, 2);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ },+ );+});diff --git a/packages/studio-server/src/helpers/peakMap.ts b/packages/studio-server/src/helpers/peakMap.tsnew file mode 100644index 0000000000..743a94b2c4--- /dev/null+++ b/packages/studio-server/src/helpers/peakMap.ts@@ -0,0 +1,131 @@+import { execFile, spawn } from "node:child_process";+import { promisify } from "node:util";+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";++export const PEAK_BIN_SECONDS = 0.05;+const PEAK_SAMPLE_RATE = 48_000;+const execFileAsync = promisify(execFile);+const PEAK_MAP_CACHE_VERSION = "peaks-v2";++/**+ * Running max-abs per bin over interleaved f32le at the source's own channel+ * count. Native channels, not `-ac 2`: ffmpeg's mono upmix pans at -3 dB,+ * which would hide a clip that really does reach 0 dBFS.+ */+export interface PeakAccumulator {+ bins: number[];+ frame: number;+ carry: Buffer;+ framesPerBin: number;+ channels: number;+}++export function createPeakAccumulator(+ sampleRate = PEAK_SAMPLE_RATE,+ binSeconds = PEAK_BIN_SECONDS,+ channels = 2,+): PeakAccumulator {+ return {+ bins: [],+ frame: 0,+ carry: Buffer.alloc(0),+ framesPerBin: Math.max(1, Math.round(sampleRate * binSeconds)),+ channels: Math.max(1, channels),+ };+}++function framePeak(data: Buffer, offset: number, channels: number): number {+ let peak = 0;+ for (let ch = 0; ch < channels; ch++) {+ peak = Math.max(peak, Math.abs(data.readFloatLE(offset + ch * 4)));+ }+ return peak;+}++export function accumulatePeaks(acc: PeakAccumulator, chunk: Buffer): void {+ const data = acc.carry.length > 0 ? Buffer.concat([acc.carry, chunk]) : chunk;+ const frameBytes = 4 * acc.channels;+ const whole = data.length - (data.length % frameBytes);+ for (let offset = 0; offset < whole; offset += frameBytes) {+ const peak = framePeak(data, offset, acc.channels);+ const bin = Math.floor(acc.frame / acc.framesPerBin);+ if (acc.bins.length <= bin) acc.bins.push(0);+ if (peak > (acc.bins[bin] ?? 0)) acc.bins[bin] = peak;+ acc.frame += 1;+ }+ acc.carry = Buffer.from(data.subarray(whole));+}++/** Linear sample peak per bin, NOT normalized — the level is the point. */+export function finishPeaks(acc: PeakAccumulator): number[] {+ return acc.bins.map((peak) => Number(peak.toFixed(4)));+}++async function probeChannels(mediaPath: string): Promise<number> {+ const { stdout } = await execFileAsync(+ findFfBinary("ffprobe") ?? "ffprobe",+ [+ "-v",+ "error",+ "-select_streams",+ "a:0",+ "-show_entries",+ "stream=channels",+ "-of",+ "csv=p=0",+ "--",+ mediaPath,+ ],+ { encoding: "utf8", timeout: 30_000, windowsHide: true },+ );+ const channels = Number.parseInt(stdout.trim(), 10);+ if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream");+ return channels;+}++export async function decodePeakMap(mediaPath: string): Promise<number[]> {+ const channels = await probeChannels(mediaPath);+ return new Promise((resolvePromise, reject) => {+ const proc = spawn(+ findFfBinary("ffmpeg") ?? "ffmpeg",+ [+ "-v",+ "error",+ "-i",+ mediaPath,+ "-map",+ "0:a:0",+ "-vn",+ "-ar",+ String(PEAK_SAMPLE_RATE),+ "-f",+ "f32le",+ "pipe:1",+ ],+ { stdio: ["ignore", "pipe", "ignore"], windowsHide: true },+ );+ const acc = createPeakAccumulator(PEAK_SAMPLE_RATE, PEAK_BIN_SECONDS, channels);+ proc.stdout?.on("data", (chunk: Buffer) => accumulatePeaks(acc, chunk));+ proc.on("close", (code) => {+ if (code !== 0) {+ reject(new Error(`ffmpeg failed decoding audio (exit ${code})`));+ return;+ }+ if (acc.frame === 0) {+ reject(new Error("ffmpeg produced no audio samples"));+ return;+ }+ resolvePromise(finishPeaks(acc));+ });+ proc.on("error", reject);+ });+}++/** Keyed on content (size + mtime) like the waveform cache, under its own version prefix. */+export function buildPeakMapCacheKey(+ assetPath: string,+ fingerprint: { size: number; mtimeMs: number },+): string {+ const name = assetPath.replace(/[/\\]/g, "_");+ return `${PEAK_MAP_CACHE_VERSION}_${name}_${fingerprint.size}-${Math.round(fingerprint.mtimeMs)}.json`;+}diff --git a/packages/studio-server/src/helpers/waveform.test.ts b/packages/studio-server/src/helpers/waveform.test.tsindex 16ba9d1e48..45478fc25c 100644--- a/packages/studio-server/src/helpers/waveform.test.ts+++ b/packages/studio-server/src/helpers/waveform.test.ts@@ -1,5 +1,10 @@import { describe, expect, it } from "vitest";-import { buildWaveformCacheKey } from "./waveform.js";+import { execFileSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";+import { buildWaveformCacheKey, decodeAudioPeaks } from "./waveform.js";describe("buildWaveformCacheKey", () => {it("is stable for the same file", () => {@@ -34,3 +39,36 @@ describe("buildWaveformCacheKey", () => {expect(buildWaveformCacheKey("a/b.m4a", fp)).toMatch(/\.json$/);});});++describe("decodeAudioPeaks on a video file", () => {+ const ffmpeg = findFfBinary("ffmpeg");+ it.skipIf(!ffmpeg)("reads the audio track of an .mp4", async () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-wave-video-"));+ const file = join(dir, "talk.mp4");+ try {+ execFileSync(ffmpeg ?? "ffmpeg", [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "color=c=black:s=32x32:d=1",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:duration=1",+ "-shortest",+ "-c:v",+ "libx264",+ "-c:a",+ "aac",+ file,+ ]);+ const peaks = await decodeAudioPeaks(file);+ expect(peaks.length).toBeGreaterThan(0);+ expect(Math.max(...peaks)).toBeCloseTo(1, 3);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});diff --git a/packages/studio-server/src/routes/peaks.test.ts b/packages/studio-server/src/routes/peaks.test.tsnew file mode 100644index 0000000000..748a924f3b--- /dev/null+++ b/packages/studio-server/src/routes/peaks.test.ts@@ -0,0 +1,64 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { Hono } from "hono";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { registerPeakRoutes } from "./peaks";+import type { StudioApiAdapter } from "../types";++const dirs: string[] = [];+afterEach(() => {+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });+});++function setup(decode: (path: string) => Promise<number[]>) {+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-route-"));+ dirs.push(dir);+ writeFileSync(join(dir, "talk.mp4"), "video");+ const adapter: StudioApiAdapter = {+ listProjects: () => [],+ resolveProject: async (id: string) => ({ id, dir }),+ bundle: async () => null,+ lint: async () => ({ findings: [] }),+ runtimeUrl: "/api/runtime.js",+ rendersDir: () => "/tmp/renders",+ startRender: () => ({ id: "j", status: "rendering", progress: 0, outputPath: "/tmp/o.mp4" }),+ };+ const app = new Hono();+ registerPeakRoutes(app, adapter, decode);+ return app;+}++describe("GET /projects/:id/peaks/*", () => {+ it("answers absolute bins once per file and serves the cache after", async () => {+ const decode = vi.fn(async () => [0.2, 0.99]);+ const app = setup(decode);+ for (let i = 0; i < 2; i++) {+ const res = await app.request("http://localhost/projects/p/peaks/talk.mp4");+ expect(await res.json()).toEqual({ binSeconds: 0.05, bins: [0.2, 0.99] });+ }+ expect(decode).toHaveBeenCalledTimes(1);+ });++ it("decodes a file once when several clips ask at the same time", async () => {+ let finish: (bins: number[]) => void = () => {};+ const decode = vi.fn(() => new Promise<number[]>((resolve) => (finish = resolve)));+ const app = setup(decode);+ const both = Promise.all([+ app.request("http://localhost/projects/p/peaks/talk.mp4"),+ app.request("http://localhost/projects/p/peaks/talk.mp4"),+ ]);+ await vi.waitFor(() => expect(decode).toHaveBeenCalled());+ finish([0.5]);+ for (const res of await both) expect((await res.json()).bins).toEqual([0.5]);+ expect(decode).toHaveBeenCalledTimes(1);+ });++ it("404s a missing file and a path outside the project", async () => {+ const app = setup(async () => []);+ expect((await app.request("http://localhost/projects/p/peaks/nope.mp4")).status).toBe(404);+ expect(+ (await app.request("http://localhost/projects/p/peaks/..%2F..%2Fetc%2Fpasswd")).status,+ ).toBe(404);+ });+});diff --git a/packages/studio-server/src/routes/peaks.ts b/packages/studio-server/src/routes/peaks.tsnew file mode 100644index 0000000000..2697461c05--- /dev/null+++ b/packages/studio-server/src/routes/peaks.ts@@ -0,0 +1,66 @@+import { existsSync, readFileSync, statSync } from "node:fs";+import { join, resolve } from "node:path";+import { isWithinProjectRoot } from "@hyperframes/parsers/asset-resolution";+import type { Hono } from "hono";+import type { StudioApiAdapter } from "../types.js";+import { isWaveformCacheDirectory, writeWaveformCache } from "../helpers/waveform.js";+import { buildPeakMapCacheKey, decodePeakMap, PEAK_BIN_SECONDS } from "../helpers/peakMap.js";+import { requestSubPath } from "../helpers/requestSubPath.js";++type DecodePeaks = (mediaPath: string) => Promise<number[]>;++const decodesInFlight = new Map<string, Promise<number[]>>();++function decodeOnce(cachePath: string, mediaPath: string, decode: DecodePeaks): Promise<number[]> {+ const pending = decodesInFlight.get(cachePath);+ if (pending) return pending;+ const started = decode(mediaPath).finally(() => decodesInFlight.delete(cachePath));+ decodesInFlight.set(cachePath, started);+ return started;+}++function readCachedBins(cacheDir: string, cachePath: string): number[] | null {+ try {+ if (!isWaveformCacheDirectory(cacheDir) || !existsSync(cachePath)) return null;+ const parsed: unknown = JSON.parse(readFileSync(cachePath, "utf-8"));+ return Array.isArray(parsed) && parsed.every((n) => typeof n === "number") ? parsed : null;+ } catch {+ return null;+ }+}++/** Absolute per-bin sample peaks for one media file, so the timeline can mark where a clip redlines. */+export function registerPeakRoutes(+ api: Hono,+ adapter: StudioApiAdapter,+ decode: DecodePeaks = decodePeakMap,+): void {+ api.get("/projects/:id/peaks/*", async (c) => {+ const project = await adapter.resolveProject(c.req.param("id"));+ if (!project) return c.json({ error: "not found" }, 404);+ const assetPath = requestSubPath(c.req.url, "projects/:id/peaks");+ const mediaPath = resolve(project.dir, assetPath);+ if (!isWithinProjectRoot(project.dir, mediaPath))+ return c.json({ error: "file not found" }, 404);+ const stats = statSync(mediaPath, { throwIfNoEntry: false });+ if (!stats?.isFile()) return c.json({ error: "file not found" }, 404);++ const cacheDir = join(project.dir, ".waveform-cache");+ const cachePath = join(cacheDir, buildPeakMapCacheKey(assetPath, stats));+ const cached = readCachedBins(cacheDir, cachePath);+ if (cached) return c.json({ binSeconds: PEAK_BIN_SECONDS, bins: cached });++ let bins: number[];+ try {+ bins = await decodeOnce(cachePath, mediaPath, decode);+ } catch {+ return c.json({ error: "failed to decode audio" }, 500);+ }+ try {+ writeWaveformCache(cachePath, bins);+ } catch {+ // A cache that cannot be written only costs the next request a decode.+ }+ return c.json({ binSeconds: PEAK_BIN_SECONDS, bins });+ });+}diff --git a/packages/studio/src/components/nle/AudioMeterStrip.test.tsx b/packages/studio/src/components/nle/AudioMeterStrip.test.tsxindex bd5a387822..e453a81527 100644--- a/packages/studio/src/components/nle/AudioMeterStrip.test.tsx+++ b/packages/studio/src/components/nle/AudioMeterStrip.test.tsx@@ -12,6 +12,8 @@ import { SILENT_CHANNEL } from "../../utils/audioMeterMath";import {AudioMeterStrip,type AudioMeterStripProps,+ CLIP_LATCH_PEAK,+ stepClipLatch,evictGoneMeterState,followMeterHook,stepAndPaintStrips,@@ -416,3 +418,54 @@ describe("stepAndPaintStrips", () => {expect(state.has("loud")).toBe(true);});});++describe("CLIP light", () => {+ const hookAt = (peak: number) => ({+ start: vi.fn(),+ stop: vi.fn(),+ read: vi.fn(() => ({ master: { l: peak, r: 0 }, groups: {} })),+ });++ it("latches at the -1 dBFS export ceiling, stays lit when the level drops, and resets on click", () => {+ usePlayerStore.setState({ elements: [clip({})] });+ const hook = hookAt(1);+ setHook(hook);+ const { host } = mount();+ const light = host.querySelector<HTMLElement>("[data-testid=meter-clip-light]");+ expect(light?.dataset.lit).toBe("false");+ tick();+ expect(light?.dataset.lit).toBe("true");+ hook.read.mockReturnValue({ master: { l: 0.1, r: 0.1 }, groups: {} });+ tick();+ expect(light?.dataset.lit).toBe("true");+ act(() => light?.click());+ expect(light?.dataset.lit).toBe("false");+ tick();+ expect(light?.dataset.lit).toBe("false");+ });++ it("stays dark in the red band just under the ceiling", () => {+ usePlayerStore.setState({ elements: [clip({})] });+ setHook(hookAt(10 ** (-1.2 / 20)));+ const { host } = mount();+ tick();+ expect(host.querySelector<HTMLElement>("[data-testid=meter-clip-light]")?.dataset.lit).toBe(+ "false",+ );+ });+});++describe("stepClipLatch", () => {+ it("lights exactly at -1 dBFS on either channel, not below", () => {+ expect(CLIP_LATCH_PEAK).toBeCloseTo(0.891251, 6);+ expect(stepClipLatch(false, { l: CLIP_LATCH_PEAK, r: 0 })).toBe(true);+ expect(stepClipLatch(false, { l: 0, r: CLIP_LATCH_PEAK })).toBe(true);+ expect(stepClipLatch(false, { l: CLIP_LATCH_PEAK - 1e-6, r: 0 })).toBe(false);+ expect(stepClipLatch(false, undefined)).toBe(false);+ });++ it("holds once lit, whatever comes next", () => {+ expect(stepClipLatch(true, { l: 0, r: 0 })).toBe(true);+ expect(stepClipLatch(true, undefined)).toBe(true);+ });+});diff --git a/packages/studio/src/components/nle/AudioMeterStrip.tsx b/packages/studio/src/components/nle/AudioMeterStrip.tsxindex dc23f46e37..5c24c56afa 100644--- a/packages/studio/src/components/nle/AudioMeterStrip.tsx+++ b/packages/studio/src/components/nle/AudioMeterStrip.tsx@@ -5,6 +5,7 @@ import {useMemo,useRef,type PointerEvent as ReactPointerEvent,+ type ReactNode,type Ref,type RefObject,} from "react";@@ -20,6 +21,7 @@ import type { StereoLevel } from "@hyperframes/core/runtime/levelTap";import { usePlayerStore } from "../../player";import { clampNumber } from "../../utils/studioHelpers";import { useAudioMetersVisible } from "../../utils/audioMeterVisibility";+import { EXPORT_PEAK_CEILING } from "../../utils/exportPeakCeiling";import { useStudioShellContextOptional } from "../../contexts/StudioContext";import { useTimelineEditContextOptional } from "../../contexts/TimelineEditContext";import {@@ -49,6 +51,24 @@ type StripBars = [Bars, Bars];const MONITOR_LABEL = "Monitor";+/** The CLIP light latches at the export ceiling, stricter than RED_AT. */+export const CLIP_LATCH_PEAK = EXPORT_PEAK_CEILING;++export function stepClipLatch(latched: boolean, master: StereoLevel | undefined): boolean {+ if (latched || !master) return latched;+ return Math.max(master.l, master.r) >= CLIP_LATCH_PEAK;+}++function paintClipLight(el: HTMLElement | null, lit: boolean): void {+ if (!el) return;+ el.dataset.lit = String(lit);+ el.style.setProperty(+ "background-color",+ lit ? "var(--color-red-500)" : "var(--color-neutral-800)",+ );+ el.style.setProperty("color", lit ? "var(--color-white)" : "var(--color-neutral-500)");+}+/** Where the fill turns amber, then red, on the same piecewise dB scale the marks use. */const AMBER_AT = markFraction(-6);const RED_AT = markFraction(-3);@@ -178,10 +198,16 @@ export function stepAndPaintStrips(}/** One rAF loop re-reads the hook off the live preview window, so a reloaded iframe is followed. */+interface ClipLatch {+ lit: boolean;+ light: HTMLElement | null;+}+function useMeterLoop(strips: Strip[],bars: RefObject<Map<string | null, StripBars>>,iframeRef: RefObject<HTMLIFrameElement | null> | undefined,+ clip: RefObject<ClipLatch>,) {const shell = useStudioShellContextOptional();const previewIframeRef = iframeRef ?? shell?.previewIframeRef;@@ -200,6 +226,11 @@ function useMeterLoop(last = now;evictGoneMeterState(state, new Set(stripsRef.current.map((s) => s.id)));stepAndPaintStrips(stripsRef.current, state, bars.current, levels, now, dt);+ const lit = stepClipLatch(clip.current.lit, levels?.master);+ if (lit !== clip.current.lit) {+ clip.current.lit = lit;+ paintClipLight(clip.current.light, lit);+ }};raf = requestAnimationFrame(tick);return () => {@@ -210,7 +241,29 @@ function useMeterLoop(// Preview iframe already gone.}};- }, [previewIframeRef, bars]);+ }, [previewIframeRef, bars, clip]);+}++function ClipLight({ clip }: { clip: RefObject<ClipLatch> }) {+ return (+ <button+ type="button"+ data-testid="meter-clip-light"+ data-lit="false"+ title="Lights when the preview reaches −1 dBFS, the export ceiling. Click to reset."+ ref={(el) => {+ clip.current.light = el;+ paintClipLight(el, clip.current.lit);+ }}+ onClick={() => {+ clip.current.lit = false;+ paintClipLight(clip.current.light, false);+ }}+ className="rounded-[2px] px-1 font-mono text-[9px] leading-[14px] font-bold tracking-wide"+ >+ CLIP+ </button>+ );}function Bar({ maskRef, peakRef }: { maskRef: Ref<HTMLDivElement>; peakRef: Ref<HTMLDivElement> }) {@@ -341,11 +394,13 @@ function MeterStrip({register,onLive,onCommit,+ clipLight,}: {strip: Strip;register: (id: string | null, bars: StripBars | null) => void;onLive: (id: string | null, volume: number) => void;onCommit: (id: string | null, volume: number) => void;+ clipLight?: ReactNode;}) {const refs = [useRef<HTMLDivElement>(null),@@ -364,6 +419,7 @@ function MeterStrip({}, [strip.id, register]);return (<div className="flex w-[104px] shrink-0 flex-col items-center gap-1 px-1.5 pt-2 pb-1">+ {clipLight}<divclassName="flex min-h-0 flex-1 items-stretch gap-1.5"aria-label={`${strip.label} level`}@@ -421,7 +477,8 @@ function MeterStripBody({ previewIframeRef }: AudioMeterStripProps) {else bars.current.delete(id);}).current;const { onLive, onCommit } = useVolumeHandlers();- useMeterLoop(strips, bars, previewIframeRef);+ const clip = useRef<ClipLatch>({ lit: false, light: null });+ useMeterLoop(strips, bars, previewIframeRef, clip);return (<divdata-testid="audio-meter-strip"@@ -434,6 +491,7 @@ function MeterStripBody({ previewIframeRef }: AudioMeterStripProps) {register={register}onLive={onLive}onCommit={onCommit}+ clipLight={strip.id === null ? <ClipLight clip={clip} /> : undefined}/>))}</div>diff --git a/packages/studio/src/hooks/useRenderClipContent.test.ts b/packages/studio/src/hooks/useRenderClipContent.test.tsindex 87a3305305..f03b43c52c 100644--- a/packages/studio/src/hooks/useRenderClipContent.test.ts+++ b/packages/studio/src/hooks/useRenderClipContent.test.ts@@ -5,6 +5,14 @@ import { createRoot } from "react-dom/client";import { afterEach, describe, expect, it } from "vitest";import { CompositionThumbnail, VideoThumbnail } from "../player";import { AudioWaveform } from "../player/components/AudioWaveform";+import { AudibleVideoClipContent } from "../player/components/AudibleVideoClipContent";+import { ClipPeakMarks } from "../player/components/ClipPeakMarks";++function unwrapPeakMarks(node: ReactNode): ReactNode {+ return isValidElement<{ children?: ReactNode }>(node) && node.type === ClipPeakMarks+ ? node.props.children+ : node;+}import type { TimelineClipRenderContext } from "../player/components/TimelineTypes";import { usePlayerStore, type TimelineElement } from "../player/store/playerStore";import { buildCompositionThumbnailUrl } from "../player/components/CompositionThumbnail";@@ -93,9 +101,51 @@ describe("useRenderClipContent", () => {root.render(React.createElement(Harness));});act(() => root.unmount());+ return unwrapPeakMarks(content);+ }++ function renderRaw(el: TimelineElement): ReactNode {+ const host = document.createElement("div");+ document.body.append(host);+ const root = createRoot(host);+ let content: ReactNode = null;+ function Harness() {+ const render = useRenderClipContent({+ projectIdRef: { current: "my-project" },+ compIdToSrc: new Map(),+ activePreviewUrl: null,+ effectiveTimelineDuration: 12,+ });+ content = render(el, { clip: "#222", label: "#fff" });+ return null;+ }+ act(() => root.render(React.createElement(Harness)));+ act(() => root.unmount());return content;}+ it("wraps a clip's waveform in peak marks at its own gain over its played source window", () => {+ const content = renderRaw({+ id: "voiceover",+ tag: "audio",+ start: 1,+ duration: 4,+ track: 1,+ src: "assets/voiceover.mp3",+ playbackStart: 2,+ playbackRate: 1.5,+ volume: 2,+ });+ expect(isValidElement(content) && content.type).toBe(ClipPeakMarks);+ if (isValidElement<Record<string, unknown>>(content)) {+ expect(content.props).toMatchObject({+ peaksUrl: "/api/projects/my-project/peaks/assets/voiceover.mp3",+ sourceWindow: { mediaStart: 2, sourceSpan: 6 },+ gain: 2,+ });+ }+ });+it("renders audio clips as waveforms even when a composition preview URL is active", () => {const content = renderClipContent({id: "voiceover",@@ -176,6 +226,58 @@ describe("useRenderClipContent", () => {}});+ it("draws a waveform strip under the thumbnails of a video with sound", () => {+ usePlayerStore.setState({ thumbnailMode: "adaptive" });+ const content = renderClipContent(+ {+ id: "talk",+ tag: "video",+ start: 0,+ duration: 4,+ track: 0,+ src: "talk.mp4",+ hasAudio: true,+ },+ null,+ );+ expect(isValidElement<{ thumbnail: ReactNode; waveform: ReactNode }>(content)).toBe(true);+ if (!isValidElement<{ thumbnail: ReactNode; waveform: ReactNode }>(content)) return;+ expect(content.type).toBe(AudibleVideoClipContent);+ const { thumbnail } = content.props;+ const waveform = unwrapPeakMarks(content.props.waveform);+ expect(isValidElement(thumbnail) && thumbnail.type).toBe(VideoThumbnail);+ expect(isValidElement<{ waveformUrl: string }>(waveform) && waveform.props.waveformUrl).toBe(+ "/api/projects/my-project/waveform/talk.mp4",+ );+ });++ it("keeps a muted or silent video to thumbnails only", () => {+ usePlayerStore.setState({ thumbnailMode: "adaptive" });+ for (const extra of [{ hasAudio: true, muted: true }, { hasAudio: false }]) {+ const content = renderClipContent(+ { id: "b", tag: "video", start: 0, duration: 4, track: 0, src: "b.mp4", ...extra },+ null,+ );+ expect(isValidElement(content) && content.type).toBe(VideoThumbnail);+ }+ });++ it("still shows the sound strip when thumbnails are off", () => {+ const content = renderClipContent(+ {+ id: "talk",+ tag: "video",+ start: 0,+ duration: 4,+ track: 0,+ src: "talk.mp4",+ hasAudio: true,+ },+ null,+ );+ expect(isValidElement(content) && content.type).toBe(AudibleVideoClipContent);+ });+it("passes empty labels to thumbnail content so TimelineClip owns clip names", () => {usePlayerStore.setState({ thumbnailMode: "adaptive" });diff --git a/packages/studio/src/hooks/useRenderClipContent.ts b/packages/studio/src/hooks/useRenderClipContent.tsindex f0b4eac451..a929e6ac3a 100644--- a/packages/studio/src/hooks/useRenderClipContent.ts+++ b/packages/studio/src/hooks/useRenderClipContent.ts@@ -12,6 +12,9 @@ import type { TimelineClipRenderContext } from "../player/components/TimelineTypimport { audioPillFlags } from "../player/components/audioClipLink";import { AudioWaveform, rendersWaveform } from "../player/components/AudioWaveform";import { ImageThumbnail } from "../player/components/ImageThumbnail";+import { AudibleVideoClipContent } from "../player/components/AudibleVideoClipContent";+import { ClipPeakMarks } from "../player/components/ClipPeakMarks";+import { clipHasSound } from "../player/components/clipMenuNormalize";import { encodePreviewPath, resolveMediaPreviewUrl } from "../player/components/thumbnailUtils";import { usePlayerStore } from "../player/store/playerStore";import { thumbnailRevisionOf } from "../player/store/thumbnailSlice";@@ -71,7 +74,7 @@ function renderAudioClip(? buildProjectApiPath(pid, `/waveform/${encodedRelative}`): undefined;const { start, end } = trimFractions(el);- return createElement(AudioWaveform, {+ const waveform = createElement(AudioWaveform, {audioUrl,waveformUrl,label: "",@@ -83,6 +86,27 @@ function renderAudioClip(priority: context.priority,...audioPillFlags(el, elements),});+ return createElement(+ ClipPeakMarks,+ {+ peaksUrl: encodedRelative ? buildProjectApiPath(pid, `/peaks/${encodedRelative}`) : undefined,+ sourceWindow: {+ mediaStart: el.playbackStart ?? 0,+ sourceSpan: el.duration * (el.playbackRate ?? 1),+ },+ gain: el.volume ?? 1,+ },+ waveform,+ );+}++function withSoundStrip(+ el: TimelineElement,+ thumbnail: ReactNode,+ waveform: () => ReactNode,+): ReactNode {+ if (el.tag !== "video" || !clipHasSound(el)) return thumbnail;+ return createElement(AudibleVideoClipContent, { thumbnail, waveform: waveform() });}export interface UseRenderClipContentOptions {@@ -117,10 +141,9 @@ export function useRenderClipContent({// Thumbnail generation disabled (perf) -> plain clip bars. Audio still shows// its waveform (cheap, not a frame thumbnail). Toggle: timeline toolbar.+ const waveform = () => renderAudioClip(el, pid, sessionEpoch, style.label, context, elements);if (effectiveMode === "hidden") {- return rendersWaveform(el)- ? renderAudioClip(el, pid, sessionEpoch, style.label, context, elements)- : null;+ return rendersWaveform(el) ? waveform() : withSoundStrip(el, null, waveform);}let compSrc = el.compositionSrc;@@ -205,7 +228,7 @@ export function useRenderClipContent({rich: context.rich,});}- return createElement(VideoThumbnail, {+ const thumbnail = createElement(VideoThumbnail, {videoSrc: mediaSrc,label: "",labelColor: style.label,@@ -216,6 +239,7 @@ export function useRenderClipContent({sessionEpoch,priority: context.priority,});+ return withSoundStrip(el, thumbnail, waveform);}if (htmlPreviewEligible) {diff --git a/packages/studio/src/player/components/AudibleVideoClipContent.tsx b/packages/studio/src/player/components/AudibleVideoClipContent.tsxnew file mode 100644index 0000000000..cbfcc59aa0--- /dev/null+++ b/packages/studio/src/player/components/AudibleVideoClipContent.tsx@@ -0,0 +1,29 @@+import type { ReactNode } from "react";++/** Share of the clip height the sound strip takes under a video's thumbnails. */+const AUDIBLE_VIDEO_WAVE_SHARE = 0.38;++/** A video that carries sound: thumbnails on top, its waveform strip along the bottom. */+export function AudibleVideoClipContent({+ thumbnail,+ waveform,+}: {+ thumbnail: ReactNode;+ waveform: ReactNode;+}) {+ const waveHeight = `${AUDIBLE_VIDEO_WAVE_SHARE * 100}%`;+ return (+ <div className="relative h-full w-full" data-testid="audible-video-clip">+ <div className="absolute inset-x-0 top-0" style={{ bottom: waveHeight }}>+ {thumbnail}+ </div>+ <div+ className="absolute inset-x-0 bottom-0 bg-black/30"+ data-testid="audible-video-wave"+ style={{ height: waveHeight }}+ >+ {waveform}+ </div>+ </div>+ );+}diff --git a/packages/studio/src/player/components/ClipPeakMarks.test.tsx b/packages/studio/src/player/components/ClipPeakMarks.test.tsxnew file mode 100644index 0000000000..9c16f3b012--- /dev/null+++ b/packages/studio/src/player/components/ClipPeakMarks.test.tsx@@ -0,0 +1,41 @@+// @vitest-environment happy-dom+import { act } from "react";+import { describe, expect, it, vi } from "vitest";+import { createHappyDomRootHarness } from "./testRootHarness";+import { ClipPeakMarks } from "./ClipPeakMarks";++const harness = createHappyDomRootHarness();++async function render(url: string, bins: number[], gain: number) {+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => Response.json({ binSeconds: 1, bins })),+ );+ const host = document.createElement("div");+ document.body.appendChild(host);+ await act(async () => {+ harness.mount(host).render(+ <ClipPeakMarks peaksUrl={url} sourceWindow={{ mediaStart: 0, sourceSpan: 2 }} gain={gain}>+ <span>wave</span>+ </ClipPeakMarks>,+ );+ });+ await act(async () => {});+ vi.unstubAllGlobals();+ return host;+}++describe("ClipPeakMarks", () => {+ it("paints red marks and the peak on a clip that redlines", async () => {+ const host = await render("/api/projects/p/peaks/loud.mp4", [0.2, 0.98], 1);+ expect(host.textContent).toContain("wave");+ expect(host.querySelector("[data-testid=clip-peak-marks]")?.textContent).toContain(+ "▲ peaks −0.2 dBFS",+ );+ });++ it("paints nothing on a quiet clip", async () => {+ const host = await render("/api/projects/p/peaks/quiet.mp4", [0.25, 0.25], 1);+ expect(host.querySelector("[data-testid=clip-peak-marks]")).toBeNull();+ });+});diff --git a/packages/studio/src/player/components/ClipPeakMarks.tsx b/packages/studio/src/player/components/ClipPeakMarks.tsxnew file mode 100644index 0000000000..e8af8cc59c--- /dev/null+++ b/packages/studio/src/player/components/ClipPeakMarks.tsx@@ -0,0 +1,71 @@+import { useEffect, useState, type ReactNode } from "react";+import { clipPeakRuns, isPeakMap, type ClipSourceWindow, type PeakMap } from "./clipPeakRuns";++const peakMapRequests = new Map<string, Promise<PeakMap | null>>();++function loadPeakMap(url: string): Promise<PeakMap | null> {+ const pending = peakMapRequests.get(url);+ if (pending) return pending;+ const request = fetch(url)+ .then((res) => (res.ok ? res.json() : null))+ .then((body: unknown) => (isPeakMap(body) ? body : null))+ .catch(() => null);+ peakMapRequests.set(url, request);+ return request;+}++const dbText = (db: number) => `${db < 0 ? "−" : "+"}${Math.abs(db).toFixed(1)} dBFS`;++/** A clip's waveform with red marks where it, at its own volume, reaches the export ceiling. */+export function ClipPeakMarks({+ peaksUrl,+ sourceWindow,+ gain,+ children,+}: {+ peaksUrl: string | undefined;+ sourceWindow: ClipSourceWindow;+ gain: number;+ children?: ReactNode;+}) {+ const [map, setMap] = useState<PeakMap | null>(null);+ useEffect(() => {+ if (!peaksUrl) return;+ let live = true;+ void loadPeakMap(peaksUrl).then((loaded) => {+ if (live) setMap(loaded);+ });+ return () => {+ live = false;+ };+ }, [peaksUrl]);+ const { runs, peakDbfs } = map+ ? clipPeakRuns(map, sourceWindow, gain)+ : { runs: [], peakDbfs: null };+ return (+ <div className="relative h-full w-full">+ {children}+ {runs.length > 0 && peakDbfs !== null && (+ <div+ className="pointer-events-none absolute inset-0"+ data-testid="clip-peak-marks"+ title={`Peaks ${dbText(peakDbfs)} at this volume; export lowers the whole mix`}+ >+ {runs.map((run) => (+ <div+ key={run.from}+ className="absolute inset-y-0 bg-red-500/80"+ style={{+ left: `${run.from * 100}%`,+ width: `max(1px, ${(run.to - run.from) * 100}%)`,+ }}+ />+ ))}+ <span className="absolute top-0 right-1 font-mono text-[9px] leading-none text-red-400">+ ▲ peaks {dbText(peakDbfs)}+ </span>+ </div>+ )}+ </div>+ );+}diff --git a/packages/studio/src/player/components/clipMenuDuck.test.ts b/packages/studio/src/player/components/clipMenuDuck.test.tsindex 0431eb439a..3833e2d83f 100644--- a/packages/studio/src/player/components/clipMenuDuck.test.ts+++ b/packages/studio/src/player/components/clipMenuDuck.test.ts@@ -28,6 +28,13 @@ describe("isDuckableBed", () => {expect(isDuckableBed(doc.getElementById("music"))).toBe(true);expect(isDuckableBed(doc.getElementById("voiceover"))).toBe(false);});++ it("refuses a clip another bed is already carving against", () => {+ const doc = compose(+ `<audio id="music" src="music.mp3" data-fx-carve='{"enabled":true,"sources":["sting"],"strength":0.25}'></audio><audio id="sting" src="sting-bed.mp3"></audio>`,+ );+ expect(isDuckableBed(doc.getElementById("sting"))).toBe(false);+ });});describe("setDuckUnderVoice", () => {diff --git a/packages/studio/src/player/components/clipMenuDuck.ts b/packages/studio/src/player/components/clipMenuDuck.tsindex 168b060643..6b8b258ed0 100644--- a/packages/studio/src/player/components/clipMenuDuck.ts+++ b/packages/studio/src/player/components/clipMenuDuck.ts@@ -15,6 +15,7 @@ import { resolveCarveSourceIds } from "@hyperframes/core/audio-groups";import { parseAutomation, type HfAutomation } from "@hyperframes/core/audio-automation";import {carveBedRoles,+ carverAgainst,collectCarveCandidates,CARVE_ABORTED,isPromiseLike,@@ -65,7 +66,8 @@ function readAutomation(bed: Element): HfAutomation {}export function isDuckableBed(bed: Element | null): bed is Element {- return bed !== null && carveBedRoles(bed.id, bed).couldBeBed;+ if (bed === null || !carveBedRoles(bed.id, bed).couldBeBed) return false;+ return carverAgainst(bed.ownerDocument, bed.id) === null;}/** Every overlapping clip with speech, including a video's own sound; never the bed itself. */diff --git a/packages/studio/src/player/components/clipPeakRuns.test.ts b/packages/studio/src/player/components/clipPeakRuns.test.tsnew file mode 100644index 0000000000..5aed064218--- /dev/null+++ b/packages/studio/src/player/components/clipPeakRuns.test.ts@@ -0,0 +1,38 @@+import { describe, expect, it } from "vitest";+import { clipPeakRuns } from "./clipPeakRuns";++const map = (bins: number[]) => ({ binSeconds: 1, bins });++describe("clipPeakRuns", () => {+ it("marks bins at or above -1 dBFS after the clip's gain and names the loudest", () => {+ const { runs, peakDbfs } = clipPeakRuns(+ map([0.1, 0.95, 0.99, 0.2, 0.9]),+ { mediaStart: 0, sourceSpan: 5 },+ 1,+ );+ expect(runs).toEqual([+ { from: 0.2, to: 0.6 },+ { from: 0.8, to: 1 },+ ]);+ expect(peakDbfs).toBeCloseTo(20 * Math.log10(0.99), 6);+ });++ it("puts nothing on a -12 dBFS clip, and marks it once boosted by +12 dB", () => {+ const quiet = map([0.25, 0.25]);+ expect(clipPeakRuns(quiet, { mediaStart: 0, sourceSpan: 2 }, 1)).toEqual({+ runs: [],+ peakDbfs: null,+ });+ expect(clipPeakRuns(quiet, { mediaStart: 0, sourceSpan: 2 }, 4).runs).toHaveLength(1);+ });++ it("only reads the trimmed window, positioned within the clip", () => {+ const { runs } = clipPeakRuns(map([1, 0, 0, 1, 0]), { mediaStart: 2, sourceSpan: 2 }, 1);+ expect(runs).toEqual([{ from: 0.5, to: 1 }]);+ });++ it("is empty for a muted clip or an empty window", () => {+ expect(clipPeakRuns(map([1]), { mediaStart: 0, sourceSpan: 1 }, 0).runs).toEqual([]);+ expect(clipPeakRuns(map([1]), { mediaStart: 0, sourceSpan: 0 }, 1).runs).toEqual([]);+ });+});diff --git a/packages/studio/src/player/components/clipPeakRuns.ts b/packages/studio/src/player/components/clipPeakRuns.tsnew file mode 100644index 0000000000..5478d231fc--- /dev/null+++ b/packages/studio/src/player/components/clipPeakRuns.ts@@ -0,0 +1,59 @@+import { EXPORT_PEAK_CEILING } from "../../utils/exportPeakCeiling";++export interface PeakMap {+ binSeconds: number;+ bins: number[];+}++export interface ClipSourceWindow {+ mediaStart: number;+ sourceSpan: number;+}++export interface ClipPeakMarkRuns {+ runs: Array<{ from: number; to: number }>;+ peakDbfs: number | null;+}++/**+ * Where, as fractions of the clip, its source at the clip's own gain reaches the+ * export ceiling, and the loudest point when it does. Adjacent hot bins merge.+ */+export function clipPeakRuns(+ map: PeakMap,+ window: ClipSourceWindow,+ gain: number,+): ClipPeakMarkRuns {+ const runs: ClipPeakMarkRuns["runs"] = [];+ if (!(window.sourceSpan > 0) || !(map.binSeconds > 0) || !(gain > 0))+ return { runs, peakDbfs: null };+ const first = Math.max(0, Math.floor(window.mediaStart / map.binSeconds));+ const last = Math.min(+ map.bins.length,+ Math.ceil((window.mediaStart + window.sourceSpan) / map.binSeconds),+ );+ const toFraction = (bin: number) =>+ Math.min(1, Math.max(0, (bin * map.binSeconds - window.mediaStart) / window.sourceSpan));+ let loudest = 0;+ for (let bin = first; bin < last; bin++) {+ const level = (map.bins[bin] ?? 0) * gain;+ if (level < EXPORT_PEAK_CEILING) continue;+ loudest = Math.max(loudest, level);+ const previous = runs.at(-1);+ const from = toFraction(bin);+ const to = toFraction(bin + 1);+ if (previous && previous.to >= from) previous.to = to;+ else runs.push({ from, to });+ }+ return { runs, peakDbfs: runs.length > 0 ? 20 * Math.log10(loudest) : null };+}++export function isPeakMap(value: unknown): value is PeakMap {+ if (typeof value !== "object" || value === null) return false;+ const bins: unknown = Reflect.get(value, "bins");+ return (+ typeof Reflect.get(value, "binSeconds") === "number" &&+ Array.isArray(bins) &&+ bins.every((bin) => typeof bin === "number")+ );+}diff --git a/packages/studio/src/utils/exportPeakCeiling.ts b/packages/studio/src/utils/exportPeakCeiling.tsnew file mode 100644index 0000000000..9cbbf97f36--- /dev/null+++ b/packages/studio/src/utils/exportPeakCeiling.ts@@ -0,0 +1 @@+export const EXPORT_PEAK_CEILING = 10 ** (-1 / 20);diff --git a/packages/studio/src/utils/timelineInspector.test.ts b/packages/studio/src/utils/timelineInspector.test.tsindex fb5cef0153..27263980a8 100644--- a/packages/studio/src/utils/timelineInspector.test.ts+++ b/packages/studio/src/utils/timelineInspector.test.ts@@ -4,6 +4,7 @@ import {canHideSelections,isAudioDomElement,isAudioTimelineElement,+ isMusicSourceElement,isMusicTrack,resolveBeatSourceTrack,} from "./timelineInspector";@@ -12,9 +13,15 @@ import type { TimelineElement } from "../player";// Minimal element factory for testsfunction el(overrides: Partial<- Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "duration">+ Pick<+ TimelineElement,+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"+ >>,-): Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "duration"> {+): Pick<+ TimelineElement,+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"+> {return {tag: "audio",src: "assets/track.mp3",@@ -153,3 +160,27 @@ describe("isAudioDomElement / canHideSelections", () => {expect(canHideSelections([{ element: el("hf-audio-group") }])).toBe(false);});});++describe("beat source from a video with sound", () => {+ const talk = { tag: "video", src: "assets/clip.mp4", hasAudio: true };++ it("counts an audible video as a music source but never a muted or silent one", () => {+ expect(isMusicSourceElement(el(talk))).toBe(true);+ expect(isMusicSourceElement(el({ ...talk, muted: true }))).toBe(false);+ expect(isMusicSourceElement(el({ ...talk, hasAudio: false }))).toBe(false);+ expect(isAudioTimelineElement(el(talk))).toBe(false);+ });++ it("takes a video tagged as music as the beat source", () => {+ const video = el({ ...talk, timelineRole: "music" });+ expect(isMusicTrack(video)).toBe(true);+ expect(resolveBeatSourceTrack([video])).toEqual({ element: video, isFallback: false });+ });++ it("falls back to the longest audible video when there is no audio", () => {+ const short = el({ ...talk, id: "a", duration: 3 });+ const long = el({ ...talk, id: "b", duration: 9 });+ const muted = el({ ...talk, id: "c", duration: 20, muted: true });+ expect(resolveBeatSourceTrack([short, long, muted])?.element).toBe(long);+ });+});diff --git a/packages/studio/src/utils/timelineInspector.ts b/packages/studio/src/utils/timelineInspector.tsindex 44faaeb8db..7bae863084 100644--- a/packages/studio/src/utils/timelineInspector.ts+++ b/packages/studio/src/utils/timelineInspector.ts@@ -44,17 +44,29 @@ export function isAudioOrVideoTimelineElement(return isAudioTimelineElement(element) || element.tag.trim().toLowerCase() === "video";}+type MusicSourceFacts = Pick<TimelineElement, "tag" | "src" | "hasAudio" | "muted">;++/** Can carry the music: an audio clip, or a video whose own sound plays. Lane zoning is unaffected. */+export function isMusicSourceElement(element: MusicSourceFacts): boolean {+ if (isAudioTimelineElement(element)) return true;+ return (+ element.tag.trim().toLowerCase() === "video" &&+ element.hasAudio === true &&+ element.muted !== true+ );+}+/** True for the music track: an audio element with data-timeline-role="music",* or — when no role is set — an id matching the music regex. Voiceover/other* audio (explicit non-music role) is excluded. */export function isMusicTrack(element:- | Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole">+ | Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "hasAudio" | "muted">| null| undefined,): boolean {if (!element) return false;- if (!isAudioTimelineElement(element)) return false;+ if (!isMusicSourceElement(element)) return false;if (element.timelineRole === "music") return true;if (element.timelineRole && element.timelineRole !== "music") return false;const id = element.domId ?? element.id ?? "";@@ -75,7 +87,7 @@ export function isMusicTrack(export function resolveBeatSourceTrack(elements: readonly Pick<TimelineElement,- "tag" | "src" | "id" | "domId" | "timelineRole" | "duration"+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted">[],): { element: (typeof elements)[number]; isFallback: boolean } | null {const explicit = elements.find(isMusicTrack);@@ -85,7 +97,7 @@ export function resolveBeatSourceTrack(// like "sfx" or "voiceover" to avoid triggering beat analysis on those).let best: (typeof elements)[number] | null = null;for (const el of elements) {- if (!isAudioTimelineElement(el)) continue;+ if (!isMusicSourceElement(el)) continue;if (el.timelineRole && el.timelineRole !== "music") continue;if (!best || el.duration > best.duration) best = el;}