Skip to content

Write HTML. Render video. Built for agents.

feat(studio): waveform strip, clip light, peak marks and video beat source (5/8) (#4817)

Vance Ingalls824e4100c636
diff --git a/packages/cli/src/commands/beats.ts b/packages/cli/src/commands/beats.ts
index 536811479f..9ae8ff1d65 100644
--- a/packages/cli/src/commands/beats.ts
+++ b/packages/cli/src/commands/beats.ts
@@ -23,7 +23,8 @@ function resolveMusicTarget(project: ProjectDir): { rel: string; audioPath: stri
const src = findMusicAudioSrc(readFileSync(project.indexPath, "utf-8"));
if (!src) {
fail(
- 'No music track found. Add data-timeline-role="music" to the <audio> element ' +
+ 'No music track found. Add data-timeline-role="music" to the <audio> element, or to a ' +
+ '<video data-has-audio="true"> whose own sound is the music ' +
"(or give it an id like music/bgm/soundtrack).",
);
}
diff --git a/packages/core/src/beats/beatFile.test.ts b/packages/core/src/beats/beatFile.test.ts
new file mode 100644
index 0000000000..38a79991bc
--- /dev/null
+++ b/packages/core/src/beats/beatFile.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import { findMusicAudioSrc } from "./beatFile";
+
+describe("findMusicAudioSrc", () => {
+ it("finds a video with sound tagged as music", () => {
+ const html = `<video id="clip" src="band.mp4" data-has-audio="true" data-timeline-role="music"></video>`;
+ expect(findMusicAudioSrc(html)).toBe("band.mp4");
+ });
+
+ it("finds an audible video by a music id", () => {
+ expect(findMusicAudioSrc(`<video id="soundtrack" src="s.mp4" data-has-audio="true">`)).toBe(
+ "s.mp4",
+ );
+ });
+
+ it("skips a muted video and one without declared sound", () => {
+ const html = [
+ `<video id="music" src="muted.mp4" data-has-audio="true" muted>`,
+ `<video id="bgm" src="undeclared.mp4">`,
+ `<audio id="music-bed" src="bed.mp3">`,
+ ].join("");
+ expect(findMusicAudioSrc(html)).toBe("bed.mp3");
+ });
+
+ it("keeps document order across audio and video", () => {
+ const html = `<audio id="music" src="first.mp3"></audio><video id="bgm" src="v.mp4" data-has-audio="true">`;
+ expect(findMusicAudioSrc(html)).toBe("first.mp3");
+ });
+});
diff --git a/packages/core/src/beats/beatFile.ts b/packages/core/src/beats/beatFile.ts
index 31f2e0b71b..3235fffb58 100644
--- a/packages/core/src/beats/beatFile.ts
+++ b/packages/core/src/beats/beatFile.ts
@@ -87,18 +87,25 @@ function attr(tag: string, name: string): string | null {
return m ? m[1]! : null;
}
+function isSoundingVideoTag(tag: string): boolean {
+ const unquoted = tag.replace(/"[^"]*"|'[^']*'/g, '""');
+ return attr(tag, "data-has-audio") === "true" && !/\smuted(?=[\s/>=])/i.test(unquoted);
+}
+
/**
* Find the music track's src in composition HTML, applying the SAME rules as the
- * Studio's `isMusicTrack` so the CLI and Studio agree on which `<audio>` is music:
- * the FIRST `<audio>` (in document order) where data-timeline-role="music", or —
+ * Studio's `isMusicTrack` so the CLI and Studio agree on which clip is music:
+ * the FIRST `<audio>`, or `<video data-has-audio="true">` without `muted` (in
+ * document order) where data-timeline-role="music", or —
* when no role is set — whose id matches the music regex. An explicit non-music
* role excludes the element. Returns the raw src attribute, or null.
*/
export function findMusicAudioSrc(html: string): string | null {
// `[^>]*` spans newlines (it's a negated class, not `.`), so multi-line opening
// tags are handled. HyperFrames authors src as an attribute on <audio>.
- const tags = html.match(/<audio\b[^>]*>/gi) ?? [];
+ const tags = html.match(/<(?:audio|video)\b[^>]*>/gi) ?? [];
for (const tag of tags) {
+ if (/^<video/i.test(tag) && !isSoundingVideoTag(tag)) continue;
const src = attr(tag, "src");
if (!src) continue;
const role = attr(tag, "data-timeline-role");
diff --git a/packages/studio-server/src/createStudioApi.ts b/packages/studio-server/src/createStudioApi.ts
index d39576c446..3c1e3cde3e 100644
--- a/packages/studio-server/src/createStudioApi.ts
+++ b/packages/studio-server/src/createStudioApi.ts
@@ -10,6 +10,7 @@ import { registerThumbnailRoutes } from "./routes/thumbnail.js";
import { registerWaveformRoutes } from "./routes/waveform.js";
import { registerFreezeFrameRoutes } from "./routes/freezeFrame.js";
import { registerLoudnessRoutes } from "./routes/loudness.js";
+import { registerPeakRoutes } from "./routes/peaks.js";
import { registerFontRoutes } from "./routes/fonts.js";
import { registerRegistryRoutes } from "./routes/registry.js";
import { registerSelectionRoutes } from "./routes/selection.js";
@@ -65,6 +66,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
registerWaveformRoutes(api, adapter);
registerFreezeFrameRoutes(api, adapter);
registerLoudnessRoutes(api, adapter);
+ registerPeakRoutes(api, adapter);
registerFontRoutes(api);
registerRegistryRoutes(api, adapter);
registerGlobalAssetRoutes(api);
diff --git a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts
new file mode 100644
index 0000000000..a2f24c8c40
--- /dev/null
+++ b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts
@@ -0,0 +1,29 @@
+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it, vi } from "vitest";
+
+// Stand-in binaries: ffprobe reports a stereo stream, ffmpeg writes samples and then fails.
+const fakeBin = vi.hoisted(() => ({ dir: "" }));
+vi.mock("@hyperframes/parsers/ff-binaries", () => ({
+ findFfBinary: (name: string) => join(fakeBin.dir, name),
+}));
+
+import { decodePeakMap } from "./peakMap";
+
+describe("decodePeakMap, decode failure", () => {
+ it.skipIf(process.platform === "win32")(
+ "rejects partial output when ffmpeg exits nonzero",
+ async () => {
+ fakeBin.dir = mkdtempSync(join(tmpdir(), "hf-fake-ff-"));
+ const script = (name: string, body: string) => {
+ const file = join(fakeBin.dir, name);
+ writeFileSync(file, `#!/bin/sh\n${body}\n`);
+ chmodSync(file, 0o755);
+ };
+ script("ffprobe", "echo 2");
+ script("ffmpeg", "head -c 38400 /dev/zero; exit 1");
+ await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/);
+ },
+ );
+});
diff --git a/packages/studio-server/src/helpers/peakMap.test.ts b/packages/studio-server/src/helpers/peakMap.test.ts
new file mode 100644
index 0000000000..6299574e75
--- /dev/null
+++ b/packages/studio-server/src/helpers/peakMap.test.ts
@@ -0,0 +1,149 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import {
+ accumulatePeaks,
+ buildPeakMapCacheKey,
+ createPeakAccumulator,
+ decodePeakMap,
+ finishPeaks,
+} from "./peakMap.js";
+
+function stereo(frames: Array<[number, number]>): Buffer {
+ const buf = Buffer.alloc(frames.length * 8);
+ frames.forEach(([l, r], i) => {
+ buf.writeFloatLE(l, i * 8);
+ buf.writeFloatLE(r, i * 8 + 4);
+ });
+ return buf;
+}
+
+describe("peak accumulation", () => {
+ it("keeps the absolute max of either channel per bin, un-normalized", () => {
+ const acc = createPeakAccumulator(4, 0.5, 2);
+ accumulatePeaks(
+ acc,
+ stereo([
+ [0.1, -0.3],
+ [0.2, 0],
+ [0, 0.05],
+ [-0.04, 0],
+ ]),
+ );
+ expect(finishPeaks(acc)).toEqual([0.3, 0.05]);
+ });
+
+ it("carries a frame split across chunks", () => {
+ const acc = createPeakAccumulator(2, 1, 2);
+ const whole = stereo([
+ [0.5, 0],
+ [0, -0.9],
+ ]);
+ accumulatePeaks(acc, whole.subarray(0, 5));
+ accumulatePeaks(acc, whole.subarray(5));
+ expect(finishPeaks(acc)).toEqual([0.9]);
+ expect(acc.frame).toBe(2);
+ });
+});
+
+describe("peak accumulation, mono", () => {
+ it("reads one channel per frame", () => {
+ const acc = createPeakAccumulator(2, 1, 1);
+ const buf = Buffer.alloc(8);
+ buf.writeFloatLE(-1, 0);
+ buf.writeFloatLE(0.5, 4);
+ accumulatePeaks(acc, buf);
+ expect(finishPeaks(acc)).toEqual([1]);
+ });
+});
+
+describe("buildPeakMapCacheKey", () => {
+ it("changes when the file is replaced and never collides with a waveform entry", () => {
+ const a = buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 });
+ expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 }));
+ expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 }));
+ expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 }));
+ expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/);
+ });
+});
+
+describe("decodePeakMap", () => {
+ const ffmpeg = findFfBinary("ffmpeg");
+ it.skipIf(!ffmpeg)(
+ "reads a 0 dBFS video track near 1.0 and a -12 dBFS one near 0.25",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
+ try {
+ const make = (name: string, volume: string) => {
+ const file = join(dir, name);
+ execFileSync(ffmpeg ?? "ffmpeg", [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=black:s=32x32:d=1",
+ "-f",
+ "lavfi",
+ "-i",
+ `sine=frequency=440:duration=1,volume=${volume}`,
+ "-shortest",
+ "-c:v",
+ "libx264",
+ "-c:a",
+ "pcm_s16le",
+ file,
+ ]);
+ return file;
+ };
+ const loud = await decodePeakMap(make("loud.mov", "8"));
+ const quiet = await decodePeakMap(make("quiet.mov", "2"));
+ expect(Math.max(...loud)).toBeGreaterThan(0.95);
+ expect(Math.max(...quiet)).toBeLessThan(0.3);
+ expect(loud.length).toBeGreaterThanOrEqual(19);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+
+ it.skipIf(!ffmpeg)(
+ "decodes the probed first audio stream when a louder stereo stream follows it",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
+ try {
+ const file = join(dir, "multi.mkv");
+ execFileSync(ffmpeg ?? "ffmpeg", [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "aevalsrc=0.1*sin(2*PI*440*t):s=48000:d=1",
+ "-f",
+ "lavfi",
+ "-i",
+ "aevalsrc=0.9*sin(2*PI*440*t)|0.9*sin(2*PI*440*t):s=48000:d=1",
+ "-map",
+ "0:a",
+ "-map",
+ "1:a",
+ "-c:a",
+ "pcm_f32le",
+ "-disposition:a",
+ "0",
+ file,
+ ]);
+ const peaks = await decodePeakMap(file);
+ expect(peaks.length).toBeGreaterThanOrEqual(19);
+ expect(peaks.length).toBeLessThanOrEqual(21);
+ expect(Math.max(...peaks)).toBeCloseTo(0.1, 2);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+});
diff --git a/packages/studio-server/src/helpers/peakMap.ts b/packages/studio-server/src/helpers/peakMap.ts
new file mode 100644
index 0000000000..743a94b2c4
--- /dev/null
+++ b/packages/studio-server/src/helpers/peakMap.ts
@@ -0,0 +1,131 @@
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+
+export const PEAK_BIN_SECONDS = 0.05;
+const PEAK_SAMPLE_RATE = 48_000;
+const execFileAsync = promisify(execFile);
+const PEAK_MAP_CACHE_VERSION = "peaks-v2";
+
+/**
+ * Running max-abs per bin over interleaved f32le at the source's own channel
+ * count. Native channels, not `-ac 2`: ffmpeg's mono upmix pans at -3 dB,
+ * which would hide a clip that really does reach 0 dBFS.
+ */
+export interface PeakAccumulator {
+ bins: number[];
+ frame: number;
+ carry: Buffer;
+ framesPerBin: number;
+ channels: number;
+}
+
+export function createPeakAccumulator(
+ sampleRate = PEAK_SAMPLE_RATE,
+ binSeconds = PEAK_BIN_SECONDS,
+ channels = 2,
+): PeakAccumulator {
+ return {
+ bins: [],
+ frame: 0,
+ carry: Buffer.alloc(0),
+ framesPerBin: Math.max(1, Math.round(sampleRate * binSeconds)),
+ channels: Math.max(1, channels),
+ };
+}
+
+function framePeak(data: Buffer, offset: number, channels: number): number {
+ let peak = 0;
+ for (let ch = 0; ch < channels; ch++) {
+ peak = Math.max(peak, Math.abs(data.readFloatLE(offset + ch * 4)));
+ }
+ return peak;
+}
+
+export function accumulatePeaks(acc: PeakAccumulator, chunk: Buffer): void {
+ const data = acc.carry.length > 0 ? Buffer.concat([acc.carry, chunk]) : chunk;
+ const frameBytes = 4 * acc.channels;
+ const whole = data.length - (data.length % frameBytes);
+ for (let offset = 0; offset < whole; offset += frameBytes) {
+ const peak = framePeak(data, offset, acc.channels);
+ const bin = Math.floor(acc.frame / acc.framesPerBin);
+ if (acc.bins.length <= bin) acc.bins.push(0);
+ if (peak > (acc.bins[bin] ?? 0)) acc.bins[bin] = peak;
+ acc.frame += 1;
+ }
+ acc.carry = Buffer.from(data.subarray(whole));
+}
+
+/** Linear sample peak per bin, NOT normalized — the level is the point. */
+export function finishPeaks(acc: PeakAccumulator): number[] {
+ return acc.bins.map((peak) => Number(peak.toFixed(4)));
+}
+
+async function probeChannels(mediaPath: string): Promise<number> {
+ const { stdout } = await execFileAsync(
+ findFfBinary("ffprobe") ?? "ffprobe",
+ [
+ "-v",
+ "error",
+ "-select_streams",
+ "a:0",
+ "-show_entries",
+ "stream=channels",
+ "-of",
+ "csv=p=0",
+ "--",
+ mediaPath,
+ ],
+ { encoding: "utf8", timeout: 30_000, windowsHide: true },
+ );
+ const channels = Number.parseInt(stdout.trim(), 10);
+ if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream");
+ return channels;
+}
+
+export async function decodePeakMap(mediaPath: string): Promise<number[]> {
+ const channels = await probeChannels(mediaPath);
+ return new Promise((resolvePromise, reject) => {
+ const proc = spawn(
+ findFfBinary("ffmpeg") ?? "ffmpeg",
+ [
+ "-v",
+ "error",
+ "-i",
+ mediaPath,
+ "-map",
+ "0:a:0",
+ "-vn",
+ "-ar",
+ String(PEAK_SAMPLE_RATE),
+ "-f",
+ "f32le",
+ "pipe:1",
+ ],
+ { stdio: ["ignore", "pipe", "ignore"], windowsHide: true },
+ );
+ const acc = createPeakAccumulator(PEAK_SAMPLE_RATE, PEAK_BIN_SECONDS, channels);
+ proc.stdout?.on("data", (chunk: Buffer) => accumulatePeaks(acc, chunk));
+ proc.on("close", (code) => {
+ if (code !== 0) {
+ reject(new Error(`ffmpeg failed decoding audio (exit ${code})`));
+ return;
+ }
+ if (acc.frame === 0) {
+ reject(new Error("ffmpeg produced no audio samples"));
+ return;
+ }
+ resolvePromise(finishPeaks(acc));
+ });
+ proc.on("error", reject);
+ });
+}
+
+/** Keyed on content (size + mtime) like the waveform cache, under its own version prefix. */
+export function buildPeakMapCacheKey(
+ assetPath: string,
+ fingerprint: { size: number; mtimeMs: number },
+): string {
+ const name = assetPath.replace(/[/\\]/g, "_");
+ return `${PEAK_MAP_CACHE_VERSION}_${name}_${fingerprint.size}-${Math.round(fingerprint.mtimeMs)}.json`;
+}
diff --git a/packages/studio-server/src/helpers/waveform.test.ts b/packages/studio-server/src/helpers/waveform.test.ts
index 16ba9d1e48..45478fc25c 100644
--- a/packages/studio-server/src/helpers/waveform.test.ts
+++ b/packages/studio-server/src/helpers/waveform.test.ts
@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
-import { buildWaveformCacheKey } from "./waveform.js";
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import { buildWaveformCacheKey, decodeAudioPeaks } from "./waveform.js";
describe("buildWaveformCacheKey", () => {
it("is stable for the same file", () => {
@@ -34,3 +39,36 @@ describe("buildWaveformCacheKey", () => {
expect(buildWaveformCacheKey("a/b.m4a", fp)).toMatch(/\.json$/);
});
});
+
+describe("decodeAudioPeaks on a video file", () => {
+ const ffmpeg = findFfBinary("ffmpeg");
+ it.skipIf(!ffmpeg)("reads the audio track of an .mp4", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-wave-video-"));
+ const file = join(dir, "talk.mp4");
+ try {
+ execFileSync(ffmpeg ?? "ffmpeg", [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=black:s=32x32:d=1",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:duration=1",
+ "-shortest",
+ "-c:v",
+ "libx264",
+ "-c:a",
+ "aac",
+ file,
+ ]);
+ const peaks = await decodeAudioPeaks(file);
+ expect(peaks.length).toBeGreaterThan(0);
+ expect(Math.max(...peaks)).toBeCloseTo(1, 3);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/packages/studio-server/src/routes/peaks.test.ts b/packages/studio-server/src/routes/peaks.test.ts
new file mode 100644
index 0000000000..748a924f3b
--- /dev/null
+++ b/packages/studio-server/src/routes/peaks.test.ts
@@ -0,0 +1,64 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Hono } from "hono";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { registerPeakRoutes } from "./peaks";
+import type { StudioApiAdapter } from "../types";
+
+const dirs: string[] = [];
+afterEach(() => {
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function setup(decode: (path: string) => Promise<number[]>) {
+ const dir = mkdtempSync(join(tmpdir(), "hf-peaks-route-"));
+ dirs.push(dir);
+ writeFileSync(join(dir, "talk.mp4"), "video");
+ const adapter: StudioApiAdapter = {
+ listProjects: () => [],
+ resolveProject: async (id: string) => ({ id, dir }),
+ bundle: async () => null,
+ lint: async () => ({ findings: [] }),
+ runtimeUrl: "/api/runtime.js",
+ rendersDir: () => "/tmp/renders",
+ startRender: () => ({ id: "j", status: "rendering", progress: 0, outputPath: "/tmp/o.mp4" }),
+ };
+ const app = new Hono();
+ registerPeakRoutes(app, adapter, decode);
+ return app;
+}
+
+describe("GET /projects/:id/peaks/*", () => {
+ it("answers absolute bins once per file and serves the cache after", async () => {
+ const decode = vi.fn(async () => [0.2, 0.99]);
+ const app = setup(decode);
+ for (let i = 0; i < 2; i++) {
+ const res = await app.request("http://localhost/projects/p/peaks/talk.mp4");
+ expect(await res.json()).toEqual({ binSeconds: 0.05, bins: [0.2, 0.99] });
+ }
+ expect(decode).toHaveBeenCalledTimes(1);
+ });
+
+ it("decodes a file once when several clips ask at the same time", async () => {
+ let finish: (bins: number[]) => void = () => {};
+ const decode = vi.fn(() => new Promise<number[]>((resolve) => (finish = resolve)));
+ const app = setup(decode);
+ const both = Promise.all([
+ app.request("http://localhost/projects/p/peaks/talk.mp4"),
+ app.request("http://localhost/projects/p/peaks/talk.mp4"),
+ ]);
+ await vi.waitFor(() => expect(decode).toHaveBeenCalled());
+ finish([0.5]);
+ for (const res of await both) expect((await res.json()).bins).toEqual([0.5]);
+ expect(decode).toHaveBeenCalledTimes(1);
+ });
+
+ it("404s a missing file and a path outside the project", async () => {
+ const app = setup(async () => []);
+ expect((await app.request("http://localhost/projects/p/peaks/nope.mp4")).status).toBe(404);
+ expect(
+ (await app.request("http://localhost/projects/p/peaks/..%2F..%2Fetc%2Fpasswd")).status,
+ ).toBe(404);
+ });
+});
diff --git a/packages/studio-server/src/routes/peaks.ts b/packages/studio-server/src/routes/peaks.ts
new file mode 100644
index 0000000000..2697461c05
--- /dev/null
+++ b/packages/studio-server/src/routes/peaks.ts
@@ -0,0 +1,66 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { isWithinProjectRoot } from "@hyperframes/parsers/asset-resolution";
+import type { Hono } from "hono";
+import type { StudioApiAdapter } from "../types.js";
+import { isWaveformCacheDirectory, writeWaveformCache } from "../helpers/waveform.js";
+import { buildPeakMapCacheKey, decodePeakMap, PEAK_BIN_SECONDS } from "../helpers/peakMap.js";
+import { requestSubPath } from "../helpers/requestSubPath.js";
+
+type DecodePeaks = (mediaPath: string) => Promise<number[]>;
+
+const decodesInFlight = new Map<string, Promise<number[]>>();
+
+function decodeOnce(cachePath: string, mediaPath: string, decode: DecodePeaks): Promise<number[]> {
+ const pending = decodesInFlight.get(cachePath);
+ if (pending) return pending;
+ const started = decode(mediaPath).finally(() => decodesInFlight.delete(cachePath));
+ decodesInFlight.set(cachePath, started);
+ return started;
+}
+
+function readCachedBins(cacheDir: string, cachePath: string): number[] | null {
+ try {
+ if (!isWaveformCacheDirectory(cacheDir) || !existsSync(cachePath)) return null;
+ const parsed: unknown = JSON.parse(readFileSync(cachePath, "utf-8"));
+ return Array.isArray(parsed) && parsed.every((n) => typeof n === "number") ? parsed : null;
+ } catch {
+ return null;
+ }
+}
+
+/** Absolute per-bin sample peaks for one media file, so the timeline can mark where a clip redlines. */
+export function registerPeakRoutes(
+ api: Hono,
+ adapter: StudioApiAdapter,
+ decode: DecodePeaks = decodePeakMap,
+): void {
+ api.get("/projects/:id/peaks/*", async (c) => {
+ const project = await adapter.resolveProject(c.req.param("id"));
+ if (!project) return c.json({ error: "not found" }, 404);
+ const assetPath = requestSubPath(c.req.url, "projects/:id/peaks");
+ const mediaPath = resolve(project.dir, assetPath);
+ if (!isWithinProjectRoot(project.dir, mediaPath))
+ return c.json({ error: "file not found" }, 404);
+ const stats = statSync(mediaPath, { throwIfNoEntry: false });
+ if (!stats?.isFile()) return c.json({ error: "file not found" }, 404);
+
+ const cacheDir = join(project.dir, ".waveform-cache");
+ const cachePath = join(cacheDir, buildPeakMapCacheKey(assetPath, stats));
+ const cached = readCachedBins(cacheDir, cachePath);
+ if (cached) return c.json({ binSeconds: PEAK_BIN_SECONDS, bins: cached });
+
+ let bins: number[];
+ try {
+ bins = await decodeOnce(cachePath, mediaPath, decode);
+ } catch {
+ return c.json({ error: "failed to decode audio" }, 500);
+ }
+ try {
+ writeWaveformCache(cachePath, bins);
+ } catch {
+ // A cache that cannot be written only costs the next request a decode.
+ }
+ return c.json({ binSeconds: PEAK_BIN_SECONDS, bins });
+ });
+}
diff --git a/packages/studio/src/components/nle/AudioMeterStrip.test.tsx b/packages/studio/src/components/nle/AudioMeterStrip.test.tsx
index bd5a387822..e453a81527 100644
--- a/packages/studio/src/components/nle/AudioMeterStrip.test.tsx
+++ b/packages/studio/src/components/nle/AudioMeterStrip.test.tsx
@@ -12,6 +12,8 @@ import { SILENT_CHANNEL } from "../../utils/audioMeterMath";
import {
AudioMeterStrip,
type AudioMeterStripProps,
+ CLIP_LATCH_PEAK,
+ stepClipLatch,
evictGoneMeterState,
followMeterHook,
stepAndPaintStrips,
@@ -416,3 +418,54 @@ describe("stepAndPaintStrips", () => {
expect(state.has("loud")).toBe(true);
});
});
+
+describe("CLIP light", () => {
+ const hookAt = (peak: number) => ({
+ start: vi.fn(),
+ stop: vi.fn(),
+ read: vi.fn(() => ({ master: { l: peak, r: 0 }, groups: {} })),
+ });
+
+ it("latches at the -1 dBFS export ceiling, stays lit when the level drops, and resets on click", () => {
+ usePlayerStore.setState({ elements: [clip({})] });
+ const hook = hookAt(1);
+ setHook(hook);
+ const { host } = mount();
+ const light = host.querySelector<HTMLElement>("[data-testid=meter-clip-light]");
+ expect(light?.dataset.lit).toBe("false");
+ tick();
+ expect(light?.dataset.lit).toBe("true");
+ hook.read.mockReturnValue({ master: { l: 0.1, r: 0.1 }, groups: {} });
+ tick();
+ expect(light?.dataset.lit).toBe("true");
+ act(() => light?.click());
+ expect(light?.dataset.lit).toBe("false");
+ tick();
+ expect(light?.dataset.lit).toBe("false");
+ });
+
+ it("stays dark in the red band just under the ceiling", () => {
+ usePlayerStore.setState({ elements: [clip({})] });
+ setHook(hookAt(10 ** (-1.2 / 20)));
+ const { host } = mount();
+ tick();
+ expect(host.querySelector<HTMLElement>("[data-testid=meter-clip-light]")?.dataset.lit).toBe(
+ "false",
+ );
+ });
+});
+
+describe("stepClipLatch", () => {
+ it("lights exactly at -1 dBFS on either channel, not below", () => {
+ expect(CLIP_LATCH_PEAK).toBeCloseTo(0.891251, 6);
+ expect(stepClipLatch(false, { l: CLIP_LATCH_PEAK, r: 0 })).toBe(true);
+ expect(stepClipLatch(false, { l: 0, r: CLIP_LATCH_PEAK })).toBe(true);
+ expect(stepClipLatch(false, { l: CLIP_LATCH_PEAK - 1e-6, r: 0 })).toBe(false);
+ expect(stepClipLatch(false, undefined)).toBe(false);
+ });
+
+ it("holds once lit, whatever comes next", () => {
+ expect(stepClipLatch(true, { l: 0, r: 0 })).toBe(true);
+ expect(stepClipLatch(true, undefined)).toBe(true);
+ });
+});
diff --git a/packages/studio/src/components/nle/AudioMeterStrip.tsx b/packages/studio/src/components/nle/AudioMeterStrip.tsx
index dc23f46e37..5c24c56afa 100644
--- a/packages/studio/src/components/nle/AudioMeterStrip.tsx
+++ b/packages/studio/src/components/nle/AudioMeterStrip.tsx
@@ -5,6 +5,7 @@ import {
useMemo,
useRef,
type PointerEvent as ReactPointerEvent,
+ type ReactNode,
type Ref,
type RefObject,
} from "react";
@@ -20,6 +21,7 @@ import type { StereoLevel } from "@hyperframes/core/runtime/levelTap";
import { usePlayerStore } from "../../player";
import { clampNumber } from "../../utils/studioHelpers";
import { useAudioMetersVisible } from "../../utils/audioMeterVisibility";
+import { EXPORT_PEAK_CEILING } from "../../utils/exportPeakCeiling";
import { useStudioShellContextOptional } from "../../contexts/StudioContext";
import { useTimelineEditContextOptional } from "../../contexts/TimelineEditContext";
import {
@@ -49,6 +51,24 @@ type StripBars = [Bars, Bars];
const MONITOR_LABEL = "Monitor";
+/** The CLIP light latches at the export ceiling, stricter than RED_AT. */
+export const CLIP_LATCH_PEAK = EXPORT_PEAK_CEILING;
+
+export function stepClipLatch(latched: boolean, master: StereoLevel | undefined): boolean {
+ if (latched || !master) return latched;
+ return Math.max(master.l, master.r) >= CLIP_LATCH_PEAK;
+}
+
+function paintClipLight(el: HTMLElement | null, lit: boolean): void {
+ if (!el) return;
+ el.dataset.lit = String(lit);
+ el.style.setProperty(
+ "background-color",
+ lit ? "var(--color-red-500)" : "var(--color-neutral-800)",
+ );
+ el.style.setProperty("color", lit ? "var(--color-white)" : "var(--color-neutral-500)");
+}
+
/** Where the fill turns amber, then red, on the same piecewise dB scale the marks use. */
const AMBER_AT = markFraction(-6);
const RED_AT = markFraction(-3);
@@ -178,10 +198,16 @@ export function stepAndPaintStrips(
}
/** One rAF loop re-reads the hook off the live preview window, so a reloaded iframe is followed. */
+interface ClipLatch {
+ lit: boolean;
+ light: HTMLElement | null;
+}
+
function useMeterLoop(
strips: Strip[],
bars: RefObject<Map<string | null, StripBars>>,
iframeRef: RefObject<HTMLIFrameElement | null> | undefined,
+ clip: RefObject<ClipLatch>,
) {
const shell = useStudioShellContextOptional();
const previewIframeRef = iframeRef ?? shell?.previewIframeRef;
@@ -200,6 +226,11 @@ function useMeterLoop(
last = now;
evictGoneMeterState(state, new Set(stripsRef.current.map((s) => s.id)));
stepAndPaintStrips(stripsRef.current, state, bars.current, levels, now, dt);
+ const lit = stepClipLatch(clip.current.lit, levels?.master);
+ if (lit !== clip.current.lit) {
+ clip.current.lit = lit;
+ paintClipLight(clip.current.light, lit);
+ }
};
raf = requestAnimationFrame(tick);
return () => {
@@ -210,7 +241,29 @@ function useMeterLoop(
// Preview iframe already gone.
}
};
- }, [previewIframeRef, bars]);
+ }, [previewIframeRef, bars, clip]);
+}
+
+function ClipLight({ clip }: { clip: RefObject<ClipLatch> }) {
+ return (
+ <button
+ type="button"
+ data-testid="meter-clip-light"
+ data-lit="false"
+ title="Lights when the preview reaches −1 dBFS, the export ceiling. Click to reset."
+ ref={(el) => {
+ clip.current.light = el;
+ paintClipLight(el, clip.current.lit);
+ }}
+ onClick={() => {
+ clip.current.lit = false;
+ paintClipLight(clip.current.light, false);
+ }}
+ className="rounded-[2px] px-1 font-mono text-[9px] leading-[14px] font-bold tracking-wide"
+ >
+ CLIP
+ </button>
+ );
}
function Bar({ maskRef, peakRef }: { maskRef: Ref<HTMLDivElement>; peakRef: Ref<HTMLDivElement> }) {
@@ -341,11 +394,13 @@ function MeterStrip({
register,
onLive,
onCommit,
+ clipLight,
}: {
strip: Strip;
register: (id: string | null, bars: StripBars | null) => void;
onLive: (id: string | null, volume: number) => void;
onCommit: (id: string | null, volume: number) => void;
+ clipLight?: ReactNode;
}) {
const refs = [
useRef<HTMLDivElement>(null),
@@ -364,6 +419,7 @@ function MeterStrip({
}, [strip.id, register]);
return (
<div className="flex w-[104px] shrink-0 flex-col items-center gap-1 px-1.5 pt-2 pb-1">
+ {clipLight}
<div
className="flex min-h-0 flex-1 items-stretch gap-1.5"
aria-label={`${strip.label} level`}
@@ -421,7 +477,8 @@ function MeterStripBody({ previewIframeRef }: AudioMeterStripProps) {
else bars.current.delete(id);
}).current;
const { onLive, onCommit } = useVolumeHandlers();
- useMeterLoop(strips, bars, previewIframeRef);
+ const clip = useRef<ClipLatch>({ lit: false, light: null });
+ useMeterLoop(strips, bars, previewIframeRef, clip);
return (
<div
data-testid="audio-meter-strip"
@@ -434,6 +491,7 @@ function MeterStripBody({ previewIframeRef }: AudioMeterStripProps) {
register={register}
onLive={onLive}
onCommit={onCommit}
+ clipLight={strip.id === null ? <ClipLight clip={clip} /> : undefined}
/>
))}
</div>
diff --git a/packages/studio/src/hooks/useRenderClipContent.test.ts b/packages/studio/src/hooks/useRenderClipContent.test.ts
index 87a3305305..f03b43c52c 100644
--- a/packages/studio/src/hooks/useRenderClipContent.test.ts
+++ b/packages/studio/src/hooks/useRenderClipContent.test.ts
@@ -5,6 +5,14 @@ import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it } from "vitest";
import { CompositionThumbnail, VideoThumbnail } from "../player";
import { AudioWaveform } from "../player/components/AudioWaveform";
+import { AudibleVideoClipContent } from "../player/components/AudibleVideoClipContent";
+import { ClipPeakMarks } from "../player/components/ClipPeakMarks";
+
+function unwrapPeakMarks(node: ReactNode): ReactNode {
+ return isValidElement<{ children?: ReactNode }>(node) && node.type === ClipPeakMarks
+ ? node.props.children
+ : node;
+}
import type { TimelineClipRenderContext } from "../player/components/TimelineTypes";
import { usePlayerStore, type TimelineElement } from "../player/store/playerStore";
import { buildCompositionThumbnailUrl } from "../player/components/CompositionThumbnail";
@@ -93,9 +101,51 @@ describe("useRenderClipContent", () => {
root.render(React.createElement(Harness));
});
act(() => root.unmount());
+ return unwrapPeakMarks(content);
+ }
+
+ function renderRaw(el: TimelineElement): ReactNode {
+ const host = document.createElement("div");
+ document.body.append(host);
+ const root = createRoot(host);
+ let content: ReactNode = null;
+ function Harness() {
+ const render = useRenderClipContent({
+ projectIdRef: { current: "my-project" },
+ compIdToSrc: new Map(),
+ activePreviewUrl: null,
+ effectiveTimelineDuration: 12,
+ });
+ content = render(el, { clip: "#222", label: "#fff" });
+ return null;
+ }
+ act(() => root.render(React.createElement(Harness)));
+ act(() => root.unmount());
return content;
}
+ it("wraps a clip's waveform in peak marks at its own gain over its played source window", () => {
+ const content = renderRaw({
+ id: "voiceover",
+ tag: "audio",
+ start: 1,
+ duration: 4,
+ track: 1,
+ src: "assets/voiceover.mp3",
+ playbackStart: 2,
+ playbackRate: 1.5,
+ volume: 2,
+ });
+ expect(isValidElement(content) && content.type).toBe(ClipPeakMarks);
+ if (isValidElement<Record<string, unknown>>(content)) {
+ expect(content.props).toMatchObject({
+ peaksUrl: "/api/projects/my-project/peaks/assets/voiceover.mp3",
+ sourceWindow: { mediaStart: 2, sourceSpan: 6 },
+ gain: 2,
+ });
+ }
+ });
+
it("renders audio clips as waveforms even when a composition preview URL is active", () => {
const content = renderClipContent({
id: "voiceover",
@@ -176,6 +226,58 @@ describe("useRenderClipContent", () => {
}
});
+ it("draws a waveform strip under the thumbnails of a video with sound", () => {
+ usePlayerStore.setState({ thumbnailMode: "adaptive" });
+ const content = renderClipContent(
+ {
+ id: "talk",
+ tag: "video",
+ start: 0,
+ duration: 4,
+ track: 0,
+ src: "talk.mp4",
+ hasAudio: true,
+ },
+ null,
+ );
+ expect(isValidElement<{ thumbnail: ReactNode; waveform: ReactNode }>(content)).toBe(true);
+ if (!isValidElement<{ thumbnail: ReactNode; waveform: ReactNode }>(content)) return;
+ expect(content.type).toBe(AudibleVideoClipContent);
+ const { thumbnail } = content.props;
+ const waveform = unwrapPeakMarks(content.props.waveform);
+ expect(isValidElement(thumbnail) && thumbnail.type).toBe(VideoThumbnail);
+ expect(isValidElement<{ waveformUrl: string }>(waveform) && waveform.props.waveformUrl).toBe(
+ "/api/projects/my-project/waveform/talk.mp4",
+ );
+ });
+
+ it("keeps a muted or silent video to thumbnails only", () => {
+ usePlayerStore.setState({ thumbnailMode: "adaptive" });
+ for (const extra of [{ hasAudio: true, muted: true }, { hasAudio: false }]) {
+ const content = renderClipContent(
+ { id: "b", tag: "video", start: 0, duration: 4, track: 0, src: "b.mp4", ...extra },
+ null,
+ );
+ expect(isValidElement(content) && content.type).toBe(VideoThumbnail);
+ }
+ });
+
+ it("still shows the sound strip when thumbnails are off", () => {
+ const content = renderClipContent(
+ {
+ id: "talk",
+ tag: "video",
+ start: 0,
+ duration: 4,
+ track: 0,
+ src: "talk.mp4",
+ hasAudio: true,
+ },
+ null,
+ );
+ expect(isValidElement(content) && content.type).toBe(AudibleVideoClipContent);
+ });
+
it("passes empty labels to thumbnail content so TimelineClip owns clip names", () => {
usePlayerStore.setState({ thumbnailMode: "adaptive" });
diff --git a/packages/studio/src/hooks/useRenderClipContent.ts b/packages/studio/src/hooks/useRenderClipContent.ts
index f0b4eac451..a929e6ac3a 100644
--- a/packages/studio/src/hooks/useRenderClipContent.ts
+++ b/packages/studio/src/hooks/useRenderClipContent.ts
@@ -12,6 +12,9 @@ import type { TimelineClipRenderContext } from "../player/components/TimelineTyp
import { audioPillFlags } from "../player/components/audioClipLink";
import { AudioWaveform, rendersWaveform } from "../player/components/AudioWaveform";
import { ImageThumbnail } from "../player/components/ImageThumbnail";
+import { AudibleVideoClipContent } from "../player/components/AudibleVideoClipContent";
+import { ClipPeakMarks } from "../player/components/ClipPeakMarks";
+import { clipHasSound } from "../player/components/clipMenuNormalize";
import { encodePreviewPath, resolveMediaPreviewUrl } from "../player/components/thumbnailUtils";
import { usePlayerStore } from "../player/store/playerStore";
import { thumbnailRevisionOf } from "../player/store/thumbnailSlice";
@@ -71,7 +74,7 @@ function renderAudioClip(
? buildProjectApiPath(pid, `/waveform/${encodedRelative}`)
: undefined;
const { start, end } = trimFractions(el);
- return createElement(AudioWaveform, {
+ const waveform = createElement(AudioWaveform, {
audioUrl,
waveformUrl,
label: "",
@@ -83,6 +86,27 @@ function renderAudioClip(
priority: context.priority,
...audioPillFlags(el, elements),
});
+ return createElement(
+ ClipPeakMarks,
+ {
+ peaksUrl: encodedRelative ? buildProjectApiPath(pid, `/peaks/${encodedRelative}`) : undefined,
+ sourceWindow: {
+ mediaStart: el.playbackStart ?? 0,
+ sourceSpan: el.duration * (el.playbackRate ?? 1),
+ },
+ gain: el.volume ?? 1,
+ },
+ waveform,
+ );
+}
+
+function withSoundStrip(
+ el: TimelineElement,
+ thumbnail: ReactNode,
+ waveform: () => ReactNode,
+): ReactNode {
+ if (el.tag !== "video" || !clipHasSound(el)) return thumbnail;
+ return createElement(AudibleVideoClipContent, { thumbnail, waveform: waveform() });
}
export interface UseRenderClipContentOptions {
@@ -117,10 +141,9 @@ export function useRenderClipContent({
// Thumbnail generation disabled (perf) -> plain clip bars. Audio still shows
// its waveform (cheap, not a frame thumbnail). Toggle: timeline toolbar.
+ const waveform = () => renderAudioClip(el, pid, sessionEpoch, style.label, context, elements);
if (effectiveMode === "hidden") {
- return rendersWaveform(el)
- ? renderAudioClip(el, pid, sessionEpoch, style.label, context, elements)
- : null;
+ return rendersWaveform(el) ? waveform() : withSoundStrip(el, null, waveform);
}
let compSrc = el.compositionSrc;
@@ -205,7 +228,7 @@ export function useRenderClipContent({
rich: context.rich,
});
}
- return createElement(VideoThumbnail, {
+ const thumbnail = createElement(VideoThumbnail, {
videoSrc: mediaSrc,
label: "",
labelColor: style.label,
@@ -216,6 +239,7 @@ export function useRenderClipContent({
sessionEpoch,
priority: context.priority,
});
+ return withSoundStrip(el, thumbnail, waveform);
}
if (htmlPreviewEligible) {
diff --git a/packages/studio/src/player/components/AudibleVideoClipContent.tsx b/packages/studio/src/player/components/AudibleVideoClipContent.tsx
new file mode 100644
index 0000000000..cbfcc59aa0
--- /dev/null
+++ b/packages/studio/src/player/components/AudibleVideoClipContent.tsx
@@ -0,0 +1,29 @@
+import type { ReactNode } from "react";
+
+/** Share of the clip height the sound strip takes under a video's thumbnails. */
+const AUDIBLE_VIDEO_WAVE_SHARE = 0.38;
+
+/** A video that carries sound: thumbnails on top, its waveform strip along the bottom. */
+export function AudibleVideoClipContent({
+ thumbnail,
+ waveform,
+}: {
+ thumbnail: ReactNode;
+ waveform: ReactNode;
+}) {
+ const waveHeight = `${AUDIBLE_VIDEO_WAVE_SHARE * 100}%`;
+ return (
+ <div className="relative h-full w-full" data-testid="audible-video-clip">
+ <div className="absolute inset-x-0 top-0" style={{ bottom: waveHeight }}>
+ {thumbnail}
+ </div>
+ <div
+ className="absolute inset-x-0 bottom-0 bg-black/30"
+ data-testid="audible-video-wave"
+ style={{ height: waveHeight }}
+ >
+ {waveform}
+ </div>
+ </div>
+ );
+}
diff --git a/packages/studio/src/player/components/ClipPeakMarks.test.tsx b/packages/studio/src/player/components/ClipPeakMarks.test.tsx
new file mode 100644
index 0000000000..9c16f3b012
--- /dev/null
+++ b/packages/studio/src/player/components/ClipPeakMarks.test.tsx
@@ -0,0 +1,41 @@
+// @vitest-environment happy-dom
+import { act } from "react";
+import { describe, expect, it, vi } from "vitest";
+import { createHappyDomRootHarness } from "./testRootHarness";
+import { ClipPeakMarks } from "./ClipPeakMarks";
+
+const harness = createHappyDomRootHarness();
+
+async function render(url: string, bins: number[], gain: number) {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => Response.json({ binSeconds: 1, bins })),
+ );
+ const host = document.createElement("div");
+ document.body.appendChild(host);
+ await act(async () => {
+ harness.mount(host).render(
+ <ClipPeakMarks peaksUrl={url} sourceWindow={{ mediaStart: 0, sourceSpan: 2 }} gain={gain}>
+ <span>wave</span>
+ </ClipPeakMarks>,
+ );
+ });
+ await act(async () => {});
+ vi.unstubAllGlobals();
+ return host;
+}
+
+describe("ClipPeakMarks", () => {
+ it("paints red marks and the peak on a clip that redlines", async () => {
+ const host = await render("/api/projects/p/peaks/loud.mp4", [0.2, 0.98], 1);
+ expect(host.textContent).toContain("wave");
+ expect(host.querySelector("[data-testid=clip-peak-marks]")?.textContent).toContain(
+ "▲ peaks −0.2 dBFS",
+ );
+ });
+
+ it("paints nothing on a quiet clip", async () => {
+ const host = await render("/api/projects/p/peaks/quiet.mp4", [0.25, 0.25], 1);
+ expect(host.querySelector("[data-testid=clip-peak-marks]")).toBeNull();
+ });
+});
diff --git a/packages/studio/src/player/components/ClipPeakMarks.tsx b/packages/studio/src/player/components/ClipPeakMarks.tsx
new file mode 100644
index 0000000000..e8af8cc59c
--- /dev/null
+++ b/packages/studio/src/player/components/ClipPeakMarks.tsx
@@ -0,0 +1,71 @@
+import { useEffect, useState, type ReactNode } from "react";
+import { clipPeakRuns, isPeakMap, type ClipSourceWindow, type PeakMap } from "./clipPeakRuns";
+
+const peakMapRequests = new Map<string, Promise<PeakMap | null>>();
+
+function loadPeakMap(url: string): Promise<PeakMap | null> {
+ const pending = peakMapRequests.get(url);
+ if (pending) return pending;
+ const request = fetch(url)
+ .then((res) => (res.ok ? res.json() : null))
+ .then((body: unknown) => (isPeakMap(body) ? body : null))
+ .catch(() => null);
+ peakMapRequests.set(url, request);
+ return request;
+}
+
+const dbText = (db: number) => `${db < 0 ? "−" : "+"}${Math.abs(db).toFixed(1)} dBFS`;
+
+/** A clip's waveform with red marks where it, at its own volume, reaches the export ceiling. */
+export function ClipPeakMarks({
+ peaksUrl,
+ sourceWindow,
+ gain,
+ children,
+}: {
+ peaksUrl: string | undefined;
+ sourceWindow: ClipSourceWindow;
+ gain: number;
+ children?: ReactNode;
+}) {
+ const [map, setMap] = useState<PeakMap | null>(null);
+ useEffect(() => {
+ if (!peaksUrl) return;
+ let live = true;
+ void loadPeakMap(peaksUrl).then((loaded) => {
+ if (live) setMap(loaded);
+ });
+ return () => {
+ live = false;
+ };
+ }, [peaksUrl]);
+ const { runs, peakDbfs } = map
+ ? clipPeakRuns(map, sourceWindow, gain)
+ : { runs: [], peakDbfs: null };
+ return (
+ <div className="relative h-full w-full">
+ {children}
+ {runs.length > 0 && peakDbfs !== null && (
+ <div
+ className="pointer-events-none absolute inset-0"
+ data-testid="clip-peak-marks"
+ title={`Peaks ${dbText(peakDbfs)} at this volume; export lowers the whole mix`}
+ >
+ {runs.map((run) => (
+ <div
+ key={run.from}
+ className="absolute inset-y-0 bg-red-500/80"
+ style={{
+ left: `${run.from * 100}%`,
+ width: `max(1px, ${(run.to - run.from) * 100}%)`,
+ }}
+ />
+ ))}
+ <span className="absolute top-0 right-1 font-mono text-[9px] leading-none text-red-400">
+ ▲ peaks {dbText(peakDbfs)}
+ </span>
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/packages/studio/src/player/components/clipMenuDuck.test.ts b/packages/studio/src/player/components/clipMenuDuck.test.ts
index 0431eb439a..3833e2d83f 100644
--- a/packages/studio/src/player/components/clipMenuDuck.test.ts
+++ b/packages/studio/src/player/components/clipMenuDuck.test.ts
@@ -28,6 +28,13 @@ describe("isDuckableBed", () => {
expect(isDuckableBed(doc.getElementById("music"))).toBe(true);
expect(isDuckableBed(doc.getElementById("voiceover"))).toBe(false);
});
+
+ it("refuses a clip another bed is already carving against", () => {
+ const doc = compose(
+ `<audio id="music" src="music.mp3" data-fx-carve='{"enabled":true,"sources":["sting"],"strength":0.25}'></audio><audio id="sting" src="sting-bed.mp3"></audio>`,
+ );
+ expect(isDuckableBed(doc.getElementById("sting"))).toBe(false);
+ });
});
describe("setDuckUnderVoice", () => {
diff --git a/packages/studio/src/player/components/clipMenuDuck.ts b/packages/studio/src/player/components/clipMenuDuck.ts
index 168b060643..6b8b258ed0 100644
--- a/packages/studio/src/player/components/clipMenuDuck.ts
+++ b/packages/studio/src/player/components/clipMenuDuck.ts
@@ -15,6 +15,7 @@ import { resolveCarveSourceIds } from "@hyperframes/core/audio-groups";
import { parseAutomation, type HfAutomation } from "@hyperframes/core/audio-automation";
import {
carveBedRoles,
+ carverAgainst,
collectCarveCandidates,
CARVE_ABORTED,
isPromiseLike,
@@ -65,7 +66,8 @@ function readAutomation(bed: Element): HfAutomation {
}
export function isDuckableBed(bed: Element | null): bed is Element {
- return bed !== null && carveBedRoles(bed.id, bed).couldBeBed;
+ if (bed === null || !carveBedRoles(bed.id, bed).couldBeBed) return false;
+ return carverAgainst(bed.ownerDocument, bed.id) === null;
}
/** Every overlapping clip with speech, including a video's own sound; never the bed itself. */
diff --git a/packages/studio/src/player/components/clipPeakRuns.test.ts b/packages/studio/src/player/components/clipPeakRuns.test.ts
new file mode 100644
index 0000000000..5aed064218
--- /dev/null
+++ b/packages/studio/src/player/components/clipPeakRuns.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import { clipPeakRuns } from "./clipPeakRuns";
+
+const map = (bins: number[]) => ({ binSeconds: 1, bins });
+
+describe("clipPeakRuns", () => {
+ it("marks bins at or above -1 dBFS after the clip's gain and names the loudest", () => {
+ const { runs, peakDbfs } = clipPeakRuns(
+ map([0.1, 0.95, 0.99, 0.2, 0.9]),
+ { mediaStart: 0, sourceSpan: 5 },
+ 1,
+ );
+ expect(runs).toEqual([
+ { from: 0.2, to: 0.6 },
+ { from: 0.8, to: 1 },
+ ]);
+ expect(peakDbfs).toBeCloseTo(20 * Math.log10(0.99), 6);
+ });
+
+ it("puts nothing on a -12 dBFS clip, and marks it once boosted by +12 dB", () => {
+ const quiet = map([0.25, 0.25]);
+ expect(clipPeakRuns(quiet, { mediaStart: 0, sourceSpan: 2 }, 1)).toEqual({
+ runs: [],
+ peakDbfs: null,
+ });
+ expect(clipPeakRuns(quiet, { mediaStart: 0, sourceSpan: 2 }, 4).runs).toHaveLength(1);
+ });
+
+ it("only reads the trimmed window, positioned within the clip", () => {
+ const { runs } = clipPeakRuns(map([1, 0, 0, 1, 0]), { mediaStart: 2, sourceSpan: 2 }, 1);
+ expect(runs).toEqual([{ from: 0.5, to: 1 }]);
+ });
+
+ it("is empty for a muted clip or an empty window", () => {
+ expect(clipPeakRuns(map([1]), { mediaStart: 0, sourceSpan: 1 }, 0).runs).toEqual([]);
+ expect(clipPeakRuns(map([1]), { mediaStart: 0, sourceSpan: 0 }, 1).runs).toEqual([]);
+ });
+});
diff --git a/packages/studio/src/player/components/clipPeakRuns.ts b/packages/studio/src/player/components/clipPeakRuns.ts
new file mode 100644
index 0000000000..5478d231fc
--- /dev/null
+++ b/packages/studio/src/player/components/clipPeakRuns.ts
@@ -0,0 +1,59 @@
+import { EXPORT_PEAK_CEILING } from "../../utils/exportPeakCeiling";
+
+export interface PeakMap {
+ binSeconds: number;
+ bins: number[];
+}
+
+export interface ClipSourceWindow {
+ mediaStart: number;
+ sourceSpan: number;
+}
+
+export interface ClipPeakMarkRuns {
+ runs: Array<{ from: number; to: number }>;
+ peakDbfs: number | null;
+}
+
+/**
+ * Where, as fractions of the clip, its source at the clip's own gain reaches the
+ * export ceiling, and the loudest point when it does. Adjacent hot bins merge.
+ */
+export function clipPeakRuns(
+ map: PeakMap,
+ window: ClipSourceWindow,
+ gain: number,
+): ClipPeakMarkRuns {
+ const runs: ClipPeakMarkRuns["runs"] = [];
+ if (!(window.sourceSpan > 0) || !(map.binSeconds > 0) || !(gain > 0))
+ return { runs, peakDbfs: null };
+ const first = Math.max(0, Math.floor(window.mediaStart / map.binSeconds));
+ const last = Math.min(
+ map.bins.length,
+ Math.ceil((window.mediaStart + window.sourceSpan) / map.binSeconds),
+ );
+ const toFraction = (bin: number) =>
+ Math.min(1, Math.max(0, (bin * map.binSeconds - window.mediaStart) / window.sourceSpan));
+ let loudest = 0;
+ for (let bin = first; bin < last; bin++) {
+ const level = (map.bins[bin] ?? 0) * gain;
+ if (level < EXPORT_PEAK_CEILING) continue;
+ loudest = Math.max(loudest, level);
+ const previous = runs.at(-1);
+ const from = toFraction(bin);
+ const to = toFraction(bin + 1);
+ if (previous && previous.to >= from) previous.to = to;
+ else runs.push({ from, to });
+ }
+ return { runs, peakDbfs: runs.length > 0 ? 20 * Math.log10(loudest) : null };
+}
+
+export function isPeakMap(value: unknown): value is PeakMap {
+ if (typeof value !== "object" || value === null) return false;
+ const bins: unknown = Reflect.get(value, "bins");
+ return (
+ typeof Reflect.get(value, "binSeconds") === "number" &&
+ Array.isArray(bins) &&
+ bins.every((bin) => typeof bin === "number")
+ );
+}
diff --git a/packages/studio/src/utils/exportPeakCeiling.ts b/packages/studio/src/utils/exportPeakCeiling.ts
new file mode 100644
index 0000000000..9cbbf97f36
--- /dev/null
+++ b/packages/studio/src/utils/exportPeakCeiling.ts
@@ -0,0 +1 @@
+export const EXPORT_PEAK_CEILING = 10 ** (-1 / 20);
diff --git a/packages/studio/src/utils/timelineInspector.test.ts b/packages/studio/src/utils/timelineInspector.test.ts
index fb5cef0153..27263980a8 100644
--- a/packages/studio/src/utils/timelineInspector.test.ts
+++ b/packages/studio/src/utils/timelineInspector.test.ts
@@ -4,6 +4,7 @@ import {
canHideSelections,
isAudioDomElement,
isAudioTimelineElement,
+ isMusicSourceElement,
isMusicTrack,
resolveBeatSourceTrack,
} from "./timelineInspector";
@@ -12,9 +13,15 @@ import type { TimelineElement } from "../player";
// Minimal element factory for tests
function el(
overrides: Partial<
- Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "duration">
+ Pick<
+ TimelineElement,
+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
+ >
>,
-): Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "duration"> {
+): Pick<
+ TimelineElement,
+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
+> {
return {
tag: "audio",
src: "assets/track.mp3",
@@ -153,3 +160,27 @@ describe("isAudioDomElement / canHideSelections", () => {
expect(canHideSelections([{ element: el("hf-audio-group") }])).toBe(false);
});
});
+
+describe("beat source from a video with sound", () => {
+ const talk = { tag: "video", src: "assets/clip.mp4", hasAudio: true };
+
+ it("counts an audible video as a music source but never a muted or silent one", () => {
+ expect(isMusicSourceElement(el(talk))).toBe(true);
+ expect(isMusicSourceElement(el({ ...talk, muted: true }))).toBe(false);
+ expect(isMusicSourceElement(el({ ...talk, hasAudio: false }))).toBe(false);
+ expect(isAudioTimelineElement(el(talk))).toBe(false);
+ });
+
+ it("takes a video tagged as music as the beat source", () => {
+ const video = el({ ...talk, timelineRole: "music" });
+ expect(isMusicTrack(video)).toBe(true);
+ expect(resolveBeatSourceTrack([video])).toEqual({ element: video, isFallback: false });
+ });
+
+ it("falls back to the longest audible video when there is no audio", () => {
+ const short = el({ ...talk, id: "a", duration: 3 });
+ const long = el({ ...talk, id: "b", duration: 9 });
+ const muted = el({ ...talk, id: "c", duration: 20, muted: true });
+ expect(resolveBeatSourceTrack([short, long, muted])?.element).toBe(long);
+ });
+});
diff --git a/packages/studio/src/utils/timelineInspector.ts b/packages/studio/src/utils/timelineInspector.ts
index 44faaeb8db..7bae863084 100644
--- a/packages/studio/src/utils/timelineInspector.ts
+++ b/packages/studio/src/utils/timelineInspector.ts
@@ -44,17 +44,29 @@ export function isAudioOrVideoTimelineElement(
return isAudioTimelineElement(element) || element.tag.trim().toLowerCase() === "video";
}
+type MusicSourceFacts = Pick<TimelineElement, "tag" | "src" | "hasAudio" | "muted">;
+
+/** Can carry the music: an audio clip, or a video whose own sound plays. Lane zoning is unaffected. */
+export function isMusicSourceElement(element: MusicSourceFacts): boolean {
+ if (isAudioTimelineElement(element)) return true;
+ return (
+ element.tag.trim().toLowerCase() === "video" &&
+ element.hasAudio === true &&
+ element.muted !== true
+ );
+}
+
/** True for the music track: an audio element with data-timeline-role="music",
* or — when no role is set — an id matching the music regex. Voiceover/other
* audio (explicit non-music role) is excluded. */
export function isMusicTrack(
element:
- | Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole">
+ | Pick<TimelineElement, "tag" | "src" | "id" | "domId" | "timelineRole" | "hasAudio" | "muted">
| null
| undefined,
): boolean {
if (!element) return false;
- if (!isAudioTimelineElement(element)) return false;
+ if (!isMusicSourceElement(element)) return false;
if (element.timelineRole === "music") return true;
if (element.timelineRole && element.timelineRole !== "music") return false;
const id = element.domId ?? element.id ?? "";
@@ -75,7 +87,7 @@ export function isMusicTrack(
export function resolveBeatSourceTrack(
elements: readonly Pick<
TimelineElement,
- "tag" | "src" | "id" | "domId" | "timelineRole" | "duration"
+ "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
>[],
): { element: (typeof elements)[number]; isFallback: boolean } | null {
const explicit = elements.find(isMusicTrack);
@@ -85,7 +97,7 @@ export function resolveBeatSourceTrack(
// like "sfx" or "voiceover" to avoid triggering beat analysis on those).
let best: (typeof elements)[number] | null = null;
for (const el of elements) {
- if (!isAudioTimelineElement(el)) continue;
+ if (!isMusicSourceElement(el)) continue;
if (el.timelineRole && el.timelineRole !== "music") continue;
if (!best || el.duration > best.duration) best = el;
}