Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Run the audio path under test: real wav, real graph, real aac track

Bitti586a549e1e14
diff --git a/fixtures/project-v1-audio-mix.json b/fixtures/project-v1-audio-mix.json
new file mode 100644
index 0000000..92cbf6d
--- /dev/null
+++ b/fixtures/project-v1-audio-mix.json
@@ -0,0 +1,129 @@
+{
+ "schemaVersion": 1,
+ "id": "project-audio-mix",
+ "name": "Project with sound to mix",
+ "createdAt": "2026-01-01T00:00:00.000Z",
+ "updatedAt": "2026-01-01T00:00:00.000Z",
+ "thumbnail": null,
+ "settings": {
+ "aspectRatio": "16:9",
+ "resolution": "1080p",
+ "frameRate": 30,
+ "backgroundColor": "#10131a"
+ },
+ "brief": {
+ "prompt": "a harbour at first light, with a score under it",
+ "status": "draft",
+ "targetDurationSeconds": 5,
+ "aspectRatio": "16:9",
+ "resolution": "1080p"
+ },
+ "assets": [
+ {
+ "id": "asset-rush",
+ "kind": "video",
+ "name": "Harbour rush",
+ "sourcePath": "D:\\tmp\\harbour\\rush-01.mp4",
+ "mimeType": "video/mp4",
+ "durationMs": 4000,
+ "width": 320,
+ "height": 180,
+ "createdAt": "2026-01-01T00:00:00.000Z"
+ },
+ {
+ "id": "asset-score",
+ "kind": "audio",
+ "name": "Score",
+ "sourcePath": "D:\\tmp\\harbour\\score.wav",
+ "mimeType": "audio/wav",
+ "durationMs": 3000,
+ "createdAt": "2026-01-01T00:00:00.000Z"
+ },
+ {
+ "id": "asset-voice",
+ "kind": "audio",
+ "name": "Voice-over take 3",
+ "sourcePath": "D:\\tmp\\harbour\\vo-take-3.wav",
+ "mimeType": "audio/wav",
+ "durationMs": 1000,
+ "createdAt": "2026-01-01T00:00:00.000Z"
+ }
+ ],
+ "timeline": {
+ "tracks": [
+ {
+ "id": "track-story",
+ "kind": "video",
+ "name": "Story",
+ "locked": false,
+ "muted": false,
+ "clips": [
+ {
+ "id": "clip-picture",
+ "assetId": "asset-rush",
+ "trackId": "track-story",
+ "startMs": 0,
+ "durationMs": 4000,
+ "sourceStartMs": 0,
+ "label": "01 - Harbour",
+ "status": "approved"
+ }
+ ]
+ },
+ {
+ "id": "track-music",
+ "kind": "audio",
+ "name": "Music",
+ "locked": false,
+ "muted": false,
+ "clips": [
+ {
+ "id": "clip-score-a",
+ "assetId": "asset-score",
+ "trackId": "track-music",
+ "startMs": 0,
+ "durationMs": 2000,
+ "sourceStartMs": 0,
+ "label": "Score - head",
+ "status": "approved"
+ },
+ {
+ "id": "clip-score-b",
+ "assetId": "asset-score",
+ "trackId": "track-music",
+ "startMs": 1000,
+ "durationMs": 2000,
+ "sourceStartMs": 500,
+ "label": "Score - reprise",
+ "status": "approved"
+ }
+ ]
+ },
+ {
+ "id": "track-voice-over",
+ "kind": "audio",
+ "name": "Voice-over",
+ "locked": false,
+ "muted": true,
+ "clips": [
+ {
+ "id": "clip-vo",
+ "assetId": "asset-voice",
+ "trackId": "track-voice-over",
+ "startMs": 0,
+ "durationMs": 1000,
+ "sourceStartMs": 0,
+ "label": "VO - take 3",
+ "status": "approved"
+ }
+ ]
+ }
+ ]
+ },
+ "references": [],
+ "generationJobs": [],
+ "agentConversation": {
+ "messages": []
+ },
+ "providerSettings": {}
+}
diff --git a/src/lib/exportPipeline.test.ts b/src/lib/exportPipeline.test.ts
index f035eb9..11116c7 100644
--- a/src/lib/exportPipeline.test.ts
+++ b/src/lib/exportPipeline.test.ts
@@ -2,8 +2,9 @@
import { invoke } from "@tauri-apps/api/core";
import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest";
+import audioFixture from "../../fixtures/project-v1-audio-mix.json";
import externalFixture from "../../fixtures/project-v1-external-media.json";
-import { buildExportPlan, defaultExportSettings } from "./export";
+import { audioMixBytes, buildExportPlan, defaultExportSettings } from "./export";
import { parseProjectConfig, type ProjectAsset } from "./project";
import { demux, ExportCancelled, PreviewSources, probeCompositor, runExport } from "./exportPipeline";
@@ -188,42 +189,235 @@ describe("planning an export of external media", () => {
});
});
-/* WebCodecs and OffscreenCanvas do not exist in jsdom, so the run below stands
- them up. They do only what the pipeline asks of them and nothing more: the
- decoder deliberately emits no frames, which makes the run fail at a known
- point AFTER the source file has been read. What is asserted is the read. */
-function installWebCodecs(): () => void {
- const scope = globalThis as unknown as Record<string, unknown>;
- const saved = {
- VideoEncoder: scope.VideoEncoder,
- VideoDecoder: scope.VideoDecoder,
- VideoFrame: scope.VideoFrame,
- EncodedVideoChunk: scope.EncodedVideoChunk,
- OffscreenCanvas: scope.OffscreenCanvas,
+/* ---------------------------------------------------------------------------
+ A real .wav, built here.
+
+ The sound half of the pipeline decodes through `decodeAudioData` rather than
+ through mp4box, so what it has to survive is a container it did not write.
+ These bytes are a genuine RIFF/WAVE file — header, `fmt ` chunk, interleaved
+ 16-bit samples — and the harness decoder below parses them as one, including
+ the detach `decodeAudioData` performs on its input. Nothing here hands the
+ pipeline a pre-decoded buffer.
+ --------------------------------------------------------------------------- */
+
+const WAV_SAMPLE_RATE = 48_000;
+
+function wavFile(options: {
+ seconds: number;
+ channels?: number;
+ sampleRate?: number;
+ /** The value of channel `channel` at frame `frame`, in -1..1. */
+ sample: (frame: number, channel: number) => number;
+}): ArrayBuffer {
+ const channels = options.channels ?? 2;
+ const sampleRate = options.sampleRate ?? WAV_SAMPLE_RATE;
+ const frames = Math.round(options.seconds * sampleRate);
+ const dataBytes = frames * channels * 2;
+ const buffer = new ArrayBuffer(44 + dataBytes);
+ const view = new DataView(buffer);
+ const ascii = (offset: number, text: string) => {
+ for (let index = 0; index < text.length; index += 1) view.setUint8(offset + index, text.charCodeAt(index));
};
+ ascii(0, "RIFF");
+ view.setUint32(4, 36 + dataBytes, true);
+ ascii(8, "WAVE");
+ ascii(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true); // PCM
+ view.setUint16(22, channels, true);
+ view.setUint32(24, sampleRate, true);
+ view.setUint32(28, sampleRate * channels * 2, true); // byte rate
+ view.setUint16(32, channels * 2, true); // block align
+ view.setUint16(34, 16, true);
+ ascii(36, "data");
+ view.setUint32(40, dataBytes, true);
+ let offset = 44;
+ for (let frame = 0; frame < frames; frame += 1) {
+ for (let channel = 0; channel < channels; channel += 1) {
+ const value = Math.max(-1, Math.min(1, options.sample(frame, channel)));
+ view.setInt16(offset, Math.max(-32768, Math.min(32767, Math.round(value * 32768))), true);
+ offset += 2;
+ }
+ }
+ return buffer;
+}
+
+/** 16-bit PCM back to planar floats, the way an audio decoder does it. Throws
+ * on anything that is not a WAVE, which is how the "could not be decoded"
+ * branch gets exercised with a real failure rather than a stubbed one. */
+function parseWav(bytes: ArrayBuffer): { channels: Float32Array[]; sampleRate: number } {
+ const view = new DataView(bytes);
+ const tag = (offset: number) =>
+ String.fromCharCode(view.getUint8(offset), view.getUint8(offset + 1), view.getUint8(offset + 2), view.getUint8(offset + 3));
+ if (bytes.byteLength < 44 || tag(0) !== "RIFF" || tag(8) !== "WAVE") {
+ throw new Error("The data is not a RIFF/WAVE file.");
+ }
+ let cursor = 12;
+ let channelCount = 0;
+ let sampleRate = 0;
+ let data: { start: number; length: number } | null = null;
+ while (cursor + 8 <= bytes.byteLength) {
+ const name = tag(cursor);
+ const size = view.getUint32(cursor + 4, true);
+ if (name === "fmt ") {
+ channelCount = view.getUint16(cursor + 10, true);
+ sampleRate = view.getUint32(cursor + 12, true);
+ } else if (name === "data") {
+ data = { start: cursor + 8, length: size };
+ }
+ cursor += 8 + size + (size % 2);
+ }
+ if (!data || channelCount === 0) throw new Error("The WAVE file has no fmt or data chunk.");
+ const frames = Math.floor(data.length / (channelCount * 2));
+ const channels = Array.from({ length: channelCount }, () => new Float32Array(frames));
+ for (let frame = 0; frame < frames; frame += 1) {
+ for (let channel = 0; channel < channelCount; channel += 1) {
+ channels[channel][frame] = view.getInt16(data.start + (frame * channelCount + channel) * 2, true) / 32768;
+ }
+ }
+ return { channels, sampleRate };
+}
+
+/* ---------------------------------------------------------------------------
+ WebCodecs, OffscreenCanvas and the offline audio graph, stood up for jsdom.
+
+ These do what the browser does and nothing more — the audio graph really
+ sums overlapping sources, really refuses to play past the end of a buffer,
+ and `decodeAudioData` really detaches the ArrayBuffer it is handed. The
+ numbers the mixing tests assert were measured in headless Chrome against the
+ same timeline first; this harness reproduces them.
+ --------------------------------------------------------------------------- */
+
+interface HarnessRecord {
+ /** Every AudioData the pipeline handed the encoder, with its samples. */
+ audioData: Array<{ format: string; timestamp: number; numberOfFrames: number; numberOfChannels: number; samples: Float32Array }>;
+ audioConfig: AudioEncoderConfig | null;
+ audioChunks: number;
+ videoChunks: number;
+}
+
+class HarnessAudioBuffer {
+ constructor(
+ readonly data: Float32Array[],
+ readonly sampleRate: number,
+ ) {}
+ get numberOfChannels() {
+ return this.data.length;
+ }
+ get length() {
+ return this.data[0]?.length ?? 0;
+ }
+ get duration() {
+ return this.length / this.sampleRate;
+ }
+ getChannelData(index: number) {
+ return this.data[index];
+ }
+}
+
+/** Options the harness needs to stand in for a browser. */
+interface HarnessOptions {
+ /** True: the decoder emits a frame per chunk, so a run reaches the muxer and
+ * produces a real file. False: it emits nothing, which is what the read
+ * tests want — they assert on the IPC log, not on a picture. */
+ decodes?: boolean;
+ /** False: no AudioEncoder/AudioData/OfflineAudioContext at all, which is the
+ * webview `detectExportSupport().audio` has to answer false for. */
+ audio?: boolean;
+}
+
+function installWebCodecs(options: HarnessOptions = {}): { restore: () => void; recorded: HarnessRecord } {
+ const { decodes = false, audio = true } = options;
+ const scope = globalThis as unknown as Record<string, unknown>;
+ const names = [
+ "VideoEncoder",
+ "VideoDecoder",
+ "VideoFrame",
+ "EncodedVideoChunk",
+ "OffscreenCanvas",
+ "AudioEncoder",
+ "AudioData",
+ "EncodedAudioChunk",
+ "OfflineAudioContext",
+ ] as const;
+ const saved: Record<string, unknown> = {};
+ for (const name of names) saved[name] = scope[name];
+ const recorded: HarnessRecord = { audioData: [], audioConfig: null, audioChunks: 0, videoChunks: 0 };
+
+ class HarnessChunk {
+ readonly type: "key" | "delta";
+ readonly timestamp: number;
+ readonly duration: number;
+ private readonly payload: Uint8Array;
+ constructor(init: { type: "key" | "delta"; timestamp: number; duration?: number; data: BufferSource }) {
+ this.type = init.type;
+ this.timestamp = init.timestamp;
+ this.duration = init.duration ?? 0;
+ this.payload =
+ init.data instanceof Uint8Array ? new Uint8Array(init.data) : new Uint8Array(init.data as ArrayBuffer);
+ }
+ get byteLength() {
+ return this.payload.byteLength;
+ }
+ copyTo(destination: Uint8Array) {
+ destination.set(this.payload);
+ }
+ }
+ scope.EncodedVideoChunk = HarnessChunk;
+ scope.EncodedAudioChunk = class extends HarnessChunk {};
+ const EncodedVideoChunkClass = scope.EncodedVideoChunk as typeof HarnessChunk;
+ const EncodedAudioChunkClass = scope.EncodedAudioChunk as typeof HarnessChunk;
+
scope.VideoEncoder = class {
static async isConfigSupported(config: unknown) {
return { supported: true, config };
}
state = "configured";
encodeQueueSize = 0;
+ private readonly emit: (chunk: unknown, meta?: unknown) => void;
+ private first = true;
+ constructor(init: { output: (chunk: unknown, meta?: unknown) => void; error: (reason: unknown) => void }) {
+ this.emit = init.output;
+ }
configure() {}
- encode() {}
+ encode(frame: { timestamp: number; duration: number }, options?: { keyFrame?: boolean }) {
+ // A payload derived from the frame time, so the muxed file is real bytes
+ // rather than a run of zeros nothing could tell apart.
+ const chunk = new EncodedVideoChunkClass({
+ type: options?.keyFrame ? "key" : "delta",
+ timestamp: frame.timestamp,
+ duration: frame.duration,
+ data: new Uint8Array([0, 0, 0, 3, 0x65, frame.timestamp & 0xff, 0]),
+ });
+ recorded.videoChunks += 1;
+ this.emit(chunk, this.first ? { decoderConfig: { codec: "avc1.42001f", codedWidth: 320, codedHeight: 180, description } } : undefined);
+ this.first = false;
+ }
async flush() {}
close() {
this.state = "closed";
}
};
+
scope.VideoDecoder = class {
state = "configured";
decodeQueueSize = 0;
+ private readonly emit: (frame: unknown) => void;
+ constructor(init: { output: (frame: unknown) => void; error: (reason: unknown) => void }) {
+ this.emit = init.output;
+ }
configure() {}
- decode() {}
+ decode(chunk: { timestamp: number; duration: number }) {
+ if (!decodes) return;
+ const Frame = scope.VideoFrame as new (source: unknown, init: { timestamp: number; duration?: number }) => unknown;
+ this.emit(new Frame(null, { timestamp: chunk.timestamp, duration: chunk.duration }));
+ }
async flush() {}
close() {
this.state = "closed";
}
};
+
scope.VideoFrame = class {
timestamp: number;
duration: number;
@@ -235,11 +429,7 @@ function installWebCodecs(): () => void {
}
close() {}
};
- scope.EncodedVideoChunk = class {
- constructor(init: Record<string, unknown>) {
- Object.assign(this, init);
- }
- };
+
scope.OffscreenCanvas = class {
constructor(
public width: number,
@@ -249,12 +439,163 @@ function installWebCodecs(): () => void {
return kind === "2d" ? { fillStyle: "", fillRect() {}, drawImage() {} } : null;
}
};
- return () => Object.assign(scope, saved);
+
+ if (audio) {
+ scope.AudioData = class {
+ readonly format: string;
+ readonly sampleRate: number;
+ readonly numberOfFrames: number;
+ readonly numberOfChannels: number;
+ readonly timestamp: number;
+ /** A real AudioData copies its samples; so does this, which is what lets
+ * the assertions read them after `close()`. */
+ readonly samples: Float32Array;
+ constructor(init: {
+ format: string;
+ sampleRate: number;
+ numberOfFrames: number;
+ numberOfChannels: number;
+ timestamp: number;
+ data: Float32Array;
+ }) {
+ this.format = init.format;
+ this.sampleRate = init.sampleRate;
+ this.numberOfFrames = init.numberOfFrames;
+ this.numberOfChannels = init.numberOfChannels;
+ this.timestamp = init.timestamp;
+ this.samples = new Float32Array(init.data);
+ }
+ close() {}
+ };
+
+ scope.AudioEncoder = class {
+ static async isConfigSupported(config: AudioEncoderConfig) {
+ return { supported: true, config };
+ }
+ state = "unconfigured";
+ encodeQueueSize = 0;
+ private readonly emit: (chunk: unknown, meta?: unknown) => void;
+ private first = true;
+ constructor(init: { output: (chunk: unknown, meta?: unknown) => void; error: (reason: unknown) => void }) {
+ this.emit = init.output;
+ }
+ configure(config: AudioEncoderConfig) {
+ recorded.audioConfig = config;
+ this.state = "configured";
+ }
+ encode(data: { format: string; timestamp: number; numberOfFrames: number; numberOfChannels: number; sampleRate: number; samples: Float32Array }) {
+ recorded.audioData.push({
+ format: data.format,
+ timestamp: data.timestamp,
+ numberOfFrames: data.numberOfFrames,
+ numberOfChannels: data.numberOfChannels,
+ samples: data.samples.slice(),
+ });
+ recorded.audioChunks += 1;
+ this.emit(
+ new EncodedAudioChunkClass({
+ type: "key",
+ timestamp: data.timestamp,
+ duration: Math.round((data.numberOfFrames * 1_000_000) / data.sampleRate),
+ data: new Uint8Array([0x21, data.numberOfFrames & 0xff, 0x00, 0x00]),
+ }),
+ // What a real AAC encoder emits alongside its first packet.
+ this.first
+ ? {
+ decoderConfig: {
+ codec: "mp4a.40.2",
+ sampleRate: 48_000,
+ numberOfChannels: 2,
+ description: new Uint8Array([0x11, 0x90]),
+ },
+ }
+ : undefined,
+ );
+ this.first = false;
+ }
+ async flush() {}
+ close() {
+ this.state = "closed";
+ }
+ };
+
+ scope.OfflineAudioContext = class {
+ readonly destination = { node: "destination" };
+ readonly numberOfChannels: number;
+ readonly length: number;
+ readonly sampleRate: number;
+ private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number }> = [];
+ constructor(init: { numberOfChannels: number; length: number; sampleRate: number }) {
+ this.numberOfChannels = init.numberOfChannels;
+ this.length = init.length;
+ this.sampleRate = init.sampleRate;
+ }
+ async decodeAudioData(bytes: ArrayBuffer): Promise<HarnessAudioBuffer> {
+ /* The real one DETACHES its input. Reproduced, because the pipeline
+ copies the read result before handing it over and nothing else would
+ notice if that copy were removed. */
+ const owned = structuredClone(bytes, { transfer: [bytes] });
+ const { channels, sampleRate } = parseWav(owned);
+ if (sampleRate !== this.sampleRate) {
+ throw new Error(`This harness does not resample: the file is ${sampleRate} Hz, the graph is ${this.sampleRate} Hz.`);
+ }
+ return new HarnessAudioBuffer(channels, sampleRate);
+ }
+ createBufferSource() {
+ const context = this;
+ return {
+ buffer: null as HarnessAudioBuffer | null,
+ connected: false,
+ connect(destination: unknown) {
+ this.connected = destination === context.destination;
+ },
+ start(when: number, offset: number, duration: number) {
+ if (!this.buffer || !this.connected) return; // an unconnected source is silence
+ context.started.push({ buffer: this.buffer, when, offset, duration });
+ },
+ };
+ }
+ async startRendering(): Promise<HarnessAudioBuffer> {
+ const out = Array.from({ length: this.numberOfChannels }, () => new Float32Array(this.length));
+ for (const source of this.started) {
+ const startFrame = Math.round(source.when * this.sampleRate);
+ const offsetFrame = Math.round(source.offset * this.sampleRate);
+ const count = Math.min(
+ Math.round(source.duration * this.sampleRate),
+ source.buffer.length - offsetFrame,
+ this.length - startFrame,
+ );
+ for (let channel = 0; channel < this.numberOfChannels; channel += 1) {
+ // Mono up-mixes onto both channels, as the browser's graph does.
+ const from = source.buffer.getChannelData(Math.min(channel, source.buffer.numberOfChannels - 1));
+ const into = out[channel];
+ for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + index];
+ }
+ }
+ return new HarnessAudioBuffer(out, this.sampleRate);
+ }
+ };
+ } else {
+ delete scope.AudioEncoder;
+ delete scope.AudioData;
+ delete scope.OfflineAudioContext;
+ delete scope.EncodedAudioChunk;
+ }
+
+ return {
+ recorded,
+ restore: () => {
+ for (const name of names) {
+ if (saved[name] === undefined) delete scope[name];
+ else scope[name] = saved[name];
+ }
+ },
+ };
}
describe("running an export of external media", () => {
it("dispatches the external read, not a project-relative one", async () => {
- const restore = installWebCodecs();
+ const { restore } = installWebCodecs();
try {
invoked.mockResolvedValue(await sampleFile(5, 30));
const config = externalProject();
@@ -282,7 +623,7 @@ describe("running an export of external media", () => {
});
it("stops on cancel without ever touching the file", async () => {
- const restore = installWebCodecs();
+ const { restore } = installWebCodecs();
try {
const config = externalProject();
const settings = defaultExportSettings(config);
@@ -304,6 +645,346 @@ describe("running an export of external media", () => {
});
});
+/* ---------------------------------------------------------------------------
+ The sound, actually running.
+
+ Everything above this line is the picture. The audio path had six tests of
+ its PLANNING and none of its execution: no AudioEncoder in the harness, no
+ audio clips in the fixture, so `mixAudio` had never been called once. These
+ run it — from the .wav bytes the IPC layer hands back, through the graph, the
+ encoder and into a file that is taken apart again to see the track.
+
+ The mixing numbers below were measured first in headless Chrome against this
+ same timeline: 192000 rendered frames in 2 channels, 0.80 at 500 ms where one
+ clip plays alone, 1.60 at 1200 ms where two overlap and sum, 0 at 3500 ms
+ where nothing covers, and a clamp to 1 at the peak.
+ --------------------------------------------------------------------------- */
+
+const AUDIO_FOLDER = "D:\\tmp\\harbour";
+const PATHS = {
+ video: "D:\\tmp\\harbour\\rush-01.mp4",
+ score: "D:\\tmp\\harbour\\score.wav",
+ voice: "D:\\tmp\\harbour\\vo-take-3.wav",
+};
+
+const audioProject = () => parseProjectConfig(audioFixture);
+
+/** The IPC layer, answering with real files per path. Anything unasked-for
+ * throws, so a read the export should not have made shows up as a failure
+ * rather than as a convenient default. */
+function serveFiles(files: Partial<Record<keyof typeof PATHS, ArrayBuffer>>) {
+ invoked.mockImplementation(async (_command: string, args: { sourcePath?: string }) => {
+ for (const [name, path] of Object.entries(PATHS)) {
+ if (args?.sourcePath === path) {
+ const bytes = files[name as keyof typeof PATHS];
+ if (!bytes) throw new Error(`the test served no ${name}, so this read should not have happened`);
+ return bytes;
+ }
+ }
+ throw new Error(`unexpected read of ${String(args?.sourcePath)}`);
+ });
+}
+
+/** A three-second stereo score at 0.8 of full scale — loud, but not clipping
+ * until two copies of it overlap. */
+const scoreFile = () => wavFile({ seconds: 3, sample: () => 0.8 });
+
+async function runAudioExport(
+ harness: ReturnType<typeof installWebCodecs>,
+ config = audioProject(),
+) {
+ const settings = defaultExportSettings(config);
+ return runExport({
+ folderPath: AUDIO_FOLDER,
+ config,
+ settings,
+ plan: buildExportPlan(config, settings),
+ bitrate: 5_000_000,
+ onProgress: () => {},
+ cancelled: () => false,
+ }).finally(() => void harness);
+}
+
+/** The samples the encoder was actually handed, put back in channel order —
+ * which only works if the planar blocks are laid out the way the muxer and the
+ * encoder expect: all of channel 0, then all of channel 1, per block. */
+function encoderChannels(recorded: HarnessRecord): Float32Array[] {
+ const total = recorded.audioData.reduce((sum, block) => sum + block.numberOfFrames, 0);
+ const channels = [new Float32Array(total), new Float32Array(total)];
+ let offset = 0;
+ for (const block of recorded.audioData) {
+ for (let channel = 0; channel < 2; channel += 1) {
+ channels[channel].set(
+ block.samples.subarray(channel * block.numberOfFrames, (channel + 1) * block.numberOfFrames),
+ offset,
+ );
+ }
+ offset += block.numberOfFrames;
+ }
+ return channels;
+}
+
+/** The sample index of a moment in the finished soundtrack. */
+const at = (ms: number) => Math.round((ms / 1000) * 48_000);
+
+/** The audio track of a finished file, read back out of the muxed bytes with
+ * mp4box — the same demuxer a player uses, not the muxer's own bookkeeping. */
+async function audioTrackOf(bytes: Uint8Array) {
+ const { createFile, MP4BoxBuffer } = await import("mp4box");
+ type Movie = import("mp4box").Movie;
+ const file = createFile();
+ const found: { movie: Movie | null } = { movie: null };
+ file.onReady = (info) => {
+ found.movie = info;
+ };
+ const copy = bytes.slice().buffer;
+ file.appendBuffer(MP4BoxBuffer.fromArrayBuffer(copy, 0), true);
+ file.flush();
+ if (!found.movie) throw new Error("the exported file has no readable MP4 header");
+ return { movie: found.movie, track: found.movie.audioTracks[0] ?? null };
+}
+
+describe("mixing real sound into a real export", () => {
+ it("reads each audio ASSET once, before the file is opened, and never a muted one", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });
+ await runAudioExport(harness);
+
+ /* Two clips share one asset, so there is one read for them, not two —
+ and the muted voice-over is never fetched at all. The score comes
+ FIRST: the mix has to be settled before the muxer can be told whether
+ the file has an audio track. */
+ expect(callLog()).toEqual([
+ { cmd: "read_external_media_file", args: { folderPath: AUDIO_FOLDER, sourcePath: PATHS.score } },
+ { cmd: "read_external_media_file", args: { folderPath: AUDIO_FOLDER, sourcePath: PATHS.video } },
+ ]);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("sums overlapping clips and clips the sum flat, saying how far over it went", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });
+ const result = await runAudioExport(harness);
+ const [left, right] = encoderChannels(harness.recorded);
+
+ // 4 s of picture at 48 kHz, in two channels: the whole soundtrack.
+ expect(left).toHaveLength(192_000);
+ expect(right).toHaveLength(192_000);
+ // One clip alone; two overlapping, summed and then cut to full scale;
+ // a stretch no clip covers, which is real silence.
+ expect(left[at(500)]).toBeCloseTo(0.8, 4);
+ expect(left[at(1200)]).toBe(1);
+ expect(right[at(1200)]).toBe(1);
+ expect(left[at(2500)]).toBeCloseTo(0.8, 4);
+ expect(left[at(3500)]).toBe(0);
+
+ /* The overlap is timeline 1000-2000 ms: 48000 frames in two channels,
+ every one of them over full scale. "Limited" would have been the wrong
+ word for it — nothing rode the gain down, the tops were cut off. */
+ expect(result.audioDetail).toMatch(/peaked at 1\.60 of full scale/);
+ // The thousands separator is whatever the running locale uses.
+ expect(result.audioDetail).toMatch(/96.?000 samples \(25\.0%\) were clipped flat/);
+ expect(result.audioDetail).toMatch(/down by about 4\.1 dB/);
+ expect(result.audioDetail).not.toMatch(/limited/i);
+ expect(result.audioProblems).toEqual([]);
+ expect(result.audioShortfalls).toEqual([]);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("feeds the encoder f32-planar blocks on a microsecond clock", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });
+ await runAudioExport(harness);
+ const blocks = harness.recorded.audioData;
+
+ expect(harness.recorded.audioConfig).toEqual({
+ codec: "mp4a.40.2",
+ sampleRate: 48_000,
+ numberOfChannels: 2,
+ bitrate: 192_000,
+ });
+ // 192000 samples in AAC's own 1024-sample frame: 187 full blocks and a
+ // 512-sample remainder. A block of 0 would be an encoder error.
+ expect(blocks).toHaveLength(188);
+ expect(blocks.every((block) => block.format === "f32-planar")).toBe(true);
+ expect(blocks.every((block) => block.numberOfChannels === 2)).toBe(true);
+ expect(blocks.slice(0, -1).every((block) => block.numberOfFrames === 1024)).toBe(true);
+ expect(blocks[blocks.length - 1].numberOfFrames).toBe(512);
+ // Each block carries twice its frame count: channel 0 then channel 1.
+ expect(blocks.every((block) => block.samples.length === block.numberOfFrames * 2)).toBe(true);
+
+ /* Timestamps are microseconds computed from the sample offset, not
+ accumulated — 1024 samples is 21333.33 µs, which no running total in
+ integers can hold without drifting. */
+ let offset = 0;
+ for (const block of blocks) {
+ expect(block.timestamp).toBe(Math.round((offset * 1_000_000) / 48_000));
+ offset += block.numberOfFrames;
+ }
+ expect(offset).toBe(192_000);
+ expect(blocks[1].timestamp).toBe(21_333);
+ expect(blocks[187].timestamp).toBe(3_989_333);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("puts an AAC track in the file, and the file says so when it is read back", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });
+ const result = await runAudioExport(harness);
+ expect(result.audio).toBe(true);
+
+ const { movie, track } = await audioTrackOf(result.bytes);
+ expect(movie.videoTracks).toHaveLength(1);
+ expect(track).not.toBeNull();
+ expect(track?.codec).toBe("mp4a.40.2");
+ expect(track?.audio?.sample_rate).toBe(48_000);
+ expect(track?.audio?.channel_count).toBe(2);
+ // One sample per encoded block, all 188 of them in the file.
+ expect(track?.nb_samples).toBe(188);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("opens the file with no audio track at all when this webview cannot encode sound", async () => {
+ const harness = installWebCodecs({ decodes: true, audio: false });
+ try {
+ // The score is deliberately not served: a webview with no AudioEncoder
+ // must not read an audio file it has no use for.
+ serveFiles({ video: await sampleFile(120, 30) });
+ const result = await runAudioExport(harness);
+
+ expect(result.audio).toBe(false);
+ expect(result.audioDetail).toMatch(/no WebCodecs AudioEncoder/i);
+ const { track } = await audioTrackOf(result.bytes);
+ expect(track).toBeNull();
+ expect(callLog().map((call) => call.args.sourcePath)).toEqual([PATHS.video]);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("says a clip outlasted its sound instead of leaving the silence unmentioned", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ /* Both clips ask for 2 s of a file that is only 1.5 s long — the head
+ from 0 s in, the reprise from 0.5 s in — so each one runs out early.
+ The graph already refused to invent the missing audio; nothing said
+ so, which is the whole of this test. */
+ serveFiles({ video: await sampleFile(120, 30), score: wavFile({ seconds: 1.5, sample: () => 0.5 }) });
+ const result = await runAudioExport(harness);
+
+ expect(result.audio).toBe(true);
+ expect(result.audioShortfalls).toEqual([
+ "“Score - head” runs for 2.00s but only 1.50s of Score is left from 0.00s in, so its last 0.50s is silence.",
+ "“Score - reprise” runs for 2.00s but only 1.00s of Score is left from 0.50s in, so its last 1.00s is silence.",
+ ]);
+ expect(result.audioDetail).toMatch(/2 clips run past the end of their sound/);
+ expect(result.audioProblems).toEqual([]);
+
+ /* And the silence is really there. The head sounds 0-1500 ms of its
+ 2000 ms; the reprise runs 1000-3000 ms and its sound stops at 2000. */
+ const [left] = encoderChannels(harness.recorded);
+ expect(left[at(1200)]).toBeCloseTo(1, 4); // head + reprise, both sounding
+ expect(left[at(1500)]).toBeCloseTo(0.5, 4); // the head has just run out
+ expect(left[at(1900)]).toBeCloseTo(0.5, 4); // reprise alone, still sounding
+ expect(left[at(2500)]).toBe(0); // reprise still on the timeline, out of sound
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("plays a clip from where it is trimmed to, and lays a mono file across both channels", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ /* One mono channel whose value IS its position in the file, so where the
+ mix reads from is legible in the samples themselves. */
+ const ramp = wavFile({
+ seconds: 3,
+ channels: 1,
+ sample: (frame) => (frame / (3 * 48_000)) * 0.5,
+ });
+ serveFiles({ video: await sampleFile(120, 30), score: ramp });
+ await runAudioExport(harness);
+ const [left, right] = encoderChannels(harness.recorded);
+
+ /* At 250 ms only "Score - head" plays, untrimmed, so the file is being
+ read at 250 ms: 0.25/3 × 0.5. */
+ expect(left[at(250)]).toBeCloseTo((0.25 / 3) * 0.5, 4);
+ /* At 2500 ms only "Score - reprise" plays, and it is trimmed to start
+ 500 ms into the file, so the file is at 2000 ms — not 2500. */
+ expect(left[at(2500)]).toBeCloseTo((2.0 / 3) * 0.5, 4);
+ // A mono source lands on both output channels, as the graph up-mixes it.
+ expect(right[at(2500)]).toBeCloseTo(left[at(2500)], 6);
+ } finally {
+ harness.restore();
+ }
+ });
+
+ it("names an audio file it could not decode rather than shipping a silent file", async () => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ serveFiles({ video: await sampleFile(120, 30), score: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]).buffer });
+ const result = await runAudioExport(harness);
+
+ expect(result.audio).toBe(false);
+ expect(result.audioProblems).toHaveLength(2);
+ // Once per clip, but the decode itself was attempted once — the failed
+ // promise is cached with the asset, so the file is read a single time.
+ expect(result.audioProblems[0]).toMatch(/“Score - head” \(Score\) could not be decoded/);
+ expect(result.audioProblems[0]).toMatch(/RIFF\/WAVE/);
+ expect(result.audioDetail).toMatch(/none of the audio clips could be decoded/);
+ expect(callLog()).toHaveLength(2);
+ const { track } = await audioTrackOf(result.bytes);
+ expect(track).toBeNull();
+ } finally {
+ harness.restore();
+ }
+ });
+});
+
+describe("what the mix costs in memory", () => {
+ it("is one float per channel per sample, and the plan says so on a long timeline", () => {
+ // 4 s: 192000 frames × 2 channels × 4 bytes.
+ expect(audioMixBytes(4_000)).toBe(1_536_000);
+ // The numbers the review measured: five minutes and an hour.
+ expect(audioMixBytes(5 * 60_000)).toBe(115_200_000);
+ expect(audioMixBytes(60 * 60_000)).toBe(1_382_400_000);
+
+ const config = audioProject();
+ const short = buildExportPlan(config, defaultExportSettings(config));
+ expect(short.notes.some((note) => /held in memory|mixed in memory/i.test(note))).toBe(false);
+
+ /* Stretch the same project to twenty minutes and the mix is 461 MB held
+ for the length of the render. That is worth a sentence. */
+ const long = {
+ ...config,
+ timeline: {
+ tracks: config.timeline.tracks.map((track) => ({
+ ...track,
+ clips: track.clips.map((clip) => ({ ...clip, durationMs: clip.durationMs * 300 })),
+ })),
+ },
+ };
+ const plan = buildExportPlan(long, defaultExportSettings(config));
+ const note = plan.notes.find((entry) => /mixed in memory/i.test(entry));
+ expect(note).toBeDefined();
+ expect(note).toMatch(/461 MB/);
+ expect(note).toMatch(/20:00\.000/);
+ expect(note).toMatch(/three times that/);
+ });
+});
+
describe("which compositor will really run", () => {
it("answers from the adapter, and says why when there is none", async () => {
// jsdom has no navigator.gpu at all — the honest answer is canvas2d with