AI media harness for planning, generating, and editing locally
TypeScript
Run the audio path under test: real wav, real graph, real aac track
Bitti586a549e1e14
diff --git a/fixtures/project-v1-audio-mix.json b/fixtures/project-v1-audio-mix.jsonnew file mode 100644index 0000000..92cbf6d--- /dev/null+++ b/fixtures/project-v1-audio-mix.json@@ -0,0 +1,129 @@+{+ "schemaVersion": 1,+ "id": "project-audio-mix",+ "name": "Project with sound to mix",+ "createdAt": "2026-01-01T00:00:00.000Z",+ "updatedAt": "2026-01-01T00:00:00.000Z",+ "thumbnail": null,+ "settings": {+ "aspectRatio": "16:9",+ "resolution": "1080p",+ "frameRate": 30,+ "backgroundColor": "#10131a"+ },+ "brief": {+ "prompt": "a harbour at first light, with a score under it",+ "status": "draft",+ "targetDurationSeconds": 5,+ "aspectRatio": "16:9",+ "resolution": "1080p"+ },+ "assets": [+ {+ "id": "asset-rush",+ "kind": "video",+ "name": "Harbour rush",+ "sourcePath": "D:\\tmp\\harbour\\rush-01.mp4",+ "mimeType": "video/mp4",+ "durationMs": 4000,+ "width": 320,+ "height": 180,+ "createdAt": "2026-01-01T00:00:00.000Z"+ },+ {+ "id": "asset-score",+ "kind": "audio",+ "name": "Score",+ "sourcePath": "D:\\tmp\\harbour\\score.wav",+ "mimeType": "audio/wav",+ "durationMs": 3000,+ "createdAt": "2026-01-01T00:00:00.000Z"+ },+ {+ "id": "asset-voice",+ "kind": "audio",+ "name": "Voice-over take 3",+ "sourcePath": "D:\\tmp\\harbour\\vo-take-3.wav",+ "mimeType": "audio/wav",+ "durationMs": 1000,+ "createdAt": "2026-01-01T00:00:00.000Z"+ }+ ],+ "timeline": {+ "tracks": [+ {+ "id": "track-story",+ "kind": "video",+ "name": "Story",+ "locked": false,+ "muted": false,+ "clips": [+ {+ "id": "clip-picture",+ "assetId": "asset-rush",+ "trackId": "track-story",+ "startMs": 0,+ "durationMs": 4000,+ "sourceStartMs": 0,+ "label": "01 - Harbour",+ "status": "approved"+ }+ ]+ },+ {+ "id": "track-music",+ "kind": "audio",+ "name": "Music",+ "locked": false,+ "muted": false,+ "clips": [+ {+ "id": "clip-score-a",+ "assetId": "asset-score",+ "trackId": "track-music",+ "startMs": 0,+ "durationMs": 2000,+ "sourceStartMs": 0,+ "label": "Score - head",+ "status": "approved"+ },+ {+ "id": "clip-score-b",+ "assetId": "asset-score",+ "trackId": "track-music",+ "startMs": 1000,+ "durationMs": 2000,+ "sourceStartMs": 500,+ "label": "Score - reprise",+ "status": "approved"+ }+ ]+ },+ {+ "id": "track-voice-over",+ "kind": "audio",+ "name": "Voice-over",+ "locked": false,+ "muted": true,+ "clips": [+ {+ "id": "clip-vo",+ "assetId": "asset-voice",+ "trackId": "track-voice-over",+ "startMs": 0,+ "durationMs": 1000,+ "sourceStartMs": 0,+ "label": "VO - take 3",+ "status": "approved"+ }+ ]+ }+ ]+ },+ "references": [],+ "generationJobs": [],+ "agentConversation": {+ "messages": []+ },+ "providerSettings": {}+}diff --git a/src/lib/exportPipeline.test.ts b/src/lib/exportPipeline.test.tsindex f035eb9..11116c7 100644--- a/src/lib/exportPipeline.test.ts+++ b/src/lib/exportPipeline.test.ts@@ -2,8 +2,9 @@import { invoke } from "@tauri-apps/api/core";import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from "vitest";+import audioFixture from "../../fixtures/project-v1-audio-mix.json";import externalFixture from "../../fixtures/project-v1-external-media.json";-import { buildExportPlan, defaultExportSettings } from "./export";+import { audioMixBytes, buildExportPlan, defaultExportSettings } from "./export";import { parseProjectConfig, type ProjectAsset } from "./project";import { demux, ExportCancelled, PreviewSources, probeCompositor, runExport } from "./exportPipeline";@@ -188,42 +189,235 @@ describe("planning an export of external media", () => {});});-/* WebCodecs and OffscreenCanvas do not exist in jsdom, so the run below stands- them up. They do only what the pipeline asks of them and nothing more: the- decoder deliberately emits no frames, which makes the run fail at a known- point AFTER the source file has been read. What is asserted is the read. */-function installWebCodecs(): () => void {- const scope = globalThis as unknown as Record<string, unknown>;- const saved = {- VideoEncoder: scope.VideoEncoder,- VideoDecoder: scope.VideoDecoder,- VideoFrame: scope.VideoFrame,- EncodedVideoChunk: scope.EncodedVideoChunk,- OffscreenCanvas: scope.OffscreenCanvas,+/* ---------------------------------------------------------------------------+ A real .wav, built here.++ The sound half of the pipeline decodes through `decodeAudioData` rather than+ through mp4box, so what it has to survive is a container it did not write.+ These bytes are a genuine RIFF/WAVE file — header, `fmt ` chunk, interleaved+ 16-bit samples — and the harness decoder below parses them as one, including+ the detach `decodeAudioData` performs on its input. Nothing here hands the+ pipeline a pre-decoded buffer.+ --------------------------------------------------------------------------- */++const WAV_SAMPLE_RATE = 48_000;++function wavFile(options: {+ seconds: number;+ channels?: number;+ sampleRate?: number;+ /** The value of channel `channel` at frame `frame`, in -1..1. */+ sample: (frame: number, channel: number) => number;+}): ArrayBuffer {+ const channels = options.channels ?? 2;+ const sampleRate = options.sampleRate ?? WAV_SAMPLE_RATE;+ const frames = Math.round(options.seconds * sampleRate);+ const dataBytes = frames * channels * 2;+ const buffer = new ArrayBuffer(44 + dataBytes);+ const view = new DataView(buffer);+ const ascii = (offset: number, text: string) => {+ for (let index = 0; index < text.length; index += 1) view.setUint8(offset + index, text.charCodeAt(index));};+ ascii(0, "RIFF");+ view.setUint32(4, 36 + dataBytes, true);+ ascii(8, "WAVE");+ ascii(12, "fmt ");+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true); // PCM+ view.setUint16(22, channels, true);+ view.setUint32(24, sampleRate, true);+ view.setUint32(28, sampleRate * channels * 2, true); // byte rate+ view.setUint16(32, channels * 2, true); // block align+ view.setUint16(34, 16, true);+ ascii(36, "data");+ view.setUint32(40, dataBytes, true);+ let offset = 44;+ for (let frame = 0; frame < frames; frame += 1) {+ for (let channel = 0; channel < channels; channel += 1) {+ const value = Math.max(-1, Math.min(1, options.sample(frame, channel)));+ view.setInt16(offset, Math.max(-32768, Math.min(32767, Math.round(value * 32768))), true);+ offset += 2;+ }+ }+ return buffer;+}++/** 16-bit PCM back to planar floats, the way an audio decoder does it. Throws+ * on anything that is not a WAVE, which is how the "could not be decoded"+ * branch gets exercised with a real failure rather than a stubbed one. */+function parseWav(bytes: ArrayBuffer): { channels: Float32Array[]; sampleRate: number } {+ const view = new DataView(bytes);+ const tag = (offset: number) =>+ String.fromCharCode(view.getUint8(offset), view.getUint8(offset + 1), view.getUint8(offset + 2), view.getUint8(offset + 3));+ if (bytes.byteLength < 44 || tag(0) !== "RIFF" || tag(8) !== "WAVE") {+ throw new Error("The data is not a RIFF/WAVE file.");+ }+ let cursor = 12;+ let channelCount = 0;+ let sampleRate = 0;+ let data: { start: number; length: number } | null = null;+ while (cursor + 8 <= bytes.byteLength) {+ const name = tag(cursor);+ const size = view.getUint32(cursor + 4, true);+ if (name === "fmt ") {+ channelCount = view.getUint16(cursor + 10, true);+ sampleRate = view.getUint32(cursor + 12, true);+ } else if (name === "data") {+ data = { start: cursor + 8, length: size };+ }+ cursor += 8 + size + (size % 2);+ }+ if (!data || channelCount === 0) throw new Error("The WAVE file has no fmt or data chunk.");+ const frames = Math.floor(data.length / (channelCount * 2));+ const channels = Array.from({ length: channelCount }, () => new Float32Array(frames));+ for (let frame = 0; frame < frames; frame += 1) {+ for (let channel = 0; channel < channelCount; channel += 1) {+ channels[channel][frame] = view.getInt16(data.start + (frame * channelCount + channel) * 2, true) / 32768;+ }+ }+ return { channels, sampleRate };+}++/* ---------------------------------------------------------------------------+ WebCodecs, OffscreenCanvas and the offline audio graph, stood up for jsdom.++ These do what the browser does and nothing more — the audio graph really+ sums overlapping sources, really refuses to play past the end of a buffer,+ and `decodeAudioData` really detaches the ArrayBuffer it is handed. The+ numbers the mixing tests assert were measured in headless Chrome against the+ same timeline first; this harness reproduces them.+ --------------------------------------------------------------------------- */++interface HarnessRecord {+ /** Every AudioData the pipeline handed the encoder, with its samples. */+ audioData: Array<{ format: string; timestamp: number; numberOfFrames: number; numberOfChannels: number; samples: Float32Array }>;+ audioConfig: AudioEncoderConfig | null;+ audioChunks: number;+ videoChunks: number;+}++class HarnessAudioBuffer {+ constructor(+ readonly data: Float32Array[],+ readonly sampleRate: number,+ ) {}+ get numberOfChannels() {+ return this.data.length;+ }+ get length() {+ return this.data[0]?.length ?? 0;+ }+ get duration() {+ return this.length / this.sampleRate;+ }+ getChannelData(index: number) {+ return this.data[index];+ }+}++/** Options the harness needs to stand in for a browser. */+interface HarnessOptions {+ /** True: the decoder emits a frame per chunk, so a run reaches the muxer and+ * produces a real file. False: it emits nothing, which is what the read+ * tests want — they assert on the IPC log, not on a picture. */+ decodes?: boolean;+ /** False: no AudioEncoder/AudioData/OfflineAudioContext at all, which is the+ * webview `detectExportSupport().audio` has to answer false for. */+ audio?: boolean;+}++function installWebCodecs(options: HarnessOptions = {}): { restore: () => void; recorded: HarnessRecord } {+ const { decodes = false, audio = true } = options;+ const scope = globalThis as unknown as Record<string, unknown>;+ const names = [+ "VideoEncoder",+ "VideoDecoder",+ "VideoFrame",+ "EncodedVideoChunk",+ "OffscreenCanvas",+ "AudioEncoder",+ "AudioData",+ "EncodedAudioChunk",+ "OfflineAudioContext",+ ] as const;+ const saved: Record<string, unknown> = {};+ for (const name of names) saved[name] = scope[name];+ const recorded: HarnessRecord = { audioData: [], audioConfig: null, audioChunks: 0, videoChunks: 0 };++ class HarnessChunk {+ readonly type: "key" | "delta";+ readonly timestamp: number;+ readonly duration: number;+ private readonly payload: Uint8Array;+ constructor(init: { type: "key" | "delta"; timestamp: number; duration?: number; data: BufferSource }) {+ this.type = init.type;+ this.timestamp = init.timestamp;+ this.duration = init.duration ?? 0;+ this.payload =+ init.data instanceof Uint8Array ? new Uint8Array(init.data) : new Uint8Array(init.data as ArrayBuffer);+ }+ get byteLength() {+ return this.payload.byteLength;+ }+ copyTo(destination: Uint8Array) {+ destination.set(this.payload);+ }+ }+ scope.EncodedVideoChunk = HarnessChunk;+ scope.EncodedAudioChunk = class extends HarnessChunk {};+ const EncodedVideoChunkClass = scope.EncodedVideoChunk as typeof HarnessChunk;+ const EncodedAudioChunkClass = scope.EncodedAudioChunk as typeof HarnessChunk;+scope.VideoEncoder = class {static async isConfigSupported(config: unknown) {return { supported: true, config };}state = "configured";encodeQueueSize = 0;+ private readonly emit: (chunk: unknown, meta?: unknown) => void;+ private first = true;+ constructor(init: { output: (chunk: unknown, meta?: unknown) => void; error: (reason: unknown) => void }) {+ this.emit = init.output;+ }configure() {}- encode() {}+ encode(frame: { timestamp: number; duration: number }, options?: { keyFrame?: boolean }) {+ // A payload derived from the frame time, so the muxed file is real bytes+ // rather than a run of zeros nothing could tell apart.+ const chunk = new EncodedVideoChunkClass({+ type: options?.keyFrame ? "key" : "delta",+ timestamp: frame.timestamp,+ duration: frame.duration,+ data: new Uint8Array([0, 0, 0, 3, 0x65, frame.timestamp & 0xff, 0]),+ });+ recorded.videoChunks += 1;+ this.emit(chunk, this.first ? { decoderConfig: { codec: "avc1.42001f", codedWidth: 320, codedHeight: 180, description } } : undefined);+ this.first = false;+ }async flush() {}close() {this.state = "closed";}};+scope.VideoDecoder = class {state = "configured";decodeQueueSize = 0;+ private readonly emit: (frame: unknown) => void;+ constructor(init: { output: (frame: unknown) => void; error: (reason: unknown) => void }) {+ this.emit = init.output;+ }configure() {}- decode() {}+ decode(chunk: { timestamp: number; duration: number }) {+ if (!decodes) return;+ const Frame = scope.VideoFrame as new (source: unknown, init: { timestamp: number; duration?: number }) => unknown;+ this.emit(new Frame(null, { timestamp: chunk.timestamp, duration: chunk.duration }));+ }async flush() {}close() {this.state = "closed";}};+scope.VideoFrame = class {timestamp: number;duration: number;@@ -235,11 +429,7 @@ function installWebCodecs(): () => void {}close() {}};- scope.EncodedVideoChunk = class {- constructor(init: Record<string, unknown>) {- Object.assign(this, init);- }- };+scope.OffscreenCanvas = class {constructor(public width: number,@@ -249,12 +439,163 @@ function installWebCodecs(): () => void {return kind === "2d" ? { fillStyle: "", fillRect() {}, drawImage() {} } : null;}};- return () => Object.assign(scope, saved);++ if (audio) {+ scope.AudioData = class {+ readonly format: string;+ readonly sampleRate: number;+ readonly numberOfFrames: number;+ readonly numberOfChannels: number;+ readonly timestamp: number;+ /** A real AudioData copies its samples; so does this, which is what lets+ * the assertions read them after `close()`. */+ readonly samples: Float32Array;+ constructor(init: {+ format: string;+ sampleRate: number;+ numberOfFrames: number;+ numberOfChannels: number;+ timestamp: number;+ data: Float32Array;+ }) {+ this.format = init.format;+ this.sampleRate = init.sampleRate;+ this.numberOfFrames = init.numberOfFrames;+ this.numberOfChannels = init.numberOfChannels;+ this.timestamp = init.timestamp;+ this.samples = new Float32Array(init.data);+ }+ close() {}+ };++ scope.AudioEncoder = class {+ static async isConfigSupported(config: AudioEncoderConfig) {+ return { supported: true, config };+ }+ state = "unconfigured";+ encodeQueueSize = 0;+ private readonly emit: (chunk: unknown, meta?: unknown) => void;+ private first = true;+ constructor(init: { output: (chunk: unknown, meta?: unknown) => void; error: (reason: unknown) => void }) {+ this.emit = init.output;+ }+ configure(config: AudioEncoderConfig) {+ recorded.audioConfig = config;+ this.state = "configured";+ }+ encode(data: { format: string; timestamp: number; numberOfFrames: number; numberOfChannels: number; sampleRate: number; samples: Float32Array }) {+ recorded.audioData.push({+ format: data.format,+ timestamp: data.timestamp,+ numberOfFrames: data.numberOfFrames,+ numberOfChannels: data.numberOfChannels,+ samples: data.samples.slice(),+ });+ recorded.audioChunks += 1;+ this.emit(+ new EncodedAudioChunkClass({+ type: "key",+ timestamp: data.timestamp,+ duration: Math.round((data.numberOfFrames * 1_000_000) / data.sampleRate),+ data: new Uint8Array([0x21, data.numberOfFrames & 0xff, 0x00, 0x00]),+ }),+ // What a real AAC encoder emits alongside its first packet.+ this.first+ ? {+ decoderConfig: {+ codec: "mp4a.40.2",+ sampleRate: 48_000,+ numberOfChannels: 2,+ description: new Uint8Array([0x11, 0x90]),+ },+ }+ : undefined,+ );+ this.first = false;+ }+ async flush() {}+ close() {+ this.state = "closed";+ }+ };++ scope.OfflineAudioContext = class {+ readonly destination = { node: "destination" };+ readonly numberOfChannels: number;+ readonly length: number;+ readonly sampleRate: number;+ private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number }> = [];+ constructor(init: { numberOfChannels: number; length: number; sampleRate: number }) {+ this.numberOfChannels = init.numberOfChannels;+ this.length = init.length;+ this.sampleRate = init.sampleRate;+ }+ async decodeAudioData(bytes: ArrayBuffer): Promise<HarnessAudioBuffer> {+ /* The real one DETACHES its input. Reproduced, because the pipeline+ copies the read result before handing it over and nothing else would+ notice if that copy were removed. */+ const owned = structuredClone(bytes, { transfer: [bytes] });+ const { channels, sampleRate } = parseWav(owned);+ if (sampleRate !== this.sampleRate) {+ throw new Error(`This harness does not resample: the file is ${sampleRate} Hz, the graph is ${this.sampleRate} Hz.`);+ }+ return new HarnessAudioBuffer(channels, sampleRate);+ }+ createBufferSource() {+ const context = this;+ return {+ buffer: null as HarnessAudioBuffer | null,+ connected: false,+ connect(destination: unknown) {+ this.connected = destination === context.destination;+ },+ start(when: number, offset: number, duration: number) {+ if (!this.buffer || !this.connected) return; // an unconnected source is silence+ context.started.push({ buffer: this.buffer, when, offset, duration });+ },+ };+ }+ async startRendering(): Promise<HarnessAudioBuffer> {+ const out = Array.from({ length: this.numberOfChannels }, () => new Float32Array(this.length));+ for (const source of this.started) {+ const startFrame = Math.round(source.when * this.sampleRate);+ const offsetFrame = Math.round(source.offset * this.sampleRate);+ const count = Math.min(+ Math.round(source.duration * this.sampleRate),+ source.buffer.length - offsetFrame,+ this.length - startFrame,+ );+ for (let channel = 0; channel < this.numberOfChannels; channel += 1) {+ // Mono up-mixes onto both channels, as the browser's graph does.+ const from = source.buffer.getChannelData(Math.min(channel, source.buffer.numberOfChannels - 1));+ const into = out[channel];+ for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + index];+ }+ }+ return new HarnessAudioBuffer(out, this.sampleRate);+ }+ };+ } else {+ delete scope.AudioEncoder;+ delete scope.AudioData;+ delete scope.OfflineAudioContext;+ delete scope.EncodedAudioChunk;+ }++ return {+ recorded,+ restore: () => {+ for (const name of names) {+ if (saved[name] === undefined) delete scope[name];+ else scope[name] = saved[name];+ }+ },+ };}describe("running an export of external media", () => {it("dispatches the external read, not a project-relative one", async () => {- const restore = installWebCodecs();+ const { restore } = installWebCodecs();try {invoked.mockResolvedValue(await sampleFile(5, 30));const config = externalProject();@@ -282,7 +623,7 @@ describe("running an export of external media", () => {});it("stops on cancel without ever touching the file", async () => {- const restore = installWebCodecs();+ const { restore } = installWebCodecs();try {const config = externalProject();const settings = defaultExportSettings(config);@@ -304,6 +645,346 @@ describe("running an export of external media", () => {});});+/* ---------------------------------------------------------------------------+ The sound, actually running.++ Everything above this line is the picture. The audio path had six tests of+ its PLANNING and none of its execution: no AudioEncoder in the harness, no+ audio clips in the fixture, so `mixAudio` had never been called once. These+ run it — from the .wav bytes the IPC layer hands back, through the graph, the+ encoder and into a file that is taken apart again to see the track.++ The mixing numbers below were measured first in headless Chrome against this+ same timeline: 192000 rendered frames in 2 channels, 0.80 at 500 ms where one+ clip plays alone, 1.60 at 1200 ms where two overlap and sum, 0 at 3500 ms+ where nothing covers, and a clamp to 1 at the peak.+ --------------------------------------------------------------------------- */++const AUDIO_FOLDER = "D:\\tmp\\harbour";+const PATHS = {+ video: "D:\\tmp\\harbour\\rush-01.mp4",+ score: "D:\\tmp\\harbour\\score.wav",+ voice: "D:\\tmp\\harbour\\vo-take-3.wav",+};++const audioProject = () => parseProjectConfig(audioFixture);++/** The IPC layer, answering with real files per path. Anything unasked-for+ * throws, so a read the export should not have made shows up as a failure+ * rather than as a convenient default. */+function serveFiles(files: Partial<Record<keyof typeof PATHS, ArrayBuffer>>) {+ invoked.mockImplementation(async (_command: string, args: { sourcePath?: string }) => {+ for (const [name, path] of Object.entries(PATHS)) {+ if (args?.sourcePath === path) {+ const bytes = files[name as keyof typeof PATHS];+ if (!bytes) throw new Error(`the test served no ${name}, so this read should not have happened`);+ return bytes;+ }+ }+ throw new Error(`unexpected read of ${String(args?.sourcePath)}`);+ });+}++/** A three-second stereo score at 0.8 of full scale — loud, but not clipping+ * until two copies of it overlap. */+const scoreFile = () => wavFile({ seconds: 3, sample: () => 0.8 });++async function runAudioExport(+ harness: ReturnType<typeof installWebCodecs>,+ config = audioProject(),+) {+ const settings = defaultExportSettings(config);+ return runExport({+ folderPath: AUDIO_FOLDER,+ config,+ settings,+ plan: buildExportPlan(config, settings),+ bitrate: 5_000_000,+ onProgress: () => {},+ cancelled: () => false,+ }).finally(() => void harness);+}++/** The samples the encoder was actually handed, put back in channel order —+ * which only works if the planar blocks are laid out the way the muxer and the+ * encoder expect: all of channel 0, then all of channel 1, per block. */+function encoderChannels(recorded: HarnessRecord): Float32Array[] {+ const total = recorded.audioData.reduce((sum, block) => sum + block.numberOfFrames, 0);+ const channels = [new Float32Array(total), new Float32Array(total)];+ let offset = 0;+ for (const block of recorded.audioData) {+ for (let channel = 0; channel < 2; channel += 1) {+ channels[channel].set(+ block.samples.subarray(channel * block.numberOfFrames, (channel + 1) * block.numberOfFrames),+ offset,+ );+ }+ offset += block.numberOfFrames;+ }+ return channels;+}++/** The sample index of a moment in the finished soundtrack. */+const at = (ms: number) => Math.round((ms / 1000) * 48_000);++/** The audio track of a finished file, read back out of the muxed bytes with+ * mp4box — the same demuxer a player uses, not the muxer's own bookkeeping. */+async function audioTrackOf(bytes: Uint8Array) {+ const { createFile, MP4BoxBuffer } = await import("mp4box");+ type Movie = import("mp4box").Movie;+ const file = createFile();+ const found: { movie: Movie | null } = { movie: null };+ file.onReady = (info) => {+ found.movie = info;+ };+ const copy = bytes.slice().buffer;+ file.appendBuffer(MP4BoxBuffer.fromArrayBuffer(copy, 0), true);+ file.flush();+ if (!found.movie) throw new Error("the exported file has no readable MP4 header");+ return { movie: found.movie, track: found.movie.audioTracks[0] ?? null };+}++describe("mixing real sound into a real export", () => {+ it("reads each audio ASSET once, before the file is opened, and never a muted one", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });+ await runAudioExport(harness);++ /* Two clips share one asset, so there is one read for them, not two —+ and the muted voice-over is never fetched at all. The score comes+ FIRST: the mix has to be settled before the muxer can be told whether+ the file has an audio track. */+ expect(callLog()).toEqual([+ { cmd: "read_external_media_file", args: { folderPath: AUDIO_FOLDER, sourcePath: PATHS.score } },+ { cmd: "read_external_media_file", args: { folderPath: AUDIO_FOLDER, sourcePath: PATHS.video } },+ ]);+ } finally {+ harness.restore();+ }+ });++ it("sums overlapping clips and clips the sum flat, saying how far over it went", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });+ const result = await runAudioExport(harness);+ const [left, right] = encoderChannels(harness.recorded);++ // 4 s of picture at 48 kHz, in two channels: the whole soundtrack.+ expect(left).toHaveLength(192_000);+ expect(right).toHaveLength(192_000);+ // One clip alone; two overlapping, summed and then cut to full scale;+ // a stretch no clip covers, which is real silence.+ expect(left[at(500)]).toBeCloseTo(0.8, 4);+ expect(left[at(1200)]).toBe(1);+ expect(right[at(1200)]).toBe(1);+ expect(left[at(2500)]).toBeCloseTo(0.8, 4);+ expect(left[at(3500)]).toBe(0);++ /* The overlap is timeline 1000-2000 ms: 48000 frames in two channels,+ every one of them over full scale. "Limited" would have been the wrong+ word for it — nothing rode the gain down, the tops were cut off. */+ expect(result.audioDetail).toMatch(/peaked at 1\.60 of full scale/);+ // The thousands separator is whatever the running locale uses.+ expect(result.audioDetail).toMatch(/96.?000 samples \(25\.0%\) were clipped flat/);+ expect(result.audioDetail).toMatch(/down by about 4\.1 dB/);+ expect(result.audioDetail).not.toMatch(/limited/i);+ expect(result.audioProblems).toEqual([]);+ expect(result.audioShortfalls).toEqual([]);+ } finally {+ harness.restore();+ }+ });++ it("feeds the encoder f32-planar blocks on a microsecond clock", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });+ await runAudioExport(harness);+ const blocks = harness.recorded.audioData;++ expect(harness.recorded.audioConfig).toEqual({+ codec: "mp4a.40.2",+ sampleRate: 48_000,+ numberOfChannels: 2,+ bitrate: 192_000,+ });+ // 192000 samples in AAC's own 1024-sample frame: 187 full blocks and a+ // 512-sample remainder. A block of 0 would be an encoder error.+ expect(blocks).toHaveLength(188);+ expect(blocks.every((block) => block.format === "f32-planar")).toBe(true);+ expect(blocks.every((block) => block.numberOfChannels === 2)).toBe(true);+ expect(blocks.slice(0, -1).every((block) => block.numberOfFrames === 1024)).toBe(true);+ expect(blocks[blocks.length - 1].numberOfFrames).toBe(512);+ // Each block carries twice its frame count: channel 0 then channel 1.+ expect(blocks.every((block) => block.samples.length === block.numberOfFrames * 2)).toBe(true);++ /* Timestamps are microseconds computed from the sample offset, not+ accumulated — 1024 samples is 21333.33 µs, which no running total in+ integers can hold without drifting. */+ let offset = 0;+ for (const block of blocks) {+ expect(block.timestamp).toBe(Math.round((offset * 1_000_000) / 48_000));+ offset += block.numberOfFrames;+ }+ expect(offset).toBe(192_000);+ expect(blocks[1].timestamp).toBe(21_333);+ expect(blocks[187].timestamp).toBe(3_989_333);+ } finally {+ harness.restore();+ }+ });++ it("puts an AAC track in the file, and the file says so when it is read back", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ serveFiles({ video: await sampleFile(120, 30), score: scoreFile() });+ const result = await runAudioExport(harness);+ expect(result.audio).toBe(true);++ const { movie, track } = await audioTrackOf(result.bytes);+ expect(movie.videoTracks).toHaveLength(1);+ expect(track).not.toBeNull();+ expect(track?.codec).toBe("mp4a.40.2");+ expect(track?.audio?.sample_rate).toBe(48_000);+ expect(track?.audio?.channel_count).toBe(2);+ // One sample per encoded block, all 188 of them in the file.+ expect(track?.nb_samples).toBe(188);+ } finally {+ harness.restore();+ }+ });++ it("opens the file with no audio track at all when this webview cannot encode sound", async () => {+ const harness = installWebCodecs({ decodes: true, audio: false });+ try {+ // The score is deliberately not served: a webview with no AudioEncoder+ // must not read an audio file it has no use for.+ serveFiles({ video: await sampleFile(120, 30) });+ const result = await runAudioExport(harness);++ expect(result.audio).toBe(false);+ expect(result.audioDetail).toMatch(/no WebCodecs AudioEncoder/i);+ const { track } = await audioTrackOf(result.bytes);+ expect(track).toBeNull();+ expect(callLog().map((call) => call.args.sourcePath)).toEqual([PATHS.video]);+ } finally {+ harness.restore();+ }+ });++ it("says a clip outlasted its sound instead of leaving the silence unmentioned", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ /* Both clips ask for 2 s of a file that is only 1.5 s long — the head+ from 0 s in, the reprise from 0.5 s in — so each one runs out early.+ The graph already refused to invent the missing audio; nothing said+ so, which is the whole of this test. */+ serveFiles({ video: await sampleFile(120, 30), score: wavFile({ seconds: 1.5, sample: () => 0.5 }) });+ const result = await runAudioExport(harness);++ expect(result.audio).toBe(true);+ expect(result.audioShortfalls).toEqual([+ "“Score - head” runs for 2.00s but only 1.50s of Score is left from 0.00s in, so its last 0.50s is silence.",+ "“Score - reprise” runs for 2.00s but only 1.00s of Score is left from 0.50s in, so its last 1.00s is silence.",+ ]);+ expect(result.audioDetail).toMatch(/2 clips run past the end of their sound/);+ expect(result.audioProblems).toEqual([]);++ /* And the silence is really there. The head sounds 0-1500 ms of its+ 2000 ms; the reprise runs 1000-3000 ms and its sound stops at 2000. */+ const [left] = encoderChannels(harness.recorded);+ expect(left[at(1200)]).toBeCloseTo(1, 4); // head + reprise, both sounding+ expect(left[at(1500)]).toBeCloseTo(0.5, 4); // the head has just run out+ expect(left[at(1900)]).toBeCloseTo(0.5, 4); // reprise alone, still sounding+ expect(left[at(2500)]).toBe(0); // reprise still on the timeline, out of sound+ } finally {+ harness.restore();+ }+ });++ it("plays a clip from where it is trimmed to, and lays a mono file across both channels", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ /* One mono channel whose value IS its position in the file, so where the+ mix reads from is legible in the samples themselves. */+ const ramp = wavFile({+ seconds: 3,+ channels: 1,+ sample: (frame) => (frame / (3 * 48_000)) * 0.5,+ });+ serveFiles({ video: await sampleFile(120, 30), score: ramp });+ await runAudioExport(harness);+ const [left, right] = encoderChannels(harness.recorded);++ /* At 250 ms only "Score - head" plays, untrimmed, so the file is being+ read at 250 ms: 0.25/3 × 0.5. */+ expect(left[at(250)]).toBeCloseTo((0.25 / 3) * 0.5, 4);+ /* At 2500 ms only "Score - reprise" plays, and it is trimmed to start+ 500 ms into the file, so the file is at 2000 ms — not 2500. */+ expect(left[at(2500)]).toBeCloseTo((2.0 / 3) * 0.5, 4);+ // A mono source lands on both output channels, as the graph up-mixes it.+ expect(right[at(2500)]).toBeCloseTo(left[at(2500)], 6);+ } finally {+ harness.restore();+ }+ });++ it("names an audio file it could not decode rather than shipping a silent file", async () => {+ const harness = installWebCodecs({ decodes: true });+ try {+ serveFiles({ video: await sampleFile(120, 30), score: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]).buffer });+ const result = await runAudioExport(harness);++ expect(result.audio).toBe(false);+ expect(result.audioProblems).toHaveLength(2);+ // Once per clip, but the decode itself was attempted once — the failed+ // promise is cached with the asset, so the file is read a single time.+ expect(result.audioProblems[0]).toMatch(/“Score - head” \(Score\) could not be decoded/);+ expect(result.audioProblems[0]).toMatch(/RIFF\/WAVE/);+ expect(result.audioDetail).toMatch(/none of the audio clips could be decoded/);+ expect(callLog()).toHaveLength(2);+ const { track } = await audioTrackOf(result.bytes);+ expect(track).toBeNull();+ } finally {+ harness.restore();+ }+ });+});++describe("what the mix costs in memory", () => {+ it("is one float per channel per sample, and the plan says so on a long timeline", () => {+ // 4 s: 192000 frames × 2 channels × 4 bytes.+ expect(audioMixBytes(4_000)).toBe(1_536_000);+ // The numbers the review measured: five minutes and an hour.+ expect(audioMixBytes(5 * 60_000)).toBe(115_200_000);+ expect(audioMixBytes(60 * 60_000)).toBe(1_382_400_000);++ const config = audioProject();+ const short = buildExportPlan(config, defaultExportSettings(config));+ expect(short.notes.some((note) => /held in memory|mixed in memory/i.test(note))).toBe(false);++ /* Stretch the same project to twenty minutes and the mix is 461 MB held+ for the length of the render. That is worth a sentence. */+ const long = {+ ...config,+ timeline: {+ tracks: config.timeline.tracks.map((track) => ({+ ...track,+ clips: track.clips.map((clip) => ({ ...clip, durationMs: clip.durationMs * 300 })),+ })),+ },+ };+ const plan = buildExportPlan(long, defaultExportSettings(config));+ const note = plan.notes.find((entry) => /mixed in memory/i.test(entry));+ expect(note).toBeDefined();+ expect(note).toMatch(/461 MB/);+ expect(note).toMatch(/20:00\.000/);+ expect(note).toMatch(/three times that/);+ });+});+describe("which compositor will really run", () => {it("answers from the adapter, and says why when there is none", async () => {// jsdom has no navigator.gpu at all — the honest answer is canvas2d with