Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Mux the sound the timeline already has

Bitti0ed5033ade85
diff --git a/src/components/workspace/ExportView.test.tsx b/src/components/workspace/ExportView.test.tsx
index 8515d8c..e014e74 100644
--- a/src/components/workspace/ExportView.test.tsx
+++ b/src/components/workspace/ExportView.test.tsx
@@ -57,9 +57,9 @@ describe("the export view where nothing can encode", () => {
expect(screen.getByText("1920 × 1080")).toBeTruthy();
});
- it("says outright that the file will have no sound", () => {
+ it("says outright that a timeline with no audio clips produces no sound", () => {
render(<ExportView config={project([clip("a", 0, 2_000)])} folderPath="/tmp/project" />);
- expect(screen.getByText("video only — see below")).toBeTruthy();
+ expect(screen.getByText("no audio clips on the timeline")).toBeTruthy();
});
it("blocks on an empty timeline rather than offering to render nothing", () => {
diff --git a/src/components/workspace/ExportView.tsx b/src/components/workspace/ExportView.tsx
index 72b4713..f7485fd 100644
--- a/src/components/workspace/ExportView.tsx
+++ b/src/components/workspace/ExportView.tsx
@@ -49,6 +49,9 @@ interface FinishedExport {
compositor: CompositorKind;
compositorDetail: string;
codecString: string;
+ audio: boolean;
+ audioDetail: string;
+ audioProblems: string[];
}
export function ExportView({ config, folderPath }: { config: ProjectConfig; folderPath: string }) {
@@ -233,6 +236,9 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
compositor: outcome.compositor,
compositorDetail: outcome.compositorDetail,
codecString: outcome.codecString,
+ audio: outcome.audio,
+ audioDetail: outcome.audioDetail,
+ audioProblems: outcome.audioProblems,
});
} catch (reason) {
if (reason instanceof ExportCancelled) setCancelled(true);
@@ -257,7 +263,21 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
formatBytes(estimatedBytes(bitrate, plan.durationMs)),
"target bitrate × duration, not a measurement",
],
- ["Sound", "None", "video only — see below"],
+ [
+ "Sound",
+ plan.audio.length === 0
+ ? "None"
+ : support.audio
+ ? `AAC · ${plan.audio.length} ${plan.audio.length === 1 ? "clip" : "clips"} mixed`
+ : "None",
+ plan.audio.length === 0
+ ? plan.audioClipCount > 0
+ ? "every audio clip falls outside the picture"
+ : "no audio clips on the timeline"
+ : support.audio
+ ? "48 kHz stereo, gaps filled with silence"
+ : "this webview has no AudioEncoder",
+ ],
[
"Compositor",
compositor ? (compositor.kind === "webgpu" ? "WebGPU" : "2D canvas") : "asking this computer…",
@@ -457,8 +477,15 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
<p>
{formatBytes(result.bytes)} written · {result.codecString} · composited with{" "}
{result.compositor === "webgpu" ? "WebGPU" : "a 2D canvas"}
- {result.compositorDetail ? <> — {result.compositorDetail}</> : null} · no audio track.
+ {result.compositorDetail ? <> — {result.compositorDetail}</> : null}.
</p>
+ <p>{result.audio ? "Sound: " : ""}{result.audioDetail}</p>
+ {result.audioProblems.length > 0 && <>
+ {/* Named, not dropped in silence: a soundtrack that vanished without
+ a word is the failure worth shouting about. */}
+ <p><TriangleAlert size={14} aria-hidden="true" /> Left out of the soundtrack:</p>
+ <ul>{result.audioProblems.map((problem) => <li key={problem}>{problem}</li>)}</ul>
+ </>}
</div>}
</section>
</div>
diff --git a/src/lib/export.test.ts b/src/lib/export.test.ts
index d8f4008..97ac656 100644
--- a/src/lib/export.test.ts
+++ b/src/lib/export.test.ts
@@ -200,7 +200,7 @@ describe("the plan", () => {
expect(plan.clipCount).toBe(2);
});
- it("says nothing about a soundtrack it cannot mux, but does not stay silent either", () => {
+ it("mixes the audio that lands inside the picture, and cuts it where the picture ends", () => {
const config = project(
[clip("a", "track-story", 0, 1_000), clip("score", "track-a2", 0, 5_000)],
[asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],
@@ -208,7 +208,70 @@ describe("the plan", () => {
const plan = buildExportPlan(config, settings());
expect(plan.audioClipCount).toBe(1);
expect(plan.durationMs).toBe(1_000);
- expect(plan.notes.join(" ")).toContain("no sound");
+ // The clip is 5s long over a 1s picture: 1s of it is in the file, and the
+ // page says where the rest went rather than dropping it in silence.
+ expect(plan.audio).toEqual([
+ { clipId: "score", assetId: "asset-score", label: "score", startMs: 0, durationMs: 1_000, sourceStartMs: 0 },
+ ]);
+ expect(plan.notes.join(" ")).toContain("mixed into an AAC track");
+ expect(plan.notes.join(" ")).toContain("stops with the picture at 00:01.000");
+ });
+
+ it("keeps its own trim: a clip starting late plays from where it was cut, not from zero", () => {
+ const config = project(
+ [
+ clip("a", "track-story", 0, 8_000),
+ clip("vo", "track-a1", 2_000, 3_000, { sourceStartMs: 4_500 }),
+ ],
+ [asset("asset-a"), asset("asset-vo", { kind: "audio", mimeType: "audio/wav" })],
+ );
+ expect(buildExportPlan(config, settings()).audio).toEqual([
+ { clipId: "vo", assetId: "asset-vo", label: "vo", startMs: 2_000, durationMs: 3_000, sourceStartMs: 4_500 },
+ ]);
+ });
+
+ it("leaves out audio that starts after the last frame, and says how much", () => {
+ const config = project(
+ [clip("a", "track-story", 0, 1_000), clip("late", "track-a1", 4_000, 2_000)],
+ [asset("asset-a"), asset("asset-late", { kind: "audio", mimeType: "audio/wav" })],
+ );
+ const plan = buildExportPlan(config, settings());
+ expect(plan.audio).toEqual([]);
+ expect(plan.notes.join(" ")).toContain("after the last video frame");
+ });
+
+ it("does not mix a muted track, and says that too", () => {
+ const config = project(
+ [clip("a", "track-story", 0, 4_000), clip("score", "track-a2", 0, 4_000)],
+ [asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],
+ );
+ const muted = {
+ ...config,
+ timeline: {
+ tracks: config.timeline.tracks.map((track) => (track.id === "track-a2" ? { ...track, muted: true } : track)),
+ },
+ };
+ const plan = buildExportPlan(muted, settings());
+ expect(plan.audio).toEqual([]);
+ expect(plan.notes.join(" ")).toContain("muted track");
+ });
+
+ it("counts overlapping audio as summed rather than picking a winner", () => {
+ const config = project(
+ [
+ clip("a", "track-story", 0, 6_000),
+ clip("vo", "track-a1", 0, 4_000),
+ clip("score", "track-a2", 2_000, 4_000),
+ ],
+ [
+ asset("asset-a"),
+ asset("asset-vo", { kind: "audio", mimeType: "audio/wav" }),
+ asset("asset-score", { kind: "audio", mimeType: "audio/wav" }),
+ ],
+ );
+ const plan = buildExportPlan(config, settings());
+ expect(plan.audio).toHaveLength(2);
+ expect(plan.notes.join(" ")).toContain("overlaps are summed");
});
it("blocks on an empty timeline, on missing media and on a container it cannot demux", () => {
diff --git a/src/lib/export.ts b/src/lib/export.ts
index 2cc0e50..b4fe94b 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -168,6 +168,51 @@ export type ExportSegment =
}
| { kind: "gap"; startFrame: number; endFrame: number };
+/** One audio clip, as much of it as lands inside the file.
+ *
+ * The picture decides how long the file is, so sound past the last frame is
+ * not in it. Rather than dropping such a clip silently or stretching the file
+ * to fit it, the audible part is worked out here and the remainder is reported
+ * as a note the user reads before pressing Export. */
+export interface AudioSegment {
+ clipId: string;
+ assetId: string;
+ label: string;
+ /** Where this lands in the OUTPUT, in ms from the start of the file. */
+ startMs: number;
+ /** How much of it lands in the output, in ms. */
+ durationMs: number;
+ /** Where inside the source file the audible part begins, in ms. */
+ sourceStartMs: number;
+}
+
+/** Every audio clip's audible part, in timeline order.
+ *
+ * Overlaps are NOT resolved here: two clips over the same moment are two
+ * segments, and the mixer sums them, the way a mixing desk does. Silence is
+ * not a segment — a gap is simply a stretch no segment covers. */
+export function audioSegments(config: ProjectConfig, durationMs: number): AudioSegment[] {
+ const segments: AudioSegment[] = [];
+ for (const track of config.timeline.tracks) {
+ if (track.kind !== "audio" || track.muted) continue;
+ for (const clip of track.clips) {
+ const start = Math.max(0, clip.startMs);
+ const end = Math.min(durationMs, clip.startMs + clip.durationMs);
+ if (end <= start) continue;
+ segments.push({
+ clipId: clip.id,
+ assetId: clip.assetId,
+ label: clip.label,
+ startMs: start,
+ durationMs: end - start,
+ // A clip dragged to a negative start would begin further into its file.
+ sourceStartMs: clip.sourceStartMs + (start - clip.startMs),
+ });
+ }
+ }
+ return segments.sort((a, b) => a.startMs - b.startMs);
+}
+
export interface ExportPlan {
width: number;
height: number;
@@ -180,8 +225,11 @@ export interface ExportPlan {
clipCount: number;
/** Frames with nothing over them; they get the project's background colour. */
gapFrames: number;
- /** Clips on audio tracks. They are NOT in the file — see `notes`. */
+ /** Clips on audio tracks, muted tracks included. */
audioClipCount: number;
+ /** The audible part of each one. Empty means the file has no sound, and the
+ * notes say why. */
+ audio: AudioSegment[];
/** Reasons this export cannot run at all. Empty means it can. */
blockers: string[];
/** True things about the output the user has to be told before they press go. */
@@ -314,12 +362,47 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
);
}
+ const audio = audioSegments(config, durationMs);
const notes: string[] = [];
- if (audioClipCount > 0) {
+ if (audio.length > 0) {
+ const overlapping = audio.filter((segment, index) =>
+ audio.some(
+ (other, otherIndex) =>
+ otherIndex !== index &&
+ other.startMs < segment.startMs + segment.durationMs &&
+ segment.startMs < other.startMs + other.durationMs,
+ ),
+ ).length;
+ notes.push(
+ `${audio.length} audio ${audio.length === 1 ? "clip is" : "clips are"} mixed into an AAC track${
+ overlapping > 0 ? `, ${overlapping} of them overlapping — overlaps are summed, as on a mixing desk` : ""
+ }. Anything no clip covers is silence.`,
+ );
+ }
+ /* Audio the file cannot hold, counted rather than dropped in silence. The
+ picture decides the length; sound past the last frame has nowhere to go. */
+ const audible = new Set(audio.map((segment) => segment.clipId));
+ const beyondPicture = tracks
+ .filter((track) => track.kind === "audio" && !track.muted)
+ .flatMap((track) => track.clips)
+ .filter((clip) => !audible.has(clip.id)).length;
+ if (beyondPicture > 0) {
notes.push(
- `The file will have no sound. ${audioClipCount} audio ${audioClipCount === 1 ? "clip is" : "clips are"} on the timeline and none of them are muxed yet.`,
+ `${beyondPicture} audio ${beyondPicture === 1 ? "clip starts" : "clips start"} after the last video frame, so ${beyondPicture === 1 ? "it is" : "they are"} not in the file: the picture decides how long it is.`,
);
}
+ const trimmed = audio.filter((segment) => segment.startMs + segment.durationMs >= durationMs).length;
+ if (trimmed > 0 && durationMs > 0) {
+ notes.push(
+ `The sound stops with the picture at ${formatDuration(durationMs)}, where ${trimmed === 1 ? "one clip is" : `${trimmed} clips are`} cut off.`,
+ );
+ }
+ const mutedClips = tracks
+ .filter((track) => track.kind === "audio" && track.muted)
+ .reduce((total, track) => total + track.clips.length, 0);
+ if (mutedClips > 0) {
+ notes.push(`${mutedClips} audio ${mutedClips === 1 ? "clip is" : "clips are"} on a muted track and will not be heard.`);
+ }
if (gapFrames > 0) {
notes.push(
`${gapFrames} of ${frameCount} frames have no clip over them and will be solid ${config.settings.backgroundColor}.`,
@@ -345,6 +428,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
clipCount: usedClipIds.size,
gapFrames,
audioClipCount,
+ audio,
blockers,
notes,
};
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
index 5271365..31793f1 100644
--- a/src/lib/exportPipeline.ts
+++ b/src/lib/exportPipeline.ts
@@ -22,6 +22,7 @@ import {
fitRect,
outputCodec,
sourceTimeMsForFrame,
+ type AudioSegment,
type ExportPlan,
type ExportSegment,
type ExportSettings,
@@ -37,6 +38,9 @@ export interface ExportSupport {
/** WebCodecs encode. Present in WebView2/Chromium, absent in older engines. */
encoder: boolean;
decoder: boolean;
+ /** WebCodecs AudioEncoder plus an OfflineAudioContext to mix into it. Without
+ * both, the file is video only and the page says so. */
+ audio: boolean;
/** `navigator.gpu` EXISTS. That is all this says, and it is not enough to
* promise a GPU compositor: on a VM, over RDP, on a blocklisted driver, the
* object is there and `requestAdapter()` still returns null. Ask
@@ -52,6 +56,10 @@ export function detectExportSupport(): ExportSupport {
return {
encoder: typeof scope.VideoEncoder === "function",
decoder: typeof scope.VideoDecoder === "function",
+ audio:
+ typeof scope.AudioEncoder === "function" &&
+ typeof scope.AudioData === "function" &&
+ typeof scope.OfflineAudioContext === "function",
webgpuApi: typeof navigator !== "undefined" && "gpu" in navigator && Boolean(navigator.gpu),
desktop: isTauri(),
};
@@ -433,11 +441,214 @@ export async function demux(bytes: ArrayBuffer, name: string): Promise<DemuxedSo
};
}
+/* ---------------------------------------------------------------------------
+ Sound — the same five stages, on the other kind of track.
+
+ Decode: the webview's own audio decoders, reached through
+ `decodeAudioData`. That is deliberate. mp4box takes ISOBMFF apart and nothing
+ else, and audio arrives as .wav, .mp3, .m4a and .flac far more often than as
+ .mp4 — routing it through AudioDecoder would mean writing a demuxer per
+ container to reach decoders the engine already exposes. Same hardware, same
+ no-FFmpeg rule.
+
+ Mix: an OfflineAudioContext, one buffer source per clip, started at the
+ clip's own position with its own trim. Overlapping clips SUM, which is what
+ two faders up on a desk do; stretches no clip covers render as the silence
+ they already are; and sources at another sample rate are resampled by the
+ graph rather than by arithmetic here.
+
+ Encode and mux: AudioEncoder to AAC, into the same mp4-muxer file as the
+ picture.
+ --------------------------------------------------------------------------- */
+
+/** 48 kHz stereo: the rate every AAC encoder accepts and the one nothing on a
+ * timeline has to be downsampled to reach. */
+const AUDIO_SAMPLE_RATE = 48_000;
+const AUDIO_CHANNELS = 2;
+const AUDIO_BITRATE = 192_000;
+/* AAC-LC. Not Opus: mp4-muxer will put Opus in an .mp4 and a large share of
+ players will then refuse the file, which is a worse outcome than a stated
+ absence of sound. */
+const AUDIO_CODEC = "mp4a.40.2";
+/** One AAC frame is 1024 samples; feeding the encoder in that unit keeps every
+ * packet full and the timestamps exact. */
+const AUDIO_BLOCK = 1024;
+
+export interface AudioMix {
+ /** One Float32Array per channel at AUDIO_SAMPLE_RATE, already limited. */
+ channels: Float32Array[];
+ frames: number;
+ /** Clips that are in the mix. */
+ used: number;
+ /** Clips that are NOT, each named with the reason. Never silently dropped. */
+ problems: string[];
+ /** True when summing overlaps drove the mix past full scale and it was
+ * limited to fit. Reported, because it is audible. */
+ limited: boolean;
+}
+
+/** Decodes and mixes every audio segment in the plan into one stereo buffer.
+ * Null when the plan has no sound in it at all. */
+async function mixAudio(options: {
+ folderPath: string;
+ config: ProjectConfig;
+ segments: AudioSegment[];
+ durationMs: number;
+ report: (detail: string) => void;
+}): Promise<AudioMix | null> {
+ const { folderPath, config, segments, durationMs, report } = options;
+ const frames = Math.floor((durationMs / 1000) * AUDIO_SAMPLE_RATE);
+ if (segments.length === 0 || frames <= 0) return null;
+
+ const context = new OfflineAudioContext({
+ numberOfChannels: AUDIO_CHANNELS,
+ length: frames,
+ sampleRate: AUDIO_SAMPLE_RATE,
+ });
+
+ /* One decode per ASSET, not per clip: the same music file cut into six pieces
+ is one file to read and one decode to pay for. */
+ const decoded = new Map<string, Promise<AudioBuffer>>();
+ const problems: string[] = [];
+ let used = 0;
+
+ for (const segment of segments) {
+ const asset = config.assets.find((candidate) => candidate.id === segment.assetId);
+ if (!asset) {
+ problems.push(`“${segment.label}” points at audio that is not in this project.`);
+ continue;
+ }
+ let buffer: AudioBuffer;
+ try {
+ let pending = decoded.get(asset.id);
+ if (!pending) {
+ report(`Reading ${asset.name}…`);
+ /* decodeAudioData DETACHES the buffer it is given, so the read result is
+ copied first — the same bytes may be wanted again by the picture. */
+ pending = readAssetBytes(folderPath, asset).then((bytes) => context.decodeAudioData(bytes.slice(0)));
+ decoded.set(asset.id, pending);
+ }
+ buffer = await pending;
+ } catch (reason) {
+ // Named, not swallowed. A soundtrack that vanished without a word is the
+ // failure this whole branch exists to avoid.
+ problems.push(
+ `“${segment.label}” (${asset.name}) could not be decoded by this computer: ${reason instanceof Error ? reason.message : String(reason)}`,
+ );
+ continue;
+ }
+ const offsetSeconds = segment.sourceStartMs / 1000;
+ if (offsetSeconds >= buffer.duration) {
+ problems.push(
+ `“${segment.label}” is trimmed to start ${offsetSeconds.toFixed(2)}s into ${asset.name}, which is only ${buffer.duration.toFixed(2)}s long, so it has nothing to play.`,
+ );
+ continue;
+ }
+ const source = context.createBufferSource();
+ source.buffer = buffer;
+ source.connect(context.destination);
+ source.start(
+ segment.startMs / 1000,
+ offsetSeconds,
+ Math.min(segment.durationMs / 1000, buffer.duration - offsetSeconds),
+ );
+ used += 1;
+ }
+
+ if (used === 0) return { channels: [], frames: 0, used: 0, problems, limited: false };
+
+ report(`Mixing ${used} audio ${used === 1 ? "clip" : "clips"}…`);
+ const rendered = await context.startRendering();
+ const channels: Float32Array[] = [];
+ let limited = false;
+ for (let channel = 0; channel < AUDIO_CHANNELS; channel += 1) {
+ const data = rendered.getChannelData(Math.min(channel, rendered.numberOfChannels - 1)).slice();
+ /* Summed clips can exceed full scale. Left alone, the encoder wraps them
+ into a crackle; clamped, they are the flat-topped peak every mixing desk
+ produces at the same point. Either way the user is told. */
+ for (let index = 0; index < data.length; index += 1) {
+ if (data[index] > 1 || data[index] < -1) {
+ data[index] = Math.max(-1, Math.min(1, data[index]));
+ limited = true;
+ }
+ }
+ channels.push(data);
+ }
+ return { channels, frames: rendered.length, used, problems, limited };
+}
+
+/** Whether this computer will encode AAC at all, asked of the encoder. */
+async function probeAudioCodec(): Promise<{ supported: boolean; detail: string }> {
+ if (typeof AudioEncoder === "undefined") {
+ return { supported: false, detail: "This webview has no WebCodecs AudioEncoder." };
+ }
+ try {
+ const support = await AudioEncoder.isConfigSupported({
+ codec: AUDIO_CODEC,
+ sampleRate: AUDIO_SAMPLE_RATE,
+ numberOfChannels: AUDIO_CHANNELS,
+ bitrate: AUDIO_BITRATE,
+ });
+ return support.supported
+ ? { supported: true, detail: `${AUDIO_CODEC} at ${AUDIO_SAMPLE_RATE / 1000} kHz stereo, ${AUDIO_BITRATE / 1000} kbit/s` }
+ : { supported: false, detail: `This computer's encoder refused AAC (${AUDIO_CODEC}) at 48 kHz stereo.` };
+ } catch (reason) {
+ return { supported: false, detail: reason instanceof Error ? reason.message : String(reason) };
+ }
+}
+
+/** Feeds the mixed buffer to the AAC encoder and the encoder's packets to the
+ * muxer. Returns once every packet is in the file. */
+async function encodeAudio(
+ mix: AudioMix,
+ add: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => void,
+): Promise<void> {
+ const failure: { reason: Error | null } = { reason: null };
+ const encoder = new AudioEncoder({
+ output: (chunk, meta) => add(chunk, meta),
+ error: (reason) => {
+ failure.reason = reason instanceof Error ? reason : new Error(String(reason));
+ },
+ });
+ encoder.configure({
+ codec: AUDIO_CODEC,
+ sampleRate: AUDIO_SAMPLE_RATE,
+ numberOfChannels: AUDIO_CHANNELS,
+ bitrate: AUDIO_BITRATE,
+ });
+ try {
+ for (let offset = 0; offset < mix.frames; offset += AUDIO_BLOCK) {
+ if (failure.reason) throw failure.reason;
+ const count = Math.min(AUDIO_BLOCK, mix.frames - offset);
+ // f32-planar: channel 0 then channel 1, exactly as the mix holds them.
+ const planar = new Float32Array(count * AUDIO_CHANNELS);
+ for (let channel = 0; channel < AUDIO_CHANNELS; channel += 1) {
+ planar.set(mix.channels[channel].subarray(offset, offset + count), channel * count);
+ }
+ const data = new AudioData({
+ format: "f32-planar",
+ sampleRate: AUDIO_SAMPLE_RATE,
+ numberOfFrames: count,
+ numberOfChannels: AUDIO_CHANNELS,
+ timestamp: Math.round((offset * 1_000_000) / AUDIO_SAMPLE_RATE),
+ data: planar,
+ });
+ encoder.encode(data);
+ data.close();
+ while (encoder.encodeQueueSize > 16) await tick();
+ }
+ await encoder.flush();
+ if (failure.reason) throw failure.reason;
+ } finally {
+ if (encoder.state !== "closed") encoder.close();
+ }
+}
+
/* ---------------------------------------------------------------------------
The run
--------------------------------------------------------------------------- */
-export type ExportPhase = "preparing" | "rendering" | "finishing" | "writing" | "done";
+export type ExportPhase = "preparing" | "mixing" | "rendering" | "finishing" | "writing" | "done";
export interface ExportProgress {
phase: ExportPhase;
@@ -461,6 +672,13 @@ export interface ExportResult {
* told nobody what to fix. */
compositorDetail: string;
codecString: string;
+ /** True when an AAC track went into the file. */
+ audio: boolean;
+ /** What the sound is, or why there is none. Always populated. */
+ audioDetail: string;
+ /** Audio clips that were left out, each named with its reason. Empty on a
+ * clean run; never a silent omission. */
+ audioProblems: string[];
}
export interface ExportRunOptions {
@@ -476,6 +694,31 @@ export interface ExportRunOptions {
const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+/** One sentence about the sound in the finished file, true in every case —
+ * including the cases where there is none. */
+function describeSoundtrack(
+ plan: ExportPlan,
+ support: ExportSupport,
+ probe: { supported: boolean; detail: string } | null,
+ mix: AudioMix | null,
+ soundtrack: AudioMix | null,
+): string {
+ if (plan.audio.length === 0) return "No sound: there are no audio clips inside the length of the picture.";
+ if (!support.audio) return "No sound: this webview has no WebCodecs AudioEncoder to make an audio track with.";
+ if (probe && !probe.supported) return `No sound: ${probe.detail}`;
+ if (!soundtrack) {
+ return mix && mix.problems.length > 0
+ ? "No sound: none of the audio clips could be decoded — see below."
+ : "No sound: the mix came out empty.";
+ }
+ const parts = [
+ `${soundtrack.used} audio ${soundtrack.used === 1 ? "clip" : "clips"} mixed to ${probe?.detail ?? AUDIO_CODEC}`,
+ ];
+ if (soundtrack.limited) parts.push("overlapping clips summed past full scale, so the peaks are limited");
+ if (soundtrack.problems.length > 0) parts.push(`${soundtrack.problems.length} left out — see below`);
+ return `${parts.join("; ")}.`;
+}
+
export async function runExport(options: ExportRunOptions): Promise<ExportResult> {
const { plan, settings, config, folderPath, bitrate, onProgress, cancelled } = options;
const support = detectExportSupport();
@@ -494,6 +737,26 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
if (cancelled()) throw new ExportCancelled();
};
+ /* Sound is prepared BEFORE the muxer exists, because an .mp4's track list is
+ fixed the moment the file is opened: whether there is an audio track is a
+ question that has to be answered first, not discovered at the end. */
+ report("mixing", 0, "Working out the soundtrack…");
+ const audioProbe = plan.audio.length === 0 ? null : await probeAudioCodec();
+ const mix =
+ audioProbe?.supported && support.audio
+ ? await mixAudio({
+ folderPath,
+ config,
+ segments: plan.audio,
+ durationMs: plan.durationMs,
+ report: (detail) => report("mixing", 0, detail),
+ })
+ : null;
+ const soundtrack = mix && mix.used > 0 && mix.frames > 0 ? mix : null;
+ const audioProblems = mix?.problems ?? [];
+ const audioDetail = describeSoundtrack(plan, support, audioProbe, mix, soundtrack);
+ stopIfCancelled();
+
report("preparing", 0, "Setting up the encoder…");
const { ArrayBufferTarget, Muxer } = await import("mp4-muxer");
const target = new ArrayBufferTarget();
@@ -505,6 +768,9 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
height: plan.height,
frameRate: plan.frameRate,
},
+ ...(soundtrack
+ ? { audio: { codec: "aac" as const, numberOfChannels: AUDIO_CHANNELS, sampleRate: AUDIO_SAMPLE_RATE } }
+ : {}),
fastStart: "in-memory",
});
@@ -741,6 +1007,10 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
}
}
+ if (soundtrack) {
+ report("finishing", framesDone, `Encoding the mixed soundtrack as AAC…`);
+ await encodeAudio(soundtrack, (chunk, meta) => muxer.addAudioChunk(chunk, meta));
+ }
report("finishing", framesDone, "Flushing the encoder and writing the MP4 index…");
await encoder.flush();
if (encoderFailure.reason) throw encoderFailure.reason;
@@ -751,6 +1021,9 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
compositor: stage.compositor.kind,
compositorDetail: stage.detail,
codecString: probe.codecString,
+ audio: Boolean(soundtrack),
+ audioDetail,
+ audioProblems,
};
} finally {
releaseLoaded();
diff --git a/src/styles/export.css b/src/styles/export.css
index 3bf2b40..41d72c2 100644
--- a/src/styles/export.css
+++ b/src/styles/export.css
@@ -387,3 +387,18 @@
margin-top: var(--space-2);
color: var(--text-muted);
}
+
+/* The audio clips that did NOT make it into the file, named one per line. This
+ list is the difference between a soundtrack that failed and one that vanished
+ without a word, so it is not allowed to read as a footnote. */
+.export-outcome ul {
+ margin: var(--space-2) 0 0;
+ padding-left: var(--space-4);
+ display: grid;
+ gap: var(--space-1);
+ color: var(--text-muted);
+}
+
+.export-outcome ul li {
+ line-height: var(--leading-normal);
+}