Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Name the clipping, report the silence, state the mix's memory

Bitti06ee54e184d9
diff --git a/src/components/workspace/ExportView.tsx b/src/components/workspace/ExportView.tsx
index f7485fd..a0d6fa7 100644
--- a/src/components/workspace/ExportView.tsx
+++ b/src/components/workspace/ExportView.tsx
@@ -1,6 +1,7 @@
import { CircleCheck, Download, FileVideo, TriangleAlert, X } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import {
+ audioMixBytes,
bitrateFor,
buildExportPlan,
defaultExportSettings,
@@ -52,6 +53,7 @@ interface FinishedExport {
audio: boolean;
audioDetail: string;
audioProblems: string[];
+ audioShortfalls: string[];
}
export function ExportView({ config, folderPath }: { config: ProjectConfig; folderPath: string }) {
@@ -239,6 +241,7 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
audio: outcome.audio,
audioDetail: outcome.audioDetail,
audioProblems: outcome.audioProblems,
+ audioShortfalls: outcome.audioShortfalls,
});
} catch (reason) {
if (reason instanceof ExportCancelled) setCancelled(true);
@@ -275,7 +278,10 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
? "every audio clip falls outside the picture"
: "no audio clips on the timeline"
: support.audio
- ? "48 kHz stereo, gaps filled with silence"
+ ? /* The mix is built whole before the file is opened and held until
+ the last frame, so its size is a real cost of this export and is
+ stated with the rest of them rather than discovered as a crash. */
+ `48 kHz stereo, gaps filled with silence · ${formatBytes(audioMixBytes(plan.durationMs))} of memory held while it renders`
: "this webview has no AudioEncoder",
],
[
@@ -486,6 +492,13 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold
<p><TriangleAlert size={14} aria-hidden="true" /> Left out of the soundtrack:</p>
<ul>{result.audioProblems.map((problem) => <li key={problem}>{problem}</li>)}</ul>
</>}
+ {result.audioShortfalls.length > 0 && <>
+ {/* These clips ARE in the file. Part of each one is silence because
+ the sound behind it ran out, which is exactly as inaudible as a
+ missing clip and was previously reported nowhere. */}
+ <p><TriangleAlert size={14} aria-hidden="true" /> Ran out of sound before the clip ended:</p>
+ <ul>{result.audioShortfalls.map((shortfall) => <li key={shortfall}>{shortfall}</li>)}</ul>
+ </>}
</div>}
</section>
</div>
diff --git a/src/lib/export.ts b/src/lib/export.ts
index b4fe94b..17f5901 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -168,6 +168,32 @@ export type ExportSegment =
}
| { kind: "gap"; startFrame: number; endFrame: number };
+/* The shape of the soundtrack, kept here rather than in the pipeline because
+ the export PAGE has to be able to state its cost before anything runs.
+
+ 48 kHz stereo: the rate every AAC encoder accepts and the one nothing on a
+ timeline has to be downsampled to reach. */
+export const AUDIO_SAMPLE_RATE = 48_000;
+export const AUDIO_CHANNELS = 2;
+
+/** How much memory the mixed soundtrack occupies while an export runs.
+ *
+ * The mix is one Float32 per channel per sample and it is built BEFORE the
+ * muxer opens — an .mp4's track list is fixed the moment the file is created,
+ * so whether there is sound has to be settled first — and then held until the
+ * last video frame has been encoded. That is 11.5 MB a minute, so a five
+ * minute cut holds 115 MB across the whole render and an hour holds 1.4 GB,
+ * roughly three times that at the peak of mixing while the decoded sources are
+ * still alive. Nobody should meet that number by running out of memory. */
+export function audioMixBytes(durationMs: number): number {
+ return Math.max(0, Math.floor((durationMs / 1000) * AUDIO_SAMPLE_RATE)) * AUDIO_CHANNELS * 4;
+}
+
+/** Where "this is a lot of memory" starts. 200 MB is a little under 15 minutes
+ * of timeline: below it the mix is a rounding error next to the decoded video,
+ * above it it is a number worth reading before pressing Export. */
+const AUDIO_MIX_NOTE_BYTES = 200_000_000;
+
/** One audio clip, as much of it as lands inside the file.
*
* The picture decides how long the file is, so sound past the last frame is
@@ -379,6 +405,15 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
}. Anything no clip covers is silence.`,
);
}
+ /* The mix is held whole, in memory, for the length of the render. On a short
+ cut that is not worth a sentence; on a long one it is the thing most likely
+ to end the export, and it was not being said anywhere. */
+ const mixBytes = audioMixBytes(durationMs);
+ if (audio.length > 0 && mixBytes >= AUDIO_MIX_NOTE_BYTES) {
+ notes.push(
+ `The whole soundtrack is mixed in memory before the file is opened and held until the last frame: about ${formatBytes(mixBytes)} for ${formatDuration(durationMs)}, and roughly three times that at the moment of mixing while the source files are still decoded.`,
+ );
+ }
/* Audio the file cannot hold, counted rather than dropped in silence. The
picture decides the length; sound past the last frame has nowhere to go. */
const audible = new Set(audio.map((segment) => segment.clipId));
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
index 31793f1..238ed2a 100644
--- a/src/lib/exportPipeline.ts
+++ b/src/lib/exportPipeline.ts
@@ -19,6 +19,8 @@
import { invoke } from "@tauri-apps/api/core";
import { isTauri, readMediaFileBytes } from "./persistence";
import {
+ AUDIO_CHANNELS,
+ AUDIO_SAMPLE_RATE,
fitRect,
outputCodec,
sourceTimeMsForFrame,
@@ -461,10 +463,8 @@ export async function demux(bytes: ArrayBuffer, name: string): Promise<DemuxedSo
picture.
--------------------------------------------------------------------------- */
-/** 48 kHz stereo: the rate every AAC encoder accepts and the one nothing on a
- * timeline has to be downsampled to reach. */
-const AUDIO_SAMPLE_RATE = 48_000;
-const AUDIO_CHANNELS = 2;
+/* AUDIO_SAMPLE_RATE and AUDIO_CHANNELS live in export.ts, with the arithmetic
+ that lets the page state what the mix will cost in memory before it runs. */
const AUDIO_BITRATE = 192_000;
/* AAC-LC. Not Opus: mp4-muxer will put Opus in an .mp4 and a large share of
players will then refuse the file, which is a worse outcome than a stated
@@ -475,16 +475,23 @@ const AUDIO_CODEC = "mp4a.40.2";
const AUDIO_BLOCK = 1024;
export interface AudioMix {
- /** One Float32Array per channel at AUDIO_SAMPLE_RATE, already limited. */
+ /** One Float32Array per channel at AUDIO_SAMPLE_RATE, already clipped. */
channels: Float32Array[];
frames: number;
/** Clips that are in the mix. */
used: number;
/** Clips that are NOT, each named with the reason. Never silently dropped. */
problems: string[];
- /** True when summing overlaps drove the mix past full scale and it was
- * limited to fit. Reported, because it is audible. */
- limited: boolean;
+ /** Clips that ARE in the mix but whose source file ran out before the clip
+ * did, so part of them is silence. The graph already refuses to invent the
+ * missing audio; this is the sentence that says it happened. */
+ shortfalls: string[];
+ /** How many samples the sum drove past full scale, and were therefore hard
+ * clipped to ±1. Not "limited": no gain was ridden, the tops were cut off. */
+ clippedSamples: number;
+ /** The loudest magnitude the sum reached BEFORE clipping. 1.6 means the mix
+ * was 4 dB over and says how much has to come down. */
+ peak: number;
}
/** Decodes and mixes every audio segment in the plan into one stereo buffer.
@@ -510,6 +517,7 @@ async function mixAudio(options: {
is one file to read and one decode to pay for. */
const decoded = new Map<string, Promise<AudioBuffer>>();
const problems: string[] = [];
+ const shortfalls: string[] = [];
let used = 0;
for (const segment of segments) {
@@ -544,37 +552,57 @@ async function mixAudio(options: {
);
continue;
}
+ /* A clip can be longer than what is left of its file — drop a five second
+ stand-in onto a two second recording and three seconds of it have no
+ sound behind them. The graph plays what exists and no more, which is
+ right; what was missing was anyone saying so. Below a millisecond this is
+ rounding between a millisecond timeline and a 48 kHz buffer, not a hole. */
+ const wantedSeconds = segment.durationMs / 1000;
+ const availableSeconds = buffer.duration - offsetSeconds;
+ const playSeconds = Math.min(wantedSeconds, availableSeconds);
+ if (wantedSeconds - availableSeconds > 0.001) {
+ shortfalls.push(
+ `“${segment.label}” runs for ${wantedSeconds.toFixed(2)}s but only ${availableSeconds.toFixed(2)}s of ${asset.name} is left from ${offsetSeconds.toFixed(2)}s in, so its last ${(wantedSeconds - availableSeconds).toFixed(2)}s is silence.`,
+ );
+ }
const source = context.createBufferSource();
source.buffer = buffer;
source.connect(context.destination);
- source.start(
- segment.startMs / 1000,
- offsetSeconds,
- Math.min(segment.durationMs / 1000, buffer.duration - offsetSeconds),
- );
+ source.start(segment.startMs / 1000, offsetSeconds, playSeconds);
used += 1;
}
- if (used === 0) return { channels: [], frames: 0, used: 0, problems, limited: false };
+ if (used === 0) {
+ return { channels: [], frames: 0, used: 0, problems, shortfalls, clippedSamples: 0, peak: 0 };
+ }
report(`Mixing ${used} audio ${used === 1 ? "clip" : "clips"}…`);
const rendered = await context.startRendering();
const channels: Float32Array[] = [];
- let limited = false;
+ let clippedSamples = 0;
+ let peak = 0;
for (let channel = 0; channel < AUDIO_CHANNELS; channel += 1) {
const data = rendered.getChannelData(Math.min(channel, rendered.numberOfChannels - 1)).slice();
/* Summed clips can exceed full scale. Left alone, the encoder wraps them
- into a crackle; clamped, they are the flat-topped peak every mixing desk
- produces at the same point. Either way the user is told. */
+ into a crackle; cut to ±1, they are flat-topped — which is CLIPPING, and
+ is called that here and on the page. It is deliberately not a limiter: a
+ limiter rides the gain of the whole mix over an attack and a release the
+ user never chose, and quietly reshaping someone's soundtrack is a worse
+ failure than a labelled one. What is reported instead is the measurement
+ a limiter would need anyway — how far over, and how much of it — so the
+ fix is to pull a fader down by a known amount. */
for (let index = 0; index < data.length; index += 1) {
- if (data[index] > 1 || data[index] < -1) {
- data[index] = Math.max(-1, Math.min(1, data[index]));
- limited = true;
+ const value = data[index];
+ const magnitude = value < 0 ? -value : value;
+ if (magnitude > peak) peak = magnitude;
+ if (magnitude > 1) {
+ data[index] = value > 0 ? 1 : -1;
+ clippedSamples += 1;
}
}
channels.push(data);
}
- return { channels, frames: rendered.length, used, problems, limited };
+ return { channels, frames: rendered.length, used, problems, shortfalls, clippedSamples, peak };
}
/** Whether this computer will encode AAC at all, asked of the encoder. */
@@ -679,6 +707,9 @@ export interface ExportResult {
/** Audio clips that were left out, each named with its reason. Empty on a
* clean run; never a silent omission. */
audioProblems: string[];
+ /** Audio clips that ARE in the file but ran out of sound before they ran out
+ * of clip. The silence is real and it is now named. */
+ audioShortfalls: string[];
}
export interface ExportRunOptions {
@@ -714,7 +745,17 @@ function describeSoundtrack(
const parts = [
`${soundtrack.used} audio ${soundtrack.used === 1 ? "clip" : "clips"} mixed to ${probe?.detail ?? AUDIO_CODEC}`,
];
- if (soundtrack.limited) parts.push("overlapping clips summed past full scale, so the peaks are limited");
+ if (soundtrack.clippedSamples > 0) {
+ const share = (soundtrack.clippedSamples / (soundtrack.frames * AUDIO_CHANNELS)) * 100;
+ parts.push(
+ `the mix peaked at ${soundtrack.peak.toFixed(2)} of full scale and ${soundtrack.clippedSamples.toLocaleString()} samples (${share.toFixed(share < 0.1 ? 3 : 1)}%) were clipped flat — pull the overlapping clips down by about ${(20 * Math.log10(soundtrack.peak)).toFixed(1)} dB to avoid it`,
+ );
+ }
+ if (soundtrack.shortfalls.length > 0) {
+ parts.push(
+ `${soundtrack.shortfalls.length} ${soundtrack.shortfalls.length === 1 ? "clip runs" : "clips run"} past the end of ${soundtrack.shortfalls.length === 1 ? "its" : "their"} sound — see below`,
+ );
+ }
if (soundtrack.problems.length > 0) parts.push(`${soundtrack.problems.length} left out — see below`);
return `${parts.join("; ")}.`;
}
@@ -754,6 +795,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
: null;
const soundtrack = mix && mix.used > 0 && mix.frames > 0 ? mix : null;
const audioProblems = mix?.problems ?? [];
+ const audioShortfalls = mix?.shortfalls ?? [];
const audioDetail = describeSoundtrack(plan, support, audioProbe, mix, soundtrack);
stopIfCancelled();
@@ -1024,6 +1066,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
audio: Boolean(soundtrack),
audioDetail,
audioProblems,
+ audioShortfalls,
};
} finally {
releaseLoaded();