AI media harness for planning, generating, and editing locally
TypeScript
Name the clipping, report the silence, state the mix's memory
Bitti06ee54e184d9
diff --git a/src/components/workspace/ExportView.tsx b/src/components/workspace/ExportView.tsxindex f7485fd..a0d6fa7 100644--- a/src/components/workspace/ExportView.tsx+++ b/src/components/workspace/ExportView.tsx@@ -1,6 +1,7 @@import { CircleCheck, Download, FileVideo, TriangleAlert, X } from "lucide-react";import { useEffect, useMemo, useRef, useState } from "react";import {+ audioMixBytes,bitrateFor,buildExportPlan,defaultExportSettings,@@ -52,6 +53,7 @@ interface FinishedExport {audio: boolean;audioDetail: string;audioProblems: string[];+ audioShortfalls: string[];}export function ExportView({ config, folderPath }: { config: ProjectConfig; folderPath: string }) {@@ -239,6 +241,7 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; foldaudio: outcome.audio,audioDetail: outcome.audioDetail,audioProblems: outcome.audioProblems,+ audioShortfalls: outcome.audioShortfalls,});} catch (reason) {if (reason instanceof ExportCancelled) setCancelled(true);@@ -275,7 +278,10 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold? "every audio clip falls outside the picture": "no audio clips on the timeline": support.audio- ? "48 kHz stereo, gaps filled with silence"+ ? /* The mix is built whole before the file is opened and held until+ the last frame, so its size is a real cost of this export and is+ stated with the rest of them rather than discovered as a crash. */+ `48 kHz stereo, gaps filled with silence · ${formatBytes(audioMixBytes(plan.durationMs))} of memory held while it renders`: "this webview has no AudioEncoder",],[@@ -486,6 +492,13 @@ export function ExportView({ config, folderPath }: { config: ProjectConfig; fold<p><TriangleAlert size={14} aria-hidden="true" /> Left out of the soundtrack:</p><ul>{result.audioProblems.map((problem) => <li key={problem}>{problem}</li>)}</ul></>}+ {result.audioShortfalls.length > 0 && <>+ {/* These clips ARE in the file. Part of each one is silence because+ the sound behind it ran out, which is exactly as inaudible as a+ missing clip and was previously reported nowhere. */}+ <p><TriangleAlert size={14} aria-hidden="true" /> Ran out of sound before the clip ended:</p>+ <ul>{result.audioShortfalls.map((shortfall) => <li key={shortfall}>{shortfall}</li>)}</ul>+ </>}</div>}</section></div>diff --git a/src/lib/export.ts b/src/lib/export.tsindex b4fe94b..17f5901 100644--- a/src/lib/export.ts+++ b/src/lib/export.ts@@ -168,6 +168,32 @@ export type ExportSegment =}| { kind: "gap"; startFrame: number; endFrame: number };+/* The shape of the soundtrack, kept here rather than in the pipeline because+ the export PAGE has to be able to state its cost before anything runs.++ 48 kHz stereo: the rate every AAC encoder accepts and the one nothing on a+ timeline has to be downsampled to reach. */+export const AUDIO_SAMPLE_RATE = 48_000;+export const AUDIO_CHANNELS = 2;++/** How much memory the mixed soundtrack occupies while an export runs.+ *+ * The mix is one Float32 per channel per sample and it is built BEFORE the+ * muxer opens — an .mp4's track list is fixed the moment the file is created,+ * so whether there is sound has to be settled first — and then held until the+ * last video frame has been encoded. That is 11.5 MB a minute, so a five+ * minute cut holds 115 MB across the whole render and an hour holds 1.4 GB,+ * roughly three times that at the peak of mixing while the decoded sources are+ * still alive. Nobody should meet that number by running out of memory. */+export function audioMixBytes(durationMs: number): number {+ return Math.max(0, Math.floor((durationMs / 1000) * AUDIO_SAMPLE_RATE)) * AUDIO_CHANNELS * 4;+}++/** Where "this is a lot of memory" starts. 200 MB is a little under 15 minutes+ * of timeline: below it the mix is a rounding error next to the decoded video,+ * above it it is a number worth reading before pressing Export. */+const AUDIO_MIX_NOTE_BYTES = 200_000_000;+/** One audio clip, as much of it as lands inside the file.** The picture decides how long the file is, so sound past the last frame is@@ -379,6 +405,15 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)}. Anything no clip covers is silence.`,);}+ /* The mix is held whole, in memory, for the length of the render. On a short+ cut that is not worth a sentence; on a long one it is the thing most likely+ to end the export, and it was not being said anywhere. */+ const mixBytes = audioMixBytes(durationMs);+ if (audio.length > 0 && mixBytes >= AUDIO_MIX_NOTE_BYTES) {+ notes.push(+ `The whole soundtrack is mixed in memory before the file is opened and held until the last frame: about ${formatBytes(mixBytes)} for ${formatDuration(durationMs)}, and roughly three times that at the moment of mixing while the source files are still decoded.`,+ );+ }/* Audio the file cannot hold, counted rather than dropped in silence. Thepicture decides the length; sound past the last frame has nowhere to go. */const audible = new Set(audio.map((segment) => segment.clipId));diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.tsindex 31793f1..238ed2a 100644--- a/src/lib/exportPipeline.ts+++ b/src/lib/exportPipeline.ts@@ -19,6 +19,8 @@import { invoke } from "@tauri-apps/api/core";import { isTauri, readMediaFileBytes } from "./persistence";import {+ AUDIO_CHANNELS,+ AUDIO_SAMPLE_RATE,fitRect,outputCodec,sourceTimeMsForFrame,@@ -461,10 +463,8 @@ export async function demux(bytes: ArrayBuffer, name: string): Promise<DemuxedSopicture.--------------------------------------------------------------------------- */-/** 48 kHz stereo: the rate every AAC encoder accepts and the one nothing on a- * timeline has to be downsampled to reach. */-const AUDIO_SAMPLE_RATE = 48_000;-const AUDIO_CHANNELS = 2;+/* AUDIO_SAMPLE_RATE and AUDIO_CHANNELS live in export.ts, with the arithmetic+ that lets the page state what the mix will cost in memory before it runs. */const AUDIO_BITRATE = 192_000;/* AAC-LC. Not Opus: mp4-muxer will put Opus in an .mp4 and a large share ofplayers will then refuse the file, which is a worse outcome than a stated@@ -475,16 +475,23 @@ const AUDIO_CODEC = "mp4a.40.2";const AUDIO_BLOCK = 1024;export interface AudioMix {- /** One Float32Array per channel at AUDIO_SAMPLE_RATE, already limited. */+ /** One Float32Array per channel at AUDIO_SAMPLE_RATE, already clipped. */channels: Float32Array[];frames: number;/** Clips that are in the mix. */used: number;/** Clips that are NOT, each named with the reason. Never silently dropped. */problems: string[];- /** True when summing overlaps drove the mix past full scale and it was- * limited to fit. Reported, because it is audible. */- limited: boolean;+ /** Clips that ARE in the mix but whose source file ran out before the clip+ * did, so part of them is silence. The graph already refuses to invent the+ * missing audio; this is the sentence that says it happened. */+ shortfalls: string[];+ /** How many samples the sum drove past full scale, and were therefore hard+ * clipped to ±1. Not "limited": no gain was ridden, the tops were cut off. */+ clippedSamples: number;+ /** The loudest magnitude the sum reached BEFORE clipping. 1.6 means the mix+ * was 4 dB over and says how much has to come down. */+ peak: number;}/** Decodes and mixes every audio segment in the plan into one stereo buffer.@@ -510,6 +517,7 @@ async function mixAudio(options: {is one file to read and one decode to pay for. */const decoded = new Map<string, Promise<AudioBuffer>>();const problems: string[] = [];+ const shortfalls: string[] = [];let used = 0;for (const segment of segments) {@@ -544,37 +552,57 @@ async function mixAudio(options: {);continue;}+ /* A clip can be longer than what is left of its file — drop a five second+ stand-in onto a two second recording and three seconds of it have no+ sound behind them. The graph plays what exists and no more, which is+ right; what was missing was anyone saying so. Below a millisecond this is+ rounding between a millisecond timeline and a 48 kHz buffer, not a hole. */+ const wantedSeconds = segment.durationMs / 1000;+ const availableSeconds = buffer.duration - offsetSeconds;+ const playSeconds = Math.min(wantedSeconds, availableSeconds);+ if (wantedSeconds - availableSeconds > 0.001) {+ shortfalls.push(+ `“${segment.label}” runs for ${wantedSeconds.toFixed(2)}s but only ${availableSeconds.toFixed(2)}s of ${asset.name} is left from ${offsetSeconds.toFixed(2)}s in, so its last ${(wantedSeconds - availableSeconds).toFixed(2)}s is silence.`,+ );+ }const source = context.createBufferSource();source.buffer = buffer;source.connect(context.destination);- source.start(- segment.startMs / 1000,- offsetSeconds,- Math.min(segment.durationMs / 1000, buffer.duration - offsetSeconds),- );+ source.start(segment.startMs / 1000, offsetSeconds, playSeconds);used += 1;}- if (used === 0) return { channels: [], frames: 0, used: 0, problems, limited: false };+ if (used === 0) {+ return { channels: [], frames: 0, used: 0, problems, shortfalls, clippedSamples: 0, peak: 0 };+ }report(`Mixing ${used} audio ${used === 1 ? "clip" : "clips"}…`);const rendered = await context.startRendering();const channels: Float32Array[] = [];- let limited = false;+ let clippedSamples = 0;+ let peak = 0;for (let channel = 0; channel < AUDIO_CHANNELS; channel += 1) {const data = rendered.getChannelData(Math.min(channel, rendered.numberOfChannels - 1)).slice();/* Summed clips can exceed full scale. Left alone, the encoder wraps them- into a crackle; clamped, they are the flat-topped peak every mixing desk- produces at the same point. Either way the user is told. */+ into a crackle; cut to ±1, they are flat-topped — which is CLIPPING, and+ is called that here and on the page. It is deliberately not a limiter: a+ limiter rides the gain of the whole mix over an attack and a release the+ user never chose, and quietly reshaping someone's soundtrack is a worse+ failure than a labelled one. What is reported instead is the measurement+ a limiter would need anyway — how far over, and how much of it — so the+ fix is to pull a fader down by a known amount. */for (let index = 0; index < data.length; index += 1) {- if (data[index] > 1 || data[index] < -1) {- data[index] = Math.max(-1, Math.min(1, data[index]));- limited = true;+ const value = data[index];+ const magnitude = value < 0 ? -value : value;+ if (magnitude > peak) peak = magnitude;+ if (magnitude > 1) {+ data[index] = value > 0 ? 1 : -1;+ clippedSamples += 1;}}channels.push(data);}- return { channels, frames: rendered.length, used, problems, limited };+ return { channels, frames: rendered.length, used, problems, shortfalls, clippedSamples, peak };}/** Whether this computer will encode AAC at all, asked of the encoder. */@@ -679,6 +707,9 @@ export interface ExportResult {/** Audio clips that were left out, each named with its reason. Empty on a* clean run; never a silent omission. */audioProblems: string[];+ /** Audio clips that ARE in the file but ran out of sound before they ran out+ * of clip. The silence is real and it is now named. */+ audioShortfalls: string[];}export interface ExportRunOptions {@@ -714,7 +745,17 @@ function describeSoundtrack(const parts = [`${soundtrack.used} audio ${soundtrack.used === 1 ? "clip" : "clips"} mixed to ${probe?.detail ?? AUDIO_CODEC}`,];- if (soundtrack.limited) parts.push("overlapping clips summed past full scale, so the peaks are limited");+ if (soundtrack.clippedSamples > 0) {+ const share = (soundtrack.clippedSamples / (soundtrack.frames * AUDIO_CHANNELS)) * 100;+ parts.push(+ `the mix peaked at ${soundtrack.peak.toFixed(2)} of full scale and ${soundtrack.clippedSamples.toLocaleString()} samples (${share.toFixed(share < 0.1 ? 3 : 1)}%) were clipped flat — pull the overlapping clips down by about ${(20 * Math.log10(soundtrack.peak)).toFixed(1)} dB to avoid it`,+ );+ }+ if (soundtrack.shortfalls.length > 0) {+ parts.push(+ `${soundtrack.shortfalls.length} ${soundtrack.shortfalls.length === 1 ? "clip runs" : "clips run"} past the end of ${soundtrack.shortfalls.length === 1 ? "its" : "their"} sound — see below`,+ );+ }if (soundtrack.problems.length > 0) parts.push(`${soundtrack.problems.length} left out — see below`);return `${parts.join("; ")}.`;}@@ -754,6 +795,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult: null;const soundtrack = mix && mix.used > 0 && mix.frames > 0 ? mix : null;const audioProblems = mix?.problems ?? [];+ const audioShortfalls = mix?.shortfalls ?? [];const audioDetail = describeSoundtrack(plan, support, audioProbe, mix, soundtrack);stopIfCancelled();@@ -1024,6 +1066,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResultaudio: Boolean(soundtrack),audioDetail,audioProblems,+ audioShortfalls,};} finally {releaseLoaded();