Open source CapCut alternative.
TypeScript
feat(site): master loudness and export audio stems [rebuild]
David Lefd0643dc7547
diff --git a/site/src/clients/chatgpt/contracts.ts b/site/src/clients/chatgpt/contracts.tsindex 5d6ed88f..bd280ca2 100644--- a/site/src/clients/chatgpt/contracts.ts+++ b/site/src/clients/chatgpt/contracts.ts@@ -8,7 +8,7 @@ export const projectSchema = z.object({ id: z.string(), name: z.string(), url: zexport const jobStatusSchema = z.enum(["queued", "running", "done", "error", "expired"]);export const previewSchema = z.object({ id: z.string(), status: jobStatusSchema, progress: z.number(), revision: z.string().nullish(), error: z.string().optional() });/** A finished export downloads from the widget; the model gets the project link. */-export const exportSchema = z.object({ id: z.string(), status: jobStatusSchema, progress: z.number(), name: z.string().nullish(), error: z.string().optional() });+export const exportSchema = z.object({ id: z.string(), status: jobStatusSchema, progress: z.number(), name: z.string().nullish(), error: z.string().optional(), stems: z.string().nullish() });/** Any queued work the caller may poll with get_job_status. */export const jobSchema = z.object({ id: z.string(), kind: z.string(), status: jobStatusSchema, progress: z.number(), error: z.string().optional() });/** One command's outcome inside a batch. */diff --git a/site/src/clients/chatgpt/server/guidance.ts b/site/src/clients/chatgpt/server/guidance.tsindex 588116a2..77b15ef1 100644--- a/site/src/clients/chatgpt/server/guidance.ts+++ b/site/src/clients/chatgpt/server/guidance.ts@@ -40,7 +40,7 @@ Inspect the project for asset ids, transcripts, trims, notes and existing coverawatch_video returns timestamped contact sheets, sceneChanges and coverage. Stamps are source seconds. Continue from coveredTo when truncated. For a request about the whole source, cover it until unwatchedSeconds reaches zero. For long footage, sweep coarsely to learn its structure, then inspect relevant spans closely. A source map or a transcript alone does not establish visual coverage. Use detail read or original to read text and match typography; scan is for structure. Check unreadSeconds and unreadFrom when legible coverage matters.After every watch, record what you observed through edit_project with note_source: source span, actions, cuts, and exact on-screen words. A further watch can be refused until those observations are recorded. Notes persist on the asset; returned contact sheets belong to this conversation. Read observed, recorded and unnoted to resume from existing evidence.listen_audio returns playable source audio. Use the transcript for words and listen for delivery, music and timing. Respect coveredTo; continue from unheardFrom until unheardSeconds reaches zero when judging a whole source. Record observations with note_source. A clip_id scopes listening to its source trim; from and to are source seconds.-Use signal measurements for editing decisions: detect_silence locates quiet spans; detect_beats returns a beat grid and timelineBeats for a clip; measure_level measures loudness and returns volumeToMatch against a target. read_color_stats measures exposure, tonal range and color before grading. compare_to_source checks the edit against a reference; capture_frame shows the composited picture. These read commands belong in inspect_project's commands array.+Use signal measurements for editing decisions: detect_silence locates quiet spans; detect_beats returns a beat grid and timelineBeats for a clip; measure_level measures loudness, including integratedLufs and truePeakDbtp per clip and for the whole mix, and returns volumeToMatch against a target. read_color_stats measures exposure, tonal range and color before grading. compare_to_source checks the edit against a reference; capture_frame shows the composited picture. These read commands belong in inspect_project's commands array.find_filler identifies transcript fillers; find_highlights ranks transcript spans. Their results need the relevant visual or audio check before cutting. Use edit_project for these commands unless list_commands marks them reads. Caption creation is a separate edit requested by the user.Plan cuts around complete thoughts and measured pauses. Split and delete in batches, read the resulting ids, then place clips to close intended gaps. Leave room around speech boundaries and run refine_speech_cuts with the requested pace. Listen around adjusted edges, widen a clipped word's trim, and check the changed cut visually. Source seconds drive watches, listens and trims; timeline seconds drive placement and splits. Use the clip timing data to map between them.Report only the coverage you inspected and the changes tool results confirm. render_preview produces a playable saved revision for the user to review.`,@@ -81,7 +81,7 @@ Read the resulting adjustments and placement: trims can clamp to shorter replace"publish-and-export": `# Publish and exportset_publish edits the project's caption, tags, sound title and handle. These fields describe the intended post.-export_video renders the saved cloud project with one of these presets: ${DOC_EXPORT_PRESETS.join(", ")}. Read its tool description for the preset choices. A returned export job is completed through get_export_status, and the card offers the finished download. render_preview and get_preview_status provide a playable preview for review. Only report a preview or export as ready when its result confirms completion.`,+export_video renders the saved cloud project with one of these presets: ${DOC_EXPORT_PRESETS.join(", ")}. Read its tool description for the preset choices. loudness masters the mix to a LUFS target for its destination, and stems adds a zip of one WAV per lane for mixing elsewhere. A returned export job is completed through get_export_status, and the card offers the finished download. render_preview and get_preview_status provide a playable preview for review. Only report a preview or export as ready when its result confirms completion.`,};export const SKILL_INDEX = Object.keys(SKILLS);diff --git a/site/src/clients/chatgpt/server/mcp.ts b/site/src/clients/chatgpt/server/mcp.tsindex 0417c335..4b594793 100644--- a/site/src/clients/chatgpt/server/mcp.ts+++ b/site/src/clients/chatgpt/server/mcp.ts@@ -18,6 +18,8 @@ import {} from "@/clients/chatgpt/server/catalog";import { MAX_COMMANDS_PER_BATCH } from "@/cut/server/cloud/commands";import { DOC_EXPORT_PRESETS } from "@/cut/lib/exportPresets";+import { bindCutLoudness, cutLoudness, LOUDNESS_CHOICES, LOUDNESS_IDS } from "@/cut/lib/loudnessSettings";+import { getGlobalSetting } from "@/lib/config/effective";import {chatgptConfig,RESOURCE_METADATA_PATH,@@ -414,13 +416,19 @@ export function createChatgptServer({title: "Export a project",description:- `Render the saved project to an MP4 the user downloads from the Donkey Cut card. preset: ${DOC_EXPORT_PRESETS.join(" | ")} (original matches the footage; tiktok is best quality 1080p; fast is a smaller 1080p; light is a 720p draft). Free; the file counts against the account's cloud storage and is listed in the project's Media panel. A render can outlast this call — finish it with get_export_status.`,- inputSchema: z.object({ projectId: idSchema, preset: z.enum(DOC_EXPORT_PRESETS).optional() }),+ `Render the saved project to an MP4 the user downloads from the Donkey Cut card. preset: ${DOC_EXPORT_PRESETS.join(" | ")} (original matches the footage; tiktok is best quality 1080p; fast is a smaller 1080p; light is a 720p draft). loudness masters the mix to an integrated target under a true-peak ceiling: ${LOUDNESS_CHOICES.map((c) => (c.id === "off" ? "off (the mix as it plays)" : `${c.id} ${cutLoudness().targets[c.id]} LUFS (${c.detail})`)).join(", ")}; the true-peak ceiling is ${cutLoudness().truePeakCeiling} dBTP; omitted uses the account default (${cutLoudness().defaultTarget}). stems: true also writes one 24-bit WAV per lane (dialogue from the video clips, then each soundtrack lane), full length and unmastered, zipped as "<name> stems.zip" beside the video in the project's exports. Free; the files count against the account's cloud storage and are listed in the project's exports. A render can outlast this call — finish it with get_export_status.`,+ inputSchema: z.object({+ projectId: idSchema,+ preset: z.enum(DOC_EXPORT_PRESETS).optional(),+ loudness: z.enum(LOUDNESS_IDS).optional(),+ stems: z.boolean().optional(),+ }),outputSchema: viewSchema,annotations: editAnnotations,_meta: { ...editMetadata, ...status("Exporting the video", "Export started") },},- ({ projectId, preset }) => editing(() => runTool(() => projects.exportVideo(projectId, preset ?? "original"))),+ ({ projectId, preset, loudness, stems }) =>+ editing(() => runTool(() => projects.exportVideo(projectId, preset ?? "original", { loudness, stems }))),);server.registerTool(@@ -589,6 +597,8 @@ export async function mcpEndpoint(request: Request) {return rateLimitResponse(rateLimit.retryAfterSeconds);}+ // export_video teaches the loudness targets the worker will master to.+ bindCutLoudness(await getGlobalSetting("cutLoudness"));const handler = chatgptHttpHandler(identity, config);try {const response = await handler.fetch(request);diff --git a/site/src/clients/chatgpt/server/projects.ts b/site/src/clients/chatgpt/server/projects.tsindex 74773a19..4b854b69 100644--- a/site/src/clients/chatgpt/server/projects.ts+++ b/site/src/clients/chatgpt/server/projects.ts@@ -7,6 +7,8 @@ import { normalizeAspect, type ProjectDoc } from "@/cut/lib/types";import { ADOPT_COMMAND, type AdoptedAsset } from "@/cut/lib/commandBatch";import { NO_CARD_OPEN, queueCommands, waitForJob, type CommandCall, type CommandJobResult, type JobRow } from "@/cut/server/cloud/commands";import { needsWorker, queueDocExport, queueImportUrl } from "@/cut/server/cloud/jobs";+import type { DocExportAudio } from "@/cut/lib/exportPresets";+import { stemsArchiveName } from "@/cut/lib/exportDelivery";import { cutLimitsFor } from "@/cut/server/cloud/limits";import { previewFromDoc } from "@/cut/server/cloud/previewJobs";import { mediaObjectUrl, mediaUrlLifetime } from "@/cut/server/cloud/mediaCdn";@@ -213,6 +215,20 @@ export function projectTools(view.export.error = "The export file was deleted. Export again.";return { view, playback: null, download: null };}+ // Stems land beside the video under its name, in the project's exports.+ if (row.outName) {+ const stems = await db.cutMediaObject.findFirst({+ where: {+ userId: identity.userId,+ projectId: row.projectId,+ kind: "export",+ fileName: stemsArchiveName(row.outName),+ uploadState: "complete",+ },+ select: { fileName: true },+ });+ if (stems) view.export.stems = stems.fileName;+ }return {view,playback: null,@@ -498,9 +514,9 @@ export function projectTools(undo: (projectId: string) => step(projectId, "undo"),redo: (projectId: string) => step(projectId, "redo"),- async exportVideo(projectId: string, preset: string): Promise<ProjectResult> {+ async exportVideo(projectId: string, preset: string, audio: DocExportAudio = {}): Promise<ProjectResult> {requireEdit();- const queued = await queueDocExport(identity.userId, projectId, preset);+ const queued = await queueDocExport(identity.userId, projectId, preset, audio);if (queued instanceof Response) return refusal(queued);const row = await waitForJob(identity.userId, queued.id, config.commandWaitMs);if (!row) throw new ProjectToolError("The export disappeared. Try again.");diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 6267f22f..b4ee9000 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -92,14 +92,12 @@ export const AI_PANEL_TOOLS = [{name: "measure_level",description:- "Measure how loud clips actually play — the level of each clip's audible sound over its trimmed range, in dBFS, with its current volume applied — so volumes are set from numbers, never by ear. Returns per clip: sourceDb (the media), volume, levelDb (what plays), loudestFrameDb (the loudest 20 ms frame as RMS — a sample peak sits higher, so it is no clipping margin), and for every clip except the target, volumeToMatch: the volume that lands it at the target's level; a clip whose source has no audio track comes back with noAudio instead of levels. \"Make the voiceover as loud as the clip\", \"match the levels\", \"balance the music under the speech\": call this with every clip involved and target_id = the one to match, then set_clip_volume / update_audio with volumeToMatch (for a bed under speech, a fraction of it). A shortfallDb means the volume ceiling (3) cannot reach the target; lower the target instead. A clip marked muted plays nothing until set_clip_muted / update_audio unmutes it; its level is what it would play unmuted.",- inputSchema: obj(- {- ids: { type: "array", items: { type: "string" }, description: "Clip ids to measure — video clips (their own sound) and soundtrack clips alike" },- target_id: str("The clip whose level the others should match (default: the first id)"),- },- ["ids"]- ),+ "Measure how loud clips actually play — the level of each clip's audible sound over its trimmed range, in dBFS, with its current volume applied — so volumes are set from numbers, never by ear. Returns per clip: sourceDb (the media), volume, levelDb (what plays), loudestFrameDb (the loudest 20 ms frame as RMS — a sample peak sits higher, so it is no clipping margin), and for every clip except the target, volumeToMatch: the volume that lands it at the target's level; a clip whose source has no audio track comes back with noAudio instead of levels. \"Make the voiceover as loud as the clip\", \"match the levels\", \"balance the music under the speech\": call this with every clip involved and target_id = the one to match, then set_clip_volume / update_audio with volumeToMatch (for a bed under speech, a fraction of it). A shortfallDb means the volume ceiling (3) cannot reach the target; lower the target instead. A clip marked muted plays nothing until set_clip_muted / update_audio unmutes it; its level is what it would play unmuted. Each clip also reports integratedLufs (BS.1770 loudness over its range) and truePeakDbtp, both as it plays with its volume applied; a volume change of g dB moves both by g (volume × 10^(g/20)), so \"music 10 LU under the voice\" is a volume that puts the music's integratedLufs 10 below the voice's. mix: true measures the whole mix as an unmastered export hears it: integratedLufs, truePeakDbtp, loudnessRangeLu and maxShortTermLufs — check it against the export's loudness targets or a platform's spec. null means silence.",+ inputSchema: obj({+ ids: { type: "array", items: { type: "string" }, description: "Clip ids to measure — video clips (their own sound) and soundtrack clips alike; optional with mix" },+ target_id: str("The clip whose level the others should match (default: the first id)"),+ mix: bool("Also measure the whole mix's loudness and true peak (default false)"),+ }),},{name: "detect_beats",diff --git a/site/src/cut/components/ExportDialog.tsx b/site/src/cut/components/ExportDialog.tsxindex a719a575..ae1b1db8 100644--- a/site/src/cut/components/ExportDialog.tsx+++ b/site/src/cut/components/ExportDialog.tsx@@ -46,6 +46,8 @@ import { useExports } from "@/cut/lib/exportStore";import { projectDuration, useEditor } from "@/cut/lib/store";import { subtitleFiles } from "@/cut/lib/subtitleFile";import { formatTime } from "@/cut/lib/time";+import { cutLoudness, LOUDNESS_CHOICES, loudnessLabel, loudnessSettings, type LoudnessId } from "@/cut/lib/loudnessSettings";+import { planStems } from "@/cut/lib/stems";import { cn } from "@/lib/utils";/** A file format the menu offers: a container carrying a codec. A codec that@@ -106,6 +108,12 @@ export function ExportDialog() {const [rangeMode, setRangeMode] = useState<"all" | "selection">("all");// A captions file beside the video.const [withCaptions, setWithCaptions] = useState(false);+ // The loudness the mix is mastered to, starting on the account's default.+ const [loudness, setLoudness] = useState<LoudnessId>(() => cutLoudness().defaultTarget);+ // A zip of one WAV per lane beside the video.+ const [withStems, setWithStems] = useState(false);+ const canStem = useMemo(() => planStems({ clips, audioClips, assets }).length > 0, [clips, audioClips, assets]);+ const stems = withStems && canStem;// The footage's frame rate, read from the sources once the dialog opens;// a "source" frame-rate choice resolves to it. Undefined while the probe// runs, null for a cut with no readable video. The probe re-runs only when@@ -149,9 +157,11 @@ export function ExportDialog() {...(hdr && chosen.codec === "h264" ? { codec: "hevc" as const, copySource: false } : {}),...(typedName ? { name: baseName } : {}),...(range ? { range } : {}),+ ...loudnessSettings(loudness),+ ...(stems ? { stems: true } : {}),};},- [choice, resolutions, sourceFps, probe, probing, audioClips.length, typedName, baseName, range, hdr]+ [choice, resolutions, sourceFps, probe, probing, audioClips.length, typedName, baseName, range, hdr, loudness, stems]);const set = (patch: Partial<ExportChoice>) => setChoice((c) => ({ ...c, ...patch }));const captionFiles = useMemo(@@ -398,6 +408,27 @@ export function ExportDialog() {</SelectContent></Select></Field>+ <Field+ label="Loudness"+ hint={+ loudness === "off"+ ? undefined+ : `Sets the whole mix to its target and holds true peaks under ${String(cutLoudness().truePeakCeiling).replace("-", "−")} dBTP.`+ }+ >+ <Select value={loudness} onValueChange={(v) => setLoudness(v as LoudnessId)}>+ <SelectTrigger className="w-fit min-w-32" aria-label="Loudness">+ <SelectValue />+ </SelectTrigger>+ <SelectContent align="end">+ {LOUDNESS_CHOICES.map((c) => (+ <SelectItem key={c.id} value={c.id}>+ {loudnessLabel(c.id)}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </Field></div><Collapsible open={moreOpen} onOpenChange={setMoreOpen} className="flex flex-col gap-3">@@ -436,6 +467,18 @@ export function ExportDialog() {<span className="text-xs text-muted-foreground">Mbps</span></label></Field>+ {canStem && (+ <Field+ label="Stems"+ hint="Saves one WAV per lane, dialogue and each soundtrack lane, zipped beside the video for mixing elsewhere."+ >+ <Switch+ aria-label="Export stems"+ checked={withStems}+ onCheckedChange={(v) => setWithStems(v === true)}+ />+ </Field>+ )}{captionFiles.length > 0 && (<Fieldlabel="Captions file"@@ -466,6 +509,8 @@ export function ExportDialog() {{FORMATS.find((f) => f.id === formatId)?.label ?? formatId} · {settings.audioCodec.toUpperCase()}{hdr ? ` · ${OUTPUT_SPACES.find((o) => o.id === colorSpace)?.label}` : ""}{range ? ` · ${formatTime(span)}` : ""}+ {settings.loudness !== undefined ? ` · ${String(settings.loudness).replace("-", "−")} LUFS` : ""}+ {stems ? " · stems" : ""}</span><span className="shrink-0 tabular-nums">{sizeEstimate}</span></div>diff --git a/site/src/cut/components/RequireSession.tsx b/site/src/cut/components/RequireSession.tsxindex 4ba89d92..fcaeb134 100644--- a/site/src/cut/components/RequireSession.tsx+++ b/site/src/cut/components/RequireSession.tsx@@ -1,6 +1,9 @@"use client";import { bindChatRuntime, bindChatgptPolling, bindCutClip, bindCutJudge, bindCutNotes } from "@/cut/lib/chatRuntime";import { bindCutColor } from "@/cut/lib/colorSettings";+import { bindCutSound } from "@/cut/lib/soundSettings";+import { bindCutMotion } from "@/cut/lib/motionSettings";+import { bindCutLoudness } from "@/cut/lib/loudnessSettings";import { bindCutMediaCache } from "@/cut/lib/chunkCache";import { useEffect, type ReactNode } from "react";@@ -48,6 +51,9 @@ export function RequireSession({ children }: { children: ReactNode }) {bindCutClip(config.data.settings.cutClip);bindCutNotes(config.data.settings.cutNotes);bindCutColor(config.data.settings.cutColor);+ bindCutSound(config.data.settings.cutSound);+ bindCutMotion(config.data.settings.cutMotion);+ bindCutLoudness(config.data.settings.cutLoudness);bindCutMediaCache(config.data.settings.cutMediaCache);reportExposures(config.data);}, [userId, config.data]);diff --git a/site/src/cut/components/SidePanel.tsx b/site/src/cut/components/SidePanel.tsxindex 92d0e6d8..dba8cd7f 100644--- a/site/src/cut/components/SidePanel.tsx+++ b/site/src/cut/components/SidePanel.tsx@@ -111,7 +111,7 @@ import { formatTime } from "@/cut/lib/time";import { useLocalPref } from "@/cut/lib/uiState";import type { MediaAsset, SidePanelTab } from "@/cut/lib/types";import { cn } from "@/lib/utils";-import { containerOfName } from "@/cut/lib/exportDelivery";+import { containerOfName, isStemsName } from "@/cut/lib/exportDelivery";import { useRevealEffect } from "@/cut/lib/refReveal";import { usePanelRequestEffect } from "@/cut/lib/panelRequest";import { CopyNameLabel } from "./AssetRefs";@@ -1185,6 +1185,20 @@ function ProjectFilesPanel({className="export-row group relative flex w-full items-center gap-2.5 overflow-hidden rounded-lg border border-border p-1.5 transition-colors hover:border-input hover:bg-muted/50"><ExportPulse file={it.file} />+ {isStemsName(it.file) ? (+ // A stems zip has no picture to preview; it downloads.+ <div className="flex min-w-0 flex-1 items-center gap-2.5">+ <span className="flex h-11 w-[25px] shrink-0 items-center justify-center rounded-[4px] bg-muted text-muted-foreground">+ <Music className="size-3" />+ </span>+ <span className="min-w-0 flex-1">+ <span className="block truncate text-[11.5px] font-medium">{it.file}</span>+ <span className="block text-[10.5px] text-muted-foreground">+ {(it.size / (1024 * 1024)).toFixed(1)} MB · WAV stems · ZIP+ </span>+ </span>+ </div>+ ) : (<buttontype="button"className="flex min-w-0 flex-1 items-center gap-2.5 text-left"@@ -1220,6 +1234,7 @@ function ProjectFilesPanel({</span></span></button>+ )}<div className="flex shrink-0 items-center gap-0.5 pr-0.5 opacity-0 transition-opacity group-hover:opacity-100">{caps.revealInFinder ? (<Buttondiff --git a/site/src/cut/components/TopBar.tools.ts b/site/src/cut/components/TopBar.tools.tsindex 16390764..b3c39e77 100644--- a/site/src/cut/components/TopBar.tools.ts+++ b/site/src/cut/components/TopBar.tools.ts@@ -8,6 +8,8 @@import { OUTPUT_SPACES } from "@donkeycut/effects-kit";import { obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";import { EXPORT_RESOLUTIONS } from "@/cut/lib/exportPresets";+import { LOUDNESS_CHOICES } from "@/cut/lib/loudnessSettings";+import { SETTINGS } from "@/lib/config/registry";import { GUIDE_IDS, GUIDE_PRESETS } from "@/cut/lib/guides";export const TOP_BAR_TOOLS = [@@ -55,7 +57,7 @@ export const TOP_BAR_TOOLS = [{name: "open_export",description:- `Open the export dialog: Best uses original resolution, frame rate, supported H.264/HEVC codec and audio settings. Whole files, trims and compatible sequences copy compressed video; joined audio is encoded at source settings. Effects or incompatible joins render with source settings. Other presets are Share, Social 4K, Small and Master. Controls include file name, whole video or selection range, MP4/MOV, H.264/HEVC/ProRes, resolution (${EXPORT_RESOLUTIONS.map((r) => r.label).join(", ")}, Source), frame rate, quality or custom bitrate, AAC/PCM, the project's color space (SDR, or HDR as HLG or PQ — a 10-bit HEVC or ProRes file; H.264 greys out), and an SRT captions file. Exporting itself stays a user action.`,+ `Open the export dialog: Best uses original resolution, frame rate, supported H.264/HEVC codec and audio settings. Whole files, trims and compatible sequences copy compressed video; joined audio is encoded at source settings. Effects or incompatible joins render with source settings. Other presets are Share, Social 4K, Small and Master. Controls include file name, whole video or selection range, MP4/MOV, H.264/HEVC/ProRes, resolution (${EXPORT_RESOLUTIONS.map((r) => r.label).join(", ")}, Source), frame rate, quality or custom bitrate, AAC/PCM, the project's color space (SDR, or HDR as HLG or PQ — a 10-bit HEVC or ProRes file; H.264 greys out), an SRT captions file, Loudness (${LOUDNESS_CHOICES.map((c) => (c.id === "off" ? c.label : `${c.label} ${SETTINGS.cutLoudness.default.targets[c.id]} LUFS`)).join(", ")} by default: one gain sets the mix's integrated loudness, and a limiter holds true peaks under ${SETTINGS.cutLoudness.default.truePeakCeiling} dBTP), and Stems (a zip of one 24-bit WAV per lane, the video clips' dialogue then each soundtrack lane, full length and unmastered, saved beside the video). measure_level with mix: true reads the mix's loudness first. Exporting itself stays a user action.`,inputSchema: obj({}),},] as const satisfies readonly AiToolDef[];diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 40665c3a..6cb7bf45 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1568,7 +1568,11 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {measure_level: async (s, input) => {const ids = Array.isArray(input.ids) ? input.ids.map(String) : [];- if (ids.length === 0) throw new ToolError("ids is required: the clip ids to measure.");+ const wantsMix = input.mix === true;+ if (ids.length === 0 && !wantsMix)+ throw new ToolError("ids is required: the clip ids to measure, or mix: true for the whole mix.");+ const mix = wantsMix ? await measureMix(s) : undefined;+ if (ids.length === 0) return { mix };const targetId = input.target_id === undefined || input.target_id === null ? ids[0] : String(input.target_id);if (!ids.includes(targetId)) ids.push(targetId);type Row = {@@ -1580,6 +1584,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {audibleSeconds: number;volume: number;levelDb: number;+ integratedLufs: number | null;+ truePeakDbtp: number | null;muted?: boolean;};const rows = new Map<string, Row | { id: string; name: string; kind: "video" | "soundtrack"; noAudio: string }>();@@ -1596,7 +1602,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// one silent generated clip among them.let level: Awaited<ReturnType<typeof measureSourceLevel>>;try {- level = await measureSourceLevel(asset.url, { from, ...(to !== undefined ? { to } : {}) });+ level = await sourceLevel(asset, from, to);} catch (e) {rows.set(id, { id, name: asset.name, kind, noAudio: e instanceof Error ? e.message : "Could not read the audio." });return;@@ -1612,6 +1618,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {volume: round2(volume),// A muted clip plays nothing; its level is what it would play unmuted.levelDb: round2(level.rmsDb + gainDb(volume)),+ // BS.1770 loudness and true peak as it plays, volume applied.+ ...playedLoudness(level.loudness, volume),...(muted ? { muted: true } : {}),});});@@ -5626,6 +5634,33 @@ function notesIn(/** A clip volume as decibels of gain; silence reads as the quiet floor. */const gainDb = (volume: number) => (volume > 0 ? 20 * Math.log10(volume) : -100);+/** A source's loudness and true peak moved by a clip's volume. */+function playedLoudness(+ loudness: LoudnessMeasure | null,+ volume: number+): { integratedLufs: number | null; truePeakDbtp: number | null } {+ if (!loudness) return { integratedLufs: null, truePeakDbtp: null };+ const r = roundedLoudness(loudness);+ const gain = gainDb(volume);+ return {+ integratedLufs: r.integratedLufs === null ? null : round2(r.integratedLufs + gain),+ truePeakDbtp: r.truePeakDbtp === null ? null : round2(r.truePeakDbtp + gain),+ };+}++/** The whole mix as an unmastered export hears it: the same offline mix, at+ * the export's 48 kHz stereo, measured to BS.1770. */+async function measureMix(s: ReturnType<typeof useEditor.getState>) {+ const whole = await renderMix(mixSpecFor(renderDoc(s), (a) => a.url), {+ sampleRate: 48000,+ channels: 2,+ resolve: (file) => file,+ });+ if (!whole) return { silent: true, note: "Nothing in the cut makes a sound." };+ const channels = Array.from({ length: whole.numberOfChannels }, (_, c) => whole.getChannelData(c));+ return { ...roundedLoudness(await measureLoudnessSliced(channels, whole.sampleRate)), seconds: round2(whole.duration) };+}+function resolveWatchTarget(s: ReturnType<typeof useEditor.getState>,input: Record<string, unknown>diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.tsindex 0d2c36f2..1b984e10 100644--- a/site/src/cut/lib/audioMix.ts+++ b/site/src/cut/lib/audioMix.ts@@ -188,31 +188,76 @@ export function foldClips(clips: MixClip[]) {}/** How far each side of a clip reaches into its handle for a crossing, capped- * by the ramp that crossing actually got. */+ * by the ramp that crossing actually got, or for a split edit, which an edge+ * with a crossing never carries. */const handles = (g: { clip: MixClip; crossIn: number; crossOut: number }) => {const rt = retimeOf(g.clip);return {// Never past the head of the source: a handle is what the trim left// behind. A reversed clip's head reaches toward the source's end, and- // `soundBack` already stops at it.- back: Math.min(g.clip.soundBack ?? 0, g.crossIn, rt.reverse ? Infinity : Math.max(0, -rt.tAt(0))),- ahead: Math.min(g.clip.soundAhead ?? 0, g.crossOut),+ // `soundBack` already stops at it; a bound recording's reach was measured+ // on the recording.+ back: Math.min(+ g.clip.soundBack ?? 0,+ Math.max(g.crossIn, g.clip.soundLead ?? 0),+ rt.reverse || g.clip.soundFrom ? Infinity : Math.max(0, -rt.tAt(0))+ ),+ ahead: Math.min(g.clip.soundAhead ?? 0, Math.max(g.crossOut, g.clip.soundTail ?? 0)),};};-/** The source span an entry decodes, low end first: its own trim, widened by- * the handle each crossing at its edges reaches into. Timeline seconds go- * through the entry's own map on the way into the source. */-const crossSpan = (- c: { file: string; in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean },- back = 0,- ahead = 0-) => {- const rt = retimeOf(c);- const { lo, hi } = srcSpan(rt, -back, rt.len + ahead);- return { file: c.file, in: lo, out: hi };+type Sounded = {+ file: string;+ in: number;+ out: number;+ speed?: number;+ speedCurve?: SpeedNode[];+ reverse?: boolean;+ soundFrom?: SpecSound;+};++/** What an entry decodes: its own trim widened by the reach at its edges,+ * on the file its sound reads — a bound recording through its offset, the+ * window narrowed to what that recording holds. */+const soundRead = (c: Sounded, back = 0, ahead = 0): (SoundWindow & { file: string }) | null => {+ const len = retimeOf(c).len;+ const from = c.soundFrom;+ const w = soundWindow(c, from ? { offset: from.offset, limit: from.duration } : { offset: 0, limit: Infinity }, -back, len + ahead);+ return w && { ...w, file: from ? from.file : c.file };};+/** The ramps a split edit puts on an entry's gain, in timeline seconds: up+ * from silence at the head of the sound that leads the picture, down to+ * silence at the end of the sound that trails it. Inside the picture the+ * clip's own fades and crossings shape the level, and they never reach out+ * here — so the trailing sound comes back up to `level` at the picture's end.+ * Scheduled after those fades: two events at the picture's end keep the order+ * they were set in, and the last one is the level the tail plays at. */+function scheduleSplit(+ param: AudioParam,+ level: number,+ head: number,+ end: number,+ lead: number,+ tail: number,+ fade: number+) {+ if (lead > 0) {+ const f = Math.min(fade, lead);+ const from = Math.max(0, head - lead);+ param.setValueAtTime(f > 0 ? 0 : level, from);+ if (f > 0) param.linearRampToValueAtTime(level, from + f);+ }+ if (tail > 0) {+ const f = Math.min(fade, tail);+ param.setValueAtTime(level, end);+ if (f > 0) {+ param.setValueAtTime(level, end + tail - f);+ param.linearRampToValueAtTime(0, end + tail);+ }+ }+}+/*** The windows during which a ducking item is audible, merged into a single* envelope: each entry is a stretch of timeline where everything that is not@@ -518,6 +563,15 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBgain.gain.setValueAtTime(level, end - fOut);gain.gain.linearRampToValueAtTime(0, end);}+ scheduleSplit(+ gain.gain,+ level,+ a.start,+ end,+ a.crossIn ? 0 : Math.min(a.soundLead ?? 0, back),+ a.crossOut ? 0 : Math.min(a.soundTail ?? 0, ahead),+ a.splitFade ?? 0+ );});}diff --git a/site/src/cut/lib/engineFeatures.ts b/site/src/cut/lib/engineFeatures.tsindex ef02d07a..a63f16a0 100644--- a/site/src/cut/lib/engineFeatures.ts+++ b/site/src/cut/lib/engineFeatures.ts@@ -4,6 +4,6 @@* hands an engine work an older app would silently drop. A build from before* the list existed answers with none.*/-export const ENGINE_FEATURES = ["export.range", "export.name", "export.prores4444", "color.v2", "media.proxy", "export.hdr", "doc.timelines", "doc.timelineAspect"] as const;+export const ENGINE_FEATURES = ["export.range", "export.name", "export.prores4444", "color.v2", "media.proxy", "export.hdr", "doc.timelines", "doc.timelineAspect", "export.loudness", "export.stems", "export.sound"] as const;export type EngineFeature = (typeof ENGINE_FEATURES)[number];diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex 0dedafe1..51110f47 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -57,6 +57,8 @@ import {EXPORT_CONTAINERS,exportBaseName,fixedRate,+ STEMS_MIME,+ stemsArchiveName,type SpecClipColor,specMediaFiles,type ExportAudioCodec,@@ -93,6 +95,14 @@ export interface ExportSettings {name?: string;/** The stretch of the timeline to render, seconds; absent = all of it. */range?: ExportRange;+ /** The integrated loudness the mix is mastered to, LUFS; absent = the+ * mix as it plays. */+ loudness?: number;+ /** The true peak a mastered mix is held under, dBTP. */+ truePeakCeiling?: number;+ /** Write the audio as stems too: one WAV per lane, zipped beside the+ * video. */+ stems?: boolean;}export { DELIVERY_DEFAULTS, EXPORT_PRESETS } from "./exportPresets";@@ -981,7 +991,7 @@ export async function buildExportPayload(// matte; painted pictures still travel beside it when the pose track keys// opacity, since opacity ships as coverage luma.for (let i = 0; i < spans.length; i++) {- const c = posed(spans[i].clip);+ const c = posed(cardBoxless(spans[i].clip));const dur = Math.max(0.1, retimeOf(c).len);const pictures = await renderClipMaskPictures(c,@@ -1184,7 +1194,7 @@ export async function buildExportPayload(removal: undefined as { rgb: string; alpha: string } | undefined,...ramp,};- overlayClipOf.set(entry, posed(c));+ overlayClipOf.set(entry, posed(cardBoxless(c)));return entry;});});@@ -1304,6 +1314,7 @@ export async function buildExportPayload(reverse: a.reverse,sound: a.sound,duck: a.duck,+ ...(a.lane ? { lane: a.lane } : {}),}));const overlays: {@@ -1446,6 +1457,7 @@ export async function buildExportPayload(target,...settings,...(target === "export" ? { sourceSegments: sourceExportPlan(doc, settings) ?? undefined } : {}),+ ...(target === "export" && settings.stems ? { stemPlan: stemsFor(doc) } : {}),// The hover proxy, the share card and the streaming ladder are SDR web// pictures whatever the project delivers.colorSpace: target === "export" ? doc.colorSpace ?? "sdr" : "sdr",@@ -1552,6 +1564,29 @@ async function postExport(});}+/** The stems a cut exports as. A cut with nothing audible has none, and+ * asking for them is refused before anything renders. */+function stemsFor(doc: ExportDoc): StemDef[] {+ const stems = planStems(doc);+ if (stems.length === 0) throw new ExportRefusedError("Nothing in this cut makes a sound to split into stems.");+ return stems;+}++/** The stems archive for an export that asks for one, rendered in this tab+ * from the same mix fold as the file's sound; null when it asks for none. */+async function stemsArchive(+ projectId: string,+ doc: ExportDoc,+ settings: ExportSettings,+ signal?: AbortSignal+): Promise<RenderedExport | null> {+ if (!settings.stems) return null;+ return renderStemsArchive(doc, settings, stemsFor(doc), {+ resolve: (asset) => renderAssetUrl(projectId, asset),+ signal,+ });+}+function renderAssetUrl(projectId: string, asset: MediaAsset): string {const state = useEditor.getState();return state.projectId === projectId@@ -1597,8 +1632,27 @@ export function downloadExport(jobId: string, outName: string, backend: CutBacke* not know — a range export would come back as the whole cut, a typed name* as the project's — so what the settings ask of the engine is checked* against what it says it carries, and the export refuses with the fix. */-async function assertEngineCarries(settings: ExportSettings, doc: ExportDoc): Promise<void> {+/** Whether the cut plays a bound recording or a split edit anywhere. */+function cutUsesSoundFeatures(doc: ExportDoc): boolean {+ const bound = new Set(doc.assets.filter((a) => a.soundFrom).map((a) => a.id));+ return doc.clips.some((c) => bound.has(c.assetId) || !!c.audioLead || !!c.audioTail);+}++async function assertEngineCarries(+ settings: ExportSettings,+ doc: ExportDoc,+ /** The engine encodes this one in ffmpeg; a tab render only hands it the+ * finished files. */+ engineRenders = true+): Promise<void> {const wants: [EngineFeature, boolean, string][] = [+ // An older engine would deliver the mix unmastered and drop the stems.+ ["export.loudness", engineRenders && settings.loudness !== undefined, "master the loudness"],+ ["export.stems", !!settings.stems, "export stems"],+ // An older engine plays the camera's own sound and no split edits, so a+ // cut that binds a separate recording or leads/trails its sound would+ // come back with the scratch track.+ ["export.sound", engineRenders && cutUsesSoundFeatures(doc), "play a separate sound recording or split edits"],// The color pipeline: an engine from before it grades nothing and folds// a log or HDR file the old way, so every export needs it.["color.v2", true, "export this cut"],@@ -1691,12 +1745,18 @@ export async function runBrowserExport(bytes: Math.round(estimateExportBytes(settings, deliverySpan(settings.range, projectDuration(doc)))),+ // The stems ride the same claim: their name is held with the video's,+ // and the completion registers both.+ ...(settings.stems ? { stems: true } : {}),}),});- const claimed = await apiJson<{ jobId?: string; url?: string; outName?: string; type?: string }>(claim);+ const claimed = await apiJson<{ jobId?: string; url?: string; outName?: string; type?: string; stemsUrl?: string }>(claim);if (!claim.ok || !claimed.jobId || !claimed.url) {throw cloudRefusal(claim, claimed) ?? new Error(claimed.error ?? "Export failed to start.");}+ if (settings.stems && !claimed.stemsUrl) {+ throw new Error("The cloud did not hold a place for the stems.");+ }const jobId = claimed.jobId;opts.onClaimed?.(jobId);@@ -1725,6 +1785,14 @@ export async function runBrowserExport(// back as soon as the upload is done with it.void rendered.discard();}+ const stems = await stemsArchive(projectId, doc, settings, opts.signal);+ if (stems) {+ try {+ await putSigned(claimed.stemsUrl!, stems.file, STEMS_MIME, { signal: opts.signal });+ } finally {+ void stems.discard();+ }+ }const done = await backend.fetch("/api/cut/export/client/complete", {method: "POST",@@ -1768,11 +1836,16 @@ async function runEngineExport(opts: NonNullable<Parameters<typeof runBrowserExport>[3]>,backend: CutBackend): Promise<string> {- await assertEngineCarries(settings, doc);+ await assertEngineCarries(settings, doc, false);const claim = await backend.fetch("/api/cut/export/client", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ projectId, container: settings.container, name: settings.name }),+ body: JSON.stringify({+ projectId,+ container: settings.container,+ name: settings.name,+ ...(settings.stems ? { stems: true } : {}),+ }),});const claimed = await apiJson<{ id?: string; outName?: string }>(claim);if (!claim.ok || !claimed.id) throw new Error(claimed.error ?? "Export failed to start.");@@ -1810,6 +1883,23 @@ async function runEngineExport(report(ratio * 0.95);},});+ // The stems land first: the video's hand-in settles the job.+ const stems = await stemsArchive(projectId, doc, settings, opts.signal);+ if (stems) {+ let landed: Response;+ try {+ landed = await backend.fetch(`/api/cut/export/client/${jobId}/stems`, {+ method: "PUT",+ headers: { "Content-Type": STEMS_MIME },+ body: stems.file,+ signal: opts.signal,+ });+ } finally {+ void stems.discard();+ }+ if (landed.status === 409) throw new DOMException("Export canceled.", "AbortError");+ if (!landed.ok) throw new Error((await apiJson<object>(landed)).error ?? "Could not save the stems.");+ }let done: Response;try {done = await backend.fetch(`/api/cut/export/client/${jobId}/file`, {@@ -1845,6 +1935,31 @@ async function runEngineExport(}}+/** A browser-resident project's stems, rendered in this tab and shelved+ * beside the video they go with. */+async function saveStems(+ projectId: string,+ doc: ExportDoc,+ settings: ExportSettings,+ videoName: string,+ signal?: AbortSignal+): Promise<void> {+ if (!settings.stems) return;+ holdRegistered(`/api/cut/projects/${projectId}/`);+ try {+ const stems = (await stemsArchive(projectId, doc, settings, signal))!;+ try {+ const saved = await saveExport(projectId, stems.file, stemsArchiveName(videoName));+ const stored = await readFileAt(await exportsDir(projectId), saved);+ if (stored) registerBlobFile(`/api/cut/projects/${projectId}/exports/${encodeURIComponent(saved)}`, stored);+ } finally {+ void stems.discard();+ }+ } finally {+ releaseRegistered(`/api/cut/projects/${projectId}/`);+ }+}+/** A browser-resident project's export: the same in-tab render, with the* finished file written into the project's exports shelf in the browser store.* The job is a row in the tab-local feed, so the dock tracks, cancels, and@@ -1875,6 +1990,7 @@ async function runStoreExport(} finally {void rendered.discard();}+ await saveStems(projectId, doc, settings, outName, opts.signal);// Serve the finished file from the store copy: the scratch file behind the// render is already gone, and these two paths are what the dock's download// button and the exports shelf resolve through backend.url().@@ -1956,11 +2072,14 @@ async function runBorrowedExport(const job = reserveBrowserExportJob(projectId, opts.projectName);opts.onClaimed?.(job.id);try {- const file = await renderBorrowedFile(projectId, doc, settings, "export", {+ // The worker borrows only the encode the tab cannot do; the stems are+ // sound, which this tab mixes itself, so they stay here.+ const file = await renderBorrowedFile(projectId, doc, { ...settings, stems: undefined }, "export", {...opts,onProgress: (progress) => { opts.onProgress?.(progress); updateBrowserExportJob(job.id, { progress }); },});const saved = await saveExport(projectId, file, file.name);+ await saveStems(projectId, doc, settings, saved, opts.signal);const stored = await readFileAt(await exportsDir(projectId), saved);if (stored) {registerBlobFile(`/api/cut/export/${job.id}/file`, stored);diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.tsindex 382e517b..9168fa98 100644--- a/site/src/cut/lib/exportDelivery.ts+++ b/site/src/cut/lib/exportDelivery.ts@@ -129,7 +129,24 @@ export function containerOfName(name: string): DeliveryContainer {/** Whether a file name is one the export writes. */export function isDeliveryName(name: string): boolean {const lower = name.toLowerCase();- return EXPORT_CONTAINERS.some((c) => lower.endsWith(c.ext));+ return EXPORT_CONTAINERS.some((c) => lower.endsWith(c.ext)) || isStemsName(name);+}++/** The stems archive that rides beside a video: "Trip.mp4" → "Trip stems.zip". */+export const STEMS_SUFFIX = " stems.zip";+export const STEMS_MIME = "application/zip";++export function stemsArchiveName(videoName: string): string {+ return `${exportBaseName(videoName)}${STEMS_SUFFIX}`;+}++export function isStemsName(name: string): boolean {+ return name.toLowerCase().endsWith(".zip");+}++/** The media type of a delivered file, by its name. */+export function deliveryMime(name: string): string {+ return isStemsName(name) ? STEMS_MIME : containerOfName(name).mime;}/**diff --git a/site/src/cut/lib/exportPresets.ts b/site/src/cut/lib/exportPresets.tsindex 1cdd7e0d..c25b1ff4 100644--- a/site/src/cut/lib/exportPresets.ts+++ b/site/src/cut/lib/exportPresets.ts@@ -1,5 +1,6 @@// The export sizes offered by name. Kept apart from the export client so a// server route can list them without pulling the editor in.+import type { LoudnessId } from "./loudnessSettings";/** The delivery every render that is not the user's own export uses — hover* proxies, share cards, ladder masters, the phone's presets. */@@ -41,6 +42,10 @@ export function isDocExportPreset(value: unknown): value is DocExportPreset {return typeof value === "string" && (DOC_EXPORT_PRESETS as readonly string[]).includes(value);}+/** How a queued render of the stored document treats its sound: a loudness+ * target (absent = the setting's default) and stems beside the file. */+export type DocExportAudio = { loudness?: LoudnessId; stems?: boolean };+/** Named output sizes; Source is derived from the project media. */export const EXPORT_RESOLUTIONS = [{ id: "2160", label: "4K", shortSide: 2160 },diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex f7fa6142..5af419b1 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -31,7 +31,10 @@ import {} from "mediabunny";import { drawBlock } from "./blockSource";import { audioFxSpans } from "./audioEffects";-import { renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix";+import { duckWindows, renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix";+import { masterInPlace } from "./loudness";+import { wavBytes, wavChunks, writeStoredZip, type StemDef } from "./stems";+import { specSound } from "./soundSource";import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";import { ensureClipLuts, sourceLookup } from "./lutBuild";import { colorRead, type ColorRead, type ReadFile } from "./sourceColor";@@ -1091,6 +1094,17 @@ export async function renderProjectToMp4(});const mix = whole && settings.range ? sliceAudio(whole, from, from + span) : whole;stop();+ // A mastered delivery: the mix as it will be heard is measured, moved by+ // one gain to its target, and limited only where that gain carries its+ // true peak over the ceiling — the same code the engine runs on its mix.+ if (mix && settings.loudness !== undefined) {+ if (settings.truePeakCeiling === undefined) throw new Error("A mastered export needs its true-peak ceiling.");+ await masterInPlace(+ Array.from({ length: mix.numberOfChannels }, (_, c) => mix.getChannelData(c)),+ { sampleRate: mix.sampleRate, targetLufs: settings.loudness, ceilingDbtp: settings.truePeakCeiling }+ );+ stop();+ }const canvas = createRasterCanvas(settings.width, settings.height);const painter = new FramePainter(doc, canvas, resolve);@@ -1183,6 +1197,78 @@ export async function renderProjectToMp4(}}+/**+ * Render the cut's stems into one stored zip in scratch storage: a 24-bit WAV+ * per stem, each as long as the delivery and starting where it starts, from+ * the same fold the export's mix comes from and before any master.+ *+ * One stem is rendered at a time, and its WAV streams into the archive a+ * second at a time, so what this holds is one stem's mix.+ */+export async function renderStemsArchive(+ doc: ExportDoc,+ settings: ExportSettings,+ stems: StemDef[],+ opts: Pick<RenderOptions, "resolve" | "signal">+): Promise<RenderedExport> {+ if (stems.length === 0) throw new Error("Nothing in this cut makes a sound to split into stems.");+ const duration = projectDuration(doc);+ const from = Math.max(0, settings.range?.start ?? 0);+ const span = deliverySpan(settings.range, duration);+ const rate = settings.audioSampleRate ?? AUDIO_RATE;+ const count = settings.audioChannels ?? AUDIO_CHANNELS;+ const spec = mixSpecFor(doc, opts.resolve);+ // The length the export's own mix comes out at, slice included.+ const total = Math.max(1, Math.ceil(duration * rate));+ const frames = settings.range+ ? Math.max(1, Math.min(total, Math.round((from + span) * rate)) - Math.min(total, Math.round(from * rate)))+ : total;++ const dir = await scratchDir();+ void sweepScratch(dir);+ const name = `stems-${crypto.randomUUID()}.zip`;+ const handle = await dir.getFileHandle(name, { create: true });+ const writable = await handle.createWritable();+ const discard = async () => {+ await writable.close().catch(() => {});+ await dir.removeEntry(name).catch(() => {});+ };+ try {+ // A stem's mix renders when its entry is first read; the entry is read+ // twice (checksum, then bytes), so the last one rendered is kept.+ let held: { stem: StemDef; channels: Float32Array[] } | null = null;+ const channelsOf = async (stem: StemDef): Promise<Float32Array[]> => {+ if (held?.stem === stem) return held.channels;+ held = null;+ opts.signal?.throwIfAborted();+ const whole = await renderMix(stemMixSpec(spec, stem), { sampleRate: rate, channels: count, resolve: (f) => f });+ const cut = whole && settings.range ? sliceAudio(whole, from, from + span) : whole;+ const channels = cut ? Array.from({ length: cut.numberOfChannels }, (_, c) => cut.getChannelData(c)) : [];+ held = { stem, channels };+ return channels;+ };+ await writeStoredZip(+ stems.map((stem) => ({+ name: stem.file,+ size: wavBytes(frames, count),+ read: async function* () {+ yield* wavChunks(await channelsOf(stem), frames, count, rate);+ },+ })),+ async (bytes) => {+ opts.signal?.throwIfAborted();+ await writable.write(bytes as Uint8Array<ArrayBuffer>);+ }+ );+ await writable.close();+ return { file: await handle.getFile(), discard };+ } catch (err) {+ await writable.abort().catch(() => {});+ await dir.removeEntry(name).catch(() => {});+ throw err;+ }+}+/** Draw the text layers live at `t` over the finished picture. *//** Stamped layers for one animated element: the kit plans the windows (the* same split its frame sequences use), and the render hangs what only it knowsdiff --git a/site/src/cut/lib/headless/docExport.ts b/site/src/cut/lib/headless/docExport.tsindex ecd8d335..dceee747 100644--- a/site/src/cut/lib/headless/docExport.ts+++ b/site/src/cut/lib/headless/docExport.ts@@ -14,13 +14,14 @@ import { useEditor } from "../store";import { sourceExportProfile } from "@/cut/lib/exportRender";import { bindHeadlessSession, type HeadlessSession } from "./bind";import { openCloudSnapshot, type CloudDocSnapshot } from "./docSession";+import { loudnessSettings } from "../loudnessSettings";// Queued document exports hydrate their captured revision in an isolated// worker, prepare the same payload as the editor, and stage its overlay// pictures in the render's own scratch directory.-export { DOC_EXPORT_PRESETS, isDocExportPreset, type DocExportPreset } from "../exportPresets";-import type { DocExportPreset } from "../exportPresets";+export { DOC_EXPORT_PRESETS, isDocExportPreset, type DocExportAudio, type DocExportPreset } from "../exportPresets";+import type { DocExportAudio, DocExportPreset } from "../exportPresets";async function settingsFor(preset: DocExportPreset, doc: ExportDoc): Promise<ExportSettings> {const fixed = EXPORT_PRESETS.find((p) => p.id === preset);@@ -41,12 +42,21 @@ export async function buildDocExportSpec(preset: DocExportPreset,tmpDir: string,snapshot: CloudDocSnapshot,- target: "export" | "preview" = "export"+ target: "export" | "preview" = "export",+ audio: DocExportAudio = {}): Promise<object> {bindHeadlessSession(session);await openCloudSnapshot(session, projectId, snapshot);const doc = renderDoc(useEditor.getState());- const payload = await buildExportPayload(projectId, doc, target === "preview" ? previewSettings(doc.aspect) : await settingsFor(preset, doc), target);+ const settings: ExportSettings =+ target === "preview"+ ? previewSettings(doc.aspect)+ : {+ ...(await settingsFor(preset, doc)),+ ...loudnessSettings(audio.loudness),+ ...(audio.stems ? { stems: true } : {}),+ };+ const payload = await buildExportPayload(projectId, doc, settings, target);await Promise.all(payload.pngs.map(async (p) =>writeFile(diff --git a/site/src/cut/lib/loudness.test.ts b/site/src/cut/lib/loudness.test.tsnew file mode 100644index 00000000..d1d22aa5--- /dev/null+++ b/site/src/cut/lib/loudness.test.ts@@ -0,0 +1,278 @@+import { describe, expect, test } from "bun:test";+import {+ integratedFrom,+ kWeighting,+ LoudnessMeter,+ masterInPlace,+ masterPlan,+ measureLoudness,+ measureLoudnessSliced,+ TruePeakLimiter,+} from "./loudness";++const RATE = 48000;++/** A sine at `dbfs` peak, `seconds` long. */+const sine = (hz: number, dbfs: number, seconds: number, rate = RATE, phase = 0) => {+ const a = 10 ** (dbfs / 20);+ const n = Math.round(seconds * rate);+ const x = new Float32Array(n);+ for (let i = 0; i < n; i++) x[i] = a * Math.sin((2 * Math.PI * hz * i) / rate + phase);+ return x;+};+const concat = (...parts: Float32Array[]) => {+ const out = new Float32Array(parts.reduce((s, p) => s + p.length, 0));+ let at = 0;+ for (const p of parts) {+ out.set(p, at);+ at += p.length;+ }+ return out;+};+const stereo = (x: Float32Array) => [x, x.slice()];++/** The magnitude response of the two K-weighting stages at `hz`, in dB. */+const responseDb = (hz: number, rate: number) => {+ const w = (2 * Math.PI * hz) / rate;+ let mag = 1;+ for (const { b, a } of kWeighting(rate)) {+ const re = (c: number[]) => c[0] + c[1] * Math.cos(-w) + c[2] * Math.cos(-2 * w);+ const im = (c: number[]) => c[1] * Math.sin(-w) + c[2] * Math.sin(-2 * w);+ mag *= Math.hypot(re(b), im(b)) / Math.hypot(re(a), im(a));+ }+ return 20 * Math.log10(mag);+};++describe("K-weighting", () => {+ test("reproduces the coefficients BS.1770 prints for 48 kHz", () => {+ const [shelf, hp] = kWeighting(48000);+ const close = (got: number[], want: number[]) =>+ got.forEach((v, i) => expect(Math.abs(v - want[i])).toBeLessThan(1e-8));+ close(shelf.b, [1.53512485958697, -2.69169618940638, 1.19839281085285]);+ close(shelf.a, [1, -1.69065929318241, 0.73248077421585]);+ close(hp.b, [1, -2, 1]);+ close(hp.a, [1, -1.99004745483398, 0.99007225036621]);+ });++ test("has the curve's shape at any rate: flat-ish at 1 kHz, +4 dB high, rolled off low", () => {+ for (const rate of [44100, 48000, 96000]) {+ // −0.691 in the loudness formula cancels this gain.+ expect(Math.abs(responseDb(997, rate) - 0.691)).toBeLessThan(0.02);+ expect(Math.abs(responseDb(10000, rate) - 4)).toBeLessThan(0.3);+ expect(responseDb(20, rate)).toBeLessThan(-10);+ }+ });+});++describe("integrated loudness", () => {+ test("a full-scale 997 Hz sine in one channel reads −3.01 LUFS", () => {+ const x = sine(997, 0, 5);+ const m = measureLoudness([x, new Float32Array(x.length)], RATE);+ expect(Math.abs(m.integratedLufs - -3.01)).toBeLessThan(0.05);+ });++ test("a stereo 1 kHz sine at −23 dBFS reads −23 LUFS (EBU Tech 3341 case 1)", () => {+ const m = measureLoudness(stereo(sine(1000, -23, 20)), RATE);+ expect(Math.abs(m.integratedLufs - -23)).toBeLessThan(0.1);+ });++ test("a mono source weighted as the two channels it plays on reads as that stereo pair", () => {+ const x = sine(1000, -20, 10);+ const mono = new LoudnessMeter(RATE, 1, { weights: [2] });+ mono.push([x]);+ const pair = measureLoudness(stereo(x), RATE);+ expect(Math.abs(mono.result().integratedLufs - pair.integratedLufs)).toBeLessThan(0.01);+ });++ test("the sliced measurement matches the one-shot one", async () => {+ const x = stereo(concat(sine(1000, -18, 4), sine(440, -30, 3)));+ const whole = measureLoudness(x, RATE);+ const sliced = await measureLoudnessSliced(x, RATE);+ expect(sliced.integratedLufs).toBeCloseTo(whole.integratedLufs, 6);+ expect(sliced.truePeakDbtp).toBeCloseTo(whole.truePeakDbtp, 6);+ });++ test("reads the same at 44.1 kHz", () => {+ const m = measureLoudness(stereo(sine(1000, -33, 20, 44100)), 44100);+ expect(Math.abs(m.integratedLufs - -33)).toBeLessThan(0.1);+ });++ test("the relative gate drops the quiet passages (Tech 3341 case 3)", () => {+ const x = concat(sine(1000, -36, 10), sine(1000, -23, 60), sine(1000, -36, 10));+ expect(Math.abs(measureLoudness(stereo(x), RATE).integratedLufs - -23)).toBeLessThan(0.1);+ });++ test("both gates together (Tech 3341 case 4)", () => {+ const x = concat(+ sine(1000, -72, 10),+ sine(1000, -36, 10),+ sine(1000, -23, 60),+ sine(1000, -36, 10),+ sine(1000, -72, 10)+ );+ expect(Math.abs(measureLoudness(stereo(x), RATE).integratedLufs - -23)).toBeLessThan(0.1);+ });++ test("silence has no integrated loudness", () => {+ const m = measureLoudness(stereo(new Float32Array(RATE * 3)), RATE);+ expect(m.integratedLufs).toBe(-Infinity);+ });++ test("gating keeps blocks over the absolute gate and within 10 LU of their mean", () => {+ const p = (l: number) => 10 ** ((l + 0.691) / 10);+ // −80 is silence; −35 is 15 LU under the −20 block's mean; the mean of+ // what remains is the two −20 blocks.+ expect(integratedFrom([p(-80), p(-20), p(-35), p(-20)])).toBeCloseTo(-20, 6);+ });++ test("the answer does not depend on how the audio was chunked", () => {+ const x = concat(sine(440, -18, 3), sine(3000, -30, 2));+ const whole = measureLoudness(stereo(x), RATE);+ const meter = new LoudnessMeter(RATE, 2);+ for (let i = 0; i < x.length; i += 1013) meter.push(stereo(x.subarray(i, i + 1013)));+ const chunked = meter.result();+ expect(chunked.integratedLufs).toBeCloseTo(whole.integratedLufs, 9);+ expect(chunked.truePeakDbtp).toBeCloseTo(whole.truePeakDbtp, 9);+ });+});++describe("loudness range", () => {+ test("two halves 10 LU apart read about 10 LU (Tech 3342 case 1)", () => {+ const x = concat(sine(1000, -20, 20), sine(1000, -30, 20));+ expect(Math.abs(measureLoudness(stereo(x), RATE).loudnessRangeLu - 10)).toBeLessThan(1);+ });++ test("halves 20 LU apart read about 20 LU (Tech 3342 case 3)", () => {+ const x = concat(sine(1000, -40, 20), sine(1000, -20, 20));+ expect(Math.abs(measureLoudness(stereo(x), RATE).loudnessRangeLu - 20)).toBeLessThan(1);+ });+});++describe("true peak", () => {+ test("finds the peak between samples: a quarter-rate sine sampled at ±45°", () => {+ // Every sample sits at 0.707 of the waveform's peak, so the samples read+ // −3.01 dBFS while the signal reaches 0 dBTP.+ const x = sine(RATE / 4, -1, 1, RATE, Math.PI / 4);+ const m = measureLoudness(stereo(x), RATE);+ expect(m.samplePeakDbfs).toBeCloseTo(-4.01, 1);+ expect(m.truePeakDbtp).toBeGreaterThan(-1.2);+ expect(m.truePeakDbtp).toBeLessThan(-0.8);+ });++ test("equals the sample peak where the samples already carry it", () => {+ const m = measureLoudness(stereo(sine(997, -6, 1)), RATE);+ expect(Math.abs(m.truePeakDbtp - -6)).toBeLessThan(0.05);+ });++ test("catches a peak split across two chunks", () => {+ const x = sine(RATE / 4, 0, 0.01, RATE, Math.PI / 4);+ const meter = new LoudnessMeter(RATE, 1);+ meter.push([x.subarray(0, 241)]);+ meter.push([x.subarray(241)]);+ expect(meter.result().truePeakDbtp).toBeGreaterThan(-0.2);+ });+});++describe("mastering plan", () => {+ const measure = (integratedLufs: number, truePeakDbtp: number) => ({+ integratedLufs,+ truePeakDbtp,+ samplePeakDbfs: truePeakDbtp,+ loudnessRangeLu: 0,+ maxMomentaryLufs: integratedLufs,+ maxShortTermLufs: integratedLufs,+ seconds: 10,+ });++ test("one static gain to the target, no limiter when the peak stays under", () => {+ expect(masterPlan(measure(-20, -8), -14, -1)).toEqual({ gainDb: 6, limit: false, peakAfterGainDbtp: -2 });+ });++ test("limits only when the gained peak crosses the ceiling", () => {+ const plan = masterPlan(measure(-20, -4), -14, -1);+ expect(plan.gainDb).toBe(6);+ expect(plan.limit).toBe(true);+ });++ test("a mix that is too loud comes down, and needs no limiter", () => {+ const plan = masterPlan(measure(-9, 0.5), -14, -1);+ expect(plan.gainDb).toBe(-5);+ expect(plan.limit).toBe(false);+ });++ test("silence keeps its level", () => {+ expect(masterPlan(measure(-Infinity, -90), -14, -1).gainDb).toBe(0);+ });+});++describe("true-peak limiter", () => {+ const run = (input: Float32Array[], ceiling: number) => {+ const lim = new TruePeakLimiter(RATE, input.length, ceiling);+ const parts: Float32Array[][] = [];+ for (let i = 0; i < input[0].length; i += 4096) parts.push(lim.process(input.map((c) => c.subarray(i, i + 4096))));+ parts.push(lim.flush());+ return input.map((_, c) => concat(...parts.map((p) => p[c])));+ };++ test("holds the true peak under the ceiling, inter-sample overs included", () => {+ // A loud tone with quarter-rate bursts whose peaks fall between samples.+ const tone = sine(220, 3, 2);+ const burst = sine(RATE / 4, 4, 2, RATE, Math.PI / 4);+ const x = tone.map((v, i) => v + (Math.floor(i / 4800) % 3 === 0 ? burst[i] : 0));+ const out = run(stereo(Float32Array.from(x)), -1);+ expect(out[0].length).toBe(x.length);+ expect(measureLoudness(out, RATE).truePeakDbtp).toBeLessThanOrEqual(-1);+ });++ test("passes audio under the ceiling through untouched and aligned", () => {+ const x = sine(1000, -12, 0.5);+ const [out] = run([x], -1);+ expect(out.length).toBe(x.length);+ let worst = 0;+ for (let i = 0; i < x.length; i++) worst = Math.max(worst, Math.abs(out[i] - x[i]));+ expect(worst).toBeLessThan(1e-6);+ });++ test("glides into a reduction instead of stepping: no sample changes gain abruptly", () => {+ const quiet = sine(100, -20, 0.5);+ const loud = sine(100, 0, 0.5);+ const x = concat(quiet, loud);+ const [out] = run([x], -6);+ // The gain at each pair of neighbouring samples, where it can be read+ // off the input.+ let jump = 0;+ for (let i = 1; i < x.length; i++) {+ if (Math.abs(x[i]) < 0.05 || Math.abs(x[i - 1]) < 0.05) continue;+ jump = Math.max(jump, Math.abs(out[i] / x[i] - out[i - 1] / x[i - 1]));+ }+ // A window of 5 ms at 48 kHz: the steepest step is the whole reduction+ // spread over 240 samples.+ expect(jump).toBeLessThan(0.005);+ });+});++describe("master in place", () => {+ test("lands a mix on the target with its true peak under the ceiling", async () => {+ // Speech-like: a tone that comes and goes, quiet overall with hot peaks.+ const x = new Float32Array(RATE * 6);+ for (let i = 0; i < x.length; i++) {+ const env = 0.5 + 0.5 * Math.sin((2 * Math.PI * i) / RATE);+ x[i] = 0.05 * env * Math.sin((2 * Math.PI * 300 * i) / RATE) + (i % 9600 < 200 ? 0.3 : 0) * Math.sin(i);+ }+ const channels = stereo(x);+ const report = await masterInPlace(channels, { sampleRate: RATE, targetLufs: -14, ceilingDbtp: -1 });+ const after = measureLoudness(channels, RATE);+ expect(report.plan.limit).toBe(true);+ expect(after.truePeakDbtp).toBeLessThanOrEqual(-1);+ // The limiter takes a little off the top of these hot bursts; the+ // integrated level lands within a unit of the target.+ expect(Math.abs(after.integratedLufs - -14)).toBeLessThan(1);+ });++ test("a clean gain when no limiting is needed", async () => {+ const channels = stereo(sine(1000, -30, 5));+ const report = await masterInPlace(channels, { sampleRate: RATE, targetLufs: -23, ceilingDbtp: -1 });+ expect(report.plan.limit).toBe(false);+ expect(Math.abs(measureLoudness(channels, RATE).integratedLufs - -23)).toBeLessThan(0.05);+ });+});diff --git a/site/src/cut/lib/loudness.ts b/site/src/cut/lib/loudness.tsnew file mode 100644index 00000000..84fd1089--- /dev/null+++ b/site/src/cut/lib/loudness.ts@@ -0,0 +1,725 @@+/**+ * Loudness as ITU-R BS.1770-4 and EBU R128 define it, and the mastering that+ * reads it.+ *+ * A level in dBFS answers "how much signal is there". Loudness answers how+ * loud a listener hears it, and the two part ways wherever the energy sits in+ * the spectrum: a bass line and a voice at the same RMS do not sound equally+ * loud. BS.1770 closes that gap with a fixed pre-filter, the K-weighting — a+ * high shelf that lifts what the head boosts above 2 kHz and a high-pass that+ * drops the rumble the ear barely registers — and then measures power.+ *+ * Integrated loudness is the power over the whole programme, measured in+ * 400 ms blocks every 100 ms and gated twice: blocks under −70 LUFS are+ * silence, and blocks more than 10 LU under the loudness of what remains are+ * the pauses between things. What survives is what the listener actually+ * hears as the programme. Loudness range is the spread of the 3 s short-term+ * loudness between its 10th and 95th percentile, after its own gates.+ *+ * True peak is where the waveform goes between the samples. A converter+ * draws a smooth curve through them, and that curve can rise above every+ * sample it passes through; a lossy encoder adds its own overshoot on top. So+ * the peak is read on a 4× oversampled copy, and the limiter holds that peak+ * under its ceiling.+ *+ * Everything here works on plain sample arrays and keeps its state across+ * calls, so the tab can hand it a whole mix and the engine a file read a+ * second at a time, and both reach the same answer.+ */++/** Blocks under this are silence (EBU R128 absolute gate). */+const ABSOLUTE_GATE = -70;+/** Integrated loudness drops blocks this far under the ungated mean. */+const RELATIVE_GATE = -10;+/** Loudness range drops short-term values this far under their mean. */+const RANGE_GATE = -20;+/** The gating step: 100 ms, so a 400 ms block is four steps. */+const STEP_S = 0.1;+const BLOCK_STEPS = 4;+/** Short-term loudness reads 3 s. */+const SHORT_TERM_STEPS = 30;++/** One K-weighting biquad, as a/b with a0 = 1. */+export interface Biquad {+ b: [number, number, number];+ a: [number, number, number];+}++/**+ * The K-weighting pre-filter at `sampleRate`: the high-shelf stage, then the+ * RLB high-pass stage.+ *+ * BS.1770 prints the coefficients for 48 kHz only. The analog prototypes+ * below are the ones those coefficients come from, so the bilinear+ * transform reproduces the printed values at 48 kHz and gives the same+ * response at 44.1 kHz or 96 kHz.+ */+export function kWeighting(sampleRate: number): [Biquad, Biquad] {+ // Stage 1: the high shelf, +4 dB above about 1.7 kHz.+ const f0 = 1681.974450955533;+ const gain = 3.999843853973347;+ const q = 0.7071752369554196;+ const k = Math.tan((Math.PI * f0) / sampleRate);+ const vh = 10 ** (gain / 20);+ const vb = vh ** 0.4996667741545416;+ const a0 = 1 + k / q + k * k;+ const shelf: Biquad = {+ b: [(vh + (vb * k) / q + k * k) / a0, (2 * (k * k - vh)) / a0, (vh - (vb * k) / q + k * k) / a0],+ a: [1, (2 * (k * k - 1)) / a0, (1 - k / q + k * k) / a0],+ };+ // Stage 2: the RLB high-pass at about 38 Hz.+ const f1 = 38.13547087602444;+ const q1 = 0.5003270373238773;+ const k1 = Math.tan((Math.PI * f1) / sampleRate);+ const d = 1 + k1 / q1 + k1 * k1;+ const highpass: Biquad = {+ b: [1, -2, 1],+ a: [1, (2 * (k1 * k1 - 1)) / d, (1 - k1 / q1 + k1 * k1) / d],+ };+ return [shelf, highpass];+}++/** Channel weights for a layout: front channels count once, the surrounds+ * 1.41 (+1.5 dB), and the LFE not at all. Mono and stereo are all ones. */+export function channelWeights(channels: number): number[] {+ // L R C LFE Ls Rs+ if (channels === 6) return [1, 1, 1, 0, 1.41, 1.41];+ return Array.from({ length: channels }, () => 1);+}++/** Power to LUFS: −0.691 cancels the K-weighting's gain at 997 Hz, so a+ * full-scale sine there reads −3.01 LUFS in one channel. */+const lufs = (power: number) => (power > 0 ? -0.691 + 10 * Math.log10(power) : -Infinity);+const fromLufs = (l: number) => 10 ** ((l + 0.691) / 10);++/* ------------------------------------------------------- true peak */++/** Half the interpolator's taps: each phase reads this many samples on each+ * side of the point it estimates. */+const HALF_TAPS = 8;+const TAPS = HALF_TAPS * 2;++/** How far the curve between two samples is taken to rise over the louder+ * of them. Audio rises about 3 dB there at the very worst (a quarter-rate+ * tone sampled either side of its crest); twice that is the margin. A point+ * whose neighbours could not reach the peak in hand times this is never+ * interpolated, which is what keeps the meter cheap on a dense mix. */+const OVERSHOOT = 2;++/**+ * The 4× interpolator, as three fractional-delay filters: the values a+ * quarter, a half and three quarters of the way from one sample to the next.+ * The sample itself is the fourth phase. Each filter is a Kaiser-windowed+ * sinc normalized to unity gain at DC.+ */+function interpolatorPhases(factor: number): Float64Array[] {+ const besselI0 = (x: number) => {+ let sum = 1;+ let term = 1;+ for (let k = 1; k < 32; k++) {+ term *= (x / (2 * k)) ** 2;+ sum += term;+ }+ return sum;+ };+ const beta = 6;+ const span = HALF_TAPS + 0.5;+ const phases: Float64Array[] = [];+ for (let p = 1; p < factor; p++) {+ const frac = p / factor;+ const taps = new Float64Array(TAPS);+ let total = 0;+ for (let j = 0; j < TAPS; j++) {+ // Tap j reads the sample at offset j - (HALF_TAPS - 1) from the left+ // neighbour of the point, which sits `frac` past it.+ const d = frac - (j - (HALF_TAPS - 1));+ const sinc = d === 0 ? 1 : Math.sin(Math.PI * d) / (Math.PI * d);+ const r = d / span;+ const w = Math.abs(r) >= 1 ? 0 : besselI0(beta * Math.sqrt(1 - r * r)) / besselI0(beta);+ taps[j] = sinc * w;+ total += taps[j];+ }+ for (let j = 0; j < TAPS; j++) taps[j] /= total;+ phases.push(taps);+ }+ return phases;+}++/** Oversampling that reads a true peak at a rate: 4× up to 96 kHz, 2× to+ * 192 kHz, and the samples themselves above that. */+function oversampling(sampleRate: number): number {+ return sampleRate >= 176400 ? 1 : sampleRate >= 88200 ? 2 : 4;+}++/** The oversampled curve between samples. */+class Interpolator {+ /** Up to three phases' taps; a phase the rate does not use is all zeros+ * and reads as silence, which never wins the peak. */+ private readonly h1: Float64Array;+ private readonly h2: Float64Array;+ private readonly h3: Float64Array;++ constructor(sampleRate: number) {+ const phases = interpolatorPhases(oversampling(sampleRate));+ const zero = new Float64Array(TAPS);+ // 2× has one phase, the midpoint; 4× has three.+ this.h1 = phases.length === 3 ? phases[0] : zero;+ this.h2 = phases.length === 3 ? phases[1] : (phases[0] ?? zero);+ this.h3 = phases.length === 3 ? phases[2] : zero;+ }++ /** The loudest interpolated value between x[i] and x[i + 1], where the+ * window x[i - HALF_TAPS + 1 .. i + HALF_TAPS] is all in range. Every+ * phase is read in one pass over the window. */+ between(x: Float32Array | Float64Array, i: number): number {+ const base = i - (HALF_TAPS - 1);+ const { h1, h2, h3 } = this;+ let y1 = 0;+ let y2 = 0;+ let y3 = 0;+ for (let j = 0; j < TAPS; j++) {+ const v = x[base + j];+ y1 += h1[j] * v;+ y2 += h2[j] * v;+ y3 += h3[j] * v;+ }+ const a1 = y1 < 0 ? -y1 : y1;+ const a2 = y2 < 0 ? -y2 : y2;+ const a3 = y3 < 0 ? -y3 : y3;+ return a1 > a2 ? (a1 > a3 ? a1 : a3) : a2 > a3 ? a2 : a3;+ }+}++/* ------------------------------------------------------- the meter */++/** What a stretch of audio measures. */+export interface LoudnessMeasure {+ /** Gated programme loudness, LUFS; −Infinity when nothing clears the+ * absolute gate. */+ integratedLufs: number;+ /** The loudest point of the waveform, sample or in between, dBTP. */+ truePeakDbtp: number;+ /** The loudest sample, dBFS. */+ samplePeakDbfs: number;+ /** Spread of the short-term loudness, LU (EBU Tech 3342). */+ loudnessRangeLu: number;+ /** The loudest 400 ms block, LUFS. */+ maxMomentaryLufs: number;+ /** The loudest 3 s window, LUFS. */+ maxShortTermLufs: number;+ /** Seconds measured. */+ seconds: number;+}++/**+ * A streaming BS.1770 meter. Push planar chunks of any length; read the+ * result at the end.+ *+ * The weighted power of every 100 ms step is kept — ten numbers a second —+ * and the blocks, the gates and the range are all computed from those at the+ * end, so a chunk boundary can land anywhere without changing the answer.+ */+export class LoudnessMeter {+ private readonly stages: [Biquad, Biquad];+ /** Per channel: the two biquads' state, x1 x2 y1 y2 for each. */+ private readonly state: Float64Array[];+ private readonly weights: number[];+ private readonly stepLength: number;+ private stepPower = 0;+ private stepFill = 0;+ /** Weighted mean square of every finished step. */+ private steps: number[] = [];+ private samples = 0;+ private samplePeak = 0;+ private truePeak = 0;+ private readonly interp: Interpolator;+ /** The last samples of each channel, for interpolating across chunks. */+ private tails: Float32Array[];++ constructor(+ readonly sampleRate: number,+ readonly channels: number,+ opts: { weights?: number[] } = {}+ ) {+ this.stages = kWeighting(sampleRate);+ this.state = Array.from({ length: channels }, () => new Float64Array(8));+ this.weights = opts.weights ?? channelWeights(channels);+ this.stepLength = Math.round(sampleRate * STEP_S);+ this.interp = new Interpolator(sampleRate);+ // The stream opens on silence: the first samples interpolate against+ // zeros, the way a converter starting from rest would draw them.+ this.tails = Array.from({ length: channels }, () => new Float32Array(TAPS - 1));+ }++ /** Measure `chunk`: one array per channel, all the same length. */+ push(chunk: ArrayLike<number>[]): void {+ const n = chunk[0]?.length ?? 0;+ if (n === 0) return;+ this.weigh(chunk, n);+ this.peaks(chunk, n);+ this.samples += n;+ }++ private weigh(chunk: ArrayLike<number>[], n: number) {+ const [s1, s2] = this.stages;+ const [p0, p1, p2] = s1.b;+ const [, pa1, pa2] = s1.a;+ const [q0, q1, q2] = s2.b;+ const [, qa1, qa2] = s2.a;+ let offset = 0;+ while (offset < n) {+ const take = Math.min(n - offset, this.stepLength - this.stepFill);+ let power = 0;+ for (let c = 0; c < this.channels; c++) {+ const w = this.weights[c] ?? 1;+ const x = chunk[c];+ const st = this.state[c];+ let [ax1, ax2, ay1, ay2, bx1, bx2, by1, by2] = st;+ let sum = 0;+ for (let i = offset; i < offset + take; i++) {+ const v = x[i];+ const y = p0 * v + p1 * ax1 + p2 * ax2 - pa1 * ay1 - pa2 * ay2;+ ax2 = ax1;+ ax1 = v;+ ay2 = ay1;+ ay1 = y;+ const z = q0 * y + q1 * bx1 + q2 * bx2 - qa1 * by1 - qa2 * by2;+ bx2 = bx1;+ bx1 = y;+ by2 = by1;+ by1 = z;+ sum += z * z;+ }+ st[0] = ax1;+ st[1] = ax2;+ st[2] = ay1;+ st[3] = ay2;+ st[4] = bx1;+ st[5] = bx2;+ st[6] = by1;+ st[7] = by2;+ if (w !== 0) power += w * sum;+ }+ this.stepPower += power;+ this.stepFill += take;+ offset += take;+ if (this.stepFill === this.stepLength) {+ this.steps.push(this.stepPower / this.stepLength);+ this.stepPower = 0;+ this.stepFill = 0;+ }+ }+ }++ private peaks(chunk: ArrayLike<number>[], n: number) {+ const keep = TAPS - 1;+ for (let c = 0; c < this.channels; c++) {+ const x = chunk[c];+ // The window is the previous chunk's tail followed by this chunk, so+ // the points between the two are measured too.+ const ext = new Float32Array(keep + n);+ ext.set(this.tails[c], 0);+ for (let i = 0; i < n; i++) {+ const v = x[i];+ ext[keep + i] = v;+ const a = v < 0 ? -v : v;+ if (a > this.samplePeak) this.samplePeak = a;+ }+ if (this.samplePeak > this.truePeak) this.truePeak = this.samplePeak;+ this.truePeak = Math.max(this.truePeak, scanBetween(this.interp, ext, this.truePeak));+ this.tails[c] = ext.slice(ext.length - keep);+ }+ }++ /** The measurement of everything pushed so far. */+ result(): LoudnessMeasure {+ // The curve past the last samples, as it falls back to rest.+ let truePeak = this.truePeak;+ for (const tail of this.tails) {+ const ext = new Float32Array(tail.length + HALF_TAPS + 1);+ ext.set(tail, 0);+ truePeak = Math.max(truePeak, scanBetween(this.interp, ext, truePeak));+ }+ const steps = this.steps;+ const blocks: number[] = [];+ for (let j = 0; j + BLOCK_STEPS <= steps.length; j++) {+ let p = 0;+ for (let k = 0; k < BLOCK_STEPS; k++) p += steps[j + k];+ blocks.push(p / BLOCK_STEPS);+ }+ const shortTerm: number[] = [];+ for (let j = 0; j + SHORT_TERM_STEPS <= steps.length; j++) {+ let p = 0;+ for (let k = 0; k < SHORT_TERM_STEPS; k++) p += steps[j + k];+ shortTerm.push(p / SHORT_TERM_STEPS);+ }+ return {+ integratedLufs: integratedFrom(blocks),+ truePeakDbtp: 20 * Math.log10(truePeak || 1e-10),+ samplePeakDbfs: 20 * Math.log10(this.samplePeak || 1e-10),+ loudnessRangeLu: rangeFrom(shortTerm),+ maxMomentaryLufs: lufs(blocks.reduce((m, p) => (p > m ? p : m), 0)),+ maxShortTermLufs: lufs(shortTerm.reduce((m, p) => (p > m ? p : m), 0)),+ seconds: this.samples / this.sampleRate,+ };+ }+}++/** The loudest point between samples in `x` that could beat `floor`. Points+ * whose window cannot reach it are never interpolated. */+function scanBetween(interp: Interpolator, x: Float32Array, floor: number): number {+ let peak = floor;+ const first = HALF_TAPS - 1;+ const last = x.length - HALF_TAPS - 1;+ for (let i = first; i <= last; i++) {+ const a = x[i] < 0 ? -x[i] : x[i];+ const b = x[i + 1] < 0 ? -x[i + 1] : x[i + 1];+ if ((a > b ? a : b) * OVERSHOOT <= peak) continue;+ const v = interp.between(x, i);+ if (v > peak) peak = v;+ }+ return peak;+}++/** Integrated loudness from block powers: the absolute gate, then the+ * relative gate 10 LU under what the absolute gate kept. */+export function integratedFrom(blocks: number[]): number {+ const abs = fromLufs(ABSOLUTE_GATE);+ const loud = blocks.filter((p) => p > abs);+ if (loud.length === 0) return -Infinity;+ const ungated = loud.reduce((s, p) => s + p, 0) / loud.length;+ const rel = fromLufs(lufs(ungated) + RELATIVE_GATE);+ const kept = loud.filter((p) => p > rel);+ if (kept.length === 0) return -Infinity;+ return lufs(kept.reduce((s, p) => s + p, 0) / kept.length);+}++/** Loudness range from short-term powers (EBU Tech 3342): gate at −70 LUFS+ * and 20 LU under the mean, then the 10th to the 95th percentile. */+export function rangeFrom(shortTerm: number[]): number {+ const abs = fromLufs(ABSOLUTE_GATE);+ const loud = shortTerm.filter((p) => p > abs);+ if (loud.length === 0) return 0;+ const mean = loud.reduce((s, p) => s + p, 0) / loud.length;+ const rel = fromLufs(lufs(mean) + RANGE_GATE);+ const kept = loud+ .filter((p) => p > rel)+ .map(lufs)+ .sort((a, b) => a - b);+ if (kept.length === 0) return 0;+ const at = (q: number) => {+ const pos = q * (kept.length - 1);+ const lo = Math.floor(pos);+ const hi = Math.min(kept.length - 1, lo + 1);+ return kept[lo] + (kept[hi] - kept[lo]) * (pos - lo);+ };+ return at(0.95) - at(0.1);+}++/** Measure whole planar buffers in one go. */+export function measureLoudness(channels: ArrayLike<number>[], sampleRate: number): LoudnessMeasure {+ const meter = new LoudnessMeter(sampleRate, channels.length);+ meter.push(channels);+ return meter.result();+}++/** `measureLoudness` in half-second slices with a yield between them, for a+ * whole mix measured on the page's thread. */+export async function measureLoudnessSliced(channels: Float32Array[], sampleRate: number): Promise<LoudnessMeasure> {+ const meter = new LoudnessMeter(sampleRate, channels.length);+ const slice = Math.max(1, Math.round(sampleRate / 2));+ const length = channels[0]?.length ?? 0;+ for (let i = 0; i < length; i += slice) {+ meter.push(channels.map((c) => c.subarray(i, Math.min(length, i + slice))));+ await new Promise<void>((r) => setTimeout(r, 0));+ }+ return meter.result();+}++/** A measurement rounded for a reply: two decimals, and null where there is+ * no number (silence has no integrated loudness). */+export function roundedLoudness(m: LoudnessMeasure) {+ const r = (v: number) => (Number.isFinite(v) ? Math.round(v * 100) / 100 : null);+ return {+ integratedLufs: r(m.integratedLufs),+ truePeakDbtp: r(m.truePeakDbtp),+ loudnessRangeLu: r(m.loudnessRangeLu),+ maxShortTermLufs: r(m.maxShortTermLufs),+ };+}++/* ------------------------------------------------------- mastering */++/** What a master does to a mix: one static gain, and the limiter only when+ * the gained mix would cross the ceiling. */+export interface MasterPlan {+ gainDb: number;+ limit: boolean;+ /** The true peak the gain alone would leave, dBTP. */+ peakAfterGainDbtp: number;+}++/**+ * Plan a master from a measurement. Silence has no loudness to move, so it+ * keeps its level; anything else moves by exactly the distance to the+ * target, and the limiter runs only where that gain carries the true peak+ * past the ceiling.+ */+export function masterPlan(measured: LoudnessMeasure, targetLufs: number, ceilingDbtp: number): MasterPlan {+ const gainDb = Number.isFinite(measured.integratedLufs) ? targetLufs - measured.integratedLufs : 0;+ const peakAfterGainDbtp = measured.truePeakDbtp + gainDb;+ return { gainDb, limit: peakAfterGainDbtp > ceilingDbtp, peakAfterGainDbtp };+}++/** How long the limiter looks ahead, and so how long it takes to reach a+ * reduction: long enough that the gain glides down, short enough that a+ * transient does not pull the sound before it audibly early. */+const LOOKAHEAD_S = 0.005;+/** How quickly the reduction lets go, as a time constant. Long enough to+ * avoid the flutter a fast release makes on bass, short enough that the mix+ * does not sit audibly lower after a hit. */+const RELEASE_S = 0.12;+/** Headroom kept under the ceiling for the small error between the gain the+ * limiter applies and the curve it predicts. */+const CEILING_MARGIN_DB = 0.05;++/**+ * A look-ahead true-peak limiter, linked across channels.+ *+ * For every sample it works out the gain that would keep the loudest point+ * near it — the sample, or the interpolated curve on either side — under the+ * ceiling. The gain it applies is that requirement held over the look-ahead+ * window, let go on an exponential release, and averaged over the window+ * again. Every held value that goes into a sample's average is no more than+ * that sample's requirement, so the average is no more than it either and the+ * ceiling holds; and the averaging is what makes the gain glide instead of+ * stepping, which is the difference between limiting and clipping.+ *+ * The output runs `latency` samples behind the input. `process` returns what+ * is ready and `flush` the rest, so the total comes out the same length as+ * what went in and lines up with it sample for sample.+ */+export class TruePeakLimiter {+ readonly latency: number;+ private readonly window: number;+ private readonly ceiling: number;+ private readonly release: number;+ private readonly interp: Interpolator;+ /** Input history per channel: long enough to interpolate and to delay. */+ private readonly hist: Float64Array[];+ /** The loudest channel's |x| per sample, on the same ring. */+ private readonly loudest: Float64Array;+ private readonly histMask: number;+ /** Samples fed (real and the flush's padding), real ones received, and+ * samples handed back. */+ private fed = 0;+ private received = 0;+ private emitted = 0;+ /** Required gains, a monotonic deque over them for the forward minimum,+ * and the held gains the running average reads: rings by sample index. */+ private readonly need: Float64Array;+ private readonly dq: Int32Array;+ private dqHead = 0;+ private dqTail = 0;+ private readonly held: Float64Array;+ private readonly ringMask: number;+ private heldSum: number;+ private lastHeld = 1;+ /** The interpolated peak just before the position being decided. */+ private betweenPrev = 0;+ private readonly scratch = new Float64Array(TAPS);++ constructor(+ readonly sampleRate: number,+ readonly channels: number,+ ceilingDbtp: number+ ) {+ this.window = Math.max(1, Math.round(LOOKAHEAD_S * sampleRate));+ this.ceiling = 10 ** ((ceilingDbtp - CEILING_MARGIN_DB) / 20);+ this.release = Math.exp(-1 / (RELEASE_S * sampleRate));+ this.interp = new Interpolator(sampleRate);+ // A position is decided once the interpolator can see HALF_TAPS past it,+ // and a sample's gain settles once the window after it is decided.+ this.latency = HALF_TAPS + this.window - 1;+ const histLen = 1 << Math.ceil(Math.log2(this.latency + TAPS + 2));+ this.histMask = histLen - 1;+ this.hist = Array.from({ length: channels }, () => new Float64Array(histLen));+ this.loudest = new Float64Array(histLen);+ const ring = 1 << Math.ceil(Math.log2(this.window + 2));+ this.ringMask = ring - 1;+ this.need = new Float64Array(ring);+ this.dq = new Int32Array(ring);+ // The gains before the first sample are unity.+ this.held = new Float64Array(ring).fill(1);+ this.heldSum = this.window;+ }++ /** Feed planar input; returns the planar output that is ready. */+ process(input: ArrayLike<number>[]): Float32Array[] {+ const n = input[0]?.length ?? 0;+ const out = Array.from({ length: this.channels }, () => new Float32Array(n));+ let w = 0;+ for (let i = 0; i < n; i++) {+ for (let c = 0; c < this.channels; c++) this.hist[c][this.fed & this.histMask] = input[c][i];+ this.received++;+ w = this.advance(out, w);+ }+ return w === n ? out : out.map((o) => o.subarray(0, w));+ }++ /** The output the look-ahead is still holding. */+ flush(): Float32Array[] {+ const left = this.received - this.emitted;+ const out = Array.from({ length: this.channels }, () => new Float32Array(left));+ let w = 0;+ // Silence past the end lets every held sample through.+ while (this.emitted < this.received) {+ for (let c = 0; c < this.channels; c++) this.hist[c][this.fed & this.histMask] = 0;+ w = this.advance(out, w);+ }+ return out.map((o) => o.subarray(0, w));+ }++ /** Take in the sample just written at `fed`, decide what it unlocks, and+ * emit the sample whose gain settled. */+ private advance(out: Float32Array[], w: number): number {+ const t = this.fed++;+ const m = this.histMask;+ let loud = 0;+ for (let c = 0; c < this.channels; c++) {+ const v = this.hist[c][t & m];+ const a = v < 0 ? -v : v;+ if (a > loud) loud = a;+ }+ this.loudest[t & m] = loud;+ const pos = t - HALF_TAPS;+ if (pos < 0) return w;+ const s = this.decide(pos);+ if (s < 0 || s >= this.received) return w;+ const g = this.heldSum / this.window;+ for (let c = 0; c < this.channels; c++) out[c][w] = this.hist[c][s & m] * g;+ this.emitted++;+ return w + 1;+ }++ /** Decide the requirement at `pos` and settle the held gain of the sample+ * a window before it, which is returned (negative while none has). */+ private decide(pos: number): number {+ const m = this.histMask;+ const near = Math.max(this.loudest[pos & m], this.loudest[(pos + 1) & m]);+ // The curve between pos and pos + 1, read only where it could reach the+ // ceiling.+ let between = 0;+ if (near * OVERSHOOT > this.ceiling) {+ const x = this.scratch;+ for (let c = 0; c < this.channels; c++) {+ for (let j = 0; j < TAPS; j++) {+ const i = pos - (HALF_TAPS - 1) + j;+ x[j] = i >= 0 ? this.hist[c][i & m] : 0;+ }+ const v = this.interp.between(x, HALF_TAPS - 1);+ if (v > between) between = v;+ }+ }+ const peak = Math.max(this.loudest[pos & m], between, this.betweenPrev);+ this.betweenPrev = between;+ const need = peak > this.ceiling ? this.ceiling / peak : 1;++ // The forward minimum of the requirement over [s, s + window - 1].+ const r = this.ringMask;+ this.need[pos & r] = need;+ while (this.dqTail > this.dqHead && this.need[this.dq[(this.dqTail - 1) & r] & r] >= need) this.dqTail--;+ this.dq[this.dqTail++ & r] = pos;+ const s = pos - this.window + 1;+ while (this.dq[this.dqHead & r] < s) this.dqHead++;+ if (s < 0) return s;+ const hold = this.need[this.dq[this.dqHead & r] & r];+ // Let go exponentially, never above what the window demands.+ const g = Math.min(hold, 1 - (1 - this.lastHeld) * this.release);+ this.lastHeld = g;+ if (s === 0) {+ // Nothing plays before the first sample, so the average leading into+ // it starts at the first sample's own held gain: a unity run there+ // would let the opening peaks through.+ this.held.fill(g);+ this.heldSum = this.window * g;+ } else {+ this.heldSum += g - this.held[(s - this.window) & r];+ this.held[s & r] = g;+ }+ return s;+ }+}++/** What a master did. */+export interface MasterReport {+ /** The mix as it came in. */+ measured: LoudnessMeasure;+ plan: MasterPlan;+}++/**+ * Master a stream of planar chunks to `targetLufs` under `ceilingDbtp`.+ *+ * Two reads of the same audio: the first measures it, the second applies the+ * gain and, only when the plan calls for it, the limiter. `chunks` is called+ * once per read; `write` receives the mastered audio in order, the same+ * number of samples as went in. The tab hands it its mix buffer a slice at a+ * time and the engine its rendered mix a second at a time, so both run this+ * exact code.+ */+export async function masterStream(+ chunks: () => AsyncIterable<Float32Array[]>,+ write: (chunk: Float32Array[]) => Promise<void> | void,+ opts: { sampleRate: number; channels: number; targetLufs: number; ceilingDbtp: number }+): Promise<MasterReport> {+ const meter = new LoudnessMeter(opts.sampleRate, opts.channels);+ for await (const chunk of chunks()) meter.push(chunk);+ const measured = meter.result();+ const plan = masterPlan(measured, opts.targetLufs, opts.ceilingDbtp);+ const gain = 10 ** (plan.gainDb / 20);+ const limiter = plan.limit ? new TruePeakLimiter(opts.sampleRate, opts.channels, opts.ceilingDbtp) : null;+ for await (const chunk of chunks()) {+ const gained = chunk.map((x) => {+ const y = new Float32Array(x.length);+ for (let i = 0; i < x.length; i++) y[i] = x[i] * gain;+ return y;+ });+ const out = limiter ? limiter.process(gained) : gained;+ if (out[0].length > 0) await write(out);+ }+ if (limiter) {+ const rest = limiter.flush();+ if (rest[0].length > 0) await write(rest);+ }+ return { measured, plan };+}++/** Master planar buffers in place; see `masterStream`. Reads and writes in+ * half-second slices and yields between them, so a long mix never holds the+ * thread for the whole of it. */+export async function masterInPlace(+ channels: Float32Array[],+ opts: { sampleRate: number; targetLufs: number; ceilingDbtp: number }+): Promise<MasterReport> {+ const slice = Math.max(1, Math.round(opts.sampleRate / 2));+ const length = channels[0]?.length ?? 0;+ const pause = () => new Promise<void>((r) => setTimeout(r, 0));+ let at = 0;+ return masterStream(+ async function* () {+ for (let i = 0; i < length; i += slice) {+ // A copy: the limiter's writes land behind the read, on the same+ // arrays.+ yield channels.map((c) => c.slice(i, Math.min(length, i + slice)));+ await pause();+ }+ },+ (chunk) => {+ chunk.forEach((c, k) => channels[k].set(c, at));+ at += chunk[0].length;+ },+ { ...opts, channels: channels.length }+ );+}+diff --git a/site/src/cut/lib/loudnessSettings.ts b/site/src/cut/lib/loudnessSettings.tsnew file mode 100644index 00000000..9a1b6a9f--- /dev/null+++ b/site/src/cut/lib/loudnessSettings.ts@@ -0,0 +1,46 @@+import { SETTINGS, type Settings } from "@/lib/config/registry";++// Export loudness: the LUFS each named target masters to, the target an+// export starts on, and the true-peak ceiling — bound once at session start+// from account config (the tab) or the global setting (a worker job), and read+// wherever an export's settings are made.++let loudness: Settings["cutLoudness"] = SETTINGS.cutLoudness.default;++export function bindCutLoudness(value: unknown): void {+ const parsed = SETTINGS.cutLoudness.schema.safeParse(value);+ if (parsed.success) loudness = parsed.data;+}++export function cutLoudness(): Settings["cutLoudness"] {+ return loudness;+}++/** A loudness choice: off, or one of the named targets. */+export type LoudnessId = Settings["cutLoudness"]["defaultTarget"];++/** The choices in menu order. The numbers come from the setting. */+export const LOUDNESS_CHOICES = [+ { id: "off", label: "Off", detail: "the mix as it plays" },+ { id: "social", label: "Social", detail: "streaming and social platforms" },+ { id: "podcast", label: "Podcast", detail: "spoken word" },+ { id: "broadcast", label: "Broadcast", detail: "EBU R128 television" },+] as const satisfies readonly { id: LoudnessId; label: string; detail: string }[];++export const LOUDNESS_IDS = LOUDNESS_CHOICES.map((c) => c.id) as [LoudnessId, ...LoudnessId[]];++/** The settings fields a choice adds to an export: the target and the+ * ceiling, or nothing for off. */+export function loudnessSettings(id: LoudnessId | undefined): { loudness?: number; truePeakCeiling?: number } {+ const s = cutLoudness();+ const choice = id ?? s.defaultTarget;+ if (choice === "off") return {};+ return { loudness: s.targets[choice], truePeakCeiling: s.truePeakCeiling };+}++/** A choice as the menu shows it: "Social · −14 LUFS". */+export function loudnessLabel(id: LoudnessId): string {+ const c = LOUDNESS_CHOICES.find((x) => x.id === id)!;+ if (id === "off") return c.label;+ return `${c.label} · ${String(cutLoudness().targets[id]).replace("-", "−")} LUFS`;+}diff --git a/site/src/cut/lib/measureLevelBound.test.ts b/site/src/cut/lib/measureLevelBound.test.tsnew file mode 100644index 00000000..0aefc50f--- /dev/null+++ b/site/src/cut/lib/measureLevelBound.test.ts@@ -0,0 +1,54 @@+import { expect, test } from "bun:test";+import { stubModule } from "@/lib/testing/stubModule";+import type { MediaAsset } from "./types";++// measure_level hears a clip where it sounds: a video bound to a recording is+// measured off the recording, over the stretch the clip plays, on the+// recording's clock.+const reads: { url: string; from: number; to?: number }[] = [];++await stubModule<typeof import("./media")>("./media", import.meta.url, {+ measureSourceLevel: (async (url: string, opts: { from: number; to?: number }) => {+ reads.push({ url, ...opts });+ return {+ rmsDb: -20,+ loudestFrameDb: -10,+ audibleSeconds: 2,+ loudness: { integrated: -18, truePeak: -3, range: 4, momentaryMax: -12, shortTermMax: -14 },+ };+ }) as never,+});++const { runAiTool } = await import("./aiTools");+const { useEditor } = await import("./store");++const media = (id: string, type: "video" | "audio", over: Partial<MediaAsset> = {}): MediaAsset => ({+ id,+ fileName: `${id}.file`,+ name: id,+ type,+ duration: 60,+ url: `blob:${id}`,+ ...over,+});++test("measure_level reads the recording bound to the video, on the recording's clock", async () => {+ useEditor.setState({+ projectId: "p1",+ loaded: true,+ readOnly: false,+ clips: [{ id: "a", assetId: "cam", start: 0, in: 10, out: 14, track: 0, muted: false }],+ audioClips: [],+ overlays: [],+ transitions: [],+ assets: [media("cam", "video", { soundFrom: { assetId: "rec", offset: -4 } }), media("rec", "audio")],+ });+ const res = (await runAiTool("measure_level", { ids: ["a"] })) as { clips: { id: string; sourceDb: number }[] };+ expect(reads).toEqual([{ url: "blob:rec", from: 6, to: 10 }]);+ expect(res.clips[0].sourceDb).toBe(-20);++ reads.length = 0;+ useEditor.getState().setAssetSoundFrom("cam", undefined);+ await runAiTool("measure_level", { ids: ["a"] });+ expect(reads).toEqual([{ url: "blob:cam", from: 10, to: 14 }]);+});diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 64580ff0..e6555ed0 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1349,15 +1349,59 @@ export async function scanSourceSpeech(return scanSpeech(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });}-/** How loud a span of a source plays. Same residency story as the speech- * scan: the page decodes its own media, the headless runner installs the- * same decoders. */+/** Where a separately recorded sound lines up with a camera's own: read off+ * both files where the page or the headless runner decodes them (soundSync.ts+ * holds the arithmetic). */+export async function syncSourceSound(+ cameraUrl: string,+ recordingUrl: string,+ opts: { cameraFrom: number; cameraTo: number; recordingTo: number }+): Promise<SyncAnswer | null> {+ return syncSound(+ {+ camera: (from, to) => sourcePcm(cameraUrl, from, to),+ recording: (from, to) => sourcePcm(recordingUrl, from, to),+ },+ opts+ );+}++/** How loud a span of a source plays: its RMS level and, from the same+ * decode, its BS.1770 loudness and true peak. A mono source is measured as+ * the two channels it plays on in the stereo mix. Same residency story as the+ * speech scan: the page decodes its own media, the headless runner installs+ * the same decoders. */export async function measureSourceLevel(sourceUrl: string,opts: { from: number; to?: number }-): Promise<LevelScan> {+): Promise<LevelScan & { loudness: LoudnessMeasure | null }> {const from = Math.max(0, opts.from);- return scanLevel(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });+ const metered: { meter?: LoudnessMeter } = {};+ async function* tapped(): AsyncGenerator<PcmChunk> {+ for await (const chunk of sourcePcm(sourceUrl, from, opts.to)) {+ const n = chunk.channels[0]?.length ?? 0;+ if (n > 0) {+ const rate = chunk.sampleRate;+ const meter = (metered.meter ??= new LoudnessMeter(+ rate,+ chunk.channels.length,+ chunk.channels.length === 1 ? { weights: [2] } : {}+ ));+ // A decode opens on the packet before `from` and may run past `to`;+ // only the asked span is measured.+ const a = Math.max(0, Math.round((from - chunk.timestamp) * rate));+ const b = opts.to === undefined ? n : Math.min(n, Math.round((opts.to - chunk.timestamp) * rate));+ if (b > a) {+ meter.push(+ Array.from({ length: meter.channels }, (_, c) => (chunk.channels[c] ?? chunk.channels[0]).subarray(a, b))+ );+ }+ }+ yield chunk;+ }+ }+ const level = await scanLevel(tapped(), { ...opts, from });+ return { ...level, loudness: metered.meter ? metered.meter.result() : null };}/** The source's musical beat grid, end to end. The grid is a property of thediff --git a/site/src/cut/lib/sourceExportPlan.ts b/site/src/cut/lib/sourceExportPlan.tsindex 440c0016..c994b9cf 100644--- a/site/src/cut/lib/sourceExportPlan.ts+++ b/site/src/cut/lib/sourceExportPlan.ts@@ -43,6 +43,8 @@ export function sourceSequence(doc: ExportDoc) {export function sourceExportPlan(doc: ExportDoc, settings: ExportSettings): SourceSegment[] | null {if (!settings.copySource || settings.container !== "mp4" || settings.audioCodec !== "aac") return null;+ // A mastered mix is new audio: the sound has to render.+ if (settings.loudness !== undefined) return null;const sources = sourceSequence(doc);if (!sources || settings.width !== sources[0].asset.width || settings.height !== sources[0].asset.height) return null;const from = Math.max(0, settings.range?.start ?? 0);diff --git a/site/src/cut/lib/stems.test.ts b/site/src/cut/lib/stems.test.tsnew file mode 100644index 00000000..f63c3555--- /dev/null+++ b/site/src/cut/lib/stems.test.ts@@ -0,0 +1,110 @@+import { describe, expect, test } from "bun:test";+import { spawnSync } from "node:child_process";+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import os from "node:os";+import path from "node:path";+import { crc32, pcm24, planStems, wavBytes, wavChunks, wavHeader, writeStoredZip } from "./stems";++const asset = (id: string, over: Record<string, unknown> = {}) => ({ id, type: "audio" as const, name: `${id}.mp3`, ...over });+const video = (id: string) => ({ id, type: "video" as const, name: `${id}.mp4` });++describe("stem plan", () => {+ test("the video clips' sound first, then one stem per soundtrack lane in lane order", () => {+ const stems = planStems({+ clips: [{ assetId: "v", muted: false }],+ audioClips: [+ { assetId: "song", lane: 1 },+ { assetId: "vo", lane: 0 },+ ],+ assets: [video("v"), asset("song", { beats: { beats: [1], bpm: 120 } }), asset("vo", { origin: "voiceover" })],+ });+ expect(stems).toEqual([+ { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },+ { name: "Voiceover", lane: 0, file: "2 Voiceover.wav" },+ { name: "Music", lane: 1, file: "3 Music.wav" },+ ]);+ });++ test("a ducking clip reads as voiceover; a lane of one file takes its name; a mixed lane its number", () => {+ const stems = planStems({+ clips: [],+ audioClips: [+ { assetId: "take", lane: 0, duck: 0.3 },+ { assetId: "whoosh", lane: 2 },+ { assetId: "whoosh", lane: 2, name: "Swish" },+ { assetId: "a", lane: 3 },+ { assetId: "b", lane: 3 },+ ],+ assets: [asset("take"), asset("whoosh"), asset("a"), asset("b")],+ });+ expect(stems.map((s) => s.name)).toEqual(["Voiceover", "whoosh", "Audio 4"]);+ });++ test("nothing audible makes no stem: muted and hidden clips, stills, silent lanes", () => {+ const stems = planStems({+ clips: [{ assetId: "v", muted: true }, { assetId: "img", muted: false }, { assetId: "v", muted: false, hidden: true }],+ audioClips: [{ assetId: "a", hidden: true }],+ assets: [video("v"), { id: "img", type: "image", name: "still.png" }, asset("a")],+ });+ expect(stems).toEqual([]);+ });++ test("two lanes of music are told apart", () => {+ const beat = { beats: { beats: [1], bpm: 90 } };+ const stems = planStems({+ clips: [],+ audioClips: [{ assetId: "a", lane: 0 }, { assetId: "b", lane: 1 }],+ assets: [asset("a", beat), asset("b", beat)],+ });+ expect(stems.map((s) => s.file)).toEqual(["1 Music.wav", "2 Music 2.wav"]);+ });+});++describe("WAV", () => {+ test("a 24-bit header that counts its data", () => {+ const h = new DataView(wavHeader(48000, 2, 48000).buffer);+ expect(h.getUint32(40, true)).toBe(48000 * 2 * 3);+ expect(h.getUint16(34, true)).toBe(24);+ expect(wavBytes(48000, 2)).toBe(44 + 48000 * 6);+ });++ test("samples interleave, clamp past full scale, and pad a short stem with silence", () => {+ const out = pcm24([Float32Array.from([1, -2]), Float32Array.from([0.5])], 0, 3, 2);+ const s = (i: number) => (out[i * 3] | (out[i * 3 + 1] << 8) | (out[i * 3 + 2] << 16)) << 8 >> 8;+ expect([s(0), s(1), s(2), s(3), s(4), s(5)]).toEqual([8388607, 4194304, -8388608, 0, 0, 0]);+ });+});++describe("stored zip", () => {+ test("CRC-32 matches the standard check value", () => {+ expect(crc32(new TextEncoder().encode("123456789"))).toBe(0xcbf43926);+ });++ test("unzip reads the archive back, every entry intact", async () => {+ const tone = Float32Array.from({ length: 4800 }, (_, i) => Math.sin(i / 10) * 0.5);+ const entries = [+ { name: "1 Dialogue.wav", channels: [tone, tone] },+ { name: "2 Music.wav", channels: [tone.map((v) => -v), tone] },+ ].map((e) => ({+ name: e.name,+ size: wavBytes(4800, 2),+ read: () => wavChunks(e.channels, 4800, 2, 48000),+ }));+ const chunks: Uint8Array[] = [];+ await writeStoredZip(entries, (b) => void chunks.push(b.slice()));+ const dir = mkdtempSync(path.join(os.tmpdir(), "stems-zip-"));+ try {+ const file = path.join(dir, "stems.zip");+ writeFileSync(file, Buffer.concat(chunks));+ const test = spawnSync("unzip", ["-t", file]);+ expect(test.status).toBe(0);+ const out = path.join(dir, "out");+ expect(spawnSync("unzip", ["-q", file, "-d", out]).status).toBe(0);+ const wav = readFileSync(path.join(out, "2 Music.wav"));+ expect(wav.length).toBe(wavBytes(4800, 2));+ expect(wav.subarray(0, 4).toString()).toBe("RIFF");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});diff --git a/site/src/cut/lib/stems.ts b/site/src/cut/lib/stems.tsnew file mode 100644index 00000000..e032427e--- /dev/null+++ b/site/src/cut/lib/stems.ts@@ -0,0 +1,343 @@+/**+ * Stems: the cut's audio split by where it lives on the timeline, one WAV per+ * part, packed in one zip beside the video.+ *+ * A stem is a lane of sound printed on its own over the whole delivery: every+ * file starts at the delivery's first sample and runs to its last, so they+ * line up when dropped side by side in a mixer. They are printed before the+ * master, with every volume, fade, duck and effect element the mix has, so+ * adding them back together gives the mix as it was before its loudness gain.+ *+ * Pure, so the tab, the engine and the worker all plan the same stems and+ * write the same files.+ */++import { assetIsSilent, type AudioClip, type MediaAsset, type VideoClip } from "./types";++/** One stem: what it is called, the file it lands in, and what it carries —+ * the sound of every video clip (`lane` null), or one soundtrack lane. */+export interface StemDef {+ name: string;+ file: string;+ lane: number | null;+}++type StemDoc = {+ clips: Pick<VideoClip, "assetId" | "muted" | "hidden">[];+ audioClips: Pick<AudioClip, "assetId" | "hidden" | "lane" | "duck" | "name">[];+ assets: Pick<MediaAsset, "id" | "type" | "name" | "origin" | "beats">[];+};++/** A name as a file name takes it. */+const fileSafe = (name: string) => name.replace(/[/\\:*?"<>|]/g, "").trim().slice(0, 40);++/** A media file's name without its extension. */+const bare = (name: string) => name.replace(/\.[a-z0-9]{2,4}$/i, "");++/**+ * What a soundtrack lane holds, as a name: the voiceover, the music, the one+ * file it plays, or its number when it mixes several things.+ */+function laneName(lane: number, clips: StemDoc["audioClips"], assets: Map<string, StemDoc["assets"][number]>): string {+ const sources = clips.map((c) => assets.get(c.assetId)!);+ if (sources.every((a, i) => a.origin === "voiceover" || (clips[i].duck ?? 1) < 1)) return "Voiceover";+ if (sources.every((a) => (a.beats?.bpm ?? 0) > 0)) return "Music";+ const ids = new Set(clips.map((c) => c.assetId));+ if (ids.size === 1) {+ const named = fileSafe(clips[0].name?.trim() || bare(sources[0].name));+ if (named) return named;+ }+ return `Audio ${lane + 1}`;+}++/**+ * The stems a cut splits into: the video clips' own sound first, when any of+ * it plays, then each soundtrack lane that has something on it, in lane+ * order. Muted and hidden clips, and lanes of nothing audible, make no stem.+ */+export function planStems(doc: StemDoc): StemDef[] {+ const assets = new Map(doc.assets.map((a) => [a.id, a]));+ const audible = (assetId: string) => {+ const a = assets.get(assetId);+ return !!a && !assetIsSilent(a);+ };+ const stems: Omit<StemDef, "file">[] = [];+ if (doc.clips.some((c) => !c.muted && !c.hidden && audible(c.assetId))) {+ stems.push({ name: "Dialogue", lane: null });+ }+ const lanes = new Map<number, StemDoc["audioClips"]>();+ for (const c of doc.audioClips) {+ if (c.hidden || !audible(c.assetId)) continue;+ const lane = c.lane ?? 0;+ lanes.set(lane, [...(lanes.get(lane) ?? []), c]);+ }+ for (const lane of [...lanes.keys()].sort((a, b) => a - b)) {+ stems.push({ name: laneName(lane, lanes.get(lane)!, assets), lane });+ }+ // Two lanes of music read "Music" and "Music 2".+ const seen = new Map<string, number>();+ return stems.map((s, i) => {+ const n = (seen.get(s.name) ?? 0) + 1;+ seen.set(s.name, n);+ const name = n === 1 ? s.name : `${s.name} ${n}`;+ return { ...s, name, file: `${i + 1} ${name}.wav` };+ });+}++/* ------------------------------------------------------- WAV */++/** Stems are 24-bit: headroom for a mix that peaks over full scale before+ * its master, and the width an audio editor works in. */+export const STEM_BYTES_PER_SAMPLE = 3;++/** The size of a WAV file holding `frames` frames. */+export function wavBytes(frames: number, channels: number): number {+ return 44 + frames * channels * STEM_BYTES_PER_SAMPLE;+}++/** A 24-bit PCM WAV header. RIFF counts in 32 bits, so a stem past 4 GB+ * (about four hours of stereo at 48 kHz) is refused. */+export function wavHeader(frames: number, channels: number, sampleRate: number): Uint8Array {+ const data = frames * channels * STEM_BYTES_PER_SAMPLE;+ if (data + 36 > 0xffffffff) throw new Error("A stem this long does not fit in a WAV file.");+ const b = new Uint8Array(44);+ const v = new DataView(b.buffer);+ const ascii = (at: number, s: string) => [...s].forEach((ch, i) => (b[at + i] = ch.charCodeAt(0)));+ ascii(0, "RIFF");+ v.setUint32(4, 36 + data, true);+ ascii(8, "WAVE");+ ascii(12, "fmt ");+ v.setUint32(16, 16, true);+ v.setUint16(20, 1, true);+ v.setUint16(22, channels, true);+ v.setUint32(24, sampleRate, true);+ v.setUint32(28, sampleRate * channels * STEM_BYTES_PER_SAMPLE, true);+ v.setUint16(32, channels * STEM_BYTES_PER_SAMPLE, true);+ v.setUint16(34, STEM_BYTES_PER_SAMPLE * 8, true);+ ascii(36, "data");+ v.setUint32(40, data, true);+ return b;+}++/** Planar float samples [from, to) as interleaved 24-bit PCM. Past ±1 is+ * clamped; a sample that is not there (a stem shorter than the delivery)+ * is silence. */+export function pcm24(channels: ArrayLike<number>[], from: number, to: number, count: number): Uint8Array {+ const frames = Math.max(0, to - from);+ const out = new Uint8Array(frames * count * STEM_BYTES_PER_SAMPLE);+ let o = 0;+ for (let i = from; i < to; i++) {+ for (let c = 0; c < count; c++) {+ const src = channels[c] ?? channels[0];+ const x = src && i < src.length ? src[i] : 0;+ const s = Math.max(-8388608, Math.min(8388607, Math.round(x * 8388607)));+ out[o++] = s & 0xff;+ out[o++] = (s >> 8) & 0xff;+ out[o++] = (s >> 16) & 0xff;+ }+ }+ return out;+}++/* ------------------------------------------------------- zip */++const CRC_TABLE = (() => {+ const t = new Uint32Array(256);+ for (let n = 0; n < 256; n++) {+ let c = n;+ for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;+ t[n] = c >>> 0;+ }+ return t;+})();++/** CRC-32 of `data`, continuing from `crc` (0 to start). */+export function crc32(data: Uint8Array, crc = 0): number {+ let c = ~crc >>> 0;+ for (let i = 0; i < data.length; i++) c = CRC_TABLE[(c ^ data[i]) & 0xff] ^ (c >>> 8);+ return ~c >>> 0;+}++/** A file in a stored (uncompressed) zip. PCM barely compresses, so every+ * entry is stored and the archive is written straight through, its size and+ * checksum known before its bytes go out. */+export interface ZipEntry {+ name: string;+ size: number;+ crc: number;+}++const MAX32 = 0xffffffff;+const UTF8_NAMES = 0x0800;+/** 1980-01-01 00:00 in DOS time: the archive carries no clock. */+const DOS_DATE = (0 << 9) | (1 << 5) | 1;++const writer = (size: number) => {+ const b = new Uint8Array(size);+ const v = new DataView(b.buffer);+ let at = 0;+ return {+ b,+ u16: (n: number) => (v.setUint16(at, n, true), (at += 2)),+ u32: (n: number) => (v.setUint32(at, n >>> 0, true), (at += 4)),+ u64: (n: number) => (v.setBigUint64(at, BigInt(n), true), (at += 8)),+ bytes: (x: Uint8Array) => (b.set(x, at), (at += x.length)),+ };+};++/** The local header that goes before an entry's bytes. A file of 4 GB or+ * more carries its sizes in a Zip64 field. */+export function zipLocalHeader(entry: ZipEntry): Uint8Array {+ const name = new TextEncoder().encode(entry.name);+ const big = entry.size >= MAX32;+ const w = writer(30 + name.length + (big ? 20 : 0));+ w.u32(0x04034b50);+ w.u16(big ? 45 : 20);+ w.u16(UTF8_NAMES);+ w.u16(0);+ w.u16(0);+ w.u16(DOS_DATE);+ w.u32(entry.crc);+ w.u32(big ? MAX32 : entry.size);+ w.u32(big ? MAX32 : entry.size);+ w.u16(name.length);+ w.u16(big ? 20 : 0);+ w.bytes(name);+ if (big) {+ w.u16(0x0001);+ w.u16(16);+ w.u64(entry.size);+ w.u64(entry.size);+ }+ return w.b;+}++/** The central directory and end records for entries whose local headers+ * start at `offsets`, the directory itself starting at `at`. Zip64 records+ * join when a size or an offset passes 4 GB. */+export function zipDirectory(entries: ZipEntry[], offsets: number[], at: number): Uint8Array {+ const enc = new TextEncoder();+ const parts: Uint8Array[] = [];+ entries.forEach((e, i) => {+ const name = enc.encode(e.name);+ const bigSize = e.size >= MAX32;+ const bigOffset = offsets[i] >= MAX32;+ const extra = (bigSize ? 16 : 0) + (bigOffset ? 8 : 0);+ const w = writer(46 + name.length + (extra ? 4 + extra : 0));+ w.u32(0x02014b50);+ w.u16(45);+ w.u16(bigSize || bigOffset ? 45 : 20);+ w.u16(UTF8_NAMES);+ w.u16(0);+ w.u16(0);+ w.u16(DOS_DATE);+ w.u32(e.crc);+ w.u32(bigSize ? MAX32 : e.size);+ w.u32(bigSize ? MAX32 : e.size);+ w.u16(name.length);+ w.u16(extra ? 4 + extra : 0);+ w.u16(0);+ w.u16(0);+ w.u16(0);+ w.u32(0);+ w.u32(bigOffset ? MAX32 : offsets[i]);+ w.bytes(name);+ if (extra) {+ w.u16(0x0001);+ w.u16(extra);+ if (bigSize) {+ w.u64(e.size);+ w.u64(e.size);+ }+ if (bigOffset) w.u64(offsets[i]);+ }+ parts.push(w.b);+ });+ const dirSize = parts.reduce((s, p) => s + p.length, 0);+ const zip64 = at >= MAX32 || dirSize >= MAX32 || entries.length >= 0xffff || entries.some((e, i) => e.size >= MAX32 || offsets[i] >= MAX32);+ if (zip64) {+ const w = writer(56 + 20);+ const end64 = at + dirSize;+ w.u32(0x06064b50);+ w.u64(44);+ w.u16(45);+ w.u16(45);+ w.u32(0);+ w.u32(0);+ w.u64(entries.length);+ w.u64(entries.length);+ w.u64(dirSize);+ w.u64(at);+ w.u32(0x07064b50);+ w.u32(0);+ w.u64(end64);+ w.u32(1);+ parts.push(w.b);+ }+ const w = writer(22);+ w.u32(0x06054b50);+ w.u16(0);+ w.u16(0);+ w.u16(zip64 ? 0xffff : entries.length);+ w.u16(zip64 ? 0xffff : entries.length);+ w.u32(zip64 ? MAX32 : dirSize);+ w.u32(zip64 ? MAX32 : at);+ w.u16(0);+ parts.push(w.b);+ const out = new Uint8Array(parts.reduce((s, p) => s + p.length, 0));+ let o = 0;+ for (const p of parts) {+ out.set(p, o);+ o += p.length;+ }+ return out;+}++/**+ * Write a stored zip through `write`, entry by entry. Each entry is read+ * twice: once for its checksum, which the local header carries, and once for+ * its bytes — so nothing holds a whole file in memory.+ */+export async function writeStoredZip(+ entries: { name: string; size: number; read: () => AsyncIterable<Uint8Array> }[],+ write: (bytes: Uint8Array) => Promise<void> | void+): Promise<void> {+ let at = 0;+ const done: ZipEntry[] = [];+ const offsets: number[] = [];+ const put = async (b: Uint8Array) => {+ await write(b);+ at += b.length;+ };+ for (const e of entries) {+ let crc = 0;+ let size = 0;+ for await (const b of e.read()) {+ crc = crc32(b, crc);+ size += b.length;+ }+ if (size !== e.size) throw new Error(`${e.name} changed size while it was being packed.`);+ const entry = { name: e.name, size, crc };+ offsets.push(at);+ await put(zipLocalHeader(entry));+ for await (const b of e.read()) await put(b);+ done.push(entry);+ }+ await put(zipDirectory(done, offsets, at));+}++/** A planar stem as WAV bytes, a slice at a time: the header, then the+ * samples. `frames` is the delivery's length, whatever the stem holds. */+export async function* wavChunks(+ channels: ArrayLike<number>[],+ frames: number,+ count: number,+ sampleRate: number+): AsyncGenerator<Uint8Array> {+ yield wavHeader(frames, count, sampleRate);+ const slice = Math.max(1, sampleRate);+ for (let i = 0; i < frames; i += slice) {+ yield pcm24(channels, i, Math.min(frames, i + slice), count);+ await new Promise<void>((r) => setTimeout(r, 0));+ }+}diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex b91ea884..c00e3e2a 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -517,9 +517,11 @@ ${textMoveCatalog()}Cleaning a transcript: find_filler names the filler words on a track with their cue ids and times; tidy the captions with update_cue, or cut the speech itself with the timeline tools.Lyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, word effect, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.Soundtrack clips: volume 0..3 (1 = unchanged, above 1 boosts), fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.-Matching levels: "as loud as", "match", "balance", "sit under" are measurements, never guesses. measure_level with every clip involved and target_id = the one to match returns each clip's playing level and the volumeToMatch that lands it on the target; set that with set_clip_volume / update_audio (a fraction of it for a bed under speech). Listening tells you how a sound feels, never how many dB it is.+Matching levels: "as loud as", "match", "balance", "sit under" are measurements, never guesses. measure_level with every clip involved and target_id = the one to match returns each clip's playing level and the volumeToMatch that lands it on the target; set that with set_clip_volume / update_audio (a fraction of it for a bed under speech). Listening tells you how a sound feels, never how many dB it is. Loudness asks in LUFS ("music 10 LU under the voice", "is this at −14 for YouTube") read integratedLufs and truePeakDbtp from measure_level; mix: true measures the whole mix, and the export's Loudness setting masters it to a target.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.Sound quality (set_clip_sound): a clip's own treatment — video clip or soundtrack clip — in three stages the Inspector's Audio tab shows as Sound quality: a ${SOUND_EQ_BANDS.length}-band equalizer (${SOUND_EQ_BANDS.map((b) => b.label).join(", ")}; dB per band), a compressor (threshold dB, ratio, attack ms, release ms) and a limiter (dBFS ceiling). It runs on that clip alone, before its fades and any ducking, in the preview and every export. "Make the voice clearer / more podcast-like / less muddy / more even in level / stop it clipping" is this tool: pick the shipped preset whose character fits the ask — ${SOUND_PRESETS.filter((p) => p.sound).map((p) => `\`${p.id}\` (${p.character})`).join(", ")} — with \`studio\` as the default for a talking-head voice; \`preset: "flat"\` clears it; the user's own saved presets apply by name; or set the stages directly, null clearing one. save_sound_preset stores a clip's treatment in the Library under a name for reuse. A treatment shapes one clip's sound; an effect element treats a stretch of the mix.+Separate recordings (sync_audio): when the user recorded sound apart from the camera (a lav, a recorder) and imported it, sync_audio lines the audio file up with the video by its scratch track and binds it to the video, so every clip of that video plays the clean recording through its own trim, speed and direction, now and after any edit. It reports the offset and a confidence and refuses an alignment that does not stand out; say so plainly and leave the camera's sound. clear: true unbinds. A bound video shows \`soundFrom\` on its media entry; listening, silence, level and transcription tools read the recording automatically. Leave the recording off the soundtrack: binding is what puts it in the cut.+Split edits (set_split_edit): a clip's sound can start before its picture (lead, a J-cut: the next voice arrives before the cut) or run past it (tail, an L-cut: the voice carries over the next shot), up to ${SPLIT_EDIT_MAX_S} s a side. Use 0.05–0.5 s to make a dialogue cut feel seamless and 1–2 s to hand one scene into the next; the extension plays source from beyond the trim with a short ramp, under the neighbour, and the picture never moves. A cut with a transition keeps the transition's handover. Clips report audioLead/audioTail in editor_state.Cutting to music: detect_beats reads a track's beat grid once — bpm plus every beat, with timeline times through the clip you pass — and the grid stays on the asset as yellow dots the user can drag, add (⌥-click the bar), or delete, with every drag and trim snapping to them. Land cuts, clip starts, and text entrances on those times: split_at at the timeline beats, or place each item to start on one.Audio effects (add_effect): the treatments over the sound — ${AUDIO_EFFECT_IDS.join(", ")} — placed on the timeline as elements, each treating everything audible under its window at once. Reach for one when the ask is how a stretch SOUNDS ("make this line sound like a phone call", "echo on that word", "muffle the music behind the voiceover"); \`amount\` says how far the treatment goes, and the effect trims and moves like any element. Loudness, fades and ducking on one clip stay with the audio tools; a dissolve across a cut stays with the transition on that cut (the audiocross style).Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.tsindex 8251cb0f..105e0645 100644--- a/site/src/cut/server/cloud/jobs.ts+++ b/site/src/cut/server/cloud/jobs.ts@@ -10,7 +10,9 @@ import { getProject } from "./projects";import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl, mediaUrlLifetime } from "./mediaCdn";import { del, head, overlayKey, overlayPrefix, presignPut, projectExportKey } from "./r2";import { contentTypeFor } from "../serveFile";-import { containerOfName, deliveryContainer, exportBaseName, specMediaFiles, type ExportContainer } from "../../lib/exportDelivery";+import { containerOfName, deliveryContainer, exportBaseName, specMediaFiles, STEMS_MIME, stemsArchiveName, type ExportContainer } from "../../lib/exportDelivery";+import { LOUDNESS_IDS, type LoudnessId } from "../../lib/loudnessSettings";+import type { DocExportAudio } from "../../lib/exportPresets";import { addUsage, outputByteCeiling, quotaCheck } from "./usage";import { caught, err, redirect } from "./util";@@ -146,7 +148,8 @@ async function dropUnregisteredExport(where: { r2Key: key },select: { id: true },});- if (!registered) await del([key]);+ // Stems land before the file, and go with it.+ if (!registered) await del([key, projectExportKey(userId, row.projectId, stemsArchiveName(row.outName))]);}/** Engine job status ("queued" | "running" | "done" | "error") from a row's@@ -183,13 +186,16 @@ async function findJob(userId: string, id: string): Promise<LeanJobRow | null> {/** Engine-style export name: the base with the container's extension and a* " 2", " 3"… suffix when taken* by an existing export object or a job still in flight. The client derives the- * base from the project name; the dedupe happens here, where the rows are. */+ * base from the project name; the dedupe happens here, where the rows are. An+ * export with stems needs its stems' name free too, and every job in flight+ * holds its stems' name. */async function exportName(userId: string,projectId: string,baseName: string,container: ExportContainer | undefined,- tx: Prisma.TransactionClient | typeof prisma = prisma+ tx: Prisma.TransactionClient | typeof prisma = prisma,+ stems = false) {// The spec's container names the extension, the same way the engine's// `containerExtension` does; a name alone (a tab's own claim) says it by its@@ -211,11 +217,11 @@ async function exportName(]);const taken = new Set<string>([...files.map((f) => f.fileName),- ...jobs.map((j) => j.outName).filter((n): n is string => !!n),+ ...jobs.flatMap((j) => (j.outName ? [j.outName, stemsArchiveName(j.outName)] : [])),]);for (let n = 1; ; n++) {const candidate = n === 1 ? `${base}${ext}` : `${base} ${n}${ext}`;- if (!taken.has(candidate)) return candidate;+ if (!taken.has(candidate) && !(stems && taken.has(stemsArchiveName(candidate)))) return candidate;}}@@ -224,7 +230,8 @@ async function exportName(export async function queueDocExport(userId: string,projectId: string,- preset: string+ preset: string,+ audio: DocExportAudio = {}): Promise<{ id: string; outName: string } | Response> {const project = await getProject(userId, projectId);if (!project) return err("Project not found.", 404);@@ -235,14 +242,19 @@ export async function queueDocExport(// What the render may land. The worker holds its own output to it, so a// queued export cannot pass this gate and then write past the account's cap.const maxBytes = await exportByteCeiling(userId);- const outName = await exportName(userId, projectId, project.name, undefined);+ const outName = await exportName(userId, projectId, project.name, undefined, prisma, audio.stems === true);const row = await prisma.cutRenderJob.create({data: {userId,projectId,kind: "export",spec: {- fromDoc: { preset, snapshot: { doc: project.doc, version: String(project.version) } },+ fromDoc: {+ preset,+ ...(audio.loudness ? { loudness: audio.loudness } : {}),+ ...(audio.stems ? { stems: true } : {}),+ snapshot: { doc: project.doc, version: String(project.version) },+ },...(maxBytes === null ? {} : { maxBytes }),} as unknown as Prisma.InputJsonValue,outName,@@ -349,6 +361,7 @@ export const jobsCloud = {clips?: { file?: string }[];overlayVideos?: { file?: string }[];audio?: { file?: string }[];+ stemPlan?: unknown[];};overlays?: { name: string; key: string }[];projectId?: string;@@ -459,7 +472,14 @@ export const jobsCloud = {}const outName =target === "export"- ? await exportName(userId, projectId, body.outName?.trim() || project?.name || "export", body.spec.container)+ ? await exportName(+ userId,+ projectId,+ body.outName?.trim() || project?.name || "export",+ body.spec.container,+ prisma,+ (body.spec.stemPlan?.length ?? 0) > 0+ ): target === "hls"? "master.m3u8": `${target}.mp4`;@@ -476,10 +496,15 @@ export const jobsCloud = {/** Capture the requested document in the queue; the worker prepares its render spec. */async exportFromDoc(userId: string, projectId: string, req: Request) {try {- const body = (await req.json().catch(() => ({}))) as { preset?: string };+ const body = (await req.json().catch(() => ({}))) as { preset?: string; loudness?: unknown; stems?: unknown };+ if (body.loudness !== undefined && !LOUDNESS_IDS.includes(body.loudness as LoudnessId))+ return err(`loudness must be one of ${LOUDNESS_IDS.join(", ")}.`, 400);// The worker owns the size list (it builds the spec), and falls back to// the source-matched original for anything it does not know.- const queued = await queueDocExport(userId, projectId, typeof body.preset === "string" ? body.preset : "original");+ const queued = await queueDocExport(userId, projectId, typeof body.preset === "string" ? body.preset : "original", {+ loudness: body.loudness as LoudnessId | undefined,+ stems: body.stems === true,+ });if (queued instanceof Response) return queued;return Response.json(queued);} catch (e) {@@ -544,7 +569,8 @@ export const jobsCloud = {*/async exportClientPresign(userId: string, req: Request) {try {- const body = (await req.json()) as { projectId?: string; outName?: string; bytes?: number };+ const body = (await req.json()) as { projectId?: string; outName?: string; bytes?: number; stems?: boolean };+ const stems = body.stems === true;const projectId = body.projectId;if (!projectId) return err("projectId is required.", 400);if (!(await getProject(userId, projectId))) return err("Project not found.", 400);@@ -565,7 +591,7 @@ export const jobsCloud = {// completion would then fail on the unique key with the render already// spent. The row is the reservation.const row = await prisma.$transaction(async (tx) => {- const outName = await exportName(userId, projectId, body.outName ?? "export.mp4", undefined, tx);+ const outName = await exportName(userId, projectId, body.outName ?? "export.mp4", undefined, tx, stems);return tx.cutRenderJob.create({data: {userId,@@ -575,19 +601,24 @@ export const jobsCloud = {// nothing ever claims it.state: "running",progress: 0,- spec: { client: true },+ spec: stems ? { client: true, stems: true } : { client: true },outName,},});});const key = projectExportKey(userId, projectId, row.outName!);const type = containerOfName(row.outName!).mime;+ // The stems zip lands beside the file, under the name claimed with it.+ const stemsUrl = stems+ ? await presignPut(projectExportKey(userId, projectId, stemsArchiveName(row.outName!)), STEMS_MIME)+ : undefined;return Response.json({jobId: row.id,key,outName: row.outName,type,url: await presignPut(key, type),+ ...(stemsUrl ? { stemsUrl } : {}),});} catch (e) {return caught(e, "Could not start the export.");@@ -620,7 +651,15 @@ export const jobsCloud = {const key = projectExportKey(userId, projectId, outName);const object = await head(key);if (!object) return err("The export was not uploaded.", 400);- const bytes = object.bytes;+ // A render that asked for stems uploaded them before the file.+ const spec = (+ await prisma.cutRenderJob.findUnique({ where: { id: row.id }, select: { spec: true } })+ )?.spec as { stems?: boolean } | null;+ const stemsName = spec?.stems ? stemsArchiveName(outName) : null;+ const stemsKey = stemsName ? projectExportKey(userId, projectId, stemsName) : null;+ const stemsObject = stemsKey ? await head(stemsKey) : null;+ if (stemsKey && !stemsObject) return err("The stems were not uploaded.", 400);+ const bytes = object.bytes + (stemsObject?.bytes ?? 0);// The account is charged what landed, so the quota is checked against it.// A file past the ceiling is dropped rather than charged: the row would// otherwise put the account over its cap for good, and the bytes it holds@@ -647,6 +686,20 @@ export const jobsCloud = {uploadState: "complete",},});+ if (stemsKey && stemsName && stemsObject) {+ await tx.cutMediaObject.create({+ data: {+ userId,+ projectId,+ r2Key: stemsKey,+ fileName: stemsName,+ mime: STEMS_MIME,+ bytes: BigInt(stemsObject.bytes),+ kind: "export",+ uploadState: "complete",+ },+ });+ }await addUsage(tx, userId, bytes);await tx.cutRenderJob.update({where: { id: row.id },diff --git a/site/src/cut/server/exportAudio.ts b/site/src/cut/server/exportAudio.tsnew file mode 100644index 00000000..49abec5b--- /dev/null+++ b/site/src/cut/server/exportAudio.ts@@ -0,0 +1,104 @@+import { open } from "node:fs/promises";+import { masterStream, type MasterReport } from "../lib/loudness";+import { writeStoredZip } from "../lib/stems";++// The two pieces of an ffmpeg export's sound that ffmpeg does not do: the+// master, which runs the tab's own loudness code over the rendered mix so the+// two renderers land the same file at the same level, and the stems zip.+// Both stream through files a second at a time, so an hour of sound costs a+// second of it in memory.++/** Raw interleaved 32-bit float PCM, read a second at a time as planar+ * chunks. */+async function* rawChunks(file: string, sampleRate: number, channels: number): AsyncGenerator<Float32Array[]> {+ const frameBytes = channels * 4;+ const bytes = new Uint8Array(sampleRate * frameBytes);+ const handle = await open(file, "r");+ try {+ let pos = 0;+ let carry = 0;+ for (;;) {+ const { bytesRead } = await handle.read(bytes, carry, bytes.length - carry, pos);+ pos += bytesRead;+ const have = carry + bytesRead;+ const frames = Math.floor(have / frameBytes);+ if (frames > 0) {+ const view = new DataView(bytes.buffer, 0, frames * frameBytes);+ const planar = Array.from({ length: channels }, () => new Float32Array(frames));+ for (let i = 0; i < frames; i++) {+ for (let c = 0; c < channels; c++) planar[c][i] = view.getFloat32((i * channels + c) * 4, true);+ }+ yield planar;+ }+ carry = have - frames * frameBytes;+ if (carry > 0) bytes.copyWithin(0, frames * frameBytes, have);+ if (bytesRead === 0) return;+ }+ } finally {+ await handle.close();+ }+}++/** Master the raw float mix at `input` into `output`, same layout. */+export async function masterRawMix(+ input: string,+ output: string,+ opts: { sampleRate: number; channels: number; targetLufs: number; ceilingDbtp: number }+): Promise<MasterReport> {+ const out = await open(output, "w");+ try {+ return await masterStream(+ () => rawChunks(input, opts.sampleRate, opts.channels),+ async (chunk) => {+ const frames = chunk[0].length;+ const buf = new Uint8Array(frames * opts.channels * 4);+ const view = new DataView(buf.buffer);+ for (let i = 0; i < frames; i++) {+ for (let c = 0; c < opts.channels; c++) view.setFloat32((i * opts.channels + c) * 4, chunk[c][i], true);+ }+ await out.write(buf);+ },+ opts+ );+ } finally {+ await out.close();+ }+}++/** A file's bytes in slices. */+async function* fileChunks(file: string): AsyncGenerator<Uint8Array> {+ const handle = await open(file, "r");+ try {+ const buf = new Uint8Array(1 << 20);+ let pos = 0;+ for (;;) {+ const { bytesRead } = await handle.read(buf, 0, buf.length, pos);+ if (bytesRead === 0) return;+ pos += bytesRead;+ yield buf.slice(0, bytesRead);+ }+ } finally {+ await handle.close();+ }+}++/** Pack rendered stem WAVs into one stored zip at `output`, each under its+ * own name. */+export async function packStems(files: { path: string; name: string }[], output: string): Promise<void> {+ const out = await open(output, "w");+ try {+ const entries = await Promise.all(+ files.map(async (f) => {+ const handle = await open(f.path, "r");+ const size = (await handle.stat()).size;+ await handle.close();+ return { name: f.name, size, read: () => fileChunks(f.path) };+ })+ );+ await writeStoredZip(entries, async (b) => {+ await out.write(b);+ });+ } finally {+ await out.close();+ }+}diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex f990c567..b57aa8c5 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -24,9 +24,15 @@ const stagedFiles = new Map<string, string>();/** Source sizes the probe reports, by staged path; absent probes as unknown. */const probedDims = new Map<string, { width: number; height: number }>();+/** The masters and stem packs the last run asked for. */+let mastered: { input: string; output: string; opts: Parameters<ExportPipelineIO["masterRawMix"]>[2] }[] = [];+let packed: { files: { path: string; name: string }[]; output: string }[] = [];+const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {const ffmpegCalls: string[][] = [];written = [];+ mastered = [];+ packed = [];// Seconds each turned file was asked to produce, summed off the `-t` of the// chunk runs that wrote its pieces, so the bake reads back a whole file.const produced = new Map<string, number>();@@ -47,6 +53,13 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {mediaDuration: async (file) => produced.get(file) ?? 0,videoEncoder: async (codec) =>codec === "hevc" ? "libx265" : codec.startsWith("prores") ? "prores_ks" : "libx264",+ masterRawMix: async (input, output, opts) => {+ mastered.push({ input, output, opts });+ return {} as Awaited<ReturnType<ExportPipelineIO["masterRawMix"]>>;+ },+ packStems: async (files, output) => {+ packed.push({ files, output });+ },runFfmpeg: async (_job, args) => {ffmpegCalls.push(args);const piece = args[args.length - 1].match(/^(.*)\.\d+\.(?:mov|wav)$/);@@ -71,6 +84,7 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {const job: RenderHandle = {tmpDir: "/tmp/graph-test",outPath: "/tmp/graph-test/out.mp4",+ stemsPath: "/tmp/graph-test/out stems.zip",progress: 0,log: [],};diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex 9e2eb602..7528ef01 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -11,6 +11,10 @@ import { assertGraphSafe, fexpr } from "./filterGraph";import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";import { bakeTurnedMedia } from "./turnMedia";import { withSpecColors } from "./fileColor";+import { alignRecording } from "./soundAlign";+import { shiftSpan, type SpecSound } from "../lib/soundSource";+import { masterRawMix, packStems } from "./exportAudio";+import type { StemDef } from "../lib/stems";import { CLIP_MAX_ZOOM, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";import { audioFxFilters, buildClipLut, buildTransferLut, CLARITY_EPS, compositeSpaceFor, detailActive, detailGain, detailRadius, effectFilterLines, graphicsToHlg, hexToHlgHex, hlgToPq, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, parseLutFile, recipeKey, retimeOf, shortestTurn, slowRuns, srcSpan, sortedKeys, soundFilters, type ChainChroma, type ClipColorRecipe, type ClipSound, type CodeFormat, type OutputSpace, type OverlayKey, type ParsedLut, type Retime, type SpeedNode } from "@donkeycut/effects-kit";@@ -103,6 +107,14 @@ export interface ExportSpec {* graph composites the whole cut and the delivery is cut from it, so fades,* captions and elements sit where the timeline has them. */range?: ExportRange;+ /** The integrated loudness the delivered mix is mastered to, LUFS; absent+ * = the mix as it plays. */+ loudness?: number;+ /** The true peak a mastered mix is held under, dBTP. */+ truePeakCeiling?: number;+ /** The stems written beside the file, zipped at the handle's+ * `stemsPath`; absent = none. */+ stemPlan?: StemDef[];duration: number;/** The frame's own color (hex): what letterboxes a fitted clip, what a gap* on track 0 plays, and what a cut of nothing but elements composites over.@@ -376,6 +388,11 @@ export interface ExportSpec {captions?: { file: string; start: number; end: number; lane?: number }[];}+/** The rate the graph mixes at: every read resamples to it. */+const MIX_RATE = 44100;+/** The rate stems are written at, the rate the tab's own stems mix at. */+const STEM_RATE = 48000;+/** What the pipeline needs from its caller's job record: the staging dir the* overlay PNGs were written into, the output path, and the mutable fields the* run reports through (progress, the live ffmpeg process for cancellation,@@ -385,6 +402,8 @@ export interface ExportSpec {export interface RenderHandle {tmpDir: string;outPath: string;+ /** Where the stems zip lands when the spec asks for stems. */+ stemsPath?: string;progress: number;error?: string;proc?: ChildProcess;@@ -762,9 +781,13 @@ export interface ExportPipelineIO {mediaDuration: typeof mediaDuration;videoEncoder: (codec: ExportVideoCodec) => Promise<string>;runFfmpeg: typeof runFfmpeg;+ masterRawMix: typeof masterRawMix;+ packStems: typeof packStems;}const realIO: ExportPipelineIO = {+ masterRawMix,+ packStems,exportSourceFiles,stat,writeFile,@@ -791,7 +814,9 @@ export async function runExport() {if (given.clips.length === 0) throw new Error("Nothing to export.");const codec = given.codec ?? "h264";- if (given.target === "export" && given.sourceSegments?.length && (codec === "h264" || codec === "hevc")) {+ // Stems are cut from the graph's own sound, so a spec that asks for them+ // renders.+ if (given.target === "export" && given.sourceSegments?.length && !given.stemPlan?.length && (codec === "h264" || codec === "hevc")) {for (const segment of given.sourceSegments) {if (!given.clips.some((clip) => segment.file === clip.file && segment.from >= clip.in && segment.to <= clip.out)) {// Adjacent splits of a source may be merged into one copy span.@@ -890,7 +915,7 @@ export async function runExport(soundOnly = false): Promise<T> => {if (!c.reverse || !c.file || c.image || c.hidden) return c;- const src = await resolveMedia(io.stat, mediaPathFor, c.file);+ const src = twinFiles.has(c.file) ? c.file : await resolveMedia(io.stat, mediaPathFor, c.file);const rt = retimeOf(c);// The span plus the handles a crossing reaches into, so the copy holds// what the sound of a cross dissolve needs on either side.@@ -1105,6 +1130,14 @@ export async function runExport(videoPresence.set(file, streams.video);if (streams.mono) monoFiles.add(file);}+ // The sound twins still read forward: stereo sound, written just now.+ for (const file of new Set([...soundTwins.values(), ...spec.audio].map((e) => e?.file ?? ""))) {+ if (!twinFiles.has(file)) continue;+ inputIndex.set(file, nInputs++);+ inputs.push("-i", file);+ audioPresence.set(file, true);+ videoPresence.set(file, false);+ }// Animated overlays: each is its own concat-demuxer slideshow (region-sized// frames with transparent filler around the element's window), the exact// mechanism the caption lanes use. Static overlays stay single PNG inputs.@@ -1570,16 +1603,17 @@ export async function runExport(image?: boolean;};const baked = new Map<Sounding, BakedAudio & { idx: number }>();- const bakeTargets: { c: Sounding; tag: string }[] = [- ...spec.clips.map((c, j) => ({ c, tag: `clip_${j}` })),- ...overlayVideos.map((oc, k) => ({ c: oc, tag: `ovl_${k}` })),+ const bakeTargets: { c: Sounding | null; tag: string }[] = [+ ...spec.clips.map((c, j) => ({ c: sounding(c), tag: `clip_${j}` })),+ ...overlayVideos.map((oc, k) => ({ c: sounding(oc), tag: `ovl_${k}` })),...spec.audio.map((a, k) => ({ c: a, tag: `snd_${k}` })),];for (const { c, tag } of bakeTargets) {+ if (!c) continue;const rt = retimeOf(c);if (rt.uniform || !c.file || c.image || c.muted || c.hidden || !audioPresence.get(c.file)) continue;const file = path.join(job.tmpDir, `retime_${tag}.wav`);- const src = await resolveMedia(io.stat, mediaPathFor, c.file);+ const src = await sourcePath(c.file);const bake = await bakeRetimedAudio({ffmpeg: (args) => io.runFfmpeg(job, args),@@ -2144,7 +2178,8 @@ export async function runExport(`color=c=${padColor}:s=${W}x${H}:r=${fps},trim=0:${num(dur)},setpts=PTS-STARTPTS,fps=${fps},format=${clipFmt}[v${j}]`);}- if (!c.muted && !c.hidden && audioPresence.get(c.file)) {+ const snd = sounding(c);+ if (!c.muted && !c.hidden && snd && audioPresence.get(snd.file)) {const vol = `${soundChain(c.sound)}${(c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : ""}`;// The picture's fade edges carry the sound with them; zoom edges don't.const afades =@@ -2277,13 +2312,24 @@ export async function runExport(const ahead = Math.min(prevC.soundAhead ?? 0, half);// Never reach past the head of the source: a handle is what the trim left// behind, and there is none before zero.- const back = Math.min(c.soundBack ?? 0, half, headRoom(c));+ const back = Math.min(c.soundBack ?? 0, half, headRoom(sounding(c) ?? c));// The outgoing clip, still sounding past its out point.const prevDur = clipDur(prevC);handleStream(prevC, `a${j}`, prevDur, prevDur + ahead, cut, crossExpr(0, half, false));// The incoming clip, already sounding before its in point.handleStream(c, `b${j}`, -back, 0, cut - back, crossExpr(back, half, true));});+ // A split edit: a clip's sound ahead of its picture (J-cut) or carried on+ // past it (L-cut), lifted out the same way and ramped from silence at its+ // far end. A transition at that cut owns the handover, so it has none.+ spec.clips.forEach((c, j) => {+ const fade = c.splitFade ?? 0;+ const lead = Math.min(c.soundLead ?? 0, c.soundBack ?? 0, headRoom(sounding(c) ?? c), clipAt[j]);+ if (lead > 0.01) handleStream(c, `l${j}`, -lead, 0, clipAt[j] - lead, splitExpr(0, lead, fade, true));+ const tail = Math.min(c.soundTail ?? 0, c.soundAhead ?? 0);+ const dur = clipDur(c);+ if (tail > 0.01) handleStream(c, `t${j}`, dur, dur + tail, clipAt[j] + dur, splitExpr(0, tail, fade, false));+ });// Join the segments. Clips abut — a transition claims no layout — so every// join keeps the accumulator's full length. A transitioned join pads the@@ -2716,6 +2762,8 @@ export async function runExport(// Soundtrack clips: trim, gain, shift into place, mix with clip audio.const soundLabels: string[] = [];+ /** The lane of each entry in soundLabels, for the stems. */+ const soundLanes: number[] = [];spec.audio.forEach((a, k) => {if (!audioPresence.get(a.file)) return;const delayMs = Math.max(0, Math.round(a.start * 1000));@@ -2735,14 +2783,29 @@ export async function runExport();// Music and other non-voiceover sound ducks under the voiceovers.soundLabels.push(a.duck !== undefined && a.duck < 1 ? `snd${k}` : duckOthers(`snd${k}`));+ soundLanes.push(a.lane ?? 0);});// The joined clip audio and overlay-video audio duck too.let aLabel = duckOthers(aAcc);+ const overlayDucked = overlaySoundLabels.map(duckOthers);+ const crossDucked = crossHandleLabels.map(duckOthers);+ // Stems hear a split of the very streams the mix sums — ducked, faded,+ // treated — so adding them back up gives the mix.+ const stemPlan = spec.stemPlan ?? [];+ const stemInputs: string[][] = stemPlan.map(() => []);+ const dialogueStem = stemPlan.findIndex((s) => s.lane === null);+ const tap = (label: string, stem: number): string => {+ if (stem < 0) return label;+ filters.push(`[${label}]asplit=2[${label}m][${label}s]`);+ stemInputs[stem].push(`${label}s`);+ return `${label}m`;+ };+ aLabel = tap(aLabel, dialogueStem);const extraSound = [- ...soundLabels,- ...overlaySoundLabels.map(duckOthers),- ...crossHandleLabels.map(duckOthers),+ ...soundLabels.map((l, i) => tap(l, stemPlan.findIndex((s) => s.lane === soundLanes[i]))),+ ...overlayDucked.map((l) => tap(l, dialogueStem)),+ ...crossDucked.map((l) => tap(l, dialogueStem)),];if (extraSound.length > 0) {const mixIn = [aLabel, ...extraSound].map((l) => `[${l}]`).join("");@@ -2834,6 +2897,42 @@ export async function runExport(aLabel = "arange";}+ // Each stem: its streams summed, treated like the mix, and laid over the+ // delivery's whole length so every file starts and ends where it does.+ const stemLabels = stemPlan.map((_, i) => {+ const ins = stemInputs[i];+ let label = `stm${i}`;+ if (ins.length === 0) filters.push(`anullsrc=r=44100:cl=stereo,atrim=0:${num(spec.duration)}[${label}]`);+ else if (ins.length === 1) label = ins[0];+ else {+ filters.push(+ `${ins.map((l) => `[${l}]`).join("")}amix=inputs=${ins.length}:duration=longest:dropout_transition=0:normalize=0[${label}]`+ );+ }+ label = treatAudio(label, `s${i}_`);+ const window = spec.range+ ? `,atrim=start=${num(Math.max(0, spec.range.start))}:end=${num(Math.min(spec.duration, spec.range.end))},asetpts=PTS-STARTPTS`+ : "";+ filters.push(`[${label}]apad=whole_dur=${num(spec.duration)},atrim=0:${num(spec.duration)}${window}[stem${i}]`);+ return `stem${i}`;+ });+ if (stemPlan.length > 0 && !job.stemsPath) throw new Error("This export has nowhere to put its stems.");+ const stemFiles = stemPlan.map((s, i) => ({ path: path.join(job.tmpDir, `stem_${i}.wav`), name: s.file }));+ const stemOutputs = stemLabels.flatMap((label, i) => [+ "-map", `[${label}]`,+ "-c:a", "pcm_s24le",+ "-ar", String(STEM_RATE),+ "-ac", "2",+ "-t", num(span),+ stemFiles[i].path,+ ]);++ // A mastered delivery takes its sound out of this pass as raw float, so the+ // master can run the tab's own loudness code over it before it is encoded.+ const master = spec.loudness !== undefined;+ if (master && spec.truePeakCeiling === undefined) throw new Error("A mastered export needs its true-peak ceiling.");+ const mixPath = path.join(job.tmpDir, "mix.f32");+const enc = await io.videoEncoder(spec.codec ?? "h264");// Encode into the tmp dir, then re-emit the container to strip a stray output@@ -2847,16 +2946,31 @@ export async function runExport(...inputs,"-filter_complex", assertGraphSafe(filters.join(";")),"-map", `[${vLabel}]`,- "-map", `[${aLabel}]`,+ ...(master ? [] : ["-map", `[${aLabel}]`]),...videoCodecArgs(enc, spec),...colorTagArgs(spec),- ...audioCodecArgs(spec),+ ...(master ? [] : audioCodecArgs(spec)),"-t", num(span),encodePath,+ ...(master+ ? ["-map", `[${aLabel}]`, "-c:a", "pcm_f32le", "-ar", String(MIX_RATE), "-ac", "2", "-f", "f32le", "-t", num(span), mixPath]+ : []),+ ...stemOutputs,],(t) => (job.progress = Math.min(0.99, t / Math.max(0.1, span))));+ const masteredPath = path.join(job.tmpDir, "master.f32");+ if (master) {+ await io.masterRawMix(mixPath, masteredPath, {+ sampleRate: MIX_RATE,+ channels: 2,+ targetLufs: spec.loudness!,+ ceilingDbtp: spec.truePeakCeiling!,+ });+ }+ if (stemPlan.length > 0) await io.packStems(stemFiles, job.stemsPath!);+// ffmpeg's autorotation already baked each source's display matrix into the// pixels, so the encode's frames are upright. But for a complex filtergraph it// ALSO copies the first input's display-matrix side data onto the output@@ -2865,12 +2979,17 @@ export async function runExport(// "desktop" frame. `-display_rotation 0` overrides that matrix to identity; a// stream copy re-emits the (already correct) pixels and audio unchanged and// writes the faststart-optimized final file.+ // A mastered mix joins the picture here, encoded for the delivery.await io.runFfmpeg(job, ["-y","-display_rotation", "0","-i", encodePath,- "-map", "0",- "-c", "copy",+ ...(master+ ? [+ "-f", "f32le", "-ar", String(MIX_RATE), "-ac", "2", "-i", masteredPath,+ "-map", "0:v", "-map", "1:a", "-c:v", "copy", ...audioCodecArgs(spec), "-t", num(span),+ ]+ : ["-map", "0", "-c", "copy"]),"-movflags", "+faststart",job.outPath,]);diff --git a/site/src/cut/server/http/export.ts b/site/src/cut/server/http/export.tsindex 0653bdf8..fc3e0aaa 100644--- a/site/src/cut/server/http/export.ts+++ b/site/src/cut/server/http/export.ts@@ -1,6 +1,7 @@import {cancelJob,completeClientJob,+ completeClientStems,createClientJob,createJob,getJob,@@ -44,9 +45,10 @@ export const exportApi = {projectId?: string;container?: ExportSpec["container"];name?: string;+ stems?: boolean;};if (!body.projectId) return Response.json({ error: "Missing project." }, { status: 400 });- const job = await createClientJob(body.projectId, body.container, body.name?.trim() || undefined);+ const job = await createClientJob(body.projectId, body.container, body.name?.trim() || undefined, body.stems === true);if (job.status === "error") return Response.json({ error: job.error }, { status: 400 });return Response.json({ id: job.id, outName: job.outName });} catch (e) {@@ -71,6 +73,17 @@ export const exportApi = {return Response.json({ id: job.id, outName: job.outName });},+ /** The tab's stems zip, streamed beside the file before the file lands. */+ async clientStems(req: Request, { jobId }: { jobId: string }) {+ try {+ const job = await completeClientStems(jobId, req.body);+ if (!job) return Response.json({ error: "This export is no longer running." }, { status: 409 });+ return Response.json({ ok: true });+ } catch (e) {+ return Response.json({ error: errorMessage(e, "Could not save the stems.") }, { status: 500 });+ }+ },+async clientRelease(_req: Request, { jobId }: { jobId: string }) {releaseClientJob(jobId);return Response.json({ ok: true });diff --git a/site/src/cut/server/http/routes.ts b/site/src/cut/server/http/routes.tsindex 2f1d16e4..63b03aae 100644--- a/site/src/cut/server/http/routes.ts+++ b/site/src/cut/server/http/routes.ts@@ -99,6 +99,7 @@ export const CUT_ROUTES: CutRoute[] = [{ method: "POST", path: "/api/cut/export/client", handler: (req) => exportApi.createClient(req) },{ method: "POST", path: "/api/cut/export/client/:jobId/progress", handler: (req, p) => exportApi.clientProgress(req, { jobId: p.jobId }) },{ method: "PUT", path: "/api/cut/export/client/:jobId/file", handler: (req, p) => exportApi.clientComplete(req, { jobId: p.jobId }) },+ { method: "PUT", path: "/api/cut/export/client/:jobId/stems", handler: (req, p) => exportApi.clientStems(req, { jobId: p.jobId }) },{ method: "POST", path: "/api/cut/export/client/:jobId/release", handler: (req, p) => exportApi.clientRelease(req, { jobId: p.jobId }) },{ method: "GET", path: "/api/cut/export/:jobId", handler: (req, p) => exportApi.status(req, { jobId: p.jobId }) },{ method: "DELETE", path: "/api/cut/export/:jobId", handler: (req, p) => exportApi.cancel(req, { jobId: p.jobId }) },diff --git a/site/src/cut/server/jobs.test.ts b/site/src/cut/server/jobs.test.tsindex 192489a2..74190935 100644--- a/site/src/cut/server/jobs.test.ts+++ b/site/src/cut/server/jobs.test.ts@@ -108,6 +108,49 @@ describe("a render the tab carries itself", () => {});});+describe("stems beside a tab render", () => {+ let projectId = "";+ beforeAll(async () => {+ projectId = (await projects.createProject("Stem Render")).id;+ });++ test("the stems land under the video's name, before the video settles the job", async () => {+ const job = await jobs.createClientJob(projectId, "mp4", undefined, true);+ expect(job.outName).toBe("Stem Render.mp4");+ expect(await jobs.completeClientStems(job.id, bytes("zip bytes"))).not.toBeNull();+ const zip = path.join(projects.exportsDir(projectId), "Stem Render stems.zip");+ expect(await readFile(zip, "utf8")).toBe("zip bytes");+ expect(await exists(`${zip}.part`)).toBe(false);+ expect((await jobs.completeClientJob(job.id, bytes("mp4 bytes")))?.status).toBe("done");+ });++ test("a name whose stems are taken is passed over by a render that asks for stems", async () => {+ // "Stem Render 2.mp4" is free, but a stray zip holds its stems' name.+ await writeFile(path.join(projects.exportsDir(projectId), "Stem Render 2 stems.zip"), "old");+ const plain = await jobs.createClientJob(projectId, "mp4");+ expect(plain.outName).toBe("Stem Render 2.mp4");+ jobs.releaseClientJob(plain.id);+ const stemmed = await jobs.createClientJob(projectId, "mp4", undefined, true);+ expect(stemmed.outName).toBe("Stem Render 3.mp4");+ jobs.releaseClientJob(stemmed.id);+ });++ test("a render that never asked for stems refuses them, and a release takes landed stems with it", async () => {+ const plain = await jobs.createClientJob(projectId, "mp4");+ expect(await jobs.completeClientStems(plain.id, bytes("zip"))).toBeNull();+ jobs.releaseClientJob(plain.id);++ const job = await jobs.createClientJob(projectId, "mov", undefined, true);+ await jobs.completeClientStems(job.id, bytes("zip"));+ const zip = path.join(projects.exportsDir(projectId), jobs.getJob(job.id)!.outName.replace(/\.mov$/, " stems.zip"));+ expect(await exists(zip)).toBe(true);+ jobs.releaseClientJob(job.id);+ // The release removes in the background.+ await new Promise((r) => setTimeout(r, 20));+ expect(await exists(zip)).toBe(false);+ });+});+describe("preview publication", () => {async function request(projectId: string) {const form = new FormData();diff --git a/site/src/cut/server/jobs.ts b/site/src/cut/server/jobs.tsindex 67f85e0a..edc1c08d 100644--- a/site/src/cut/server/jobs.ts+++ b/site/src/cut/server/jobs.ts@@ -5,7 +5,7 @@ import path from "node:path";import { assertLocalRuntime } from "./local-only";import { createJobRegistry } from "./jobRegistry";import { containerExtension, runExport, type ExportSpec } from "./exportPipeline";-import { exportBaseName } from "../lib/exportDelivery";+import { exportBaseName, stemsArchiveName } from "../lib/exportDelivery";import { exportsDir, mediaPath, projectDir, readProject, setActiveJobGuard } from "./projects";import { errorMessage } from "./util";@@ -30,6 +30,8 @@ export interface Job {tmpDir: string;outPath: string;outName: string;+ /** The stems zip beside the file, when the export asked for stems. */+ stemsPath?: string;proc?: ChildProcess;/** A render the browser tab carries itself: the row holds the file's name* and the dock's slot while the tab draws the frames, and the finished file@@ -145,6 +147,7 @@ function startRun(job: Job, spec: ExportSpec) {job.status = "error";job.error = errorMessage(err, String(err));void rm(job.outPath, { force: true }); // no half-written files in exports/+ if (job.stemsPath) void rm(job.stemsPath, { force: true });}).finally(() => {void rm(job.tmpDir, { recursive: true, force: true }); // overlay tmp, win or lose@@ -212,8 +215,9 @@ export function cancelJob(id: string) {}/** Export file named after the project, with a " 2", " 3"… suffix when the- * name is already taken by a file on disk or an export still in flight. */-async function exportName(projectId: string, projectName: string, ext: string) {+ * name is already taken by a file on disk or an export still in flight. An+ * export with stems needs its stems' name free too. */+async function exportName(projectId: string, projectName: string, ext: string, stems = false) {const base = exportBaseName(projectName);const taken = new Set(await readdir(exportsDir(projectId)).catch(() => [] as string[])@@ -221,11 +225,14 @@ async function exportName(projectId: string, projectName: string, ext: string) {// A failed job wrote nothing under its name; only live and finished ones// hold theirs.for (const j of jobs.values()) {- if (j.projectId === projectId && j.outName && j.status !== "error") taken.add(j.outName);+ if (j.projectId === projectId && j.outName && j.status !== "error") {+ taken.add(j.outName);+ if (j.stemsPath) taken.add(stemsArchiveName(j.outName));+ }}for (let n = 1; ; n++) {const candidate = n === 1 ? `${base}${ext}` : `${base} ${n}${ext}`;- if (!taken.has(candidate)) return candidate;+ if (!taken.has(candidate) && !(stems && taken.has(stemsArchiveName(candidate)))) return candidate;}}@@ -233,9 +240,10 @@ async function exportName(projectId: string, projectName: string, ext: string) {// its outName is assigned, so two jobs racing through their first awaits could// otherwise both claim "<Project>.mp4" and overwrite each other's render.let namingQueue: Promise<unknown> = Promise.resolve();-function claimExportName(job: Job, projectName: string, ext: string): Promise<void> {+function claimExportName(job: Job, projectName: string, ext: string, stems = false): Promise<void> {const claim = namingQueue.then(async () => {- job.outName = await exportName(job.projectId, projectName, ext);+ job.outName = await exportName(job.projectId, projectName, ext, stems);+ if (stems) job.stemsPath = path.join(exportsDir(job.projectId), stemsArchiveName(job.outName));});namingQueue = claim.catch(() => {});return claim;@@ -291,7 +299,7 @@ export async function createJob(form: FormData): Promise<Job> {if (!doc) throw new Error("Project not found.");job.projectName = doc.name;if (preview) job.outName = "preview.mp4";- else await claimExportName(job, spec.name ?? doc.name, containerExtension(spec));+ else await claimExportName(job, spec.name ?? doc.name, containerExtension(spec), !!spec.stemPlan?.length);job.outPath = path.join(preview ? path.join(projectDir(spec.projectId), ".previews") : exportsDir(spec.projectId),preview ? `${job.id}.mp4` : job.outName@@ -342,7 +350,9 @@ export async function createJob(form: FormData): Promise<Job> {export async function createClientJob(projectId: string,container: ExportSpec["container"],- name?: string+ name?: string,+ /** The tab hands in a stems zip beside the file. */+ stems = false): Promise<Job> {assertLocalRuntime();sweepClientJobs();@@ -367,7 +377,7 @@ export async function createClientJob(const doc = await readProject(projectId);if (!doc) throw new Error("Project not found.");job.projectName = doc.name;- await claimExportName(job, name ?? doc.name, containerExtension({ container }));+ await claimExportName(job, name ?? doc.name, containerExtension({ container }), stems);job.outPath = path.join(exportsDir(projectId), job.outName);await mkdir(path.dirname(job.outPath), { recursive: true });} catch (err) {@@ -424,6 +434,35 @@ export async function completeClientJob(return job;}+/**+ * A tab render's stems zip, streamed beside the file under the name claimed+ * with it. It lands before the file does, since the file settles the job.+ * Null when the job is no longer a running tab render that asked for stems.+ */+export async function completeClientStems(+ id: string,+ body: ReadableStream<Uint8Array> | null+): Promise<Job | null> {+ const job = jobs.get(id);+ if (!job?.client || job.status !== "running" || !job.stemsPath) return null;+ job.seenAt = Date.now();+ const partial = `${job.stemsPath}.part`;+ try {+ if (!body) throw new Error("The stems arrived without a file.");+ const file = await open(partial, "w");+ try {+ for await (const chunk of body) await file.write(chunk);+ } finally {+ await file.close();+ }+ await rename(partial, job.stemsPath);+ } catch (err) {+ await rm(partial, { force: true });+ throw err;+ }+ return job;+}+/** A tab render that stopped before its file landed gives the name and the* row back; nothing of it stays in the feed. That covers a render the tab* abandoned while still running and one whose hand-in broke off — a cancel@@ -433,6 +472,8 @@ export async function completeClientJob(export function releaseClientJob(id: string): void {const job = jobs.get(id);if (!job?.client || job.status === "done") return;+ // Stems that landed for a file that never did go with it.+ if (job.stemsPath) void rm(job.stemsPath, { force: true });jobs.delete(id);}diff --git a/site/src/cut/worker/exportJob.ts b/site/src/cut/worker/exportJob.tsindex c270f949..e835c554 100644--- a/site/src/cut/worker/exportJob.ts+++ b/site/src/cut/worker/exportJob.ts@@ -5,13 +5,16 @@ import type { CloudDocSnapshot } from "../lib/headless/docSession";import { buildDocExportSpec, isDocExportPreset } from "../lib/headless/docExport";import { runExport, type ExportSpec, type RenderHandle } from "../server/exportPipeline";import { storeCardArtifacts } from "./cardJob";-import { prisma, registerObject, type ClaimedJob } from "./db";+import { prisma, registerObjectIn, storageTransaction, type ClaimedJob } from "./db";import { deleteObjects, downloadToFile, exportKey, mediaKey, mimeFor, uploadFile } from "./r2";import { overlayKey } from "../server/cloud/r2";import { STORAGE_FULL } from "../lib/operationFailure";-import { specMediaFiles } from "../lib/exportDelivery";+import { specMediaFiles, stemsArchiveName } from "../lib/exportDelivery";import { runnerSession } from "./session";import { bindCutColor } from "../lib/colorSettings";+import { bindCutSound } from "../lib/soundSettings";+import { bindCutLoudness } from "../lib/loudnessSettings";+import type { DocExportAudio } from "../lib/exportPresets";import { getGlobalSetting } from "@/lib/config/effective";/** The stored spec of an export/preview CutRenderJob: the engine export spec@@ -139,7 +142,12 @@ export async function runExportJob(handle.tmpDir = path.join(work, "overlays");await mkdir(handle.tmpDir, { recursive: true });// A doc-built spec sizes its LUTs from the same setting the tab binds.- if (stored.fromDoc) bindCutColor(await getGlobalSetting("cutColor"));+ if (stored.fromDoc) {+ bindCutColor(await getGlobalSetting("cutColor"));+ bindCutSound(await getGlobalSetting("cutSound"));+ // Its loudness target and ceiling come from the setting the tab binds.+ bindCutLoudness(await getGlobalSetting("cutLoudness"));+ }const body: ExportJobSpec = stored.fromDoc? {...stored,@@ -149,7 +157,8 @@ export async function runExportJob(isDocExportPreset(stored.fromDoc.preset) ? stored.fromDoc.preset : "original",handle.tmpDir,stored.fromDoc.snapshot,- job.kind === "preview" ? "preview" : "export"+ job.kind === "preview" ? "preview" : "export",+ { loudness: stored.fromDoc.loudness, stems: stored.fromDoc.stems })) as ExportSpec,}: stored;@@ -166,6 +175,11 @@ export async function runExportJob(const outName = mode === "export" ? job.outName?.trim() || "export.mp4" : `${mode}.mp4`;handle.outPath = path.join(work, "out", outName);await mkdir(path.dirname(handle.outPath), { recursive: true });+ // Stems ride the export they were asked with, under the name claimed with it.+ const stemsName = mode === "export" && spec.stemPlan?.length ? stemsArchiveName(outName) : null;+ // A borrowed render hands back only the picture; its tab mixes the stems.+ if (stemsName && body.mediaFrom === "overlays") throw new Error("A borrowed render cannot carry stems.");+ if (stemsName) handle.stemsPath = path.join(work, "out", stemsName);await runExport(handle, spec, (file) => path.join(mediaDir, path.basename(file)));@@ -195,25 +209,42 @@ export async function runExportJob(// the account had when the render was queued. The file is still on the// worker's disk here: nothing goes up, and nothing is charged.if (!preview && typeof stored.maxBytes === "number") {- const size = (await stat(handle.outPath)).size;+ const size =+ (await stat(handle.outPath)).size + (handle.stemsPath ? (await stat(handle.stemsPath)).size : 0);if (size > stored.maxBytes) throw new Error(STORAGE_FULL);}- const bytes = await uploadFile(key, handle.outPath, mime);+ const stemsKey = stemsName ? exportKey(job.userId, projectId, stemsName) : null;+ const uploaded = [key, ...(stemsKey ? [stemsKey] : [])];try {- await registerObject({- userId: job.userId,- projectId,- r2Key: key,- fileName: outName,- mime,- bytes,- kind: preview ? "preview" : "export",+ const bytes = await uploadFile(key, handle.outPath, mime);+ const stemsBytes = stemsKey ? await uploadFile(stemsKey, handle.stemsPath!, mimeFor(stemsName!)) : 0;+ await storageTransaction(async (tx) => {+ await registerObjectIn(tx, {+ userId: job.userId,+ projectId,+ r2Key: key,+ fileName: outName,+ mime,+ bytes,+ kind: preview ? "preview" : "export",+ });+ if (stemsKey) {+ await registerObjectIn(tx, {+ userId: job.userId,+ projectId,+ r2Key: stemsKey,+ fileName: stemsName!,+ mime: mimeFor(stemsName!),+ bytes: stemsBytes,+ kind: "export",+ });+ }});} catch (err) {// The file is up and no row owns it. The sweep walks rows, so nothing// would ever find these bytes again — and a storage wall that leaves the// bytes behind walls the account in tighter with every retry.- await deleteObjects([key]).catch(() => {});+ await deleteObjects(uploaded).catch(() => {});throw err;}if (preview) {diff --git a/site/src/cut/worker/r2.ts b/site/src/cut/worker/r2.tsindex d48225d3..0d967468 100644--- a/site/src/cut/worker/r2.ts+++ b/site/src/cut/worker/r2.ts@@ -163,6 +163,8 @@ const MIME_BY_EXT: Record<string, string> = {".m3u8": "application/vnd.apple.mpegurl",".ts": "video/mp2t",".m4s": "video/iso.segment",+ // An export's stems.+ ".zip": "application/zip",};export function mimeFor(fileName: string): string {