Skip to content

Open source CapCut alternative.

TypeScript

feat(site): sync a separate mic and split edits [rebuild]

David Le72bdb1eedb27
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index fe8c321b..544d67bc 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -33,6 +33,7 @@ import {
} from "@donkeycut/effects-kit";
import { bool, ids, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
import { builtinLutCatalogText } from "@/cut/lib/builtinLuts";
+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";
/** Per-field slider hints; ranges interpolate the exported constants so the
* schema can never drift from the model the renderer clamps to. */
@@ -227,6 +228,27 @@ export const INSPECTOR_TOOLS = [
description: "Set the gain on a video clip's own audio (soundtrack clips use update_audio).",
inputSchema: obj({ clipId: str("Video clip id"), ids: ids("clipId"), volume: num("0..3 (1 = unchanged, up to 3 boosts the clip's own sound)") }, ["volume"]),
},
+ {
+ name: "sync_audio",
+ description:
+ "Dual-system sound: line up a separately recorded audio file (a lavalier, a recorder, a second camera's mic) with a video by matching it against the camera's own sound, then bind it to the video. Every clip of that video — the ones on the timeline now and any split, trimmed, sped, reversed, or copied later — plays the recording through its own trim and speed, and the camera's track goes quiet; the preview, every export, captions, and the listening tools all hear the recording. Name the video by clipId (one of its clips) or assetId, and the recording by audio_asset_id (an audio file in the project). The match reports `offset` (recording seconds = video seconds + offset) and `confidence`; when no single alignment stands out — a different take, or two files with no sound in common — it refuses and binds nothing, and the user can pick another file. clear: true returns the video to its own sound. editor_state shows each bound video's `soundFrom` on its media entry.",
+ inputSchema: obj({
+ clipId: str("A video clip whose source video gets the recording"),
+ assetId: str("The video asset to bind, when no clip names it"),
+ audio_asset_id: str("The audio asset holding the clean recording"),
+ clear: bool("Unbind the recording: the video plays its own sound again"),
+ }),
+ },
+ {
+ name: "set_split_edit",
+ description: `Split edit on a video clip: its sound starts ahead of its picture (lead — a J-cut, the next speaker heard before the cut) or carries on past it (tail — an L-cut, a voice running over the next shot). The extension plays the clip's own source from beyond its trim, under the neighbouring clip, ramping in from silence at the far end; pictures and clip positions stay where they are. Typical values: 0.05–0.5 s to soften a dialogue cut, 1–2 s to hand one scene into the next. Each side is capped at ${SPLIT_EDIT_MAX_S} s and at the source the clip has beyond its trim; 0 clears that side, and a side left out keeps its value. A cut carrying a transition keeps the transition's own handover, so that side plays no split edit. editor_state reports audioLead/audioTail on the clip.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ ids: ids("clipId"),
+ lead: num(`Seconds the sound starts before the picture, 0..${SPLIT_EDIT_MAX_S} (0 clears)`),
+ tail: num(`Seconds the sound carries past the picture's end, 0..${SPLIT_EDIT_MAX_S} (0 clears)`),
+ }),
+ },
{
name: "set_clip_sound",
description:
diff --git a/site/src/cut/components/Inspector.tsx b/site/src/cut/components/Inspector.tsx
index 3615b3fc..37655cda 100644
--- a/site/src/cut/components/Inspector.tsx
+++ b/site/src/cut/components/Inspector.tsx
@@ -79,9 +79,11 @@ import {
restingMaskFrame,
} from "@donkeycut/effects-kit";
import { BLOCK_COLOR } from "@/cut/lib/blockSource";
+import { bindRecording, unbindRecording } from "@/cut/lib/soundBind";
+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";
import { clipLen, clipWindow, maxClipFade, useEditor, type EditorState } from "@/cut/lib/store";
import { PANEL_GLOBAL, usePanelState, useRememberedScroll } from "@/cut/lib/panelState";
-import { usePreviewTime } from "@/cut/lib/playhead";
+import { playheadAt, usePreviewTime } from "@/cut/lib/playhead";
import { CLIP_MAX_ZOOM, clipCovers, clipKeyed, clipPoseAt, clipZoom, contentRect } from "@/cut/lib/types";
import { AnimationTiles } from "@/cut/components/AnimationTiles";
import { ColorField } from "@/cut/components/ColorField";
@@ -753,6 +755,8 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {
onCheckedChange={(v) => updateClip(clip.id, { muted: v })}
/>
</Row>
+ {asset?.type === "video" && <ClipSoundSource clip={clip} asset={asset} />}
+ {asset?.type === "video" && (!assetIsSilent(asset) || !!asset.soundFrom) && <ClipSplitEdit clip={clip} />}
{asset && !assetIsSilent(asset) && (
<Row
label="Beats"
@@ -761,7 +765,7 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {
<BeatsControl asset={asset} />
</Row>
)}
- {asset && !assetIsSilent(asset) && (
+ {asset && (!assetIsSilent(asset) || !!asset.soundFrom) && (
<SoundQualityRow sound={clip.sound} onOpen={() => setView("sound")} />
)}
<ClipGeneratedAudio clip={clip} />
@@ -770,6 +774,127 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {
);
}
+/** The Select value that stands for the video's own sound. */
+const CAMERA_SOUND = "camera";
+
+/**
+ * Where a video's sound comes from: the camera's own track, or a recording
+ * made apart from it. Picking a recording lines it up against the camera's
+ * sound and binds it to the video, so every clip of the video plays it; a
+ * recording with no clear match is refused and the reason shown.
+ */
+function ClipSoundSource({ clip, asset }: { clip: VideoClip; asset: MediaAsset }) {
+ const assets = useEditor((s) => s.assets);
+ const recordings = useMemo(
+ () => assets.filter((a) => a.type === "audio" && !a.upload && a.duration > 0),
+ [assets]
+ );
+ const audioAssetId = asset.soundFrom?.assetId;
+ const bound = audioAssetId ? recordings.find((a) => a.id === audioAssetId) : undefined;
+ const [syncing, setSyncing] = useState(false);
+ const [refusal, setRefusal] = useState<string | null>(null);
+ const pick = (id: string) => {
+ setRefusal(null);
+ if (id === CAMERA_SOUND) {
+ unbindRecording(asset.id);
+ return;
+ }
+ setSyncing(true);
+ void bindRecording(asset.id, id, clip)
+ .catch((e: unknown) => setRefusal(e instanceof Error ? e.message : "Could not line the recording up."))
+ .finally(() => setSyncing(false));
+ };
+ return (
+ <>
+ <Row
+ label="Sound"
+ info="Where this video's sound comes from. Pick an audio file recorded alongside it (a lav or a recorder) and it is lined up by matching it against the camera's sound; every clip of this video then plays the recording, through any trim, speed or split."
+ >
+ {syncing && <Loader2 className="size-3.5 animate-spin text-muted-foreground" />}
+ <Select value={bound ? bound.id : CAMERA_SOUND} onValueChange={(v) => pick(String(v))} disabled={syncing}>
+ <SelectTrigger className="clip-sound-source h-8 w-36 text-[12px]">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value={CAMERA_SOUND} className="text-[12px]">
+ Camera
+ </SelectItem>
+ {recordings.map((r) => (
+ <SelectItem key={r.id} value={r.id} className="text-[12px]">
+ {r.name}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ <ResetButton title="Use the camera's sound" show={!!bound} onClick={() => pick(CAMERA_SOUND)} />
+ </Row>
+ {bound && asset.soundFrom && (
+ <p className="pb-1 text-[11.5px] text-muted-foreground">
+ Synced {asset.soundFrom.offset >= 0 ? "+" : "−"}
+ {Math.abs(asset.soundFrom.offset).toFixed(3)} s · every clip of this video plays it
+ </p>
+ )}
+ {refusal && <p className="pb-1 text-[11.5px] text-destructive">{refusal}</p>}
+ </>
+ );
+}
+
+const formatSplit = (v: number) => (v > 0 ? `${v.toFixed(2)} s` : "Off");
+const parseSplitInput = (raw: string) =>
+ /^off$/i.test(raw.trim()) ? 0 : parseNumberInput(raw.replace(/\s*s$/i, ""));
+
+/**
+ * The clip's split edit: its sound running ahead of the picture (a J-cut)
+ * or past it (an L-cut), played from the source beyond the trim. The picture
+ * and the clip's place on the timeline stay put.
+ */
+function ClipSplitEdit({ clip }: { clip: VideoClip }) {
+ const updateClip = useEditor((s) => s.updateClip);
+ const [leadDraft, setLeadDraft] = useState<number | null>(null);
+ const [tailDraft, setTailDraft] = useState<number | null>(null);
+ const lead = leadDraft ?? clip.audioLead ?? 0;
+ const tail = tailDraft ?? clip.audioTail ?? 0;
+ const slider = (
+ label: string,
+ value: number,
+ onDraft: (v: number | null) => void,
+ write: (v: number) => Partial<VideoClip>
+ ) => (
+ <ValueSlider
+ label={label}
+ sliderClassName="clip-split-edit data-horizontal:w-24"
+ valueClassName="w-11 text-muted-foreground"
+ value={value}
+ min={0}
+ max={SPLIT_EDIT_MAX_S}
+ step={0.05}
+ format={formatSplit}
+ parse={parseSplitInput}
+ onDraft={onDraft}
+ onCommit={(v) => {
+ updateClip(clip.id, write(v));
+ onDraft(null);
+ }}
+ />
+ );
+ return (
+ <>
+ <Row
+ label="Sound lead"
+ info="J-cut: this clip's sound starts this long before its picture, under the clip before it. A tenth to half a second softens a dialogue cut; a second or two hands one scene into the next."
+ >
+ {slider("Sound lead", lead, setLeadDraft, (v) => ({ audioLead: v > 0 ? v : undefined }))}
+ </Row>
+ <Row
+ label="Sound tail"
+ info="L-cut: this clip's sound carries on this long past its picture, under the clip after it."
+ >
+ {slider("Sound tail", tail, setTailDraft, (v) => ({ audioTail: v > 0 ? v : undefined }))}
+ </Row>
+ </>
+ );
+}
+
const formatDb = (v: number) => `${v > 0 ? "+" : ""}${v.toFixed(1)} dB`;
const parseDbInput = (raw: string) =>
parseNumberInput(raw.replace(/\s*dB$/i, "").replace(/^\+/, ""));
diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsx
index 75d81298..17da3a01 100644
--- a/site/src/cut/components/Timeline.tsx
+++ b/site/src/cut/components/Timeline.tsx
@@ -132,16 +132,17 @@ const WAVE_H = 24;
* play in the band (an image, a silent video) drops the band and its box stops
* at the filmstrip's edge. */
const FILM_H = VIDEO_H - 4 - WAVE_H;
-/** Whether a clip has a sound band to draw: a video carrying audio of its own.
- * Muting greys the band down, so the clip keeps its shape and its waveform. An
- * image or a silent video has nothing to show there. */
-const hasSoundBand = (asset: MediaAsset) =>
- asset.type === "video" && !!asset.peaks?.length;
+/** Whether a clip has a sound band to draw: a video carrying audio of its own,
+ * or a recording bound to it (`sound`, which the band then draws). Muting
+ * greys the band down, so the clip keeps its shape and its waveform. An image
+ * or a silent video has nothing to show there. */
+const hasSoundBand = (asset: MediaAsset, sound?: MediaAsset) =>
+ asset.type === "video" && !!(sound ?? asset).peaks?.length;
/** A video row's height. Rows whose clips carry no audio lose the sound band's
* worth, so the row ends where the filmstrips do. An empty row keeps the full
* height: it is a drop target for clips that do have sound. */
const videoRowH = (list: ClipSpan[]) =>
- list.length > 0 && !list.some((sp) => hasSoundBand(sp.asset))
+ list.length > 0 && !list.some((sp) => hasSoundBand(sp.asset, sp.sound?.asset))
? VIDEO_H - WAVE_H
: VIDEO_H;
/** The `mt-1.5` every row carries, so a row drag knows what one row's worth of
@@ -3809,7 +3810,7 @@ function ClipView({
const { clip, asset } = span;
const spine = clip.track === 0;
const loading = useEditor((s) => s.loadingMedia.has(asset.fileName));
- const baking = useMatteBakes((s) => s.jobs[clip.id]?.status === "running");
+ const baking = useMatteBakes((s) => s.jobs[clip.id]?.status === "running" || s.jobs[cardMatteKey(clip.id)]?.status === "running");
const rt = retimeOf(clip);
// Every box is its clip's whole footprint. Clips never overlap — a
// transition is a render-time blend at the cut, drawn as the bar above the
@@ -3833,7 +3834,11 @@ function ClipView({
// The clip's own sound, drawn in the box's band under the picture. Muting —
// detach included — greys the band, so the clip's shape holds still and its
// waveform stays readable (a detached copy draws on the soundtrack row too).
- const hasWave = hasSoundBand(asset);
+ // A recording bound to the video is what plays, so it is what draws, moved
+ // onto the video's clock.
+ const hasWave = hasSoundBand(asset, span.sound?.asset);
+ const waveAsset = span.sound?.asset ?? asset;
+ const waveOff = span.sound?.offset ?? 0;
// Bumped as tile captures land, so the memo re-plans and picks them up.
const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);
// The strip draws the stretch of the box the scroller shows. The box's
@@ -3927,7 +3932,10 @@ function ClipView({
)}
style={{ height: WAVE_H }}
>
- <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={barW} h={WAVE_H - 4} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps)} className="inset-x-0 inset-y-0.5" />
+ <WaveformCanvas asset={waveAsset} from={clip.in + waveOff} to={clip.out + waveOff} w={barW} h={WAVE_H - 4} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps) + waveOff} className="inset-x-0 inset-y-0.5" />
+ {/* A split edit: the edge whose sound runs past the picture. */}
+ {(span.soundLead ?? 0) > 0 && <div className="tl-clip-split absolute inset-y-0 left-0 w-0.5 bg-white/80" />}
+ {(span.soundTail ?? 0) > 0 && <div className="tl-clip-split absolute inset-y-0 right-0 w-0.5 bg-white/80" />}
</div>
)}
{selected && (
@@ -5122,6 +5130,9 @@ function WaveTile({
const start = (from / duration) * n;
const perBar = (((to - from) / duration) * n) / bars;
for (let i = 0; i < bars; i++) {
+ // A stretch the file does not hold (a bound recording that started
+ // late or stopped early) draws nothing.
+ if (start + (i + 1) * perBar <= 0 || start + i * perBar >= n) continue;
const a = Math.max(0, Math.min(n - 1, Math.floor(start + i * perBar)));
const b = Math.max(a + 1, Math.ceil(start + (i + 1) * perBar));
let p = 0;
@@ -5333,6 +5344,13 @@ function AudioView({
onSnap: (x: number | null) => void;
}) {
const loading = useEditor((s) => (asset ? s.loadingMedia.has(asset.fileName) : false));
+ // Detached from a video bound to a recording, the clip plays the recording,
+ // so the bar draws it on the video's clock.
+ const recording = useEditor((s) => {
+ const id = asset?.soundFrom?.assetId;
+ return id ? s.assets.find((a) => a.id === id && !assetIsSilent(a)) : undefined;
+ });
+ const waveOff = recording ? (asset?.soundFrom?.offset ?? 0) : 0;
const len = clipLen(clip);
const w = Math.max(10, len * pps);
// The bar is drawn a gutter narrower than the footprint; what sits inside
@@ -5387,7 +5405,7 @@ function AudioView({
className="pointer-events-none absolute inset-0"
style={{ clipPath: fadeClipPath(fadeInPx, fadeOutPx, barW) }}
>
- <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={barW} h={AUDIO_H - 8} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps)} className="inset-x-0 inset-y-1" />
+ <WaveformCanvas asset={recording ?? asset} from={clip.in + waveOff} to={clip.out + waveOff} w={barW} h={AUDIO_H - 8} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps) + waveOff} className="inset-x-0 inset-y-1" />
</div>
<BeatDots asset={asset} lo={clip.in} hi={clip.out} retime={rt} pps={pps} w={barW} />
<FadeEnvelope inPx={fadeInPx} outPx={fadeOutPx} barW={barW} />
diff --git a/site/src/cut/components/inspectorToolFields.test.ts b/site/src/cut/components/inspectorToolFields.test.ts
index f8d51c6b..c689d79d 100644
--- a/site/src/cut/components/inspectorToolFields.test.ts
+++ b/site/src/cut/components/inspectorToolFields.test.ts
@@ -25,8 +25,11 @@ const ITEM_TOOLS = new Set([
"update_audio",
"set_clip_volume",
"set_clip_sound",
+ "sync_audio",
+ "set_split_edit",
"set_framing",
"set_clip_style",
+ "set_camera_card",
"set_speed",
"set_speed_curve",
"set_color_grade",
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 4188ba66..565637ae 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -9,11 +9,12 @@ import {
projectDuration,
useEditor,
} from "@/cut/lib/store";
-import { headSrc, matteLumaToAlpha, retimeOf, smoothsAt, srcSpan } from "@donkeycut/effects-kit";
+import { headSrc, matteLumaToAlpha, retimeOf, smoothsAt } from "@donkeycut/effects-kit";
import { blendInto, SYNTH_EDGE, synthWeight } from "@/cut/lib/frameSynth";
import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";
import { assetIsSilent, clipCovers, rectOf } from "@/cut/lib/types";
-import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
+import type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
+import { soundSourceOf, soundWindow } from "@/cut/lib/soundSource";
import { SubjectMaskCompositor } from "@/cut/lib/behindPass";
import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";
import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";
@@ -167,28 +168,40 @@ export function engineLog(msg: string): void {
*/
const SOUND_LOOKAHEAD_S = 3;
+const OWN_TRACK = { offset: 0, limit: Infinity };
+
const voiceSpan = (sp: ClipSpan) => {
- // The handles a crossing reaches into play at the pace of the footage
- // beside them, so the voice's span widens through the clip's own map and
- // its retime is rebuilt over the wider span.
- const rt = retimeOf(sp.clip);
- const { lo, hi } = srcSpan(rt, -sp.soundBack, rt.len + sp.soundAhead);
+ // The handles a crossing or a split edit reaches into play at the pace of
+ // the footage beside them, so the voice's span widens through the clip's
+ // own map and its retime is rebuilt over the wider span. A recording bound
+ // to the video is read through the same map, on its own clock, and starts
+ // late where it rolled after the camera did.
+ const w = soundWindow(sp.clip, sp.sound ?? OWN_TRACK, -sp.soundBack, sp.len + sp.soundAhead);
+ if (!w) return null;
return {
- url: sp.asset.url,
- start: sp.start - sp.soundBack,
- in: lo,
- out: hi,
- retime: retimeOf({
- in: lo,
- out: hi,
- speed: sp.clip.speed,
- speedCurve: sp.clip.speedCurve,
- reverse: sp.clip.reverse,
- }),
+ url: (sp.sound?.asset ?? sp.asset).url,
+ start: sp.start - sp.soundBack + w.at,
+ in: w.lo,
+ out: w.hi,
+ retime: retimeOf(w.span),
sound: sp.clip.sound,
};
};
+/** A span with nothing to hear: its video has no sound track and no
+ * recording bound to it. */
+const spanSilent = (sp: ClipSpan) => !sp.sound && assetIsSilent(sp.asset);
+
+/** An audio clip's sound: its own file, or the recording bound to the video
+ * it was detached from. */
+const audioVoice = (a: AudioClip, asset: MediaAsset, assets: MediaAsset[]) => {
+ const src = soundSourceOf(asset, assets);
+ if (!src.bound) return { url: asset.url, start: a.start, in: a.in, out: a.out, retime: retimeOf(a) };
+ const w = soundWindow(a, src, 0, retimeOf(a).len);
+ if (!w) return null;
+ return { url: src.asset.url, start: a.start + w.at, in: w.lo, out: w.hi, retime: retimeOf(w.span) };
+};
+
class Engine {
/** Which engine this is, for the perf eval's pool dump. */
readonly serial = ++engineSerial;
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index f4726ba6..febb9568 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -462,6 +462,17 @@ function describeState(
duration: r(a.duration),
...(a.origin ? { origin: a.origin } : {}),
...(a.folderId != null ? { folderId: a.folderId } : {}),
+ // The recording bound to this video (sync_audio): every clip of it
+ // plays that file, recording second = video second + offset.
+ ...(a.soundFrom
+ ? {
+ soundFrom: {
+ assetId: a.soundFrom.assetId,
+ name: s.assets.find((x) => x.id === a.soundFrom!.assetId)?.name ?? null,
+ offset: Math.round(a.soundFrom.offset * 1000) / 1000,
+ },
+ }
+ : {}),
// What the file's code values mean, for footage that is not plain
// Rec.709: the profile in force and, when the person overrode it, what
// the header read (set_source_color changes it).
@@ -529,6 +540,7 @@ function describeState(
// The clip's own equalizer/compressor/limiter, set through
// set_clip_sound; absent when its sound is untouched.
...(sp.clip.sound ? { sound: sp.clip.sound } : {}),
+ ...splitEditOf(sp.clip),
framing: sp.clip.fit ?? "fit",
...describeRate(sp.clip),
// The generated scene shot this clip came from — sceneShot is the
@@ -696,6 +708,12 @@ function describeAudio(
};
}
+/** A clip's split edit as set_split_edit wrote it: seconds its sound leads
+ * the picture and carries past it. Absent when it has none. */
+function splitEditOf(c: VideoClip) {
+ return c.audioLead || c.audioTail ? { audioLead: c.audioLead ?? 0, audioTail: c.audioTail ?? 0 } : {};
+}
+
function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>) {
const rect = rectOf(c);
return {
@@ -707,6 +725,7 @@ function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>
in: r(c.in),
out: r(c.out),
muted: c.muted,
+ ...splitEditOf(c),
...(c.hidden ? { hidden: true } : {}),
...(c.sound ? { sound: c.sound } : {}),
// The frame region this layer occupies: Full covers the frame; Top/Bottom/
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index a2d85839..9af7363d 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -284,6 +284,9 @@ import {
type WatchDetail,
} from "./types";
+import { soundSourceOf, SPLIT_EDIT_MAX_S } from "./soundSource";
+import { bindRecording, unbindRecording } from "./soundBind";
+
const round2 = (n: number) => Math.round(n * 100) / 100;
/** The timeline rows a mutation touched, small enough to ride every result:
@@ -1544,7 +1547,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...(wanted > 3 ? { shortfallDb: round2(20 * Math.log10(wanted / 3)) } : {}),
};
});
- return { targetId, clips };
+ return { targetId, clips, ...(mix ? { mix } : {}) };
},
listen_audio: async (s, input) => {
@@ -1559,7 +1562,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// A whole audio file that clears the cap falls through to the same
// fitting path a range takes, so length truncates instead of refusing.
const heard = whole
- ? { inline: whole, to: asset.duration }
+ ? { inline: whole, from, to: asset.duration }
: await listenSpan(projectId, asset, from, to);
// Listening shows interest in the source's sound — queue the background
// sweep so its transcript (and, for video, its visual map) fills in.
@@ -1579,7 +1582,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
source: sourceRef(asset),
duration: round2(asset.duration),
...(clip ? { clipId: clip.id } : {}),
- from: round2(from),
+ from: round2(heard.from),
+ ...(soundSourceOf(asset, s.assets).bound
+ ? { soundFrom: soundSourceOf(asset, s.assets).asset.name }
+ : {}),
// Where this play stopped, and how much of the source nobody has
// heard — the sound's answer to coveredTo and unwatchedSeconds. The
// cap is bytes, so the seconds that fit depend on the source.
@@ -1807,6 +1813,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const h = clamp(rg.h, 0.05, 1);
patch.frame = { x: clamp(rg.x, 0, 1 - w), y: clamp(rg.y, 0, 1 - h), w, h };
}
+ refuseCardFraming(c, input, ["fit", "zoom"]);
if (input.fit === "fit" || input.fit === "fill") patch.fit = input.fit;
if (isNum(input.zoom)) {
const z = clamp(input.zoom, 1, CLIP_MAX_ZOOM);
@@ -2077,6 +2084,50 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return { id: next.id, volume: next.volume ?? 1 };
},
+ sync_audio: async (s, input) => {
+ const clip = input.clipId !== undefined ? requireItem(s.clips, input.clipId, "video clip") : undefined;
+ const video = requireItem(s.assets, clip?.assetId ?? input.assetId, "video asset");
+ if (video.type !== "video") throw new ToolError(`"${video.name}" is not a video.`);
+ if (input.clear === true || input.audio_asset_id === null) {
+ unbindRecording(video.id);
+ return { assetId: video.id, soundFrom: null, note: `"${video.name}" plays its own sound again.` };
+ }
+ return syncVideoSound(s, video, input.audio_asset_id, clip);
+ },
+
+ set_split_edit: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const side = (v: unknown, name: string) => {
+ if (v === undefined) return undefined;
+ if (!isNum(v)) throw new ToolError(`${name} is seconds, 0..${SPLIT_EDIT_MAX_S}.`);
+ const secs = Math.round(clamp(v, 0, SPLIT_EDIT_MAX_S) * 1000) / 1000;
+ return secs > 0 ? secs : 0;
+ };
+ const lead = side(input.lead, "lead");
+ const tail = side(input.tail, "tail");
+ if (lead === undefined && tail === undefined) throw new ToolError("Pass lead, tail, or both, in seconds (0 clears).");
+ s.updateClip(clip.id, {
+ ...(lead !== undefined ? { audioLead: lead || undefined } : {}),
+ ...(tail !== undefined ? { audioTail: tail || undefined } : {}),
+ });
+ const next = useEditor.getState();
+ const live = next.clips.find((c) => c.id === clip.id)!;
+ const sp = getClipSpans(next.clips, next.assets, live.track).find((x) => x.clip.id === clip.id);
+ const playing = { lead: round2(sp?.soundLead ?? 0), tail: round2(sp?.soundTail ?? 0) };
+ const short = (asked: number | undefined, got: number) => asked !== undefined && asked - got > 0.01;
+ return {
+ id: clip.id,
+ audioLead: live.audioLead ?? 0,
+ audioTail: live.audioTail ?? 0,
+ playing,
+ ...(short(lead, playing.lead) || short(tail, playing.tail)
+ ? {
+ note: "Part of the split edit does not play: a transition owns that cut, the source ends there, or the timeline starts there. `playing` is what sounds.",
+ }
+ : {}),
+ };
+ },
+
rename_item: (s, input) => {
const name = String(input.name ?? "").trim().slice(0, 60) || undefined;
const id = String(input.id ?? "");
@@ -2255,7 +2306,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// puts that whole track on the soundtrack with nothing else placed.
if (input.assetId) {
const asset = requireItem(s.assets, input.assetId, "project asset");
- if (asset.type === "image" || assetIsSilent(asset))
+ if (asset.type === "image" || (assetIsSilent(asset) && !soundSourceOf(asset, s.assets).bound))
throw new ToolError(`"${asset.name}" carries no audio.`);
const start = isNum(input.start) ? Math.max(0, input.start) : 0;
s.addAudioFromAsset(asset.id, start);
@@ -2707,6 +2758,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
set_framing: (s, input) => {
const clip = requireItem(s.clips, input.clipId, "video clip");
+ refuseCardFraming(clip, input, ["mode", "zoom", "panX", "panY"]);
// Only what the call names changes; the rest of the framing stays as
// the user left it.
const mode = input.mode === "fill" || input.mode === "fit" ? input.mode : clip.fit ?? "fit";
@@ -5009,6 +5061,7 @@ export const MEDIA_RUNTIME_TOOLS: ReadonlySet<string> = new Set([
"detect_silence",
"detect_beats",
"refine_speech_cuts",
+ "sync_audio",
"capture_frame",
"compare_to_source",
"render_preview",
@@ -5525,9 +5578,38 @@ async function eachScanned<T>(items: T[], run: (item: T) => Promise<void>) {
* threshold cannot answer it, because a quiet room and a loud one disagree
* about what silence is and a cut placed on the wrong answer lands in a word. */
async function fetchSpeech(asset: MediaAsset, from: number, to: number) {
- return scanSourceSpeech(asset.url, { from, to }).catch((e) => {
+ const heard = soundSpan(asset, from, to);
+ const scan = await scanSourceSpeech(heard.asset.url, { from: heard.from, to: heard.to }).catch((e) => {
throw new ToolError(e instanceof Error ? e.message : "Could not read the audio.");
});
+ return {
+ ...scan,
+ from: heard.back(scan.from),
+ to: heard.back(scan.to),
+ segments: scan.segments.map((g) => ({ ...g, start: heard.back(g.start), end: heard.back(g.end) })),
+ };
+}
+
+/**
+ * Where a source range's sound is read. A video bound to a recording sounds
+ * from the recording: the range moves onto its clock and narrows to what it
+ * holds, and `back` carries a second on that file back to the video's source
+ * time, so every answer comes back in the seconds the caller asked in.
+ */
+function soundSpan(asset: MediaAsset, from: number, to: number | undefined) {
+ const src = soundSourceOf(asset, useEditor.getState().assets);
+ if (!src.bound) return { asset, from, to, back: (t: number) => t };
+ const lo = Math.max(0, from + src.offset);
+ const hi = Math.min(src.limit, (to ?? asset.duration) + src.offset);
+ if (!(hi - lo > 0.05))
+ throw new ToolError(`"${src.asset.name}", the sound bound to "${asset.name}", holds nothing of that stretch.`);
+ return { asset: src.asset, from: lo, to: hi as number | undefined, back: (t: number) => t - src.offset };
+}
+
+/** How loud a source range plays, read where its sound is. */
+async function sourceLevel(asset: MediaAsset, from: number, to: number | undefined) {
+ const heard = soundSpan(asset, from, to);
+ return measureSourceLevel(heard.asset.url, { from: heard.from, ...(heard.to !== undefined ? { to: heard.to } : {}) });
}
/** Silent stretches of a source range, in source seconds — the engine's
@@ -5542,23 +5624,28 @@ async function fetchSilences(
// Same defaults and clamps as the engine's silence handler.
const thresholdDb = clamp(opts.thresholdDb ?? -30, -90, 0);
const minSilence = clamp(opts.minSilence ?? 0.35, 0.05, 10);
+ const heard = soundSpan(asset, opts.from, opts.to);
+ const back = (list: { start: number; end: number; duration: number }[]) =>
+ list.map((x) => ({ ...x, start: heard.back(x.start), end: heard.back(x.end) }));
if (getBackend().kind !== "local") {
- return detectSilenceClientSide(asset.url, {
- from: opts.from,
- ...(opts.to !== undefined ? { to: opts.to } : {}),
- thresholdDb,
- minSilence,
- }).catch((e) => {
- throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");
- });
+ return back(
+ await detectSilenceClientSide(heard.asset.url, {
+ from: heard.from,
+ ...(heard.to !== undefined ? { to: heard.to } : {}),
+ thresholdDb,
+ minSilence,
+ }).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");
+ })
+ );
}
const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
- file: asset.fileName,
- from: opts.from,
- ...(opts.to !== undefined ? { to: opts.to } : {}),
+ file: heard.asset.fileName,
+ from: heard.from,
+ ...(heard.to !== undefined ? { to: heard.to } : {}),
threshold_db: thresholdDb,
min_silence: minSilence,
}),
@@ -5568,7 +5655,7 @@ async function fetchSilences(
error?: string;
}>(res);
if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");
- return body.silences;
+ return back(body.silences);
}
/** Pull a source's audio track off (video and audio alike) and inline it for
@@ -5612,15 +5699,20 @@ const LISTEN_MAX_S = 600;
async function listenSpan(
projectId: string,
asset: MediaAsset,
- from: number,
- to: number | undefined,
-): Promise<{ inline: InlineImage; to: number }> {
+ askedFrom: number,
+ askedTo: number | undefined,
+): Promise<{ inline: InlineImage; from: number; to: number }> {
+ // A bound recording is what is heard, from where it holds
+ // sound; the span reports back in the video's own seconds.
+ const heard = soundSpan(asset, askedFrom, askedTo ?? Math.min(asset.duration, askedFrom + LISTEN_MAX_S));
+ const from = heard.from;
+ const to = heard.to;
const end = Math.min(to ?? from + LISTEN_MAX_S, from + LISTEN_MAX_S);
let span = end - from;
for (let attempt = 0; attempt < 3 && span > 0.5; attempt++) {
- const blob = await audioSpanBlob(projectId, asset, from, round2(from + span));
+ const blob = await audioSpanBlob(projectId, heard.asset, from, round2(from + span));
const inline = await blobToInlineAudio(blob);
- if (inline) return { inline, to: round2(from + span) };
+ if (inline) return { inline, from: round2(heard.back(from)), to: round2(heard.back(from + span)) };
const perSecond = blob.size / span;
// Leave room for the base64 inflation the cap already accounts for.
const fits = (MAX_AUDIO_BYTES * 0.9) / perSecond;
@@ -5695,6 +5787,7 @@ const SWEEP_TARGETS: Readonly<Record<string, string>> = {
set_clip_muted: "clipId",
set_clip_hidden: "clipId",
set_clip_volume: "clipId",
+ set_split_edit: "clipId",
set_color_preset: "clipId",
set_color_grade: "clipId",
set_color_curves: "clipId",
diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.ts
index e602df5f..4aafe7be 100644
--- a/site/src/cut/lib/audioMix.ts
+++ b/site/src/cut/lib/audioMix.ts
@@ -23,12 +23,12 @@ import {
buildAudioFx,
retimeOf,
soundRecipe,
- srcSpan,
type ClipSound,
type SpeedNode,
} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { fitSpan, retimeFits } from "./retimeFit";
+import { shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";
/** One track-0 clip's audio in the sequential fold. A spacer (no file) only
* shapes time. */
@@ -61,6 +61,15 @@ export interface MixClip {
* are what make the two clips audible at once over a hard picture cut. */
soundAhead?: number;
soundBack?: number;
+ /** A split edit: seconds of `soundBack` the clip's sound plays ahead of its
+ * picture and of `soundAhead` past its end, each ramping from silence over
+ * `splitFade` at its outer end. */
+ soundLead?: number;
+ soundTail?: number;
+ splitFade?: number;
+ /** The video's sound recorded apart from it: the sound reads this file,
+ * `offset` seconds later in the source (see soundSource.ts). */
+ soundFrom?: SpecSound;
}
/** A clip placed at an absolute time: an upper-track video's audio, a
@@ -87,6 +96,13 @@ export interface MixItem {
sound?: ClipSound;
/** While this item is audible, everything else drops to this gain. */
duck?: number;
+ soundLead?: number;
+ soundTail?: number;
+ splitFade?: number;
+ soundFrom?: SpecSound;
+ /** The soundtrack lane this item sits on; absent for a video clip's own
+ * sound. What a stem render picks items by. */
+ lane?: number;
}
/** One audio effect element: the treatment it names, and the stretch of
@@ -285,21 +301,23 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
// crossing at either edge widens it into the handle it reaches for.
const folded = foldClips(spec.clips).map((g) => {
const { back, ahead } = handles(g);
- return { ...g, back, ahead, span: crossSpan(g.clip, back, ahead) };
+ const read = g.clip.file && !g.clip.muted ? soundRead(g.clip, back, ahead) : null;
+ return { ...g, back, ahead, read };
+ });
+ // Each placed item's reach: a crossing's handle, or the split edit an edge
+ // without one carries.
+ const placed = spec.items.map((a) => {
+ const back = Math.min(a.soundBack ?? 0, Math.max(a.crossIn ?? 0, a.soundLead ?? 0));
+ const ahead = Math.min(a.soundAhead ?? 0, Math.max(a.crossOut ?? 0, a.soundTail ?? 0));
+ const read = a.file && !a.muted ? soundRead(a, back, ahead) : null;
+ return { a, back, ahead, read };
});
const wanted = new Map<string, { file: string; from: number; to: number }>();
- const spanKey = (c: { file: string; in: number; out: number }) =>
- `${c.file}|${c.in.toFixed(3)}|${c.out.toFixed(3)}`;
- for (const c of folded) {
- if (!c.clip.file || c.clip.muted) continue;
- const span = c.span;
- wanted.set(spanKey(span), { file: span.file, from: Math.max(0, span.in), to: span.out });
- }
- for (const a of spec.items) {
- if (!a.file || a.muted) continue;
- const span = crossSpan(a, a.soundBack, a.soundAhead);
- wanted.set(spanKey(span), { file: span.file, from: Math.max(0, span.in), to: span.out });
+ const spanKey = (c: { file: string; lo: number; hi: number }) =>
+ `${c.file}|${c.lo.toFixed(3)}|${c.hi.toFixed(3)}`;
+ for (const { read } of [...folded, ...placed]) {
+ if (read) wanted.set(spanKey(read), { file: read.file, from: read.lo, to: read.hi });
}
const buffers = new Map<string, AudioBuffer>();
@@ -320,7 +338,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
// Everything that can be ducked passes through one node; the voiceovers doing
// the ducking connect past it, so they never duck themselves.
const ducked = ctx.createGain();
- scheduleDuck(ducked, duckWindows(spec.items));
+ scheduleDuck(ducked, spec.ducks ?? duckWindows(spec.items));
// Every sound meets on the bus, which is what the effect elements treat —
// the ffmpeg graph applies them to the mixed stream in the same place.
@@ -425,18 +443,22 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
src.connect(gain);
}
gain.connect(into);
- src.start(Math.max(0, at), 0, Math.max(0, duration));
+ // A sound reaching before the timeline's start joins part-way in.
+ const skip = Math.max(0, -at);
+ src.start(Math.max(0, at), skip, Math.max(0, duration - skip));
};
for (const g of folded) {
const c = g.clip;
- const key = spanKey(g.span);
- const buf = !c.muted && c.file ? buffers.get(key) : undefined;
- if (!buf) continue; // muted, silent, or a gap spacer: only shapes time
- const fit = fitted(key, buf, c, Math.max(0, g.span.in), g.span.out);
- const from = g.at - g.back + fit.shift;
+ const read = g.read;
+ const key = read ? spanKey(read) : "";
+ const buf = read ? buffers.get(key) : undefined;
+ if (!read || !buf) continue; // muted, silent, or a gap spacer: only shapes time
+ // Laid through the clip's own map, moved onto the file the sound reads.
+ const fit = fitted(key, buf, shiftSpan(c, c.soundFrom?.offset ?? 0), read.lo, read.hi);
+ const from = g.at - g.back + read.at + fit.shift;
const end = g.at + g.dur;
- play(fit.buffer, from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {
+ play(fit.buffer, from, read.len, ducked, c.sound, (gain) => {
const level = Math.max(0, c.volume ?? 1);
gain.gain.value = level;
// A crossing and a fade never share an edge — the cut carries one
@@ -453,23 +475,29 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
gain.gain.setValueAtTime(level, end - g.fadeOut);
gain.gain.linearRampToValueAtTime(0, end);
}
+ scheduleSplit(
+ gain.gain,
+ level,
+ g.at,
+ end,
+ g.crossIn > 0 ? 0 : Math.min(c.soundLead ?? 0, g.back),
+ g.crossOut > 0 ? 0 : Math.min(c.soundTail ?? 0, g.ahead),
+ c.splitFade ?? 0
+ );
});
}
- for (const a of spec.items) {
- const back = Math.min(a.soundBack ?? 0, a.crossIn ?? 0);
- const ahead = Math.min(a.soundAhead ?? 0, a.crossOut ?? 0);
- const key = spanKey(crossSpan(a, back, ahead));
- const buf = a.muted ? undefined : buffers.get(key);
- if (!buf) continue;
+ for (const { a, back, ahead, read } of placed) {
+ const key = read ? spanKey(read) : "";
+ const buf = read ? buffers.get(key) : undefined;
+ if (!read || !buf) continue;
const dur = itemDur(a);
const end = a.start + dur;
// A voiceover that ducks everything else is not itself ducked; it still
// lands on the bus, so the effect elements treat it with the rest.
const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;
- const reach = crossSpan(a, back, ahead);
- const fit = fitted(key, buf, a, Math.max(0, reach.in), reach.out);
- play(fit.buffer, a.start - back + fit.shift, dur + back + ahead, into, a.sound, (gain) => {
+ const fit = fitted(key, buf, shiftSpan(a, a.soundFrom?.offset ?? 0), read.lo, read.hi);
+ play(fit.buffer, a.start - back + read.at + fit.shift, read.len, into, a.sound, (gain) => {
const level = Math.max(0, a.volume);
gain.gain.value = level;
const fIn = Math.min(a.fadeIn ?? 0, dur);
diff --git a/site/src/cut/lib/audioMixSplit.test.ts b/site/src/cut/lib/audioMixSplit.test.ts
new file mode 100644
index 00000000..54ad2736
--- /dev/null
+++ b/site/src/cut/lib/audioMixSplit.test.ts
@@ -0,0 +1,233 @@
+import { describe, expect, test } from "bun:test";
+import { stubModule } from "@/lib/testing/stubModule";
+import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";
+
+// Where the offline fold lands a clip's sound when it leaves its picture: a
+// split edit reaching past either edge, and a recording bound to the video.
+// Each file reads back silence with one click at a source second of its own,
+// so where the click lands in the mix says exactly which source second played
+// at which timeline second.
+const RATE = 44100;
+
+const globals = globalThis as Record<string, unknown>;
+globals.AudioBuffer ??= AudioBuffer;
+globals.OfflineAudioContext ??= OfflineAudioContext;
+
+/** Each file's click, in its own source seconds, and its length. */
+const FILES: Record<string, { click?: number; duration: number }> = {
+ "quiet.mp4": { duration: 60 },
+ "lead.mp4": { click: 4.8, duration: 60 },
+ "tail.mp4": { click: 4.3, duration: 60 },
+ "cam.mp4": { click: 12, duration: 60 },
+ "rec.wav": { click: 8.5, duration: 60 },
+ "late.wav": { click: 1, duration: 60 },
+ "layer.mp4": { click: 4.7, duration: 60 },
+};
+
+await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {
+ decodeAudioSpan: (async (file: string, from: number, to: number) => {
+ const f = FILES[file];
+ const lo = Math.max(0, from);
+ const hi = Math.min(f.duration, to);
+ const buf = new AudioBuffer({ length: Math.max(1, Math.round((hi - lo) * RATE)), numberOfChannels: 1, sampleRate: RATE });
+ if (f.click !== undefined && f.click >= lo && f.click < hi) {
+ const at = Math.round((f.click - lo) * RATE);
+ const ch = buf.getChannelData(0);
+ for (let i = 0; i < 32 && at + i < ch.length; i++) ch[at + i] = 1;
+ }
+ return buf;
+ }) as never,
+});
+
+const { renderMix } = await import("./audioMix");
+type Spec = Parameters<typeof renderMix>[0];
+
+const render = async (spec: Spec) => {
+ const buffer = await renderMix(spec, { sampleRate: RATE, channels: 1, resolve: (f) => f });
+ expect(buffer).not.toBeNull();
+ return buffer!.getChannelData(0);
+};
+
+/** The timeline second the click starts at, or null when none sounds. */
+const clickAt = (data: Float32Array) => {
+ for (let i = 0; i < data.length; i++) if (Math.abs(data[i]) > 0.5) return i / RATE;
+ return null;
+};
+
+const near = (got: number | null, want: number) => {
+ expect(got).not.toBeNull();
+ expect(Math.abs(got! - want)).toBeLessThan(2 / RATE);
+};
+
+describe("split edits in the offline fold", () => {
+ test("a J-cut plays the incoming clip's sound ahead of its picture", async () => {
+ // b's picture starts at 4 on source 5; its sound leads by half a second,
+ // so source 4.8 plays at 3.8 — over a's picture.
+ const mix = await render({
+ duration: 8,
+ clips: [
+ { file: "quiet.mp4", in: 0, out: 4, muted: false },
+ { file: "lead.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },
+ ],
+ items: [],
+ });
+ near(clickAt(mix), 3.8);
+ });
+
+ test("the lead's ramp starts it from silence", async () => {
+ // A click inside the ramp: 0.01 s into a 0.03 s fade plays at a third.
+ FILES["ramp.mp4"] = { click: 4.51, duration: 60 };
+ const mix = await render({
+ duration: 8,
+ clips: [
+ { file: "quiet.mp4", in: 0, out: 4, muted: false },
+ { file: "ramp.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },
+ ],
+ items: [],
+ });
+ const at = Math.round(3.51 * RATE);
+ expect(mix[at]).toBeGreaterThan(0.25);
+ expect(mix[at]).toBeLessThan(0.42);
+ });
+
+ test("an L-cut carries the outgoing clip's sound past its picture", async () => {
+ // a's picture ends at 4 on source 4; its sound runs on, so source 4.3
+ // plays at 4.3 — under b's picture.
+ const mix = await render({
+ duration: 8,
+ clips: [
+ { file: "tail.mp4", in: 0, out: 4, muted: false, soundTail: 0.5, soundAhead: 0.5, splitFade: 0.03 },
+ { file: "quiet.mp4", in: 0, out: 4, muted: false },
+ ],
+ items: [],
+ });
+ near(clickAt(mix), 4.3);
+ });
+
+ test("without a split edit the trimmed-away sound stays silent", async () => {
+ const mix = await render({
+ duration: 8,
+ clips: [
+ { file: "tail.mp4", in: 0, out: 4, muted: false },
+ { file: "lead.mp4", in: 5, out: 9, muted: false },
+ ],
+ items: [],
+ });
+ expect(clickAt(mix)).toBeNull();
+ });
+
+ test("a layer clip's lead lands ahead of its start", async () => {
+ const mix = await render({
+ duration: 8,
+ clips: [{ file: "", in: 0, out: 8, muted: true }],
+ items: [{ file: "layer.mp4", in: 5, out: 7, start: 2, volume: 1, soundLead: 0.4, soundBack: 0.4, splitFade: 0.03 }],
+ });
+ near(clickAt(mix), 1.7);
+ });
+});
+
+describe("a recording bound to the video", () => {
+ test("plays the recording through the offset, and the camera's track stays out", async () => {
+ // Recording second = video second − 3, so recording 8.5 is video 11.5,
+ // which this clip (in 10, at 0) plays at 1.5. The camera's own click at
+ // video 12 must not sound.
+ const mix = await render({
+ duration: 4,
+ clips: [{ file: "cam.mp4", in: 10, out: 14, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
+ items: [],
+ });
+ near(clickAt(mix), 1.5);
+ expect(Math.abs(mix[Math.round(2 * RATE) + 4])).toBeLessThan(0.01);
+ });
+
+ test("a recording that rolled after the camera leaves silence before it, in time", async () => {
+ // The recording starts at video second 2; the clip starts on video
+ // second 1. Recording 1 is video 3, played at timeline 2.
+ const mix = await render({
+ duration: 4,
+ clips: [{ file: "cam.mp4", in: 1, out: 5, muted: false, soundFrom: { file: "late.wav", offset: -2, duration: 60 } }],
+ items: [],
+ });
+ near(clickAt(mix), 2);
+ });
+
+ test("a speed change runs the recording through the clip's own map", async () => {
+ // At 2× the clip covers video 10..14 in two seconds: video 11.5 plays at 0.75.
+ const mix = await render({
+ duration: 2,
+ clips: [{ file: "cam.mp4", in: 10, out: 14, speed: 2, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
+ items: [],
+ });
+ const got = clickAt(mix);
+ expect(got).not.toBeNull();
+ // The stretch smears a click over a few milliseconds.
+ expect(Math.abs(got! - 0.75)).toBeLessThan(0.03);
+ });
+
+ test("a reversed clip plays the recording backward from its out point", async () => {
+ // Reversed, timeline 0 shows video 14 and timeline 4 video 10: video
+ // 11.5 lands at 2.5.
+ const mix = await render({
+ duration: 4,
+ clips: [{ file: "cam.mp4", in: 10, out: 14, reverse: true, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
+ items: [],
+ });
+ let peak = 0;
+ let at = 0;
+ for (let i = 0; i < mix.length; i++) if (Math.abs(mix[i]) > peak) { peak = Math.abs(mix[i]); at = i / RATE; }
+ expect(Math.abs(at - 2.5)).toBeLessThan(0.002);
+ });
+});
+
+describe("stems from the document", () => {
+ test("the dialogue stem plays the bound recording and the split edit; a lane keeps its own sound", async () => {
+ const { mixSpecFor, stemMixSpec } = await import("./exportRender");
+ const media = (id: string, type: "video" | "audio", over: Record<string, unknown> = {}) => ({
+ id,
+ fileName: id,
+ name: id,
+ type,
+ duration: 60,
+ url: "",
+ ...over,
+ });
+ // Video second t plays recording t − 2: the recording's click at 8.5 is
+ // video 10.5, which clip a (video 10..12) plays at 0.5. Clip b starts at
+ // 2 on source 5 and leads by half a second: source 4.8 plays at 1.8. The
+ // soundtrack's click at its own 1.0 sits on lane 0.
+ const doc = {
+ aspect: "16:9",
+ assets: [
+ media("cam.mp4", "video", { soundFrom: { assetId: "rec.wav", offset: -2 } }),
+ media("rec.wav", "audio"),
+ media("lead.mp4", "video"),
+ media("late.wav", "audio"),
+ ],
+ clips: [
+ { id: "a", assetId: "cam.mp4", start: 0, in: 10, out: 12, track: 0, muted: false },
+ { id: "b", assetId: "lead.mp4", start: 2, in: 5, out: 9, track: 0, muted: false, audioLead: 0.5 },
+ ],
+ audioClips: [{ id: "s", assetId: "late.wav", start: 0, in: 0, out: 4, volume: 1, lane: 0 }],
+ overlays: [],
+ subtitles: { cues: [] },
+ } as unknown as Parameters<typeof mixSpecFor>[0];
+ const spec = mixSpecFor(doc, (a) => a.fileName);
+ const clicks = (data: Float32Array) => {
+ const at: number[] = [];
+ let quietSince = -RATE;
+ for (let i = 0; i < data.length; i++) {
+ if (Math.abs(data[i]) <= 0.5) continue;
+ if (i - quietSince > RATE * 0.1) at.push(i / RATE);
+ quietSince = i;
+ }
+ return at;
+ };
+ const dialogue = clicks(await render(stemMixSpec(spec, { name: "Dialogue", file: "1 Dialogue.wav", lane: null })));
+ expect(dialogue).toHaveLength(2);
+ near(dialogue[0], 0.5);
+ near(dialogue[1], 1.8);
+ const lane = clicks(await render(stemMixSpec(spec, { name: "late", file: "2 late.wav", lane: 0 })));
+ expect(lane).toHaveLength(1);
+ near(lane[0], 1);
+ });
+});
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 34dcabd6..576056de 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -32,10 +32,12 @@ import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, pa
import { renderElementFrames, renderElementPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";
import { liveReader } from "./liveReader";
+import { specSound } from "./soundSource";
import type {
Aspect,
AudioClip,
ClipAnim,
+ ClipSpan,
MediaAsset,
Overlay,
Selection,
@@ -785,6 +787,19 @@ function renderClipBorderPng(
return rasterCanvasToPng(canvas);
}
+/** A clip's sound as the spec carries it: the recording bound to its video,
+ * and a split edit's reach past its picture with the ramp at the far end. */
+function specSpanSound(sp: ClipSpan) {
+ return {
+ ...(sp.sound
+ ? { soundFrom: { file: sp.sound.asset.fileName, offset: sp.sound.offset, duration: sp.sound.asset.duration } }
+ : {}),
+ ...(sp.soundLead || sp.soundTail
+ ? { soundLead: sp.soundLead ?? 0, soundTail: sp.soundTail ?? 0, splitFade: sp.splitFade ?? 0 }
+ : {}),
+ };
+}
+
/** What a source's code values mean, for the spec: the profile the header
* settled (with the person's override), and the matrix and range the file
* decodes with. A still is sRGB, decoded to full-range RGB, and a block is
@@ -880,6 +895,7 @@ export async function buildExportPayload(
: undefined;
const clipEntries = spans.map((sp) => ({
+ ...specSpanSound(sp),
file: sp.asset.fileName,
in: sp.clip.in,
out: sp.clip.out,
@@ -1094,10 +1110,11 @@ export async function buildExportPayload(
r.soundBack = sp.soundBack;
});
return trackSpans
- .map((sp, i) => ({ c: sp.clip, ramp: ramps[i] }))
+ .map((sp, i) => ({ c: sp.clip, sp, ramp: ramps[i] }))
.filter(({ c }) => !c.hidden && c.start < duration)
- .map(({ c, ramp }) => {
+ .map(({ c, sp, ramp }) => {
const entry = {
+ ...specSpanSound(sp),
file: assetById.get(c.assetId)!.fileName,
in: c.in,
out: c.out,
@@ -1226,6 +1243,7 @@ export async function buildExportPayload(
const audio = doc.audioClips
.filter((a) => !a.hidden && a.start < duration && assetById.has(a.assetId))
.map((a) => ({
+ soundFrom: specSound(assetById.get(a.assetId)!, assetById, (x) => x.fileName),
file: assetById.get(a.assetId)!.fileName,
in: a.in,
out: a.out,
diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.ts
index 2a81ca14..382e517b 100644
--- a/site/src/cut/lib/exportDelivery.ts
+++ b/site/src/cut/lib/exportDelivery.ts
@@ -161,9 +161,9 @@ export function videoBitrateFor(s: {
/** Every media file a render spec plays — clips, picture-in-picture video,
* sound — by name, once each. */
export function specMediaFiles(spec: {
- clips?: { file?: string; staged?: boolean }[];
- overlayVideos?: { file?: string }[];
- audio?: { file?: string }[];
+ clips?: { file?: string; staged?: boolean; soundFrom?: { file?: string } }[];
+ overlayVideos?: { file?: string; soundFrom?: { file?: string } }[];
+ audio?: { file?: string; soundFrom?: { file?: string } }[];
}): string[] {
const names = new Set<string>();
for (const c of [...(spec.clips ?? []), ...(spec.overlayVideos ?? []), ...(spec.audio ?? [])]) {
@@ -172,6 +172,8 @@ export function specMediaFiles(spec: {
// project's folder finds nothing, and staging it there puts it where the
// render does not read.
if (c?.file && !("staged" in c && c.staged)) names.add(c.file);
+ // A recording bound to the video is read beside it.
+ if (c?.soundFrom?.file) names.add(c.soundFrom.file);
}
return [...names];
}
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index f3eb1108..d80f91d4 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -859,6 +859,14 @@ function trackZeroFades(spans: ClipSpan[], i: number): { fadeIn: number; fadeOut
* while the picture keeps them, and everything after the first gap plays early.
*/
export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => string): MixSpec {
+ // A clip's sound past its picture (a split edit) and the recording bound to
+ // its video, as the fold reads them.
+ const spanSound = (sp: ClipSpan) => ({
+ ...(sp.sound ? { soundFrom: { file: resolve(sp.sound.asset), offset: sp.sound.offset, duration: sp.sound.asset.duration } } : {}),
+ ...(sp.soundLead || sp.soundTail
+ ? { soundLead: sp.soundLead, soundTail: sp.soundTail, splitFade: sp.splitFade }
+ : {}),
+ });
const duration = projectDuration(doc);
const spans = getClipSpans(doc.clips, doc.assets, 0);
const byId = new Map(doc.assets.map((a) => [a.id, a]));
@@ -878,20 +886,24 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
speedCurve: sp.clip.speedCurve,
reverse: sp.clip.reverse,
sound: sp.clip.sound,
- muted: sp.clip.muted || assetIsSilent(sp.asset),
+ muted: sp.clip.muted || (!sp.sound && assetIsSilent(sp.asset)),
fadeIn: ramps[i].head,
fadeOut: ramps[i].tail,
crossIn: ramps[i].crossIn,
crossOut: ramps[i].crossOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
+ ...spanSound(sp),
});
});
}
for (const a of doc.audioClips) {
const asset = byId.get(a.assetId);
- if (!asset || a.hidden || a.start >= duration || assetIsSilent(asset)) continue;
+ if (!asset || a.hidden || a.start >= duration) continue;
+ const soundFrom = specSound(asset, byId, resolve);
+ if (!soundFrom && assetIsSilent(asset)) continue;
items.push({
+ ...(soundFrom ? { soundFrom } : {}),
file: resolve(asset),
in: a.in,
out: a.out,
diff --git a/site/src/cut/lib/framePlan.ts b/site/src/cut/lib/framePlan.ts
index 322cc7f6..803bc7cd 100644
--- a/site/src/cut/lib/framePlan.ts
+++ b/site/src/cut/lib/framePlan.ts
@@ -266,11 +266,32 @@ export function crossHandles(spans: ClipSpan[], t: number): { span: ClipSpan; ga
const from = sp.start - sp.soundBack;
const to = sp.start + sp.len + sp.soundAhead;
if (t < from || t >= to) continue;
- out.push({ span: sp, gain: soundCrossGain(spans, sp, t) });
+ out.push({ span: sp, gain: soundCrossGain(spans, sp, t) * splitEditGain(sp, t) });
}
return out;
}
+/**
+ * The gain a split edit puts on a clip's sound at `t`: full inside the
+ * picture, a short ramp up from silence where a J-cut's lead opens and down to
+ * silence where an L-cut's tail closes. The offline fold and the ffmpeg graph
+ * write the same ramps.
+ */
+export function splitEditGain(sp: ClipSpan, t: number): number {
+ const lead = sp.soundLead ?? 0;
+ const tail = sp.soundTail ?? 0;
+ const end = sp.start + sp.len;
+ if (lead > 0 && t < sp.start) {
+ const f = Math.min(sp.splitFade ?? 0, lead);
+ return f > 0 ? Math.max(0, Math.min(1, (t - (sp.start - lead)) / f)) : 1;
+ }
+ if (tail > 0 && t >= end) {
+ const f = Math.min(sp.splitFade ?? 0, tail);
+ return f > 0 ? Math.max(0, Math.min(1, (end + tail - t) / f)) : 1;
+ }
+ return 1;
+}
+
/** How much of a clip's animation window is still running at `rel`, for the
* sides that show a neighbour's frame behind them. */
function animLive(a: ClipAnim | undefined, side: "in" | "out", rel: number, len: number) {
diff --git a/site/src/cut/lib/itemKinds.ts b/site/src/cut/lib/itemKinds.ts
index 35e433ce..b7661794 100644
--- a/site/src/cut/lib/itemKinds.ts
+++ b/site/src/cut/lib/itemKinds.ts
@@ -96,11 +96,16 @@ export const ITEM_KINDS: { [K in ItemKind]: ItemKindDef<K> } = {
at: (c, start) => ({ ...c, start }),
split: (c, at) => {
const [left, right] = splitMedia(c, at);
- return [{ ...left, transition: undefined, transitionStyle: undefined, animOut: undefined }, { ...right, animIn: undefined }];
+ // A split edit stays on the outer edges: the new cut between the halves
+ // carries none.
+ return [
+ { ...left, transition: undefined, transitionStyle: undefined, animOut: undefined, audioTail: undefined },
+ { ...right, animIn: undefined, audioLead: undefined },
+ ];
},
clone: deep,
assetIds: (c) =>
- [c.assetId, c.removal?.matte?.assetId, c.removal?.backdrop?.assetId].filter(
+ [c.assetId, c.removal?.matte?.assetId, c.removal?.backdrop?.assetId, c.card?.matte?.assetId].filter(
(id): id is string => typeof id === "string" && id.length > 0
),
remapAssets: (c, to) => ({
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 11535b91..64580ff0 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1,6 +1,8 @@
"use client";
import { scanBeats, scanLevel, scanSilence, scanSpeech, type BeatScan, type LevelScan, type PcmChunk, type SpeechScan } from "./audioScan";
+import { syncSound, type SyncAnswer } from "./soundSync";
+import { LoudnessMeter, type LoudnessMeasure } from "./loudness";
import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";
import { registerBlobFile } from "./backend/browser/registry";
import { cloudBackend } from "./backend/cloud";
diff --git a/site/src/cut/lib/projectReference.ts b/site/src/cut/lib/projectReference.ts
index feadfb3b..3445feb7 100644
--- a/site/src/cut/lib/projectReference.ts
+++ b/site/src/cut/lib/projectReference.ts
@@ -160,7 +160,14 @@ export async function landReferenceAssets(
): Promise<LandedAsset[]> {
const out: LandedAsset[] = [];
const toCopy: StoredAsset[] = [];
+ // A video bound to a separate recording brings the recording with it.
+ const refAssets = new Map(ref.doc.assets.map((a) => [a.id, a]));
+ const wanted = [...sources];
for (const source of sources) {
+ const rec = source.soundFrom && refAssets.get(source.soundFrom.assetId);
+ if (rec && !wanted.some((w) => w.id === rec.id)) wanted.push(rec);
+ }
+ for (const source of wanted) {
const have = useEditor
.getState()
.assets.find((a) => a.copiedFrom?.projectId === ref.projectId && a.copiedFrom.assetId === source.id);
@@ -194,7 +201,15 @@ export async function landReferenceAssets(
else if (!isLinkedAssetType(asset.type)) void enrichAsset(asset).catch(() => {});
out.push({ sourceId: source.id, asset, reused: false });
}
- // In the order asked for, so a caller's index arithmetic holds.
+ // A copied video's binding points at the copy of its recording.
const bySource = new Map(out.map((l) => [l.sourceId, l]));
+ for (const l of out) {
+ const from = l.asset.soundFrom;
+ if (l.reused || !from) continue;
+ const rec = bySource.get(from.assetId)?.asset.id;
+ useEditor.getState().updateAsset(l.asset.id, { soundFrom: rec ? { ...from, assetId: rec } : undefined });
+ l.asset = useEditor.getState().assets.find((a) => a.id === l.asset.id) ?? l.asset;
+ }
+ // In the order asked for, so a caller's index arithmetic holds.
return sources.map((s) => bySource.get(s.id)!);
}
diff --git a/site/src/cut/lib/projectTemplate.test.ts b/site/src/cut/lib/projectTemplate.test.ts
index ea4e9d1b..9bcf4b61 100644
--- a/site/src/cut/lib/projectTemplate.test.ts
+++ b/site/src/cut/lib/projectTemplate.test.ts
@@ -132,6 +132,25 @@ describe("insertTemplate", () => {
const MAP: Record<string, string> = { v0: "b0", v1: "b1", m0: "bm", s0: "bs", f0: "" };
const idsFor = (assetByMedia: string[], map = MAP) => assetByMedia.map((id) => map[id] ?? "");
+ test("a video's bound recording travels with it and binds to the landed copy", () => {
+ const doc = richDoc();
+ const withRec = {
+ ...doc,
+ assets: [
+ ...(doc.assets ?? []).map((a) => (a.id === "v0" ? { ...a, soundFrom: { assetId: "r0", offset: 0.25 } } : a)),
+ { id: "r0", fileName: "lav.wav", name: "lav", type: "audio" as const, duration: 60 },
+ ],
+ };
+ const { template, assetByMedia } = templateFromDoc(withRec);
+ const v0 = assetByMedia.indexOf("v0");
+ const r0 = assetByMedia.indexOf("r0");
+ expect(r0).toBeGreaterThanOrEqual(0);
+ expect(template.media[v0].soundFrom).toEqual({ media: r0, offset: 0.25 });
+ useEditor.setState({ assets: [...useEditor.getState().assets, asset("br", "audio", 60)] });
+ useEditor.getState().insertTemplate({ ...template, id: "t", addedAt: 0 }, idsFor(assetByMedia, { ...MAP, r0: "br" }), 0);
+ expect(useEditor.getState().assets.find((a) => a.id === "b0")?.soundFrom).toEqual({ assetId: "br", offset: 0.25 });
+ });
+
test("stands the whole edit up on other footage, treatment and all", () => {
const { template, assetByMedia } = templateFromDoc(richDoc());
useEditor.getState().insertTemplate({ ...template, id: "t", addedAt: 0 }, idsFor(assetByMedia), 0);
diff --git a/site/src/cut/lib/projectTemplate.ts b/site/src/cut/lib/projectTemplate.ts
index 20594dfb..9181f182 100644
--- a/site/src/cut/lib/projectTemplate.ts
+++ b/site/src/cut/lib/projectTemplate.ts
@@ -89,6 +89,11 @@ export function templateFromDoc(doc: ProjectDoc, opts: { items?: string[] } = {}
assetByMedia.push(a.id);
sourceAssets.push(a);
indexByAsset.set(assetId, i);
+ // The recording a video plays rides along with it.
+ if (a.soundFrom) {
+ const rec = mediaFor(a.soundFrom.assetId);
+ if (rec != null) media[i] = { ...media[i], soundFrom: { media: rec, offset: a.soundFrom.offset } };
+ }
return i;
};
diff --git a/site/src/cut/lib/soundBind.ts b/site/src/cut/lib/soundBind.ts
new file mode 100644
index 00000000..aec9727a
--- /dev/null
+++ b/site/src/cut/lib/soundBind.ts
@@ -0,0 +1,88 @@
+/**
+ * Binding a separately recorded sound to a video: find where the recording
+ * lines up with the camera's own track, and only when one alignment clearly
+ * wins, write it onto the video asset (`soundFrom`). The Audio tab's Sound
+ * picker and the assistant's sync_audio both land here, so the person and the
+ * assistant hold the recording to the same bar.
+ */
+
+import { syncSourceSound } from "./media";
+import { cutSound } from "./soundSettings";
+import { useEditor } from "./store";
+import { assetIsSilent, type MediaAsset } from "./types";
+import { queueWatchSweep } from "./watch/sweep";
+
+export interface BoundSound {
+ assetId: string;
+ audioAssetId: string;
+ /** Recording seconds = video seconds + offset. */
+ offset: number;
+ /** The best alignment over the runner-up a second or more away. */
+ confidence: number;
+ /** Whether the offset sits on the sample (else on 20 ms). */
+ refined: boolean;
+ /** Clips of the video on the timeline, all now sounding from the recording. */
+ clips: number;
+}
+
+/**
+ * Line `audioId` up with `videoId` and bind it. `near` is the source span of
+ * the clip the person is working on, so the camera's stretch that is matched
+ * is the one they care about. Throws a readable Error when the files cannot
+ * be matched or no alignment stands out; nothing is bound then.
+ */
+export async function bindRecording(
+ videoId: string,
+ audioId: string,
+ near?: { in: number; out: number }
+): Promise<BoundSound> {
+ const s = useEditor.getState();
+ const video = s.assets.find((a) => a.id === videoId);
+ const rec = s.assets.find((a) => a.id === audioId);
+ if (!video || video.type !== "video") throw new Error("That video is no longer in the project.");
+ if (!rec) throw new Error("That recording is no longer in the project.");
+ if (rec.type !== "audio") throw new Error(`"${rec.name}" is not an audio file.`);
+ if (assetIsSilent(video))
+ throw new Error(`"${video.name}" has no sound of its own to line the recording up against.`);
+ if (!(video.duration > 0) || !(rec.duration > 0)) throw new Error("Both files must finish importing first.");
+ const { syncProbeSeconds, syncMaxSeconds, syncMinConfidence } = cutSound();
+ const [cameraFrom, cameraTo] = probeWindow(video, syncProbeSeconds, near);
+ const answer = await syncSourceSound(video.url, rec.url, {
+ cameraFrom,
+ cameraTo,
+ recordingTo: Math.min(rec.duration, syncMaxSeconds),
+ });
+ if (!answer || answer.confidence < syncMinConfidence) {
+ const read = answer ? ` (confidence ${answer.confidence.toFixed(2)}, needs ${syncMinConfidence})` : "";
+ throw new Error(
+ `No clear alignment between "${rec.name}" and "${video.name}"${read}. They may be different takes, or share too little sound.`
+ );
+ }
+ const offset = Math.round(answer.offset * 1e5) / 1e5;
+ useEditor.getState().setAssetSoundFrom(video.id, { assetId: rec.id, offset });
+ // Its transcript is heard again, off the recording.
+ queueWatchSweep(video.id);
+ return {
+ assetId: video.id,
+ audioAssetId: rec.id,
+ offset,
+ confidence: Math.round(answer.confidence * 100) / 100,
+ refined: answer.refined,
+ clips: useEditor.getState().clips.filter((c) => c.assetId === video.id).length,
+ };
+}
+
+/** Return a video to its own sound. */
+export function unbindRecording(videoId: string): void {
+ useEditor.getState().setAssetSoundFrom(videoId, undefined);
+ queueWatchSweep(videoId);
+}
+
+/** The camera's stretch to match: `seconds` long, centred on the clip being
+ * worked on, else on the middle of the video, inside the file. */
+function probeWindow(video: MediaAsset, seconds: number, near?: { in: number; out: number }): [number, number] {
+ const len = Math.min(seconds, video.duration);
+ const mid = near ? (near.in + near.out) / 2 : video.duration / 2;
+ const from = Math.max(0, Math.min(video.duration - len, mid - len / 2));
+ return [from, from + len];
+}
diff --git a/site/src/cut/lib/soundSettings.ts b/site/src/cut/lib/soundSettings.ts
new file mode 100644
index 00000000..0b85cd45
--- /dev/null
+++ b/site/src/cut/lib/soundSettings.ts
@@ -0,0 +1,17 @@
+import { SETTINGS, type Settings } from "@/lib/config/registry";
+
+// The sound editing tunables — the split edit's ramp and how a separate
+// recording is lined up — bound once at session start from account config (the
+// tab) or the global setting (a worker job), and read where a clip's sound is
+// laid out.
+
+let sound: Settings["cutSound"] = SETTINGS.cutSound.default;
+
+export function bindCutSound(value: unknown): void {
+ const parsed = SETTINGS.cutSound.schema.safeParse(value);
+ if (parsed.success) sound = parsed.data;
+}
+
+export function cutSound(): Settings["cutSound"] {
+ return sound;
+}
diff --git a/site/src/cut/lib/soundSource.ts b/site/src/cut/lib/soundSource.ts
new file mode 100644
index 00000000..ee7945b5
--- /dev/null
+++ b/site/src/cut/lib/soundSource.ts
@@ -0,0 +1,143 @@
+/**
+ * Where a clip's sound comes from.
+ *
+ * A camera records the picture and a scratch track beside it; a recorder on
+ * the speaker records the sound worth keeping. Binding the recording to the
+ * video asset (`soundFrom`) puts every clip of that video on the recording:
+ * the clip keeps its trim, rate and direction, and each source second of the
+ * video is read off the recording `offset` seconds later. The binding sits on
+ * the asset, so a split, a trim, a retime, a reverse
+ * or a copy cannot leave the sound behind. Every reader of a clip's sound —
+ * the preview, the fold the tab exports and transcribes through, the spec the
+ * engine and the worker render, the tools that listen — resolves it here.
+ */
+
+import { retimeOf, speedCurveOf, srcSpan, type Retimable, type SpeedNode } from "@donkeycut/effects-kit";
+import { assetIsSilent, type StoredAsset } from "./types";
+
+type Sourced = Pick<StoredAsset, "id" | "type" | "duration" | "soundFrom">;
+
+/** The longest a split edit carries a clip's sound past its picture, either
+ * side, in seconds. */
+export const SPLIT_EDIT_MAX_S = 5;
+
+/** The sound a video plays: its own file's track, or the recording bound to
+ * it, with the seconds added to its source time to land on that file. */
+export interface SoundSource<A> {
+ asset: A;
+ offset: number;
+ /** The last second the sound file holds, for a bound recording; the
+ * asset's own track is read as far as it goes. */
+ limit: number;
+ bound: boolean;
+}
+
+/** `asset`'s sound. A binding to a recording that is gone, silent, or the
+ * video itself plays the camera's own track. One hop: a recording's own
+ * binding is never followed. */
+export function soundSourceOf<A extends Sourced>(
+ asset: A,
+ assets: readonly A[] | ReadonlyMap<string, A>
+): SoundSource<A> {
+ const from = asset.soundFrom;
+ if (from && from.assetId !== asset.id && Number.isFinite(from.offset)) {
+ const rec = isMap(assets) ? assets.get(from.assetId) : assets.find((a) => a.id === from.assetId);
+ if (rec && !assetIsSilent(rec))
+ return { asset: rec, offset: from.offset, limit: rec.duration > 0 ? rec.duration : Infinity, bound: true };
+ }
+ return { asset, offset: 0, limit: Infinity, bound: false };
+}
+
+const isMap = <A>(v: readonly A[] | ReadonlyMap<string, A>): v is ReadonlyMap<string, A> => v instanceof Map;
+
+/** Source seconds of the sound file on either side of a clip's trim: what a
+ * handle, a split edit or a crossing can reach into before its head and past
+ * its tail. A reversed clip's head faces the source's end. */
+export function soundRoom(
+ c: { in: number; out: number; reverse?: boolean },
+ src: { asset: { duration: number }; offset: number }
+): { head: number; tail: number } {
+ const lo = c.in + src.offset;
+ const hi = c.out + src.offset;
+ const end = src.asset.duration;
+ return c.reverse ? { head: end - hi, tail: lo } : { head: lo, tail: end - hi };
+}
+
+/** A span moved onto its sound's clock: the same map from the timeline,
+ * `offset` seconds later in the source. Speed curve nodes sit in source
+ * seconds, so they move with it. */
+export function shiftSpan<T extends Retimable>(c: T, offset: number): T {
+ if (!offset) return c;
+ const nodes = speedCurveOf(c);
+ return {
+ ...c,
+ in: c.in + offset,
+ out: c.out + offset,
+ ...(nodes ? { speedCurve: nodes.map(([at, rate]): SpeedNode => [at + offset, rate]) } : {}),
+ };
+}
+
+/** What one stretch of a span's sound reads from its file. */
+export interface SoundWindow {
+ /** Seconds on the sound file, low end first. */
+ lo: number;
+ hi: number;
+ /** Timeline seconds into the window the sound begins: a recording that
+ * starts after the window does leaves silence ahead of it. */
+ at: number;
+ /** Timeline seconds the sound runs. */
+ len: number;
+ /** The stretch as a span of its own on the sound file — the map a renderer
+ * lays it through. */
+ span: Retimable;
+}
+
+/**
+ * The sound of span `c` over the window `[fromT, toT]`, timeline seconds from
+ * its head (negative reaches before it, past its length beyond it), read from
+ * the source `src` describes. The window narrows to what the file holds: a
+ * recording rolled after the camera did, or stopped before it, leaves the
+ * rest of the window silent. Null when nothing of the window is on the file.
+ */
+export function soundWindow(
+ c: Retimable,
+ src: { offset: number; limit: number },
+ fromT: number,
+ toT: number
+): SoundWindow | null {
+ const moved = shiftSpan(c, src.offset);
+ const rt = retimeOf(moved);
+ const want = srcSpan(rt, fromT, toT);
+ const lo = Math.max(0, want.lo);
+ const hi = Math.min(src.limit, want.hi);
+ if (!(hi - lo > 1e-4)) return null;
+ const t0 = rt.tAt(rt.reverse ? hi : lo);
+ const t1 = rt.tAt(rt.reverse ? lo : hi);
+ return {
+ lo,
+ hi,
+ at: Math.max(0, t0 - fromT),
+ len: Math.max(0, t1 - t0),
+ span: { in: lo, out: hi, speed: moved.speed, speedCurve: moved.speedCurve, reverse: moved.reverse },
+ };
+}
+
+/** A bound recording as a render spec carries it: the file the sound reads,
+ * its clock's offset from the video's, and its length. */
+export interface SpecSound {
+ file: string;
+ offset: number;
+ duration: number;
+}
+
+/** The spec entry for `asset`'s bound recording, named the way the spec
+ * names files; absent when the video plays its own track. */
+export function specSound<A extends Sourced>(
+ asset: A,
+ assets: readonly A[] | ReadonlyMap<string, A>,
+ fileOf: (a: A) => string
+): SpecSound | undefined {
+ const src = soundSourceOf(asset, assets);
+ if (!src.bound) return undefined;
+ return { file: fileOf(src.asset), offset: src.offset, duration: src.asset.duration };
+}
diff --git a/site/src/cut/lib/soundSync.test.ts b/site/src/cut/lib/soundSync.test.ts
new file mode 100644
index 00000000..bc91e804
--- /dev/null
+++ b/site/src/cut/lib/soundSync.test.ts
@@ -0,0 +1,197 @@
+import { describe, expect, test } from "bun:test";
+import {
+ coarseSync,
+ ENV_HOP_S,
+ EnvelopeBuilder,
+ envelopeOf,
+ MonoResampler,
+ SYNC_RATE,
+ syncSignals,
+ syncSound,
+ type SyncChunk,
+} from "./soundSync";
+
+/** A seeded generator, so a failure replays. */
+function rng(seed: number) {
+ let s = seed >>> 0;
+ return () => {
+ s = (s * 1664525 + 1013904223) >>> 0;
+ return s / 2 ** 32;
+ };
+}
+
+/** Something with the shape of speech: bursts of tone and noise at random
+ * levels, separated by pauses of random length. */
+function speechLike(seconds: number, seed: number): Float32Array {
+ const r = rng(seed);
+ const out = new Float32Array(Math.round(seconds * SYNC_RATE));
+ let i = 0;
+ while (i < out.length) {
+ const burst = Math.round((0.08 + r() * 0.3) * SYNC_RATE);
+ const level = 0.1 + r() * 0.8;
+ const pitch = 90 + r() * 200;
+ for (let k = 0; k < burst && i < out.length; k++, i++) {
+ const env = Math.sin((Math.PI * k) / burst);
+ out[i] = level * env * (0.6 * Math.sin((2 * Math.PI * pitch * i) / SYNC_RATE) + 0.4 * (r() * 2 - 1));
+ }
+ i += Math.round((0.03 + r() * 0.4) * SYNC_RATE);
+ }
+ return out;
+}
+
+/** The same room heard by another device: its own gain, its own hiss, and a
+ * polarity of its choosing. */
+function heardBy(signal: Float32Array, gain: number, noise: number, seed: number, flip = false): Float32Array {
+ const r = rng(seed);
+ const out = new Float32Array(signal.length);
+ for (let i = 0; i < signal.length; i++) out[i] = (flip ? -1 : 1) * gain * signal[i] + noise * (r() * 2 - 1);
+ return out;
+}
+
+describe("syncSignals", () => {
+ const room = speechLike(70, 7);
+
+ test("finds where the camera's sound sits in the recording, to the sample", () => {
+ // The recorder rolled 12.345 s before the camera.
+ const startAt = Math.round(12.345 * SYNC_RATE);
+ const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.01, 1);
+ const recording = heardBy(room, 1, 0.002, 2);
+ const answer = syncSignals(camera, 0, recording, 0)!;
+ expect(answer).not.toBeNull();
+ expect(Math.abs(answer.offset - 12.345)).toBeLessThan(1.5 / SYNC_RATE);
+ expect(answer.refined).toBe(true);
+ expect(answer.confidence).toBeGreaterThan(2);
+ });
+
+ test("a recording that started after the camera gives a negative offset", () => {
+ // Camera time 0 is recording time -4.5: the recorder rolled late.
+ const camera = heardBy(room.subarray(0, 40 * SYNC_RATE), 0.5, 0.01, 3);
+ const late = Math.round(4.5 * SYNC_RATE);
+ const recording = heardBy(room.subarray(late), 1, 0.002, 4);
+ const answer = syncSignals(camera, 0, recording, 0)!;
+ expect(Math.abs(answer.offset - -4.5)).toBeLessThan(1.5 / SYNC_RATE);
+ });
+
+ test("source times carry through: a camera window late in its file, a recording window late in its", () => {
+ const startAt = Math.round(20 * SYNC_RATE);
+ const camera = heardBy(room.subarray(startAt, startAt + 20 * SYNC_RATE), 0.4, 0.01, 5);
+ const recording = heardBy(room.subarray(5 * SYNC_RATE), 1, 0.002, 6);
+ // The camera window starts at camera second 100; the recording window at
+ // recording second 5 of its file, which is room second 5. Camera second
+ // 100 is room second 20, so recording = camera − 80.
+ const answer = syncSignals(camera, 100, recording, 5)!;
+ expect(Math.abs(answer.offset - -80)).toBeLessThan(1.5 / SYNC_RATE);
+ });
+
+ test("an inverted microphone still lines up", () => {
+ const startAt = Math.round(7.25 * SYNC_RATE);
+ const camera = heardBy(room.subarray(startAt, startAt + 25 * SYNC_RATE), 0.2, 0.005, 7, true);
+ const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 8), 0)!;
+ expect(Math.abs(answer.offset - 7.25)).toBeLessThan(1.5 / SYNC_RATE);
+ });
+
+ test("survives a noisy camera track", () => {
+ const startAt = Math.round(30.01 * SYNC_RATE);
+ const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.08, 9);
+ const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 10), 0)!;
+ // Within a frame at 60 fps even if the fine pass gives up.
+ expect(Math.abs(answer.offset - 30.01)).toBeLessThan(0.017);
+ expect(answer.confidence).toBeGreaterThan(1.4);
+ });
+
+ test("two unrelated recordings have no clear peak", () => {
+ const camera = speechLike(30, 11);
+ const recording = speechLike(70, 12);
+ const answer = syncSignals(camera, 0, recording, 0);
+ expect(answer === null || answer.confidence < 1.4).toBe(true);
+ });
+
+ test("digital silence on either side has nothing to agree on", () => {
+ const answer = syncSignals(new Float32Array(20 * SYNC_RATE), 0, room, 0);
+ expect(answer === null || answer.confidence < 1.4).toBe(true);
+ });
+});
+
+describe("coarseSync", () => {
+ test("the lag is in envelope hops", () => {
+ const room = speechLike(40, 21);
+ const startAt = Math.round(10 * SYNC_RATE);
+ const env = envelopeOf(room);
+ const cam = envelopeOf(room.subarray(startAt, startAt + 15 * SYNC_RATE));
+ const c = coarseSync(cam, env)!;
+ expect(c.lag * ENV_HOP_S).toBeCloseTo(10, 1);
+ });
+});
+
+describe("streaming pieces", () => {
+ test("the resampler folds 48 kHz stereo to 16 kHz mono at the right length", () => {
+ const r = new MonoResampler(1);
+ const left = new Float32Array(48000).fill(0.5);
+ const right = new Float32Array(48000).fill(-0.1);
+ r.push([left.subarray(0, 20000), right.subarray(0, 20000)], 48000);
+ r.push([left.subarray(20000), right.subarray(20000)], 48000);
+ expect(Math.abs(r.length - 16000)).toBeLessThanOrEqual(1);
+ expect(r.samples()[100]).toBeCloseTo(0.2, 5);
+ });
+
+ test("the resampler keeps 44.1 kHz timing", () => {
+ const r = new MonoResampler(2);
+ r.push([new Float32Array(88200)], 44100);
+ expect(Math.abs(r.length - 32000)).toBeLessThanOrEqual(1);
+ });
+
+ test("an envelope built in pieces equals one built whole", () => {
+ const room = speechLike(5, 31);
+ const b = new EnvelopeBuilder(1);
+ for (let i = 0; i < room.length; i += 777) b.push(room.subarray(i, i + 777));
+ expect([...b.envelope()]).toEqual([...envelopeOf(room)]);
+ });
+});
+
+describe("syncSound", () => {
+ /** A file read the way the decoders hand it over: 48 kHz stereo, in chunks
+ * that start wherever the read was asked to. */
+ const fileOf = (signal: Float32Array) => {
+ let reads = 0;
+ async function* read(from: number, to: number): AsyncGenerator<SyncChunk> {
+ reads++;
+ const lo = Math.max(0, Math.round(from * SYNC_RATE));
+ const hi = Math.min(signal.length, Math.round(to * SYNC_RATE));
+ for (let i = lo; i < hi; i += 1500) {
+ const n = Math.min(1500, hi - i);
+ const ch = new Float32Array(n * 3);
+ for (let k = 0; k < n * 3; k++) ch[k] = signal[i + Math.floor(k / 3)];
+ yield { channels: [ch, ch], timestamp: i / SYNC_RATE, sampleRate: SYNC_RATE * 3 };
+ }
+ }
+ return { read, reads: () => reads };
+ };
+
+ test("streams the recording and lands the offset on the sample", async () => {
+ const room = speechLike(80, 41);
+ // The recorder rolled 7.25 s before the camera; the camera's file is the
+ // room from 7.25 on.
+ const startAt = Math.round(7.25 * SYNC_RATE);
+ const camera = fileOf(heardBy(room.subarray(startAt), 0.4, 0.01, 42));
+ const recording = fileOf(heardBy(room, 1, 0.002, 43));
+ const answer = (await syncSound(
+ { camera: camera.read, recording: recording.read },
+ { cameraFrom: 10, cameraTo: 40, recordingTo: 80 }
+ ))!;
+ expect(answer).not.toBeNull();
+ expect(Math.abs(answer.offset - 7.25)).toBeLessThan(1.5 / SYNC_RATE);
+ expect(answer.refined).toBe(true);
+ // The whole recording once for its envelope, then the fine window.
+ expect(recording.reads()).toBe(2);
+ });
+
+ test("an unrelated recording is not confident", async () => {
+ const camera = fileOf(speechLike(40, 51));
+ const recording = fileOf(speechLike(80, 52));
+ const answer = await syncSound(
+ { camera: camera.read, recording: recording.read },
+ { cameraFrom: 0, cameraTo: 30, recordingTo: 80 }
+ );
+ expect(answer === null || answer.confidence < 1.4).toBe(true);
+ });
+});
diff --git a/site/src/cut/lib/soundSync.ts b/site/src/cut/lib/soundSync.ts
new file mode 100644
index 00000000..a9b07a39
--- /dev/null
+++ b/site/src/cut/lib/soundSync.ts
@@ -0,0 +1,452 @@
+/**
+ * Lining up a separately recorded sound with a camera's own.
+ *
+ * Both devices heard the same room, so the camera's scratch track and the
+ * clean recording carry the same events — words, claps, a door — at times
+ * that differ by one constant: when each device started rolling. That
+ * constant is where the two signals agree best, which is a cross-correlation.
+ *
+ * Correlating raw waveforms over a long recording costs too much and trips on
+ * what the two microphones disagree about (distance, polarity, the room), so
+ * the search runs in two passes. The coarse pass correlates loudness
+ * envelopes — log energy per 20 ms, with the slow level drift taken out —
+ * which survive any difference in gain or tone and turn an hour of sound into
+ * a few hundred thousand numbers. The fine pass correlates the waveforms
+ * themselves at 16 kHz, a few seconds of each, within a hair of where the
+ * coarse pass landed, to put the offset on the sample.
+ *
+ * How sure the answer is reads off the coarse pass: the best agreement over
+ * the runner-up a second or more away from it. A recording of a different
+ * take, or two tracks with nothing loud in common, has no clear winner, and
+ * the caller refuses it.
+ *
+ * Pure: arrays in, numbers out. The decoding happens where the file lives.
+ */
+
+/** The rate both signals are compared at. */
+export const SYNC_RATE = 16000;
+/** Samples per envelope hop: 20 ms. */
+export const ENV_HOP = 320;
+export const ENV_HOP_S = ENV_HOP / SYNC_RATE;
+/** Hops the slow level drift is measured over, taken out of the envelope. */
+const DETREND_HOPS = 50;
+/** A runner-up closer than this to the best lag is the same peak's shoulder. */
+const PEAK_EXCLUDE_S = 1;
+/** The least of either signal that has to overlap at a lag for it to count. */
+const MIN_OVERLAP_SHARE = 0.5;
+/** How far the fine pass searches either side of the coarse answer. */
+export const FINE_REACH_S = 0.06;
+/** How much of the camera's sound the fine pass compares. */
+export const FINE_SECONDS = 4;
+/** Confidence reported when the runner-up does not agree at all. */
+const CONFIDENCE_CAP = 99;
+
+/**
+ * Folds decoded chunks of any rate and channel count into one mono signal at
+ * `SYNC_RATE`: channels averaged, then each output sample the mean of the
+ * input samples it covers — a box filter, enough to keep the fold from
+ * aliasing what the correlation compares.
+ */
+export class MonoResampler {
+ private out: Float32Array;
+ private n = 0;
+ private sum = 0;
+ private count = 0;
+ /** Input position, in output samples. */
+ private pos = 0;
+
+ constructor(expectedSeconds = 60) {
+ this.out = new Float32Array(Math.max(1024, Math.ceil(expectedSeconds * SYNC_RATE)));
+ }
+
+ push(channels: Float32Array[], sampleRate: number): void {
+ const step = SYNC_RATE / sampleRate;
+ const len = channels[0]?.length ?? 0;
+ const k = channels.length;
+ for (let i = 0; i < len; i++) {
+ let v = 0;
+ for (let c = 0; c < k; c++) v += channels[c][i];
+ this.sum += v / k;
+ this.count++;
+ const next = this.pos + step;
+ if (Math.floor(next) > Math.floor(this.pos)) {
+ this.emit(this.sum / this.count);
+ this.sum = 0;
+ this.count = 0;
+ }
+ this.pos = next;
+ }
+ }
+
+ private emit(v: number): void {
+ if (this.n === this.out.length) {
+ const grown = new Float32Array(this.out.length * 2);
+ grown.set(this.out);
+ this.out = grown;
+ }
+ this.out[this.n++] = v;
+ }
+
+ /** The signal so far. */
+ samples(): Float32Array {
+ return this.out.subarray(0, this.n);
+ }
+
+ /** Hand the samples gathered since the last drain to `into` and let them
+ * go, so a long signal streams through in one chunk's worth of memory. */
+ drain(into: (mono: Float32Array) => void): void {
+ if (this.n > 0) into(this.out.subarray(0, this.n));
+ this.n = 0;
+ }
+
+ get length(): number {
+ return this.n;
+ }
+}
+
+/**
+ * The loudness envelope the coarse pass compares, built from mono samples at
+ * `SYNC_RATE` as they arrive: log RMS per hop. Kept separate from the
+ * samples, so an hour of recording costs an hour of envelope.
+ */
+export class EnvelopeBuilder {
+ private out: Float32Array;
+ private n = 0;
+ private acc = 0;
+ private fill = 0;
+
+ constructor(expectedSeconds = 60) {
+ this.out = new Float32Array(Math.max(256, Math.ceil(expectedSeconds / ENV_HOP_S) + 1));
+ }
+
+ push(mono: Float32Array): void {
+ for (let i = 0; i < mono.length; i++) {
+ this.acc += mono[i] * mono[i];
+ if (++this.fill === ENV_HOP) {
+ if (this.n === this.out.length) {
+ const grown = new Float32Array(this.out.length * 2);
+ grown.set(this.out);
+ this.out = grown;
+ }
+ this.out[this.n++] = Math.log(1e-8 + this.acc / ENV_HOP);
+ this.acc = 0;
+ this.fill = 0;
+ }
+ }
+ }
+
+ envelope(): Float32Array {
+ return this.out.subarray(0, this.n);
+ }
+}
+
+/** The envelope of a whole mono signal at `SYNC_RATE`. */
+export function envelopeOf(mono: Float32Array): Float32Array {
+ const b = new EnvelopeBuilder(mono.length / SYNC_RATE);
+ b.push(mono);
+ return b.envelope();
+}
+
+/** The envelope with its slow drift taken out and scaled to unit variance,
+ * so the correlation weighs where loudness moves, whatever level a device
+ * recorded. */
+export function normalizeEnvelope(env: Float32Array): Float64Array {
+ const n = env.length;
+ const out = new Float64Array(n);
+ if (n === 0) return out;
+ // Running mean over a centered window, by prefix sums.
+ const prefix = new Float64Array(n + 1);
+ for (let i = 0; i < n; i++) prefix[i + 1] = prefix[i] + env[i];
+ const half = Math.floor(DETREND_HOPS / 2);
+ let sq = 0;
+ for (let i = 0; i < n; i++) {
+ const a = Math.max(0, i - half);
+ const b = Math.min(n, i + half + 1);
+ const v = env[i] - (prefix[b] - prefix[a]) / (b - a);
+ out[i] = v;
+ sq += v * v;
+ }
+ const sd = Math.sqrt(sq / n);
+ if (sd > 1e-9) for (let i = 0; i < n; i++) out[i] /= sd;
+ return out;
+}
+
+/** In-place radix-2 FFT; `inverse` runs it backward, unscaled. */
+function fft(re: Float64Array, im: Float64Array, inverse: boolean): void {
+ const n = re.length;
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ for (; j & bit; bit >>= 1) j ^= bit;
+ j ^= bit;
+ if (i < j) {
+ let t = re[i];
+ re[i] = re[j];
+ re[j] = t;
+ t = im[i];
+ im[i] = im[j];
+ im[j] = t;
+ }
+ }
+ for (let len = 2; len <= n; len <<= 1) {
+ const ang = ((inverse ? 2 : -2) * Math.PI) / len;
+ const wr = Math.cos(ang);
+ const wi = Math.sin(ang);
+ const half = len >> 1;
+ for (let i = 0; i < n; i += len) {
+ let cr = 1;
+ let ci = 0;
+ for (let k = 0; k < half; k++) {
+ const ar = re[i + k + half] * cr - im[i + k + half] * ci;
+ const ai = re[i + k + half] * ci + im[i + k + half] * cr;
+ re[i + k + half] = re[i + k] - ar;
+ im[i + k + half] = im[i + k] - ai;
+ re[i + k] += ar;
+ im[i + k] += ai;
+ const nr = cr * wr - ci * wi;
+ ci = cr * wi + ci * wr;
+ cr = nr;
+ }
+ }
+ }
+}
+
+/**
+ * Cross-correlation of `a` against `b` at every lag where they overlap:
+ * `at(k)` is Σ a[i]·b[i + k], for k from −(a.length − 1) to b.length − 1.
+ */
+export function crossCorrelate(
+ a: ArrayLike<number>,
+ b: ArrayLike<number>
+): { at: (k: number) => number; min: number; max: number } {
+ const need = a.length + b.length - 1;
+ let n = 1;
+ while (n < need) n <<= 1;
+ const ar = new Float64Array(n);
+ const ai = new Float64Array(n);
+ const br = new Float64Array(n);
+ const bi = new Float64Array(n);
+ for (let i = 0; i < a.length; i++) ar[i] = a[i];
+ for (let i = 0; i < b.length; i++) br[i] = b[i];
+ fft(ar, ai, false);
+ fft(br, bi, false);
+ // conj(A)·B
+ for (let i = 0; i < n; i++) {
+ const r = ar[i] * br[i] + ai[i] * bi[i];
+ const im = ar[i] * bi[i] - ai[i] * br[i];
+ ar[i] = r;
+ ai[i] = im;
+ }
+ fft(ar, ai, true);
+ const scale = 1 / n;
+ return {
+ at: (k) => ar[(k + n) % n] * scale,
+ min: -(a.length - 1),
+ max: b.length - 1,
+ };
+}
+
+/** The coarse answer: the lag, in hops, at which the camera's envelope sits
+ * best inside the recording's, and how clearly it beats the runner-up. */
+export interface CoarseSync {
+ lag: number;
+ confidence: number;
+}
+
+/**
+ * Where `camera` (an envelope) sits inside `recording` (another): camera hop
+ * i lines up with recording hop i + lag. Lags where less than half of the
+ * shorter signal overlaps are never considered — a sliver of overlap agrees
+ * with anything.
+ */
+export function coarseSync(camera: Float32Array, recording: Float32Array): CoarseSync | null {
+ const a = normalizeEnvelope(camera);
+ const b = normalizeEnvelope(recording);
+ if (a.length < 4 || b.length < 4) return null;
+ const xc = crossCorrelate(a, b);
+ const need = Math.ceil(Math.min(a.length, b.length) * MIN_OVERLAP_SHARE);
+ // Overlap at lag k: the hops of a that land inside b.
+ const overlap = (k: number) => Math.min(a.length, b.length - k) - Math.max(0, -k);
+ let best = -Infinity;
+ let lag = 0;
+ for (let k = xc.min; k <= xc.max; k++) {
+ if (overlap(k) < need) continue;
+ const v = xc.at(k);
+ if (v > best) {
+ best = v;
+ lag = k;
+ }
+ }
+ if (!Number.isFinite(best)) return null;
+ const exclude = Math.round(PEAK_EXCLUDE_S / ENV_HOP_S);
+ let second = -Infinity;
+ for (let k = xc.min; k <= xc.max; k++) {
+ if (Math.abs(k - lag) <= exclude || overlap(k) < need) continue;
+ second = Math.max(second, xc.at(k));
+ }
+ const confidence =
+ best <= 0 ? 0 : second <= 0 ? CONFIDENCE_CAP : Math.min(CONFIDENCE_CAP, best / second);
+ return { lag, confidence };
+}
+
+/**
+ * The fine pass: `camera` and `recording` are waveforms at `SYNC_RATE`, the
+ * recording starting `reach` samples earlier than where the coarse pass put
+ * the camera's first sample and running `reach` samples longer. Returns the
+ * shift, in samples, that lines them up best (negative = the recording sits
+ * earlier than the coarse answer), or null when no shift stands out. Polarity
+ * is ignored: two microphones can face the source from opposite sides.
+ */
+export function fineSync(camera: Float32Array, recording: Float32Array, reach: number): number | null {
+ if (camera.length < 64 || recording.length < camera.length + 2 * reach) return null;
+ const xc = crossCorrelate(camera, recording);
+ let best = -1;
+ let at = 0;
+ for (let k = 0; k <= 2 * reach; k++) {
+ const v = Math.abs(xc.at(k));
+ if (v > best) {
+ best = v;
+ at = k;
+ }
+ }
+ // The runner-up outside the peak's own cycle, a millisecond either side.
+ const exclude = Math.round(SYNC_RATE / 1000);
+ let second = 0;
+ for (let k = 0; k <= 2 * reach; k++) {
+ if (Math.abs(k - at) <= exclude) continue;
+ second = Math.max(second, Math.abs(xc.at(k)));
+ }
+ if (!(best > 0) || best < second * 1.05) return null;
+ return at - reach;
+}
+
+/** The loudest `seconds` of a signal at `SYNC_RATE`, by hop energy: where the
+ * fine pass has the most to compare. Returns the first sample. */
+export function loudestStretch(mono: Float32Array, seconds: number): number {
+ const len = Math.min(mono.length, Math.round(seconds * SYNC_RATE));
+ if (len >= mono.length) return 0;
+ let sum = 0;
+ for (let i = 0; i < len; i++) sum += mono[i] * mono[i];
+ let best = sum;
+ let at = 0;
+ // Slide a hop at a time; exact enough and cheap.
+ for (let s = ENV_HOP; s + len <= mono.length; s += ENV_HOP) {
+ for (let i = s - ENV_HOP; i < s; i++) sum -= mono[i] * mono[i];
+ for (let i = s + len - ENV_HOP; i < s + len; i++) sum += mono[i] * mono[i];
+ if (sum > best) {
+ best = sum;
+ at = s;
+ }
+ }
+ return at;
+}
+
+export interface SyncAnswer {
+ /** Recording seconds = camera seconds + offset. */
+ offset: number;
+ confidence: number;
+ /** Whether the fine pass put the offset on the sample; false leaves it at
+ * the coarse pass's 20 ms grain. */
+ refined: boolean;
+}
+
+/**
+ * Both passes over two whole signals in hand, each at `SYNC_RATE` and each
+ * with the source second its first sample sits at. The tools stream the
+ * recording through `EnvelopeBuilder` instead, since a long one cannot be
+ * held whole, and read the fine window on its own; this is the same
+ * arithmetic for signals already decoded.
+ */
+export function syncSignals(
+ camera: Float32Array,
+ cameraStart: number,
+ recording: Float32Array,
+ recordingStart: number
+): SyncAnswer | null {
+ const coarse = coarseSync(envelopeOf(camera), envelopeOf(recording));
+ if (!coarse) return null;
+ const offset = recordingStart - cameraStart + coarse.lag * ENV_HOP_S;
+ const reach = Math.round(FINE_REACH_S * SYNC_RATE);
+ const from = loudestStretch(camera, FINE_SECONDS);
+ const cam = camera.subarray(from, from + Math.round(FINE_SECONDS * SYNC_RATE));
+ // Where the camera's stretch starts in the recording, by the coarse answer.
+ const recAt = Math.round((cameraStart + from / SYNC_RATE + offset - recordingStart) * SYNC_RATE) - reach;
+ const shift =
+ recAt >= 0 && recAt + cam.length + 2 * reach <= recording.length
+ ? fineSync(cam, recording.subarray(recAt, recAt + cam.length + 2 * reach), reach)
+ : null;
+ return {
+ offset: shift === null ? offset : offset + shift / SYNC_RATE,
+ confidence: coarse.confidence,
+ refined: shift !== null,
+ };
+}
+
+/** Decoded audio of any rate and channel count, chunk by chunk, starting at
+ * `timestamp` source seconds. */
+export interface SyncChunk {
+ channels: Float32Array[];
+ timestamp: number;
+ sampleRate: number;
+}
+
+/** Reads a span of each file's sound, wherever it lives. */
+export interface SyncReader {
+ camera: (from: number, to: number) => AsyncIterable<SyncChunk>;
+ recording: (from: number, to: number) => AsyncIterable<SyncChunk>;
+}
+
+/** A span folded to mono at `SYNC_RATE`, with the source second its first
+ * sample sits at. */
+async function monoOf(chunks: AsyncIterable<SyncChunk>, seconds: number) {
+ const fold = new MonoResampler(seconds);
+ let start: number | null = null;
+ for await (const c of chunks) {
+ start ??= c.timestamp;
+ fold.push(c.channels, c.sampleRate);
+ }
+ return { mono: fold.samples(), start: start ?? 0 };
+}
+
+/**
+ * Both passes over two files read as they stream: `cameraFrom..cameraTo` of
+ * the camera (the stretch whose sound is matched), the recording's first
+ * `recordingTo` seconds as an envelope only, and then the few seconds of the
+ * recording the fine pass needs. Memory holds the camera's stretch, the
+ * recording's envelope, and one chunk.
+ */
+export async function syncSound(
+ read: SyncReader,
+ opts: { cameraFrom: number; cameraTo: number; recordingTo: number }
+): Promise<SyncAnswer | null> {
+ const cam = await monoOf(read.camera(opts.cameraFrom, opts.cameraTo), opts.cameraTo - opts.cameraFrom);
+ if (cam.mono.length < SYNC_RATE) return null;
+ const fold = new MonoResampler(10);
+ const env = new EnvelopeBuilder(opts.recordingTo);
+ let recStart: number | null = null;
+ for await (const c of read.recording(0, opts.recordingTo)) {
+ recStart ??= c.timestamp;
+ fold.push(c.channels, c.sampleRate);
+ fold.drain((m) => env.push(m));
+ }
+ const coarse = coarseSync(envelopeOf(cam.mono), env.envelope());
+ if (!coarse) return null;
+ const offset = (recStart ?? 0) - cam.start + coarse.lag * ENV_HOP_S;
+ // The fine pass: the camera's loudest few seconds against the recording
+ // around where the coarse pass put them.
+ const reach = Math.round(FINE_REACH_S * SYNC_RATE);
+ const from = loudestStretch(cam.mono, FINE_SECONDS);
+ const piece = cam.mono.subarray(from, from + Math.round(FINE_SECONDS * SYNC_RATE));
+ const at = cam.start + from / SYNC_RATE + offset - reach / SYNC_RATE;
+ const window = Math.max(0, at - 0.25);
+ const rec = await monoOf(read.recording(window, at + (piece.length + 2 * reach) / SYNC_RATE + 0.25), FINE_SECONDS + 1);
+ const recAt = Math.round((at - rec.start) * SYNC_RATE);
+ const shift =
+ recAt >= 0 && recAt + piece.length + 2 * reach <= rec.mono.length
+ ? fineSync(piece, rec.mono.subarray(recAt, recAt + piece.length + 2 * reach), reach)
+ : null;
+ return {
+ offset: shift === null ? offset : offset + shift / SYNC_RATE,
+ confidence: coarse.confidence,
+ refined: shift !== null,
+ };
+}
diff --git a/site/src/cut/lib/sourceExportPlan.ts b/site/src/cut/lib/sourceExportPlan.ts
index cb48699b..440c0016 100644
--- a/site/src/cut/lib/sourceExportPlan.ts
+++ b/site/src/cut/lib/sourceExportPlan.ts
@@ -15,7 +15,7 @@ export function sourceSequence(doc: ExportDoc) {
const neutral: Partial<Record<keyof VideoClip, unknown>> = {
track: 0, muted: false, volume: 1, fit: "fit", zoom: 1,
panX: 0, panY: 0, flipH: false, flipV: false, rotation: 0, opacity: 1,
- speed: 1, reverse: false, smoothSlow: false, transition: 0, hidden: false,
+ speed: 1, reverse: false, smoothSlow: false, transition: 0, hidden: false, audioLead: 0, audioTail: 0,
};
const metadata = new Set(["id", "assetId", "name", "groupId", "in", "out", "start"]);
for (const clip of clips) {
@@ -25,7 +25,7 @@ export function sourceSequence(doc: ExportDoc) {
if (!(key in neutral) || value !== neutral[key as keyof VideoClip]) return null;
}
const asset = doc.assets.find((a) => a.id === clip.assetId);
- if (!asset || asset.type !== "video" || asset.block || !asset.width || !asset.height ||
+ if (!asset || asset.type !== "video" || asset.block || asset.soundFrom || !asset.width || !asset.height ||
!Number.isFinite(clip.in) || !Number.isFinite(clip.out) || clip.in < 0 || clip.out <= clip.in || clip.out > asset.duration + 0.000001) return null;
// A file whose code values need converting (a log source, or an HDR file
// into another delivery) has to render; an ungraded file already in the
diff --git a/site/src/cut/lib/splitEdit.test.ts b/site/src/cut/lib/splitEdit.test.ts
new file mode 100644
index 00000000..23a0f101
--- /dev/null
+++ b/site/src/cut/lib/splitEdit.test.ts
@@ -0,0 +1,166 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { runAiTool } from "./aiTools";
+import { soundWindow } from "./soundSource";
+import { getClipSpans, useEditor } from "./store";
+import type { MediaAsset, VideoClip } from "./types";
+
+/** Split edits and bound recordings as the document holds them: what the
+ * spans every renderer reads make of a clip's lead, tail and recording, and
+ * how the edits that reshape clips carry them. */
+
+const video = (over: Partial<MediaAsset> = {}): MediaAsset => ({
+ id: "cam",
+ fileName: "cam.mp4",
+ name: "cam.mp4",
+ type: "video",
+ duration: 60,
+ url: "",
+ ...over,
+});
+const recording: MediaAsset = { id: "rec", fileName: "rec.wav", name: "rec.wav", type: "audio", duration: 50, url: "" };
+
+const clip = (id: string, start: number, inS: number, outS: number, over: Partial<VideoClip> = {}): VideoClip => ({
+ id,
+ assetId: "cam",
+ start,
+ in: inS,
+ out: outS,
+ track: 0,
+ muted: false,
+ ...over,
+});
+
+const spanOf = (id: string) => {
+ const s = useEditor.getState();
+ return getClipSpans(s.clips, s.assets).find((sp) => sp.clip.id === id)!;
+};
+
+beforeEach(() => {
+ useEditor.setState({
+ projectId: "p1",
+ loaded: true,
+ readOnly: false,
+ clips: [clip("a", 0, 10, 14), clip("b", 4, 20, 24)],
+ transitions: [],
+ audioClips: [],
+ overlays: [],
+ assets: [video(), recording],
+ selection: null,
+ multiSelection: [],
+ });
+});
+
+describe("set_split_edit", () => {
+ test("a lead and a tail reach past the picture and nothing moves", async () => {
+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4 });
+ await runAiTool("set_split_edit", { clipId: "a", tail: 0.6 });
+ const b = spanOf("b");
+ const a = spanOf("a");
+ expect(b.soundLead).toBeCloseTo(0.4, 5);
+ expect(b.soundBack).toBeCloseTo(0.4, 5);
+ expect(a.soundTail).toBeCloseTo(0.6, 5);
+ expect(a.soundAhead).toBeCloseTo(0.6, 5);
+ expect(b.start).toBe(4);
+ expect(b.len).toBe(4);
+ });
+
+ test("a lead is capped by the source before the trim and by the timeline's start", async () => {
+ useEditor.setState({ clips: [clip("a", 1, 0.3, 4)] });
+ const res = (await runAiTool("set_split_edit", { clipId: "a", lead: 2 })) as { playing: { lead: number }; note?: string };
+ expect(spanOf("a").soundLead).toBeCloseTo(0.3, 5);
+ expect(res.playing.lead).toBeCloseTo(0.3, 2);
+ expect(res.note).toBeDefined();
+ });
+
+ test("a cut carrying a transition keeps no split edit on that side", async () => {
+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4 });
+ expect(spanOf("b").soundLead).toBeCloseTo(0.4, 5);
+ await runAiTool("set_transition", { clipId: "a", seconds: 0.5 });
+ expect(spanOf("a").transitionOut).toBeGreaterThan(0);
+ expect(spanOf("b").soundLead ?? 0).toBe(0);
+ // The setting stays on the clip, and plays again once the cut is plain.
+ await runAiTool("set_transition", { clipId: "a", seconds: 0 });
+ expect(spanOf("b").soundLead).toBeCloseTo(0.4, 5);
+ });
+
+ test("0 clears a side and a side left out keeps its value", async () => {
+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4, tail: 0.3 });
+ await runAiTool("set_split_edit", { clipId: "b", lead: 0 });
+ const b = useEditor.getState().clips.find((c) => c.id === "b")!;
+ expect(b.audioLead).toBeUndefined();
+ expect(b.audioTail).toBeCloseTo(0.3, 5);
+ });
+
+ test("a split keeps the lead on the left half and the tail on the right", async () => {
+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4, tail: 0.3 });
+ await runAiTool("split_at", { t: 6 });
+ const halves = useEditor
+ .getState()
+ .clips.filter((c) => c.start >= 4)
+ .sort((x, y) => x.start - y.start);
+ expect(halves).toHaveLength(2);
+ expect(halves[0].audioLead).toBeCloseTo(0.4, 5);
+ expect(halves[0].audioTail).toBeUndefined();
+ expect(halves[1].audioLead).toBeUndefined();
+ expect(halves[1].audioTail).toBeCloseTo(0.3, 5);
+ });
+
+ test("ids lands one split edit on several clips", async () => {
+ await runAiTool("set_split_edit", { ids: ["a", "b"], tail: 0.2 });
+ const s = useEditor.getState();
+ expect(s.clips.every((c) => c.audioTail === 0.2)).toBe(true);
+ });
+});
+
+describe("a recording bound to the video", () => {
+ test("every clip of the video reads it, a split's halves included", async () => {
+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: -5 });
+ await runAiTool("split_at", { t: 2 });
+ const s = useEditor.getState();
+ const spans = getClipSpans(s.clips, s.assets);
+ expect(spans).toHaveLength(3);
+ for (const sp of spans) {
+ expect(sp.sound?.asset.id).toBe("rec");
+ expect(sp.sound?.offset).toBe(-5);
+ }
+ });
+
+ test("the recording's own room bounds the handles", () => {
+ // The recording starts at video second 5; clip a starts on video 10, so
+ // five seconds of it sit before the trim.
+ useEditor.setState({ clips: [clip("a", 8, 10, 14, { audioLead: 9 })] });
+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: -5 });
+ expect(spanOf("a").soundLead).toBeCloseTo(5, 5);
+ });
+
+ test("a window the recording does not reach narrows and starts late", () => {
+ // Recording second = video second − 12: video 10..12 is before it rolled.
+ const w = soundWindow({ in: 10, out: 14 }, { offset: -12, limit: 50 }, 0, 4)!;
+ expect(w.lo).toBe(0);
+ expect(w.hi).toBeCloseTo(2, 5);
+ expect(w.at).toBeCloseTo(2, 5);
+ });
+
+ test("binding is one undo step, and removing the recording unbinds", async () => {
+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: 1.5 });
+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toEqual({ assetId: "rec", offset: 1.5 });
+ useEditor.getState().undo();
+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();
+ useEditor.getState().redo();
+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom?.assetId).toBe("rec");
+ useEditor.getState().removeAsset("rec");
+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();
+ });
+
+ test("sync_audio clear returns the video to its own sound", async () => {
+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: 1 });
+ await runAiTool("sync_audio", { clipId: "a", clear: true });
+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();
+ expect(spanOf("a").sound).toBeUndefined();
+ });
+
+ test("sync_audio refuses a file that is not audio", async () => {
+ await expect(runAiTool("sync_audio", { clipId: "a", audio_asset_id: "cam" })).rejects.toThrow();
+ });
+});
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 7659b0c8..bf090f40 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -394,6 +394,9 @@ interface DocSnapshot {
* chat; it lives on the asset like the beat grid, so the checkpoint
* carries it the same way. */
sourceColor: { id: string; profile: SourceProfile | undefined }[];
+ /** Each video's bound recording, set from the Audio tab and the chat; on
+ * the asset like the source-colour override, carried the same way. */
+ soundFrom: { id: string; from: SoundFrom | undefined }[];
}
export type SubtitleStatus = "idle" | "running" | "ready" | "empty" | "error";
@@ -1015,6 +1018,7 @@ const sharedFields = (s: Pick<EditorState, "guideLines" | "colorSpace" | "assets
colorSpace: s.colorSpace,
beats: new Map(s.assets.map((a) => [a.id, a.beats])),
profiles: new Map(s.assets.map((a) => [a.id, a.colorProfile])),
+ sounds: new Map(s.assets.map((a) => [a.id, a.soundFrom])),
});
/** A project-wide setting changed on another timeline while this one was
@@ -1033,6 +1037,8 @@ function rebaseShared(snaps: DocSnapshot[], then: SharedFields, now: SharedField
snap.sourceColor = snap.sourceColor.map((c) =>
profiles.has(c.id) ? { id: c.id, profile: now.profiles.get(c.id) } : c
);
+ if (sounds.size)
+ snap.soundFrom = snap.soundFrom.map((c) => (sounds.has(c.id) ? { id: c.id, from: now.sounds.get(c.id) } : c));
}
}
@@ -1495,47 +1501,70 @@ export function cutTranscribeSpec(
const spans = getClipSpans(s.clips, s.assets);
const duration = projectDuration(s);
const assetById = new Map(s.assets.map((a) => [a.id, a]));
- const audio = s.audioClips
- .filter(
- (a) =>
- !a.hidden &&
- !a.reverse &&
- a.start < duration &&
- assetById.has(a.assetId) &&
- !assetIsSilent(assetById.get(a.assetId)!)
- )
- .map((a) => ({
- file: assetById.get(a.assetId)!.fileName,
- in: a.in,
- out: a.out,
- start: a.start,
- volume: a.volume,
- speed: a.speed,
- speedCurve: a.speedCurve,
- reverse: a.reverse,
- }))
- .concat(
- overlayLayers(s.clips)
- .filter(
- (c) =>
- !c.hidden &&
- !c.muted &&
- !c.reverse &&
- c.start < duration &&
- assetById.has(c.assetId) &&
- !assetIsSilent(assetById.get(c.assetId)!)
- )
- .map((c) => ({
- file: assetById.get(c.assetId)!.fileName,
- in: c.in,
- out: c.out,
- start: c.start,
- volume: 1,
- speed: c.speed,
- speedCurve: c.speedCurve,
- reverse: c.reverse,
- }))
- );
+ type Item = CloudTranscribeSpec["audio"][number];
+ /** The stretch `[fromT, toT]` of a span's sound (timeline seconds from its
+ * head at `start`) as a placed item, read off its sound file: the video's
+ * own track or the recording bound to it. Null when that file holds none
+ * of it. */
+ const placed = (
+ c: Parameters<typeof soundWindow>[0],
+ src: { asset: MediaAsset; offset: number; limit: number },
+ start: number,
+ fromT: number,
+ toT: number,
+ volume: number
+ ): Item | null => {
+ const w = soundWindow(c, src, fromT, toT);
+ return w
+ ? {
+ file: src.asset.fileName,
+ in: w.lo,
+ out: w.hi,
+ start: start + fromT + w.at,
+ volume,
+ speed: w.span.speed,
+ speedCurve: w.span.speedCurve,
+ reverse: w.span.reverse,
+ }
+ : null;
+ };
+ const own = (asset: MediaAsset) => ({ asset, offset: 0, limit: Infinity });
+ const audio: Item[] = [];
+ for (const a of s.audioClips) {
+ const asset = assetById.get(a.assetId);
+ if (!asset || a.hidden || a.reverse || a.start >= duration) continue;
+ const src = soundSourceOf(asset, assetById);
+ if (!src.bound && assetIsSilent(asset)) continue;
+ const item = placed(a, src, a.start, 0, retimeOf(a).len, a.volume);
+ if (item) audio.push(item);
+ }
+ // Every upper-track clip's sound, split edit included; on track 0 the fold
+ // carries the clip itself, so only what plays off its own footprint — a
+ // bound recording, a split edit's lead and tail — rides here.
+ const audible = (sp: ClipSpan) =>
+ !sp.clip.hidden && !sp.clip.muted && !sp.clip.reverse && (!!sp.sound || !assetIsSilent(sp.asset));
+ for (const track of new Set(overlayLayers(s.clips).map((c) => c.track))) {
+ for (const sp of getClipSpans(s.clips, s.assets, track)) {
+ if (!audible(sp) || sp.start >= duration) continue;
+ const item = placed(sp.clip, sp.sound ?? own(sp.asset), sp.start, -(sp.soundLead ?? 0), sp.len + (sp.soundTail ?? 0), 1);
+ if (item) audio.push(item);
+ }
+ }
+ for (const sp of spans) {
+ if (!audible(sp)) continue;
+ const src = sp.sound ?? own(sp.asset);
+ const pieces: [number, number][] = sp.sound
+ ? [[-(sp.soundLead ?? 0), sp.len + (sp.soundTail ?? 0)]]
+ : [
+ [-(sp.soundLead ?? 0), 0],
+ [sp.len, sp.len + (sp.soundTail ?? 0)],
+ ];
+ for (const [from, to] of pieces) {
+ if (to - from <= 1e-3) continue;
+ const item = placed(sp.clip, src, sp.start, from, to, 1);
+ if (item) audio.push(item);
+ }
+ }
if (spans.length === 0 && audio.length === 0) return null;
const silentSpacer = (len: number) => ({ file: "", in: 0, out: len, muted: true, speed: 1, transition: 0 });
return {
@@ -1554,8 +1583,9 @@ export function cutTranscribeSpec(
in: sp.clip.in,
out: sp.clip.out,
// Sound played backward is not speech; the clip keeps its
- // time in the mix as silence.
- muted: sp.clip.muted || assetIsSilent(sp.asset) || !!sp.clip.reverse,
+ // time in the mix as silence. A bound recording plays from
+ // the items above, so the slot holds silence too.
+ muted: sp.clip.muted || assetIsSilent(sp.asset) || !!sp.clip.reverse || !!sp.sound,
speed: clipSpeed(sp.clip),
speedCurve: sp.clip.speedCurve,
// The clamped cross-dissolve overlap, so the mix overlaps clip
@@ -1611,6 +1641,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
return {
beats: assets.map((a) => ({ id: a.id, grid: a.beats })),
sourceColor: assets.map((a) => ({ id: a.id, profile: a.colorProfile })),
+ soundFrom: assets.map((a) => ({ id: a.id, from: a.soundFrom })),
guideLines: { v: [...guideLines.v], h: [...guideLines.h] },
colorSpace,
// Render-owned clips are excluded — history captures the user's timeline,
@@ -1633,7 +1664,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
* doc — an undo mid-session (a brush stroke, a panel edit) keeps the panel
* and its stage gizmos on the clip being worked. */
const restoreDoc = (snap: DocSnapshot) => {
- const { beats, sourceColor, ...doc } = snap;
+ const { beats, sourceColor, soundFrom, ...doc } = snap;
const { clips, audioClips, assets, selection, multiSelection } = get();
const genClips = clips.filter((c) => genClipIds.has(c.id));
const genAudio = audioClips.filter((c) => genAudioIds.has(c.id));
@@ -1641,11 +1672,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// since the checkpoint was taken is not in it and keeps the grid it has.
const grids = new Map(beats.map((b) => [b.id, b.grid]));
const profiles = new Map(sourceColor.map((c) => [c.id, c.profile]));
+ const sounds = new Map(soundFrom.map((c) => [c.id, c.from]));
const withBeats = assets.map((a) => {
let next = a;
if (grids.has(a.id) && grids.get(a.id) !== a.beats) next = { ...next, beats: grids.get(a.id) };
if (profiles.has(a.id) && profiles.get(a.id) !== a.colorProfile)
next = withColorProfile(next, profiles.get(a.id));
+ if (sounds.has(a.id) && sounds.get(a.id) !== a.soundFrom) next = withSoundFrom(next, sounds.get(a.id));
return next;
});
const beatsMoved = withBeats.some((a, i) => a !== assets[i]);
@@ -2556,6 +2589,18 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
}));
},
+ setAssetSoundFrom: (id, from) => {
+ const asset = get().assets.find((a) => a.id === id);
+ if (!asset || get().readOnly) return;
+ if (asset.soundFrom?.assetId === from?.assetId && asset.soundFrom?.offset === from?.offset) return;
+ push();
+ // The source's transcript was heard off the sound it had; the sweep
+ // hears the new one.
+ set((s) => ({
+ assets: s.assets.map((a) => (a.id === id ? { ...withSoundFrom(a, from), speech: undefined } : a)),
+ }));
+ },
+
setAssetProxy: (id, proxy, proxyUrl) => {
if (get().readOnly) return;
set((s) => ({
@@ -2697,7 +2742,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const scrub = (snap: DocSnapshot) => {
snap.clips = snap.clips.filter((c) => c.assetId !== id);
snap.audioClips = snap.audioClips.filter((c) => c.assetId !== id);
+ snap.soundFrom = snap.soundFrom.map((c) => (c.from?.assetId === id ? { id: c.id, from: undefined } : c));
};
+ // A video bound to this recording goes back to its own track.
+ const remaining = (assets: MediaAsset[]) =>
+ assets
+ .filter((a) => a.id !== id)
+ .map((a) => (a.soundFrom?.assetId === id ? withSoundFrom(a, undefined) : a));
for (const snap of history) scrub(snap);
for (const snap of future) scrub(snap);
if (pending) scrub(pending.snap);
@@ -2842,7 +2893,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// A video's own track plays on the soundtrack the same way a sound file
// does — it is what detaching a clip's audio leaves behind — so a cut
// laid out as empty shots can still carry the reference's sound.
- if (!asset || asset.type === "image" || assetIsSilent(asset)) return;
+ if (!asset || asset.type === "image" || (assetIsSilent(asset) && !soundSourceOf(asset, get().assets).bound)) return;
push();
// Within its lane the clip slides to the next free slot at or after the
// target so it never lands on top of an existing sound.
@@ -3704,7 +3755,8 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const clip = clips.find((c) => c.id === selection.id);
if (!clip || clip.muted) return; // no sound to detach
const span = clipWindow(clips, assets, clip.id);
- if (!span || assetIsSilent(span.asset)) return; // a still has none either
+ // A still has none either; a bound recording counts as the clip's sound.
+ if (!span || (assetIsSilent(span.asset) && !soundSourceOf(span.asset, assets).bound)) return;
push();
const audio: AudioClip = {
id: uid(),
@@ -4105,7 +4157,15 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
style: t.style,
...(t.hidden ? { hidden: true } : {}),
}));
+ // A video's separate recording binds to the landed copy of it.
+ const bound = new Map<string, SoundFrom>();
+ template.media.forEach((m, i) => {
+ const video = assetIds[i];
+ const rec = m.soundFrom ? assetIds[m.soundFrom.media] : undefined;
+ if (video && rec && m.soundFrom) bound.set(video, { assetId: rec, offset: m.soundFrom.offset });
+ });
set((s) => ({
+ ...(bound.size ? { assets: s.assets.map((a) => (bound.has(a.id) ? { ...a, soundFrom: bound.get(a.id) } : a)) } : {}),
clips: [...s.clips, ...newClips, ...newLayers].sort((a, b) => a.start - b.start),
audioClips: [...s.audioClips, ...newAudio],
overlays: [...s.overlays, ...newTexts],
@@ -4280,27 +4340,46 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const projectId = s.projectId;
const sp = clipWindow(s.clips, s.assets, clipId);
if (!sp) throw new Error("The clip is no longer on the timeline.");
- if (assetIsSilent(sp.asset)) throw new Error("This clip is a still — it has no sound.");
+ const src = soundSourceOf(sp.asset, s.assets);
+ if (assetIsSilent(sp.asset) && !src.bound) throw new Error("This clip is a still — it has no sound.");
if (sp.clip.reverse) throw new Error("This clip plays backward — its sound is not speech.");
+ // A recording bound to the video is what the clip says; it is read on
+ // the video's clock, starting late where it rolled late.
+ const heard = src.bound ? soundWindow(sp.clip, src, 0, sp.len) : null;
+ if (src.bound && !heard) throw new Error("The recording bound to this video holds none of this clip.");
const lane = s.subtitleLane;
const epoch = laneEpoch;
// The clip's own sound, deliberately unmuted: this transcribes what the
// clip says even when its timeline audio is muted.
- const spec = {
+ const spec: CloudTranscribeSpec = {
duration: sp.len,
locale: trackLocale(s.subtitles, lane),
clips: [
- {
- file: sp.asset.fileName,
- in: sp.clip.in,
- out: sp.clip.out,
- muted: false,
- speed: clipSpeed(sp.clip),
- speedCurve: sp.clip.speedCurve,
- transition: 0,
- },
+ heard
+ ? { file: "", in: 0, out: sp.len, muted: true, speed: 1, transition: 0 }
+ : {
+ file: sp.asset.fileName,
+ in: sp.clip.in,
+ out: sp.clip.out,
+ muted: false,
+ speed: clipSpeed(sp.clip),
+ speedCurve: sp.clip.speedCurve,
+ transition: 0,
+ },
],
- audio: [],
+ audio: heard
+ ? [
+ {
+ file: src.asset.fileName,
+ in: heard.lo,
+ out: heard.hi,
+ start: heard.at,
+ volume: 1,
+ speed: heard.span.speed,
+ speedCurve: heard.span.speedCurve,
+ },
+ ]
+ : [],
};
set({ subtitleStatus: "running", subtitleError: null, subtitleStartedAt: Date.now() });
try {
@@ -5043,7 +5122,7 @@ function beatsEdited(prev: MediaAsset[], next: MediaAsset[]): boolean {
const before = new Map(prev.map((a) => [a.id, a]));
return next.some((a) => {
const was = before.get(a.id);
- return !!was && (was.beats !== a.beats || was.colorProfile !== a.colorProfile);
+ return !!was && (was.beats !== a.beats || was.colorProfile !== a.colorProfile || was.soundFrom !== a.soundFrom);
});
}
@@ -5055,6 +5134,13 @@ function withColorProfile<A extends StoredAsset>(asset: A, profile: SourceProfil
return (profile ? { ...rest, colorProfile: profile } : rest) as A;
}
+/** The asset with its bound recording set or cleared. */
+function withSoundFrom<A extends StoredAsset>(asset: A, from: SoundFrom | undefined): A {
+ const { soundFrom: _was, ...rest } = asset;
+ void _was;
+ return (from ? { ...rest, soundFrom: from } : rest) as A;
+}
+
/** A stored asset in the current shape. Projects saved while the override
* rode inside the header record (`color.profile`) carry it out to
* `colorProfile` once, on load. */
@@ -5174,7 +5260,7 @@ export const docTimelines = (() => {
export function storedAssets(assets: MediaAsset[]): StoredAsset[] {
return assets
.filter((a) => !tabOnlyUpload(a))
- .map(({ id, fileName, name, type, duration, sizeBytes, width, height, origin, chatId, folderId, language, watch, speech, beats, sceneCuts, copiedFrom, block, color, colorProfile, proxy }) => ({
+ .map(({ id, fileName, name, type, duration, sizeBytes, width, height, origin, chatId, folderId, language, watch, speech, beats, sceneCuts, copiedFrom, block, color, colorProfile, proxy, soundFrom }) => ({
id,
fileName,
name,
@@ -5196,6 +5282,7 @@ export function storedAssets(assets: MediaAsset[]): StoredAsset[] {
...(color !== undefined ? { color } : {}),
...(colorProfile !== undefined ? { colorProfile } : {}),
...(proxy !== undefined ? { proxy } : {}),
+ ...(soundFrom !== undefined ? { soundFrom } : {}),
}));
}
@@ -5772,6 +5859,18 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)
// and they are what lets the two clips be audible at once over a cut
// their pictures hard-join.
const prevSound = spans[spans.length - 1]?.soundOut ?? 0;
+ // The handles are measured on the file the sound reads: a recording
+ // bound to the video holds its own stretch on either side of the trim.
+ const src = soundSourceOf(asset, byId);
+ const room = soundRoom(clip, src);
+ // A split edit carries the sound past an edge the picture cuts on. A
+ // transition at that cut owns the handover, so the edge keeps none; the
+ // lead never reaches before the timeline's start.
+ const prevJoin = present[i - 1] ? transitionOverlap(present[i - 1].clip, clip) > 0 : false;
+ const lead = prevJoin
+ ? 0
+ : Math.min(clip.start, handleSeconds(Math.min(SPLIT_EDIT_MAX_S, clip.audioLead ?? 0), room.head, clip, "head"));
+ const tail = overlap > 0 ? 0 : handleSeconds(Math.min(SPLIT_EDIT_MAX_S, clip.audioTail ?? 0), room.tail, clip, "tail");
spans.push({
clip,
asset,
@@ -5780,8 +5879,12 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)
transitionOut: onSound ? 0 : overlap,
soundOut,
// A reversed clip's head faces the source's end and its tail its start.
- soundAhead: handleSeconds(soundOut, clip.reverse ? clip.in : asset.duration - clip.out, clip, "tail"),
- soundBack: handleSeconds(prevSound, clip.reverse ? asset.duration - clip.out : clip.in, clip, "head"),
+ soundAhead: Math.max(tail, handleSeconds(soundOut, room.tail, clip, "tail")),
+ soundBack: Math.max(lead, handleSeconds(prevSound, room.head, clip, "head")),
+ ...(lead > 0 || tail > 0
+ ? { soundLead: lead, soundTail: tail, splitFade: cutSound().splitFadeSeconds }
+ : {}),
+ ...(src.bound ? { sound: { asset: src.asset, offset: src.offset, limit: src.limit } } : {}),
});
}
return spans;
@@ -6216,9 +6319,17 @@ export function assetIdsOnAnyTimeline(s: {
transitions: TimelineTransition[];
subtitles: SubtitlesBlock;
timelines: Partial<Record<TimelineId, TimelineBody>>;
+ assets?: readonly StoredAsset[];
}): Set<string> {
const used = assetIdsInUse(s);
for (const body of Object.values(s.timelines)) for (const id of assetIdsInUse(body)) used.add(id);
+ return withBoundSound(used, s.assets);
+}
+
+/** `used` plus the recordings bound to the videos in it: a bound recording
+ * plays wherever its video does. */
+function withBoundSound(used: Set<string>, assets: readonly StoredAsset[] | undefined): Set<string> {
+ for (const a of assets ?? []) if (a.soundFrom && used.has(a.id)) used.add(a.soundFrom.assetId);
return used;
}
@@ -6240,7 +6351,7 @@ export function assetIdsInUse(s: {
});
const captionFont = s.subtitles?.font ? fontAssetId(s.subtitles.font) : null;
if (captionFont) used.add(captionFont);
- return used;
+ return withBoundSound(used, s.assets);
}
export function projectDuration(s: {
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index ca70bf75..0509d3f1 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -279,6 +279,18 @@ export interface StoredAsset {
* file beside the master in the project's media. The preview reads it; an
* export reads the master. */
proxy?: { fileName: string; sizeBytes: number };
+ /** A video's sound recorded on a separate device: every clip of this video
+ * plays that recording, and the camera's own track stays out (see
+ * soundSource.ts). Absent = the camera's sound. */
+ soundFrom?: SoundFrom;
+}
+
+/** Where a video's sound comes from when it was recorded apart from the
+ * picture: the recording, and the seconds its clock runs ahead of the
+ * video's. Recording time = video source time + `offset`. */
+export interface SoundFrom {
+ assetId: string;
+ offset: number;
}
/** A folder in the Media panel's Project Files view — a flat, project-local
@@ -1122,6 +1134,9 @@ export interface TemplateMedia {
duration: number;
width?: number;
height?: number;
+ /** A video's separately recorded sound: the recording's index in `media`
+ * and its clock offset (see SoundFrom). */
+ soundFrom?: { media: number; offset: number };
}
export interface TemplateLayer {
/** Explicit timeline group; absent means independent. */
@@ -1546,6 +1561,19 @@ export interface ClipSpan {
* the incoming half of the cross dissolve at the cut behind it. Zero when
* the clip starts at its source's head. */
soundBack: number;
+ /** The split edit's share of `soundBack`: seconds this clip's sound plays
+ * ahead of its picture, ramping up over `splitFade` from silence. Zero or
+ * absent when the cut behind it carries a transition, which owns that
+ * handover. */
+ soundLead?: number;
+ /** The split edit's share of `soundAhead`: seconds the sound carries past
+ * the picture's end, ramping down over `splitFade` at its far end. */
+ soundTail?: number;
+ /** The ramp at the outer end of a split edit's extension, seconds. */
+ splitFade?: number;
+ /** The recording bound to the clip's video, when it has one: the clip's
+ * sound reads it, `offset` seconds past the video's source time. */
+ sound?: { asset: MediaAsset; offset: number; limit: number };
}
/** The document persisted as project.json inside each project folder. */
diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.ts
index 0baa0332..1fb37e7f 100644
--- a/site/src/cut/lib/watch/sweep.ts
+++ b/site/src/cut/lib/watch/sweep.ts
@@ -12,6 +12,7 @@ import { engineTranscribeSamples } from "../localStt";
import { sampleWatchFrames } from "../media";
import { runTranscription, useEditor } from "../store";
import { trackLocale } from "../subtitles";
+import { soundSourceOf, soundWindow } from "../soundSource";
import { speechOnsets } from "./fuse";
import { mergeSpeech, mergeWatch, nextUncoveredSpan } from "./merge";
@@ -129,12 +130,18 @@ async function transcribeChunk(
if (!s.projectId || !asset) return;
const locale = asset.language ?? trackLocale(s.subtitles, 0);
const specFrom = Math.max(0, span.from - (span.from > 0 ? SPEECH_PAD_S : 0));
- const cues = await transcribeFree(s.projectId, {
- duration: span.to - specFrom,
- locale,
- clips: [],
- audio: [{ file: asset.fileName, in: specFrom, out: span.to, start: 0, volume: 1 }],
- });
+ // A video bound to a recording is transcribed off the recording, laid on
+ // the video's clock; a stretch the recording does not reach is silence.
+ const src = soundSourceOf(asset, s.assets);
+ const heard = soundWindow({ in: specFrom, out: span.to }, src, 0, span.to - specFrom);
+ const cues = heard
+ ? await transcribeFree(s.projectId, {
+ duration: span.to - specFrom,
+ locale,
+ clips: [],
+ audio: [{ file: src.asset.fileName, in: heard.lo, out: heard.hi, start: heard.at, volume: 1 }],
+ })
+ : [];
if (cues === null) return; // another project took over mid-run
const segments = cues
.map((c) => ({ start: c.start + specFrom, end: c.end + specFrom, text: c.text }))
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 940570cc..477a86f9 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -12,6 +12,7 @@
import { EXPORT_RESOLUTIONS } from "@/cut/lib/exportPresets";
import { UI_TOOLS } from "@/cut/lib/projectCommands";
import { STOCK_SFX_CATEGORIES } from "@/cut/lib/stock";
+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";
import { builtinLutCatalogText } from "@/cut/lib/builtinLuts";
import { AI_PANEL_TOOLS } from "@/cut/components/AiPanel.tools";
import {
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
index 62672cb3..1174c9a9 100644
--- a/site/src/cut/server/ai/headlessTools.ts
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -35,6 +35,7 @@ const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set([
"detect_silence",
"detect_beats",
"refine_speech_cuts",
+ "sync_audio",
"freeze_frame",
"capture_frame",
"compare_to_source",
diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.ts
index e66d92c6..78098cb8 100644
--- a/site/src/cut/server/cloud/library.ts
+++ b/site/src/cut/server/cloud/library.ts
@@ -890,6 +890,7 @@ export const libraryCloud = {
duration: m.duration,
width: m.width,
height: m.height,
+ ...(m.soundFrom ? { soundFrom: m.soundFrom } : {}),
});
}
} catch (e) {
diff --git a/site/src/cut/server/cloud/sharedView.test.ts b/site/src/cut/server/cloud/sharedView.test.ts
index b450a53d..f8ad13f3 100644
--- a/site/src/cut/server/cloud/sharedView.test.ts
+++ b/site/src/cut/server/cloud/sharedView.test.ts
@@ -83,6 +83,18 @@ describe("filterDocForShare", () => {
]);
});
+ test("a placed video's bound recording travels with it", () => {
+ const bound = {
+ ...doc,
+ assets: [
+ ...doc.assets.map((a) => (a.id === "placed" ? { ...a, soundFrom: { assetId: "lav", offset: 0.4 } } : a)),
+ { id: "lav", fileName: "lav.wav", type: "audio", duration: 1 },
+ ],
+ } as unknown as ProjectDoc;
+ expect(ids(filterDocForShare(bound, features()))).toEqual(["lav", "placed"]);
+ expect(sharedFileNames(bound, features()).has("lav.wav")).toBe(true);
+ });
+
test("a caption font travels only when subtitles are shared", () => {
const captioned = {
...doc,
diff --git a/site/src/cut/server/cloud/sharedView.ts b/site/src/cut/server/cloud/sharedView.ts
index 82527084..f8823f39 100644
--- a/site/src/cut/server/cloud/sharedView.ts
+++ b/site/src/cut/server/cloud/sharedView.ts
@@ -95,7 +95,10 @@ function allowedAssets(doc: ProjectDoc, features: ShareFeatures): StoredAsset[]
for (const c of clips) {
if (c.removal?.matte) referenced.add(c.removal.matte.assetId);
if (c.removal?.backdrop?.assetId) referenced.add(c.removal.backdrop.assetId);
+ if (c.card?.matte) referenced.add(c.card.matte.assetId);
}
+ // A video bound to a separate recording plays that recording.
+ for (const a of assets) if (referenced.has(a.id) && a.soundFrom) referenced.add(a.soundFrom.assetId);
const fonts = new Set<string>();
for (const o of Array.isArray(doc.overlays) ? doc.overlays : []) {
if (o.kind === "sticker" && o.assetId) referenced.add(o.assetId);
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index eb714cd6..0c49db29 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -137,6 +137,15 @@ export interface ExportSpec {
* a cut their pictures hard-join. */
soundAhead?: number;
soundBack?: number;
+ /** A split edit: seconds of `soundBack` the sound plays ahead of the
+ * picture (J-cut) and of `soundAhead` it carries past it (L-cut), each
+ * ramping from silence over `splitFade` at its far end. */
+ soundLead?: number;
+ soundTail?: number;
+ splitFade?: number;
+ /** The recording bound to this clip's video, which its sound reads in
+ * place of the file's own track (see soundAlign.ts). */
+ soundFrom?: SpecSound;
/** Transition look id, resolved to an xfade name through the
* TRANSITION_XFADE allowlist (unknown ids render as a plain fade). Cross
* zoom renders as the fade plus zoom ramps on both segments' overlap
@@ -244,6 +253,11 @@ export interface ExportSpec {
tailSound?: number;
soundBack?: number;
soundAhead?: number;
+ /** A split edit and a bound recording, as on `clips`. */
+ soundLead?: number;
+ soundTail?: number;
+ splitFade?: number;
+ soundFrom?: SpecSound;
/** A still image: looped for the clip's length instead of trimmed. */
image?: boolean;
/** Manual color adjustments, baked into this overlay's segment. */
@@ -296,6 +310,10 @@ export interface ExportSpec {
/** Voiceover ducking: while this clip plays, every other sound drops to
* this gain (0..1). Ducking clips never duck each other. */
duck?: number;
+ /** Detached from a video bound to a recording: it plays the recording. */
+ soundFrom?: SpecSound;
+ /** The soundtrack lane it sits on, which picks its stem; absent = 0. */
+ lane?: number;
}[];
/** Overlay elements. A static element is one full-frame PNG windowed with
* `enable`. An animated element ships `frames` — a region-cropped slideshow
@@ -882,13 +900,61 @@ export async function runExport(
turned.set(file, { video, audio, mono });
return { ...mirrorRetimable(c, pivot), file };
};
+ // A video bound to a recording sounds from the recording. Each entry that
+ // reads one gets a sound twin: the recording laid on the video's clock over
+ // the stretch the entry reaches (soundAlign.ts), with the entry's trim and
+ // map moved onto that file. Picture reads keep the entry; every sound read
+ // below goes through `sounding`. A twin turns with its entry, and a
+ // recording that holds none of the stretch leaves the entry silent.
+ const twinFiles = new Set<string>();
+ const twinOf = async <T extends Parameters<typeof turn>[0] & { soundFrom?: SpecSound; muted?: boolean }>(
+ c: T,
+ tag: string
+ ): Promise<T | null | undefined> => {
+ if (!c.soundFrom || !c.file || c.image || c.hidden || c.muted) return undefined;
+ const rec = await resolveMedia(io.stat, mediaPathFor, c.soundFrom.file);
+ const rt = retimeOf(c);
+ const reach = srcSpan(rt, -(c.soundBack ?? 0), rt.len + (c.soundAhead ?? 0));
+ const file = path.join(job.tmpDir, `bound_${tag}.wav`);
+ const mono = (await io.audioChannels(rec)) === 1;
+ const laid = await alignRecording((args) => io.runFfmpeg(job, args), rec, c.soundFrom, reach.lo, reach.hi, file, {
+ mono,
+ });
+ if (!laid) return null;
+ twinFiles.add(file);
+ const { soundFrom: _bound, ...rest } = shiftSpan(c, -reach.lo);
+ void _bound;
+ return { ...rest, file } as T;
+ };
+ /** Each entry's sound twin, null for one whose recording is silent there. */
+ const soundTwins = new Map<object, Sounding | null>();
const turnedClips: ExportSpec["clips"] = [];
- for (const [j, c] of spec.clips.entries()) turnedClips.push(await turn(c, `clip_${j}`));
+ for (const [j, c] of spec.clips.entries()) {
+ const twin = await twinOf(c, `clip_${j}`);
+ const tc = await turn(c, `clip_${j}`);
+ turnedClips.push(tc);
+ if (twin !== undefined) soundTwins.set(tc, twin && (await turn(twin, `clipsnd_${j}`, true)));
+ }
const turnedOverlays: NonNullable<ExportSpec["overlayVideos"]> = [];
- for (const [k, oc] of (spec.overlayVideos ?? []).entries()) turnedOverlays.push(await turn(oc, `ovl_${k}`));
+ for (const [k, oc] of (spec.overlayVideos ?? []).entries()) {
+ const twin = await twinOf(oc, `ovl_${k}`);
+ const to = await turn(oc, `ovl_${k}`);
+ turnedOverlays.push(to);
+ if (twin !== undefined) soundTwins.set(to, twin && (await turn(twin, `ovlsnd_${k}`, true)));
+ }
const turnedAudio: ExportSpec["audio"] = [];
- for (const [k, a] of spec.audio.entries()) turnedAudio.push(await turn(a, `snd_${k}`, true));
+ for (const [k, a] of spec.audio.entries()) {
+ const twin = await twinOf(a, `snd_${k}`);
+ if (twin === null) continue;
+ turnedAudio.push(await turn(twin ?? a, `snd_${k}`, true));
+ }
spec = { ...spec, clips: turnedClips, overlayVideos: turnedOverlays, audio: turnedAudio };
+ /** What an entry's sound reads: its sound twin, or the entry itself. Null
+ * when a bound recording holds nothing over it. */
+ const sounding = <T extends Sounding>(c: T): T | Sounding | null => (soundTwins.has(c) ? soundTwins.get(c)! : c);
+ /** The path an entry's file reads from: a twin is on disk already. */
+ const sourcePath = (file: string) =>
+ twinFiles.has(file) ? Promise.resolve(file) : resolveMedia(io.stat, mediaPathFor, file);
// Tracks number 0..N bottom-up: track 0 folds sequentially into the base
// picture, the rest overlay it in track order (highest last = frontmost).
diff --git a/site/src/cut/server/library.ts b/site/src/cut/server/library.ts
index bf33c06c..639590f6 100644
--- a/site/src/cut/server/library.ts
+++ b/site/src/cut/server/library.ts
@@ -81,6 +81,8 @@ export interface TemplateMedia {
duration: number;
width?: number;
height?: number;
+ /** A video's separately recorded sound: the recording's index in `media`. */
+ soundFrom?: { media: number; offset: number };
}
export interface TemplateLayer {
media: number; // index into template.media
@@ -544,6 +546,7 @@ export interface TemplateInput {
duration: number;
width?: number;
height?: number;
+ soundFrom?: { media: number; offset: number };
}[];
layers: TemplateLayer[];
audio: TemplateAudio[];
@@ -631,6 +634,7 @@ export async function saveTemplate(
duration: m.duration,
width: m.width,
height: m.height,
+ ...(m.soundFrom ? { soundFrom: m.soundFrom } : {}),
});
}
const template: LibraryTemplate = {
diff --git a/site/src/cut/server/soundAlign.ts b/site/src/cut/server/soundAlign.ts
new file mode 100644
index 00000000..16df43e3
--- /dev/null
+++ b/site/src/cut/server/soundAlign.ts
@@ -0,0 +1,68 @@
+/**
+ * A recording laid on its video's clock, for the ffmpeg graphs.
+ *
+ * A video bound to a separately recorded sound (`soundFrom`) plays the
+ * recording `offset` seconds past its own source time. The graphs read a
+ * clip's sound through the clip's own trim and retime map, so the recording
+ * is first written out over the stretch of video source seconds the clip can
+ * reach, starting at that stretch's first second: second `τ` of the file is
+ * video second `lo + τ`. Where the recording had not started yet, or had
+ * already stopped, the file holds silence, so every read stays on time. The
+ * clip then sounds from this file like any other, through the same map.
+ */
+
+import { BAKE_RATE } from "./retimeAudio";
+
+/** Six decimals: a seek and a length to the microsecond, well under a sample. */
+const exact = (n: number) => n.toFixed(6);
+
+/**
+ * Write video source seconds `[lo, hi]` of the recording `rec` (bound with
+ * `offset`, `duration` seconds long) to `outFile`, a stereo float WAV at the
+ * bake rate, silence-padded to the whole window. False when the recording
+ * holds none of the window; nothing is written then.
+ */
+export async function alignRecording(
+ ffmpeg: (args: string[]) => Promise<void>,
+ rec: string,
+ bound: { offset: number; duration: number },
+ lo: number,
+ hi: number,
+ outFile: string,
+ /** A mono recording is laid into both channels at full level, the way the
+ * browser's mixer upmixes it. */
+ opts: { mono?: boolean } = {}
+): Promise<boolean> {
+ const len = hi - lo;
+ const start = lo + bound.offset;
+ const from = Math.max(0, start);
+ const to = Math.min(bound.duration, hi + bound.offset);
+ if (!(len > 0.001) || !(to - from > 0.001)) return false;
+ const pad = Math.round((from - start) * BAKE_RATE);
+ await ffmpeg([
+ "-y",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-ss",
+ exact(from),
+ "-i",
+ rec,
+ "-t",
+ exact(to - from),
+ "-vn",
+ "-af",
+ `aresample=${BAKE_RATE},` +
+ (opts.mono ? "pan=stereo|c0=c0|c1=c0," : "aformat=channel_layouts=stereo,") +
+ (pad > 0 ? `adelay=delays=${pad}S:all=1,` : "") +
+ `apad=whole_len=${Math.round(len * BAKE_RATE)}`,
+ "-t",
+ exact(len),
+ "-ar",
+ String(BAKE_RATE),
+ "-c:a",
+ "pcm_f32le",
+ outFile,
+ ]);
+ return true;
+}
diff --git a/site/src/cut/server/soundBound.e2e.test.ts b/site/src/cut/server/soundBound.e2e.test.ts
new file mode 100644
index 00000000..46e15f29
--- /dev/null
+++ b/site/src/cut/server/soundBound.e2e.test.ts
@@ -0,0 +1,267 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";
+
+// A clip's sound away from its own track and its own picture, through a real
+// ffmpeg: a recording bound to the video (laid on the video's clock, then
+// read through the clip's map) and a split edit's lead and tail. Each source
+// is silence with one click at a known second; where the click lands in the
+// exported file says which source second played at which timeline second.
+
+const available = (cmd: string) => spawnSync(cmd, ["-version"], { stdio: "ignore" }).status === 0;
+const tools = available("ffmpeg");
+const RATE = 44100;
+
+const ff = (args: string[]) => {
+ const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { encoding: "utf8" });
+ if (r.status !== 0) throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);
+};
+
+/** A click: 2 ms of a loud square at `at` seconds, silence elsewhere. */
+const click = (at: number, secs: number) =>
+ `aevalsrc=exprs='if(between(t\\,${at}\\,${at + 0.002})\\,0.9\\,0)':s=${RATE}:d=${secs}`;
+
+let media = "";
+let tmp = "";
+
+beforeAll(async () => {
+ if (!tools) return;
+ media = await mkdtemp(path.join(os.tmpdir(), "cut-sound-media-"));
+ tmp = await mkdtemp(path.join(os.tmpdir(), "cut-sound-job-"));
+ const video = (name: string, audio: string, secs: number) =>
+ ff([
+ "-f", "lavfi", "-i", `testsrc2=size=160x120:rate=24:duration=${secs}`,
+ "-f", "lavfi", "-i", audio,
+ "-shortest", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "pcm_s16le",
+ path.join(media, name),
+ ]);
+ // The camera's own scratch track clicks at 2.5; the recording at 3.0 of
+ // its own clock.
+ video("cam.mov", click(2.5, 8), 8);
+ ff(["-f", "lavfi", "-i", click(3, 8), "-c:a", "pcm_s16le", path.join(media, "rec.wav")]);
+ video("quiet.mov", `anullsrc=r=${RATE}:cl=stereo:d=8`, 8);
+ video("lead.mov", click(0.8, 8), 8);
+ video("tail.mov", click(2.3, 8), 8);
+});
+
+afterAll(async () => {
+ if (media) await rm(media, { recursive: true, force: true });
+ if (tmp) await rm(tmp, { recursive: true, force: true });
+});
+
+/** Render `over` and return the timeline second of the first click, or null. */
+const clickIn = async (over: Partial<ExportSpec> & Pick<ExportSpec, "clips" | "duration">, name: string) => {
+ const spec: ExportSpec = {
+ projectId: "sound",
+ width: 160,
+ height: 120,
+ fps: 24,
+ crf: 35,
+ preset: "ultrafast",
+ container: "mov",
+ audioCodec: "pcm",
+ audio: [],
+ overlays: [],
+ ...over,
+ };
+ const job: RenderHandle = { tmpDir: tmp, outPath: path.join(tmp, `${name}.mov`), progress: 0, log: [] };
+ await runExport(job, spec, (file) => path.join(media, file));
+ const r = spawnSync("ffmpeg", ["-v", "error", "-i", job.outPath, "-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"], {
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ if (r.status !== 0) throw new Error(`decode failed: ${r.stderr}`);
+ const buf = r.stdout as Buffer;
+ const data = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
+ for (let i = 0; i < data.length; i++) if (Math.abs(data[i]) > 0.3) return i / RATE;
+ return null;
+};
+
+/** Every click in a rendered file, as the timeline second it starts on. */
+const clicksOf = (file: string): number[] => {
+ const r = spawnSync("ffmpeg", ["-v", "error", "-i", file, "-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"], {
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ if (r.status !== 0) throw new Error(`decode failed: ${r.stderr}`);
+ const buf = r.stdout as Buffer;
+ const data = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
+ const at: number[] = [];
+ let quietSince = 0;
+ for (let i = 0; i < data.length; i++) {
+ if (Math.abs(data[i]) <= 0.3) continue;
+ if (i - quietSince > RATE * 0.1) at.push(i / RATE);
+ quietSince = i;
+ }
+ return at;
+};
+
+const near = (got: number | null, want: number) => {
+ expect(got).not.toBeNull();
+ expect(Math.abs(got! - want)).toBeLessThan(0.005);
+};
+
+const bound = { file: "rec.wav", offset: -1, duration: 8 };
+
+describe("a recording bound to the video, through ffmpeg", () => {
+ test.skipIf(!tools)("plays the recording on the video's clock, and the camera's track stays out", async () => {
+ // Recording 3.0 is video 4.0, which a clip from video 1 plays at 3.0.
+ near(await clickIn({ duration: 4, clips: [{ file: "cam.mov", in: 1, out: 5, muted: false, soundFrom: bound }] }, "bound"), 3);
+ });
+
+ test.skipIf(!tools)("a reversed clip turns the recording with it", async () => {
+ // Reversed over video 1..5, timeline t shows video 5 − t: video 4 at 1.
+ near(
+ await clickIn(
+ { duration: 4, clips: [{ file: "cam.mov", in: 1, out: 5, muted: false, reverse: true, soundFrom: bound }] },
+ "bound-reverse"
+ ),
+ 1
+ );
+ });
+
+ test.skipIf(!tools)("a soundtrack entry detached from the video plays the recording", async () => {
+ near(
+ await clickIn(
+ {
+ duration: 4,
+ clips: [{ file: "quiet.mov", in: 0, out: 4, muted: true }],
+ audio: [{ file: "cam.mov", in: 2, out: 5, start: 1, volume: 1, soundFrom: bound }],
+ },
+ "bound-audio"
+ ),
+ 3
+ );
+ });
+
+ test.skipIf(!tools)("a recording that holds nothing of the clip leaves it silent", async () => {
+ expect(
+ await clickIn(
+ { duration: 1, clips: [{ file: "cam.mov", in: 0, out: 1, muted: false, soundFrom: { ...bound, offset: 20 } }] },
+ "bound-gone"
+ )
+ ).toBeNull();
+ });
+});
+
+describe("a mastered export with stems, through ffmpeg", () => {
+ test.skipIf(!tools)("the dialogue stem carries the bound recording and the split edit, and the mix masters them", async () => {
+ // Recording 3.0 is video 4.0, which the first clip (video 3..5) plays at
+ // 1.0. The second clip starts at 2 on source 1; its lead plays source
+ // 0.8 at 1.8. The soundtrack's click at 3.0 belongs to its own lane.
+ const spec: ExportSpec = {
+ projectId: "sound",
+ width: 160,
+ height: 120,
+ fps: 24,
+ crf: 35,
+ preset: "ultrafast",
+ container: "mov",
+ audioCodec: "pcm",
+ overlays: [],
+ duration: 4,
+ loudness: -16,
+ truePeakCeiling: -1,
+ clips: [
+ { file: "cam.mov", in: 3, out: 5, muted: false, soundFrom: bound },
+ { file: "lead.mov", in: 1, out: 3, muted: false, soundBack: 0.5, soundLead: 0.5, splitFade: 0.03 },
+ ],
+ audio: [{ file: "rec.wav", in: 0, out: 4, start: 0, volume: 1, lane: 0 }],
+ stemPlan: [
+ { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },
+ { name: "rec", lane: 0, file: "2 rec.wav" },
+ ],
+ };
+ const job: RenderHandle = {
+ tmpDir: tmp,
+ outPath: path.join(tmp, "stems.mov"),
+ stemsPath: path.join(tmp, "stems.zip"),
+ progress: 0,
+ log: [],
+ };
+ await runExport(job, spec, (file) => path.join(media, file));
+ const unzipped = path.join(tmp, "stems");
+ expect(spawnSync("unzip", ["-o", "-q", job.stemsPath!, "-d", unzipped]).status).toBe(0);
+
+ const dialogue = clicksOf(path.join(unzipped, "1 Dialogue.wav"));
+ expect(dialogue).toHaveLength(2);
+ near(dialogue[0], 1);
+ near(dialogue[1], 1.8);
+ const lane = clicksOf(path.join(unzipped, "2 rec.wav"));
+ expect(lane).toHaveLength(1);
+ near(lane[0], 3);
+ const mix = clicksOf(job.outPath);
+ expect(mix).toHaveLength(3);
+ [1, 1.8, 3].forEach((want, i) => near(mix[i], want));
+ });
+});
+
+describe("split edits, through ffmpeg", () => {
+ test.skipIf(!tools)("a J-cut plays the incoming clip's sound ahead of its picture", async () => {
+ // The second clip starts at 2 on source 1; its lead plays source 0.8 at 1.8.
+ near(
+ await clickIn(
+ {
+ duration: 4,
+ clips: [
+ { file: "quiet.mov", in: 0, out: 2, muted: false },
+ { file: "lead.mov", in: 1, out: 3, muted: false, soundBack: 0.5, soundLead: 0.5, splitFade: 0.03 },
+ ],
+ },
+ "jcut"
+ ),
+ 1.8
+ );
+ });
+
+ test.skipIf(!tools)("an L-cut carries the outgoing clip's sound past its picture", async () => {
+ near(
+ await clickIn(
+ {
+ duration: 4,
+ clips: [
+ { file: "tail.mov", in: 0, out: 2, muted: false, soundAhead: 0.5, soundTail: 0.5, splitFade: 0.03 },
+ { file: "quiet.mov", in: 0, out: 2, muted: false },
+ ],
+ },
+ "lcut"
+ ),
+ 2.3
+ );
+ });
+
+ test.skipIf(!tools)("an upper-track clip's lead lands ahead of its start", async () => {
+ near(
+ await clickIn(
+ {
+ duration: 4,
+ clips: [{ file: "quiet.mov", in: 0, out: 4, muted: true }],
+ overlayVideos: [
+ {
+ file: "lead.mov", in: 1, out: 2, start: 2, track: 1, muted: false,
+ soundBack: 0.5, soundLead: 0.5, splitFade: 0.03,
+ },
+ ],
+ },
+ "overlay-jcut"
+ ),
+ 1.8
+ );
+ });
+
+ test.skipIf(!tools)("without a split edit the trimmed-away sound stays out", async () => {
+ expect(
+ await clickIn(
+ {
+ duration: 4,
+ clips: [
+ { file: "tail.mov", in: 0, out: 2, muted: false },
+ { file: "lead.mov", in: 1, out: 3, muted: false },
+ ],
+ },
+ "plain"
+ )
+ ).toBeNull();
+ });
+});
diff --git a/site/src/cut/worker/turnJob.ts b/site/src/cut/worker/turnJob.ts
index 790cc495..70249a5b 100644
--- a/site/src/cut/worker/turnJob.ts
+++ b/site/src/cut/worker/turnJob.ts
@@ -2,6 +2,8 @@ import type { UIMessage } from "ai";
import { getGlobalSetting } from "@/lib/config/effective";
import { bindCutClip, bindCutJudge } from "@/cut/lib/chatRuntime";
import { bindCutColor } from "@/cut/lib/colorSettings";
+import { bindCutSound } from "@/cut/lib/soundSettings";
+import { bindCutMotion } from "@/cut/lib/motionSettings";
import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import { openCloudProject, pushCloudProject } from "../lib/headless/docSession";
import { dropPiSession, streamCutChat } from "../lib/pi/cutAgent";
@@ -111,6 +113,8 @@ export async function runTurnJob(
bindCutJudge(judgeSettings);
bindCutClip(await getGlobalSetting("cutClip"));
bindCutColor(await getGlobalSetting("cutColor"));
+ bindCutSound(await getGlobalSetting("cutSound"));
+ bindCutMotion(await getGlobalSetting("cutMotion"));
deps.judgeSettings = judgeSettings;
const toolCalls: string[] = [];
const exec = deps.execTool;