Open source CapCut alternative.
TypeScript
feat(site): sync a separate mic and split edits [rebuild]
David Le72bdb1eedb27
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.tsindex fe8c321b..544d67bc 100644--- a/site/src/cut/components/Inspector.tools.ts+++ b/site/src/cut/components/Inspector.tools.ts@@ -33,6 +33,7 @@ import {} from "@donkeycut/effects-kit";import { bool, ids, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";import { builtinLutCatalogText } from "@/cut/lib/builtinLuts";+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";/** Per-field slider hints; ranges interpolate the exported constants so the* schema can never drift from the model the renderer clamps to. */@@ -227,6 +228,27 @@ export const INSPECTOR_TOOLS = [description: "Set the gain on a video clip's own audio (soundtrack clips use update_audio).",inputSchema: obj({ clipId: str("Video clip id"), ids: ids("clipId"), volume: num("0..3 (1 = unchanged, up to 3 boosts the clip's own sound)") }, ["volume"]),},+ {+ name: "sync_audio",+ description:+ "Dual-system sound: line up a separately recorded audio file (a lavalier, a recorder, a second camera's mic) with a video by matching it against the camera's own sound, then bind it to the video. Every clip of that video — the ones on the timeline now and any split, trimmed, sped, reversed, or copied later — plays the recording through its own trim and speed, and the camera's track goes quiet; the preview, every export, captions, and the listening tools all hear the recording. Name the video by clipId (one of its clips) or assetId, and the recording by audio_asset_id (an audio file in the project). The match reports `offset` (recording seconds = video seconds + offset) and `confidence`; when no single alignment stands out — a different take, or two files with no sound in common — it refuses and binds nothing, and the user can pick another file. clear: true returns the video to its own sound. editor_state shows each bound video's `soundFrom` on its media entry.",+ inputSchema: obj({+ clipId: str("A video clip whose source video gets the recording"),+ assetId: str("The video asset to bind, when no clip names it"),+ audio_asset_id: str("The audio asset holding the clean recording"),+ clear: bool("Unbind the recording: the video plays its own sound again"),+ }),+ },+ {+ name: "set_split_edit",+ description: `Split edit on a video clip: its sound starts ahead of its picture (lead — a J-cut, the next speaker heard before the cut) or carries on past it (tail — an L-cut, a voice running over the next shot). The extension plays the clip's own source from beyond its trim, under the neighbouring clip, ramping in from silence at the far end; pictures and clip positions stay where they are. Typical values: 0.05–0.5 s to soften a dialogue cut, 1–2 s to hand one scene into the next. Each side is capped at ${SPLIT_EDIT_MAX_S} s and at the source the clip has beyond its trim; 0 clears that side, and a side left out keeps its value. A cut carrying a transition keeps the transition's own handover, so that side plays no split edit. editor_state reports audioLead/audioTail on the clip.`,+ inputSchema: obj({+ clipId: str("Video clip id"),+ ids: ids("clipId"),+ lead: num(`Seconds the sound starts before the picture, 0..${SPLIT_EDIT_MAX_S} (0 clears)`),+ tail: num(`Seconds the sound carries past the picture's end, 0..${SPLIT_EDIT_MAX_S} (0 clears)`),+ }),+ },{name: "set_clip_sound",description:diff --git a/site/src/cut/components/Inspector.tsx b/site/src/cut/components/Inspector.tsxindex 3615b3fc..37655cda 100644--- a/site/src/cut/components/Inspector.tsx+++ b/site/src/cut/components/Inspector.tsx@@ -79,9 +79,11 @@ import {restingMaskFrame,} from "@donkeycut/effects-kit";import { BLOCK_COLOR } from "@/cut/lib/blockSource";+import { bindRecording, unbindRecording } from "@/cut/lib/soundBind";+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";import { clipLen, clipWindow, maxClipFade, useEditor, type EditorState } from "@/cut/lib/store";import { PANEL_GLOBAL, usePanelState, useRememberedScroll } from "@/cut/lib/panelState";-import { usePreviewTime } from "@/cut/lib/playhead";+import { playheadAt, usePreviewTime } from "@/cut/lib/playhead";import { CLIP_MAX_ZOOM, clipCovers, clipKeyed, clipPoseAt, clipZoom, contentRect } from "@/cut/lib/types";import { AnimationTiles } from "@/cut/components/AnimationTiles";import { ColorField } from "@/cut/components/ColorField";@@ -753,6 +755,8 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {onCheckedChange={(v) => updateClip(clip.id, { muted: v })}/></Row>+ {asset?.type === "video" && <ClipSoundSource clip={clip} asset={asset} />}+ {asset?.type === "video" && (!assetIsSilent(asset) || !!asset.soundFrom) && <ClipSplitEdit clip={clip} />}{asset && !assetIsSilent(asset) && (<Rowlabel="Beats"@@ -761,7 +765,7 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {<BeatsControl asset={asset} /></Row>)}- {asset && !assetIsSilent(asset) && (+ {asset && (!assetIsSilent(asset) || !!asset.soundFrom) && (<SoundQualityRow sound={clip.sound} onOpen={() => setView("sound")} />)}<ClipGeneratedAudio clip={clip} />@@ -770,6 +774,127 @@ function ClipAudioPanel({ clip }: { clip: VideoClip }) {);}+/** The Select value that stands for the video's own sound. */+const CAMERA_SOUND = "camera";++/**+ * Where a video's sound comes from: the camera's own track, or a recording+ * made apart from it. Picking a recording lines it up against the camera's+ * sound and binds it to the video, so every clip of the video plays it; a+ * recording with no clear match is refused and the reason shown.+ */+function ClipSoundSource({ clip, asset }: { clip: VideoClip; asset: MediaAsset }) {+ const assets = useEditor((s) => s.assets);+ const recordings = useMemo(+ () => assets.filter((a) => a.type === "audio" && !a.upload && a.duration > 0),+ [assets]+ );+ const audioAssetId = asset.soundFrom?.assetId;+ const bound = audioAssetId ? recordings.find((a) => a.id === audioAssetId) : undefined;+ const [syncing, setSyncing] = useState(false);+ const [refusal, setRefusal] = useState<string | null>(null);+ const pick = (id: string) => {+ setRefusal(null);+ if (id === CAMERA_SOUND) {+ unbindRecording(asset.id);+ return;+ }+ setSyncing(true);+ void bindRecording(asset.id, id, clip)+ .catch((e: unknown) => setRefusal(e instanceof Error ? e.message : "Could not line the recording up."))+ .finally(() => setSyncing(false));+ };+ return (+ <>+ <Row+ label="Sound"+ info="Where this video's sound comes from. Pick an audio file recorded alongside it (a lav or a recorder) and it is lined up by matching it against the camera's sound; every clip of this video then plays the recording, through any trim, speed or split."+ >+ {syncing && <Loader2 className="size-3.5 animate-spin text-muted-foreground" />}+ <Select value={bound ? bound.id : CAMERA_SOUND} onValueChange={(v) => pick(String(v))} disabled={syncing}>+ <SelectTrigger className="clip-sound-source h-8 w-36 text-[12px]">+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value={CAMERA_SOUND} className="text-[12px]">+ Camera+ </SelectItem>+ {recordings.map((r) => (+ <SelectItem key={r.id} value={r.id} className="text-[12px]">+ {r.name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ <ResetButton title="Use the camera's sound" show={!!bound} onClick={() => pick(CAMERA_SOUND)} />+ </Row>+ {bound && asset.soundFrom && (+ <p className="pb-1 text-[11.5px] text-muted-foreground">+ Synced {asset.soundFrom.offset >= 0 ? "+" : "−"}+ {Math.abs(asset.soundFrom.offset).toFixed(3)} s · every clip of this video plays it+ </p>+ )}+ {refusal && <p className="pb-1 text-[11.5px] text-destructive">{refusal}</p>}+ </>+ );+}++const formatSplit = (v: number) => (v > 0 ? `${v.toFixed(2)} s` : "Off");+const parseSplitInput = (raw: string) =>+ /^off$/i.test(raw.trim()) ? 0 : parseNumberInput(raw.replace(/\s*s$/i, ""));++/**+ * The clip's split edit: its sound running ahead of the picture (a J-cut)+ * or past it (an L-cut), played from the source beyond the trim. The picture+ * and the clip's place on the timeline stay put.+ */+function ClipSplitEdit({ clip }: { clip: VideoClip }) {+ const updateClip = useEditor((s) => s.updateClip);+ const [leadDraft, setLeadDraft] = useState<number | null>(null);+ const [tailDraft, setTailDraft] = useState<number | null>(null);+ const lead = leadDraft ?? clip.audioLead ?? 0;+ const tail = tailDraft ?? clip.audioTail ?? 0;+ const slider = (+ label: string,+ value: number,+ onDraft: (v: number | null) => void,+ write: (v: number) => Partial<VideoClip>+ ) => (+ <ValueSlider+ label={label}+ sliderClassName="clip-split-edit data-horizontal:w-24"+ valueClassName="w-11 text-muted-foreground"+ value={value}+ min={0}+ max={SPLIT_EDIT_MAX_S}+ step={0.05}+ format={formatSplit}+ parse={parseSplitInput}+ onDraft={onDraft}+ onCommit={(v) => {+ updateClip(clip.id, write(v));+ onDraft(null);+ }}+ />+ );+ return (+ <>+ <Row+ label="Sound lead"+ info="J-cut: this clip's sound starts this long before its picture, under the clip before it. A tenth to half a second softens a dialogue cut; a second or two hands one scene into the next."+ >+ {slider("Sound lead", lead, setLeadDraft, (v) => ({ audioLead: v > 0 ? v : undefined }))}+ </Row>+ <Row+ label="Sound tail"+ info="L-cut: this clip's sound carries on this long past its picture, under the clip after it."+ >+ {slider("Sound tail", tail, setTailDraft, (v) => ({ audioTail: v > 0 ? v : undefined }))}+ </Row>+ </>+ );+}+const formatDb = (v: number) => `${v > 0 ? "+" : ""}${v.toFixed(1)} dB`;const parseDbInput = (raw: string) =>parseNumberInput(raw.replace(/\s*dB$/i, "").replace(/^\+/, ""));diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsxindex 75d81298..17da3a01 100644--- a/site/src/cut/components/Timeline.tsx+++ b/site/src/cut/components/Timeline.tsx@@ -132,16 +132,17 @@ const WAVE_H = 24;* play in the band (an image, a silent video) drops the band and its box stops* at the filmstrip's edge. */const FILM_H = VIDEO_H - 4 - WAVE_H;-/** Whether a clip has a sound band to draw: a video carrying audio of its own.- * Muting greys the band down, so the clip keeps its shape and its waveform. An- * image or a silent video has nothing to show there. */-const hasSoundBand = (asset: MediaAsset) =>- asset.type === "video" && !!asset.peaks?.length;+/** Whether a clip has a sound band to draw: a video carrying audio of its own,+ * or a recording bound to it (`sound`, which the band then draws). Muting+ * greys the band down, so the clip keeps its shape and its waveform. An image+ * or a silent video has nothing to show there. */+const hasSoundBand = (asset: MediaAsset, sound?: MediaAsset) =>+ asset.type === "video" && !!(sound ?? asset).peaks?.length;/** A video row's height. Rows whose clips carry no audio lose the sound band's* worth, so the row ends where the filmstrips do. An empty row keeps the full* height: it is a drop target for clips that do have sound. */const videoRowH = (list: ClipSpan[]) =>- list.length > 0 && !list.some((sp) => hasSoundBand(sp.asset))+ list.length > 0 && !list.some((sp) => hasSoundBand(sp.asset, sp.sound?.asset))? VIDEO_H - WAVE_H: VIDEO_H;/** The `mt-1.5` every row carries, so a row drag knows what one row's worth of@@ -3809,7 +3810,7 @@ function ClipView({const { clip, asset } = span;const spine = clip.track === 0;const loading = useEditor((s) => s.loadingMedia.has(asset.fileName));- const baking = useMatteBakes((s) => s.jobs[clip.id]?.status === "running");+ const baking = useMatteBakes((s) => s.jobs[clip.id]?.status === "running" || s.jobs[cardMatteKey(clip.id)]?.status === "running");const rt = retimeOf(clip);// Every box is its clip's whole footprint. Clips never overlap — a// transition is a render-time blend at the cut, drawn as the bar above the@@ -3833,7 +3834,11 @@ function ClipView({// The clip's own sound, drawn in the box's band under the picture. Muting —// detach included — greys the band, so the clip's shape holds still and its// waveform stays readable (a detached copy draws on the soundtrack row too).- const hasWave = hasSoundBand(asset);+ // A recording bound to the video is what plays, so it is what draws, moved+ // onto the video's clock.+ const hasWave = hasSoundBand(asset, span.sound?.asset);+ const waveAsset = span.sound?.asset ?? asset;+ const waveOff = span.sound?.offset ?? 0;// Bumped as tile captures land, so the memo re-plans and picks them up.const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);// The strip draws the stretch of the box the scroller shows. The box's@@ -3927,7 +3932,10 @@ function ClipView({)}style={{ height: WAVE_H }}>- <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={barW} h={WAVE_H - 4} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps)} className="inset-x-0 inset-y-0.5" />+ <WaveformCanvas asset={waveAsset} from={clip.in + waveOff} to={clip.out + waveOff} w={barW} h={WAVE_H - 4} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps) + waveOff} className="inset-x-0 inset-y-0.5" />+ {/* A split edit: the edge whose sound runs past the picture. */}+ {(span.soundLead ?? 0) > 0 && <div className="tl-clip-split absolute inset-y-0 left-0 w-0.5 bg-white/80" />}+ {(span.soundTail ?? 0) > 0 && <div className="tl-clip-split absolute inset-y-0 right-0 w-0.5 bg-white/80" />}</div>)}{selected && (@@ -5122,6 +5130,9 @@ function WaveTile({const start = (from / duration) * n;const perBar = (((to - from) / duration) * n) / bars;for (let i = 0; i < bars; i++) {+ // A stretch the file does not hold (a bound recording that started+ // late or stopped early) draws nothing.+ if (start + (i + 1) * perBar <= 0 || start + i * perBar >= n) continue;const a = Math.max(0, Math.min(n - 1, Math.floor(start + i * perBar)));const b = Math.max(a + 1, Math.ceil(start + (i + 1) * perBar));let p = 0;@@ -5333,6 +5344,13 @@ function AudioView({onSnap: (x: number | null) => void;}) {const loading = useEditor((s) => (asset ? s.loadingMedia.has(asset.fileName) : false));+ // Detached from a video bound to a recording, the clip plays the recording,+ // so the bar draws it on the video's clock.+ const recording = useEditor((s) => {+ const id = asset?.soundFrom?.assetId;+ return id ? s.assets.find((a) => a.id === id && !assetIsSilent(a)) : undefined;+ });+ const waveOff = recording ? (asset?.soundFrom?.offset ?? 0) : 0;const len = clipLen(clip);const w = Math.max(10, len * pps);// The bar is drawn a gutter narrower than the footprint; what sits inside@@ -5387,7 +5405,7 @@ function AudioView({className="pointer-events-none absolute inset-0"style={{ clipPath: fadeClipPath(fadeInPx, fadeOutPx, barW) }}>- <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={barW} h={AUDIO_H - 8} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps)} className="inset-x-0 inset-y-1" />+ <WaveformCanvas asset={recording ?? asset} from={clip.in + waveOff} to={clip.out + waveOff} w={barW} h={AUDIO_H - 8} map={rt.uniform && !rt.reverse ? undefined : (px) => rt.srcAt(px / pps) + waveOff} className="inset-x-0 inset-y-1" /></div><BeatDots asset={asset} lo={clip.in} hi={clip.out} retime={rt} pps={pps} w={barW} /><FadeEnvelope inPx={fadeInPx} outPx={fadeOutPx} barW={barW} />diff --git a/site/src/cut/components/inspectorToolFields.test.ts b/site/src/cut/components/inspectorToolFields.test.tsindex f8d51c6b..c689d79d 100644--- a/site/src/cut/components/inspectorToolFields.test.ts+++ b/site/src/cut/components/inspectorToolFields.test.ts@@ -25,8 +25,11 @@ const ITEM_TOOLS = new Set(["update_audio","set_clip_volume","set_clip_sound",+ "sync_audio",+ "set_split_edit","set_framing","set_clip_style",+ "set_camera_card","set_speed","set_speed_curve","set_color_grade",diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 4188ba66..565637ae 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -9,11 +9,12 @@ import {projectDuration,useEditor,} from "@/cut/lib/store";-import { headSrc, matteLumaToAlpha, retimeOf, smoothsAt, srcSpan } from "@donkeycut/effects-kit";+import { headSrc, matteLumaToAlpha, retimeOf, smoothsAt } from "@donkeycut/effects-kit";import { blendInto, SYNTH_EDGE, synthWeight } from "@/cut/lib/frameSynth";import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";import { assetIsSilent, clipCovers, rectOf } from "@/cut/lib/types";-import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";+import type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";+import { soundSourceOf, soundWindow } from "@/cut/lib/soundSource";import { SubjectMaskCompositor } from "@/cut/lib/behindPass";import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";@@ -167,28 +168,40 @@ export function engineLog(msg: string): void {*/const SOUND_LOOKAHEAD_S = 3;+const OWN_TRACK = { offset: 0, limit: Infinity };+const voiceSpan = (sp: ClipSpan) => {- // The handles a crossing reaches into play at the pace of the footage- // beside them, so the voice's span widens through the clip's own map and- // its retime is rebuilt over the wider span.- const rt = retimeOf(sp.clip);- const { lo, hi } = srcSpan(rt, -sp.soundBack, rt.len + sp.soundAhead);+ // The handles a crossing or a split edit reaches into play at the pace of+ // the footage beside them, so the voice's span widens through the clip's+ // own map and its retime is rebuilt over the wider span. A recording bound+ // to the video is read through the same map, on its own clock, and starts+ // late where it rolled after the camera did.+ const w = soundWindow(sp.clip, sp.sound ?? OWN_TRACK, -sp.soundBack, sp.len + sp.soundAhead);+ if (!w) return null;return {- url: sp.asset.url,- start: sp.start - sp.soundBack,- in: lo,- out: hi,- retime: retimeOf({- in: lo,- out: hi,- speed: sp.clip.speed,- speedCurve: sp.clip.speedCurve,- reverse: sp.clip.reverse,- }),+ url: (sp.sound?.asset ?? sp.asset).url,+ start: sp.start - sp.soundBack + w.at,+ in: w.lo,+ out: w.hi,+ retime: retimeOf(w.span),sound: sp.clip.sound,};};+/** A span with nothing to hear: its video has no sound track and no+ * recording bound to it. */+const spanSilent = (sp: ClipSpan) => !sp.sound && assetIsSilent(sp.asset);++/** An audio clip's sound: its own file, or the recording bound to the video+ * it was detached from. */+const audioVoice = (a: AudioClip, asset: MediaAsset, assets: MediaAsset[]) => {+ const src = soundSourceOf(asset, assets);+ if (!src.bound) return { url: asset.url, start: a.start, in: a.in, out: a.out, retime: retimeOf(a) };+ const w = soundWindow(a, src, 0, retimeOf(a).len);+ if (!w) return null;+ return { url: src.asset.url, start: a.start + w.at, in: w.lo, out: w.hi, retime: retimeOf(w.span) };+};+class Engine {/** Which engine this is, for the perf eval's pool dump. */readonly serial = ++engineSerial;diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.tsindex f4726ba6..febb9568 100644--- a/site/src/cut/lib/aiContext.ts+++ b/site/src/cut/lib/aiContext.ts@@ -462,6 +462,17 @@ function describeState(duration: r(a.duration),...(a.origin ? { origin: a.origin } : {}),...(a.folderId != null ? { folderId: a.folderId } : {}),+ // The recording bound to this video (sync_audio): every clip of it+ // plays that file, recording second = video second + offset.+ ...(a.soundFrom+ ? {+ soundFrom: {+ assetId: a.soundFrom.assetId,+ name: s.assets.find((x) => x.id === a.soundFrom!.assetId)?.name ?? null,+ offset: Math.round(a.soundFrom.offset * 1000) / 1000,+ },+ }+ : {}),// What the file's code values mean, for footage that is not plain// Rec.709: the profile in force and, when the person overrode it, what// the header read (set_source_color changes it).@@ -529,6 +540,7 @@ function describeState(// The clip's own equalizer/compressor/limiter, set through// set_clip_sound; absent when its sound is untouched....(sp.clip.sound ? { sound: sp.clip.sound } : {}),+ ...splitEditOf(sp.clip),framing: sp.clip.fit ?? "fit",...describeRate(sp.clip),// The generated scene shot this clip came from — sceneShot is the@@ -696,6 +708,12 @@ function describeAudio(};}+/** A clip's split edit as set_split_edit wrote it: seconds its sound leads+ * the picture and carries past it. Absent when it has none. */+function splitEditOf(c: VideoClip) {+ return c.audioLead || c.audioTail ? { audioLead: c.audioLead ?? 0, audioTail: c.audioTail ?? 0 } : {};+}+function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>) {const rect = rectOf(c);return {@@ -707,6 +725,7 @@ function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>in: r(c.in),out: r(c.out),muted: c.muted,+ ...splitEditOf(c),...(c.hidden ? { hidden: true } : {}),...(c.sound ? { sound: c.sound } : {}),// The frame region this layer occupies: Full covers the frame; Top/Bottom/diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex a2d85839..9af7363d 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -284,6 +284,9 @@ import {type WatchDetail,} from "./types";+import { soundSourceOf, SPLIT_EDIT_MAX_S } from "./soundSource";+import { bindRecording, unbindRecording } from "./soundBind";+const round2 = (n: number) => Math.round(n * 100) / 100;/** The timeline rows a mutation touched, small enough to ride every result:@@ -1544,7 +1547,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {...(wanted > 3 ? { shortfallDb: round2(20 * Math.log10(wanted / 3)) } : {}),};});- return { targetId, clips };+ return { targetId, clips, ...(mix ? { mix } : {}) };},listen_audio: async (s, input) => {@@ -1559,7 +1562,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// A whole audio file that clears the cap falls through to the same// fitting path a range takes, so length truncates instead of refusing.const heard = whole- ? { inline: whole, to: asset.duration }+ ? { inline: whole, from, to: asset.duration }: await listenSpan(projectId, asset, from, to);// Listening shows interest in the source's sound — queue the background// sweep so its transcript (and, for video, its visual map) fills in.@@ -1579,7 +1582,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {source: sourceRef(asset),duration: round2(asset.duration),...(clip ? { clipId: clip.id } : {}),- from: round2(from),+ from: round2(heard.from),+ ...(soundSourceOf(asset, s.assets).bound+ ? { soundFrom: soundSourceOf(asset, s.assets).asset.name }+ : {}),// Where this play stopped, and how much of the source nobody has// heard — the sound's answer to coveredTo and unwatchedSeconds. The// cap is bytes, so the seconds that fit depend on the source.@@ -1807,6 +1813,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const h = clamp(rg.h, 0.05, 1);patch.frame = { x: clamp(rg.x, 0, 1 - w), y: clamp(rg.y, 0, 1 - h), w, h };}+ refuseCardFraming(c, input, ["fit", "zoom"]);if (input.fit === "fit" || input.fit === "fill") patch.fit = input.fit;if (isNum(input.zoom)) {const z = clamp(input.zoom, 1, CLIP_MAX_ZOOM);@@ -2077,6 +2084,50 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {return { id: next.id, volume: next.volume ?? 1 };},+ sync_audio: async (s, input) => {+ const clip = input.clipId !== undefined ? requireItem(s.clips, input.clipId, "video clip") : undefined;+ const video = requireItem(s.assets, clip?.assetId ?? input.assetId, "video asset");+ if (video.type !== "video") throw new ToolError(`"${video.name}" is not a video.`);+ if (input.clear === true || input.audio_asset_id === null) {+ unbindRecording(video.id);+ return { assetId: video.id, soundFrom: null, note: `"${video.name}" plays its own sound again.` };+ }+ return syncVideoSound(s, video, input.audio_asset_id, clip);+ },++ set_split_edit: (s, input) => {+ const clip = requireItem(s.clips, input.clipId, "video clip");+ const side = (v: unknown, name: string) => {+ if (v === undefined) return undefined;+ if (!isNum(v)) throw new ToolError(`${name} is seconds, 0..${SPLIT_EDIT_MAX_S}.`);+ const secs = Math.round(clamp(v, 0, SPLIT_EDIT_MAX_S) * 1000) / 1000;+ return secs > 0 ? secs : 0;+ };+ const lead = side(input.lead, "lead");+ const tail = side(input.tail, "tail");+ if (lead === undefined && tail === undefined) throw new ToolError("Pass lead, tail, or both, in seconds (0 clears).");+ s.updateClip(clip.id, {+ ...(lead !== undefined ? { audioLead: lead || undefined } : {}),+ ...(tail !== undefined ? { audioTail: tail || undefined } : {}),+ });+ const next = useEditor.getState();+ const live = next.clips.find((c) => c.id === clip.id)!;+ const sp = getClipSpans(next.clips, next.assets, live.track).find((x) => x.clip.id === clip.id);+ const playing = { lead: round2(sp?.soundLead ?? 0), tail: round2(sp?.soundTail ?? 0) };+ const short = (asked: number | undefined, got: number) => asked !== undefined && asked - got > 0.01;+ return {+ id: clip.id,+ audioLead: live.audioLead ?? 0,+ audioTail: live.audioTail ?? 0,+ playing,+ ...(short(lead, playing.lead) || short(tail, playing.tail)+ ? {+ note: "Part of the split edit does not play: a transition owns that cut, the source ends there, or the timeline starts there. `playing` is what sounds.",+ }+ : {}),+ };+ },+rename_item: (s, input) => {const name = String(input.name ?? "").trim().slice(0, 60) || undefined;const id = String(input.id ?? "");@@ -2255,7 +2306,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// puts that whole track on the soundtrack with nothing else placed.if (input.assetId) {const asset = requireItem(s.assets, input.assetId, "project asset");- if (asset.type === "image" || assetIsSilent(asset))+ if (asset.type === "image" || (assetIsSilent(asset) && !soundSourceOf(asset, s.assets).bound))throw new ToolError(`"${asset.name}" carries no audio.`);const start = isNum(input.start) ? Math.max(0, input.start) : 0;s.addAudioFromAsset(asset.id, start);@@ -2707,6 +2758,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {set_framing: (s, input) => {const clip = requireItem(s.clips, input.clipId, "video clip");+ refuseCardFraming(clip, input, ["mode", "zoom", "panX", "panY"]);// Only what the call names changes; the rest of the framing stays as// the user left it.const mode = input.mode === "fill" || input.mode === "fit" ? input.mode : clip.fit ?? "fit";@@ -5009,6 +5061,7 @@ export const MEDIA_RUNTIME_TOOLS: ReadonlySet<string> = new Set(["detect_silence","detect_beats","refine_speech_cuts",+ "sync_audio","capture_frame","compare_to_source","render_preview",@@ -5525,9 +5578,38 @@ async function eachScanned<T>(items: T[], run: (item: T) => Promise<void>) {* threshold cannot answer it, because a quiet room and a loud one disagree* about what silence is and a cut placed on the wrong answer lands in a word. */async function fetchSpeech(asset: MediaAsset, from: number, to: number) {- return scanSourceSpeech(asset.url, { from, to }).catch((e) => {+ const heard = soundSpan(asset, from, to);+ const scan = await scanSourceSpeech(heard.asset.url, { from: heard.from, to: heard.to }).catch((e) => {throw new ToolError(e instanceof Error ? e.message : "Could not read the audio.");});+ return {+ ...scan,+ from: heard.back(scan.from),+ to: heard.back(scan.to),+ segments: scan.segments.map((g) => ({ ...g, start: heard.back(g.start), end: heard.back(g.end) })),+ };+}++/**+ * Where a source range's sound is read. A video bound to a recording sounds+ * from the recording: the range moves onto its clock and narrows to what it+ * holds, and `back` carries a second on that file back to the video's source+ * time, so every answer comes back in the seconds the caller asked in.+ */+function soundSpan(asset: MediaAsset, from: number, to: number | undefined) {+ const src = soundSourceOf(asset, useEditor.getState().assets);+ if (!src.bound) return { asset, from, to, back: (t: number) => t };+ const lo = Math.max(0, from + src.offset);+ const hi = Math.min(src.limit, (to ?? asset.duration) + src.offset);+ if (!(hi - lo > 0.05))+ throw new ToolError(`"${src.asset.name}", the sound bound to "${asset.name}", holds nothing of that stretch.`);+ return { asset: src.asset, from: lo, to: hi as number | undefined, back: (t: number) => t - src.offset };+}++/** How loud a source range plays, read where its sound is. */+async function sourceLevel(asset: MediaAsset, from: number, to: number | undefined) {+ const heard = soundSpan(asset, from, to);+ return measureSourceLevel(heard.asset.url, { from: heard.from, ...(heard.to !== undefined ? { to: heard.to } : {}) });}/** Silent stretches of a source range, in source seconds — the engine's@@ -5542,23 +5624,28 @@ async function fetchSilences(// Same defaults and clamps as the engine's silence handler.const thresholdDb = clamp(opts.thresholdDb ?? -30, -90, 0);const minSilence = clamp(opts.minSilence ?? 0.35, 0.05, 10);+ const heard = soundSpan(asset, opts.from, opts.to);+ const back = (list: { start: number; end: number; duration: number }[]) =>+ list.map((x) => ({ ...x, start: heard.back(x.start), end: heard.back(x.end) }));if (getBackend().kind !== "local") {- return detectSilenceClientSide(asset.url, {- from: opts.from,- ...(opts.to !== undefined ? { to: opts.to } : {}),- thresholdDb,- minSilence,- }).catch((e) => {- throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");- });+ return back(+ await detectSilenceClientSide(heard.asset.url, {+ from: heard.from,+ ...(heard.to !== undefined ? { to: heard.to } : {}),+ thresholdDb,+ minSilence,+ }).catch((e) => {+ throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");+ })+ );}const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {method: "POST",headers: { "Content-Type": "application/json" },body: JSON.stringify({- file: asset.fileName,- from: opts.from,- ...(opts.to !== undefined ? { to: opts.to } : {}),+ file: heard.asset.fileName,+ from: heard.from,+ ...(heard.to !== undefined ? { to: heard.to } : {}),threshold_db: thresholdDb,min_silence: minSilence,}),@@ -5568,7 +5655,7 @@ async function fetchSilences(error?: string;}>(res);if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");- return body.silences;+ return back(body.silences);}/** Pull a source's audio track off (video and audio alike) and inline it for@@ -5612,15 +5699,20 @@ const LISTEN_MAX_S = 600;async function listenSpan(projectId: string,asset: MediaAsset,- from: number,- to: number | undefined,-): Promise<{ inline: InlineImage; to: number }> {+ askedFrom: number,+ askedTo: number | undefined,+): Promise<{ inline: InlineImage; from: number; to: number }> {+ // A bound recording is what is heard, from where it holds+ // sound; the span reports back in the video's own seconds.+ const heard = soundSpan(asset, askedFrom, askedTo ?? Math.min(asset.duration, askedFrom + LISTEN_MAX_S));+ const from = heard.from;+ const to = heard.to;const end = Math.min(to ?? from + LISTEN_MAX_S, from + LISTEN_MAX_S);let span = end - from;for (let attempt = 0; attempt < 3 && span > 0.5; attempt++) {- const blob = await audioSpanBlob(projectId, asset, from, round2(from + span));+ const blob = await audioSpanBlob(projectId, heard.asset, from, round2(from + span));const inline = await blobToInlineAudio(blob);- if (inline) return { inline, to: round2(from + span) };+ if (inline) return { inline, from: round2(heard.back(from)), to: round2(heard.back(from + span)) };const perSecond = blob.size / span;// Leave room for the base64 inflation the cap already accounts for.const fits = (MAX_AUDIO_BYTES * 0.9) / perSecond;@@ -5695,6 +5787,7 @@ const SWEEP_TARGETS: Readonly<Record<string, string>> = {set_clip_muted: "clipId",set_clip_hidden: "clipId",set_clip_volume: "clipId",+ set_split_edit: "clipId",set_color_preset: "clipId",set_color_grade: "clipId",set_color_curves: "clipId",diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.tsindex e602df5f..4aafe7be 100644--- a/site/src/cut/lib/audioMix.ts+++ b/site/src/cut/lib/audioMix.ts@@ -23,12 +23,12 @@ import {buildAudioFx,retimeOf,soundRecipe,- srcSpan,type ClipSound,type SpeedNode,} from "@donkeycut/effects-kit";import { decodeAudioSpan } from "./mediaRead";import { fitSpan, retimeFits } from "./retimeFit";+import { shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";/** One track-0 clip's audio in the sequential fold. A spacer (no file) only* shapes time. */@@ -61,6 +61,15 @@ export interface MixClip {* are what make the two clips audible at once over a hard picture cut. */soundAhead?: number;soundBack?: number;+ /** A split edit: seconds of `soundBack` the clip's sound plays ahead of its+ * picture and of `soundAhead` past its end, each ramping from silence over+ * `splitFade` at its outer end. */+ soundLead?: number;+ soundTail?: number;+ splitFade?: number;+ /** The video's sound recorded apart from it: the sound reads this file,+ * `offset` seconds later in the source (see soundSource.ts). */+ soundFrom?: SpecSound;}/** A clip placed at an absolute time: an upper-track video's audio, a@@ -87,6 +96,13 @@ export interface MixItem {sound?: ClipSound;/** While this item is audible, everything else drops to this gain. */duck?: number;+ soundLead?: number;+ soundTail?: number;+ splitFade?: number;+ soundFrom?: SpecSound;+ /** The soundtrack lane this item sits on; absent for a video clip's own+ * sound. What a stem render picks items by. */+ lane?: number;}/** One audio effect element: the treatment it names, and the stretch of@@ -285,21 +301,23 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB// crossing at either edge widens it into the handle it reaches for.const folded = foldClips(spec.clips).map((g) => {const { back, ahead } = handles(g);- return { ...g, back, ahead, span: crossSpan(g.clip, back, ahead) };+ const read = g.clip.file && !g.clip.muted ? soundRead(g.clip, back, ahead) : null;+ return { ...g, back, ahead, read };+ });+ // Each placed item's reach: a crossing's handle, or the split edit an edge+ // without one carries.+ const placed = spec.items.map((a) => {+ const back = Math.min(a.soundBack ?? 0, Math.max(a.crossIn ?? 0, a.soundLead ?? 0));+ const ahead = Math.min(a.soundAhead ?? 0, Math.max(a.crossOut ?? 0, a.soundTail ?? 0));+ const read = a.file && !a.muted ? soundRead(a, back, ahead) : null;+ return { a, back, ahead, read };});const wanted = new Map<string, { file: string; from: number; to: number }>();- const spanKey = (c: { file: string; in: number; out: number }) =>- `${c.file}|${c.in.toFixed(3)}|${c.out.toFixed(3)}`;- for (const c of folded) {- if (!c.clip.file || c.clip.muted) continue;- const span = c.span;- wanted.set(spanKey(span), { file: span.file, from: Math.max(0, span.in), to: span.out });- }- for (const a of spec.items) {- if (!a.file || a.muted) continue;- const span = crossSpan(a, a.soundBack, a.soundAhead);- wanted.set(spanKey(span), { file: span.file, from: Math.max(0, span.in), to: span.out });+ const spanKey = (c: { file: string; lo: number; hi: number }) =>+ `${c.file}|${c.lo.toFixed(3)}|${c.hi.toFixed(3)}`;+ for (const { read } of [...folded, ...placed]) {+ if (read) wanted.set(spanKey(read), { file: read.file, from: read.lo, to: read.hi });}const buffers = new Map<string, AudioBuffer>();@@ -320,7 +338,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB// Everything that can be ducked passes through one node; the voiceovers doing// the ducking connect past it, so they never duck themselves.const ducked = ctx.createGain();- scheduleDuck(ducked, duckWindows(spec.items));+ scheduleDuck(ducked, spec.ducks ?? duckWindows(spec.items));// Every sound meets on the bus, which is what the effect elements treat —// the ffmpeg graph applies them to the mixed stream in the same place.@@ -425,18 +443,22 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBsrc.connect(gain);}gain.connect(into);- src.start(Math.max(0, at), 0, Math.max(0, duration));+ // A sound reaching before the timeline's start joins part-way in.+ const skip = Math.max(0, -at);+ src.start(Math.max(0, at), skip, Math.max(0, duration - skip));};for (const g of folded) {const c = g.clip;- const key = spanKey(g.span);- const buf = !c.muted && c.file ? buffers.get(key) : undefined;- if (!buf) continue; // muted, silent, or a gap spacer: only shapes time- const fit = fitted(key, buf, c, Math.max(0, g.span.in), g.span.out);- const from = g.at - g.back + fit.shift;+ const read = g.read;+ const key = read ? spanKey(read) : "";+ const buf = read ? buffers.get(key) : undefined;+ if (!read || !buf) continue; // muted, silent, or a gap spacer: only shapes time+ // Laid through the clip's own map, moved onto the file the sound reads.+ const fit = fitted(key, buf, shiftSpan(c, c.soundFrom?.offset ?? 0), read.lo, read.hi);+ const from = g.at - g.back + read.at + fit.shift;const end = g.at + g.dur;- play(fit.buffer, from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {+ play(fit.buffer, from, read.len, ducked, c.sound, (gain) => {const level = Math.max(0, c.volume ?? 1);gain.gain.value = level;// A crossing and a fade never share an edge — the cut carries one@@ -453,23 +475,29 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBgain.gain.setValueAtTime(level, end - g.fadeOut);gain.gain.linearRampToValueAtTime(0, end);}+ scheduleSplit(+ gain.gain,+ level,+ g.at,+ end,+ g.crossIn > 0 ? 0 : Math.min(c.soundLead ?? 0, g.back),+ g.crossOut > 0 ? 0 : Math.min(c.soundTail ?? 0, g.ahead),+ c.splitFade ?? 0+ );});}- for (const a of spec.items) {- const back = Math.min(a.soundBack ?? 0, a.crossIn ?? 0);- const ahead = Math.min(a.soundAhead ?? 0, a.crossOut ?? 0);- const key = spanKey(crossSpan(a, back, ahead));- const buf = a.muted ? undefined : buffers.get(key);- if (!buf) continue;+ for (const { a, back, ahead, read } of placed) {+ const key = read ? spanKey(read) : "";+ const buf = read ? buffers.get(key) : undefined;+ if (!read || !buf) continue;const dur = itemDur(a);const end = a.start + dur;// A voiceover that ducks everything else is not itself ducked; it still// lands on the bus, so the effect elements treat it with the rest.const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;- const reach = crossSpan(a, back, ahead);- const fit = fitted(key, buf, a, Math.max(0, reach.in), reach.out);- play(fit.buffer, a.start - back + fit.shift, dur + back + ahead, into, a.sound, (gain) => {+ const fit = fitted(key, buf, shiftSpan(a, a.soundFrom?.offset ?? 0), read.lo, read.hi);+ play(fit.buffer, a.start - back + read.at + fit.shift, read.len, into, a.sound, (gain) => {const level = Math.max(0, a.volume);gain.gain.value = level;const fIn = Math.min(a.fadeIn ?? 0, dur);diff --git a/site/src/cut/lib/audioMixSplit.test.ts b/site/src/cut/lib/audioMixSplit.test.tsnew file mode 100644index 00000000..54ad2736--- /dev/null+++ b/site/src/cut/lib/audioMixSplit.test.ts@@ -0,0 +1,233 @@+import { describe, expect, test } from "bun:test";+import { stubModule } from "@/lib/testing/stubModule";+import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";++// Where the offline fold lands a clip's sound when it leaves its picture: a+// split edit reaching past either edge, and a recording bound to the video.+// Each file reads back silence with one click at a source second of its own,+// so where the click lands in the mix says exactly which source second played+// at which timeline second.+const RATE = 44100;++const globals = globalThis as Record<string, unknown>;+globals.AudioBuffer ??= AudioBuffer;+globals.OfflineAudioContext ??= OfflineAudioContext;++/** Each file's click, in its own source seconds, and its length. */+const FILES: Record<string, { click?: number; duration: number }> = {+ "quiet.mp4": { duration: 60 },+ "lead.mp4": { click: 4.8, duration: 60 },+ "tail.mp4": { click: 4.3, duration: 60 },+ "cam.mp4": { click: 12, duration: 60 },+ "rec.wav": { click: 8.5, duration: 60 },+ "late.wav": { click: 1, duration: 60 },+ "layer.mp4": { click: 4.7, duration: 60 },+};++await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {+ decodeAudioSpan: (async (file: string, from: number, to: number) => {+ const f = FILES[file];+ const lo = Math.max(0, from);+ const hi = Math.min(f.duration, to);+ const buf = new AudioBuffer({ length: Math.max(1, Math.round((hi - lo) * RATE)), numberOfChannels: 1, sampleRate: RATE });+ if (f.click !== undefined && f.click >= lo && f.click < hi) {+ const at = Math.round((f.click - lo) * RATE);+ const ch = buf.getChannelData(0);+ for (let i = 0; i < 32 && at + i < ch.length; i++) ch[at + i] = 1;+ }+ return buf;+ }) as never,+});++const { renderMix } = await import("./audioMix");+type Spec = Parameters<typeof renderMix>[0];++const render = async (spec: Spec) => {+ const buffer = await renderMix(spec, { sampleRate: RATE, channels: 1, resolve: (f) => f });+ expect(buffer).not.toBeNull();+ return buffer!.getChannelData(0);+};++/** The timeline second the click starts at, or null when none sounds. */+const clickAt = (data: Float32Array) => {+ for (let i = 0; i < data.length; i++) if (Math.abs(data[i]) > 0.5) return i / RATE;+ return null;+};++const near = (got: number | null, want: number) => {+ expect(got).not.toBeNull();+ expect(Math.abs(got! - want)).toBeLessThan(2 / RATE);+};++describe("split edits in the offline fold", () => {+ test("a J-cut plays the incoming clip's sound ahead of its picture", async () => {+ // b's picture starts at 4 on source 5; its sound leads by half a second,+ // so source 4.8 plays at 3.8 — over a's picture.+ const mix = await render({+ duration: 8,+ clips: [+ { file: "quiet.mp4", in: 0, out: 4, muted: false },+ { file: "lead.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },+ ],+ items: [],+ });+ near(clickAt(mix), 3.8);+ });++ test("the lead's ramp starts it from silence", async () => {+ // A click inside the ramp: 0.01 s into a 0.03 s fade plays at a third.+ FILES["ramp.mp4"] = { click: 4.51, duration: 60 };+ const mix = await render({+ duration: 8,+ clips: [+ { file: "quiet.mp4", in: 0, out: 4, muted: false },+ { file: "ramp.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },+ ],+ items: [],+ });+ const at = Math.round(3.51 * RATE);+ expect(mix[at]).toBeGreaterThan(0.25);+ expect(mix[at]).toBeLessThan(0.42);+ });++ test("an L-cut carries the outgoing clip's sound past its picture", async () => {+ // a's picture ends at 4 on source 4; its sound runs on, so source 4.3+ // plays at 4.3 — under b's picture.+ const mix = await render({+ duration: 8,+ clips: [+ { file: "tail.mp4", in: 0, out: 4, muted: false, soundTail: 0.5, soundAhead: 0.5, splitFade: 0.03 },+ { file: "quiet.mp4", in: 0, out: 4, muted: false },+ ],+ items: [],+ });+ near(clickAt(mix), 4.3);+ });++ test("without a split edit the trimmed-away sound stays silent", async () => {+ const mix = await render({+ duration: 8,+ clips: [+ { file: "tail.mp4", in: 0, out: 4, muted: false },+ { file: "lead.mp4", in: 5, out: 9, muted: false },+ ],+ items: [],+ });+ expect(clickAt(mix)).toBeNull();+ });++ test("a layer clip's lead lands ahead of its start", async () => {+ const mix = await render({+ duration: 8,+ clips: [{ file: "", in: 0, out: 8, muted: true }],+ items: [{ file: "layer.mp4", in: 5, out: 7, start: 2, volume: 1, soundLead: 0.4, soundBack: 0.4, splitFade: 0.03 }],+ });+ near(clickAt(mix), 1.7);+ });+});++describe("a recording bound to the video", () => {+ test("plays the recording through the offset, and the camera's track stays out", async () => {+ // Recording second = video second − 3, so recording 8.5 is video 11.5,+ // which this clip (in 10, at 0) plays at 1.5. The camera's own click at+ // video 12 must not sound.+ const mix = await render({+ duration: 4,+ clips: [{ file: "cam.mp4", in: 10, out: 14, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],+ items: [],+ });+ near(clickAt(mix), 1.5);+ expect(Math.abs(mix[Math.round(2 * RATE) + 4])).toBeLessThan(0.01);+ });++ test("a recording that rolled after the camera leaves silence before it, in time", async () => {+ // The recording starts at video second 2; the clip starts on video+ // second 1. Recording 1 is video 3, played at timeline 2.+ const mix = await render({+ duration: 4,+ clips: [{ file: "cam.mp4", in: 1, out: 5, muted: false, soundFrom: { file: "late.wav", offset: -2, duration: 60 } }],+ items: [],+ });+ near(clickAt(mix), 2);+ });++ test("a speed change runs the recording through the clip's own map", async () => {+ // At 2× the clip covers video 10..14 in two seconds: video 11.5 plays at 0.75.+ const mix = await render({+ duration: 2,+ clips: [{ file: "cam.mp4", in: 10, out: 14, speed: 2, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],+ items: [],+ });+ const got = clickAt(mix);+ expect(got).not.toBeNull();+ // The stretch smears a click over a few milliseconds.+ expect(Math.abs(got! - 0.75)).toBeLessThan(0.03);+ });++ test("a reversed clip plays the recording backward from its out point", async () => {+ // Reversed, timeline 0 shows video 14 and timeline 4 video 10: video+ // 11.5 lands at 2.5.+ const mix = await render({+ duration: 4,+ clips: [{ file: "cam.mp4", in: 10, out: 14, reverse: true, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],+ items: [],+ });+ let peak = 0;+ let at = 0;+ for (let i = 0; i < mix.length; i++) if (Math.abs(mix[i]) > peak) { peak = Math.abs(mix[i]); at = i / RATE; }+ expect(Math.abs(at - 2.5)).toBeLessThan(0.002);+ });+});++describe("stems from the document", () => {+ test("the dialogue stem plays the bound recording and the split edit; a lane keeps its own sound", async () => {+ const { mixSpecFor, stemMixSpec } = await import("./exportRender");+ const media = (id: string, type: "video" | "audio", over: Record<string, unknown> = {}) => ({+ id,+ fileName: id,+ name: id,+ type,+ duration: 60,+ url: "",+ ...over,+ });+ // Video second t plays recording t − 2: the recording's click at 8.5 is+ // video 10.5, which clip a (video 10..12) plays at 0.5. Clip b starts at+ // 2 on source 5 and leads by half a second: source 4.8 plays at 1.8. The+ // soundtrack's click at its own 1.0 sits on lane 0.+ const doc = {+ aspect: "16:9",+ assets: [+ media("cam.mp4", "video", { soundFrom: { assetId: "rec.wav", offset: -2 } }),+ media("rec.wav", "audio"),+ media("lead.mp4", "video"),+ media("late.wav", "audio"),+ ],+ clips: [+ { id: "a", assetId: "cam.mp4", start: 0, in: 10, out: 12, track: 0, muted: false },+ { id: "b", assetId: "lead.mp4", start: 2, in: 5, out: 9, track: 0, muted: false, audioLead: 0.5 },+ ],+ audioClips: [{ id: "s", assetId: "late.wav", start: 0, in: 0, out: 4, volume: 1, lane: 0 }],+ overlays: [],+ subtitles: { cues: [] },+ } as unknown as Parameters<typeof mixSpecFor>[0];+ const spec = mixSpecFor(doc, (a) => a.fileName);+ const clicks = (data: Float32Array) => {+ const at: number[] = [];+ let quietSince = -RATE;+ for (let i = 0; i < data.length; i++) {+ if (Math.abs(data[i]) <= 0.5) continue;+ if (i - quietSince > RATE * 0.1) at.push(i / RATE);+ quietSince = i;+ }+ return at;+ };+ const dialogue = clicks(await render(stemMixSpec(spec, { name: "Dialogue", file: "1 Dialogue.wav", lane: null })));+ expect(dialogue).toHaveLength(2);+ near(dialogue[0], 0.5);+ near(dialogue[1], 1.8);+ const lane = clicks(await render(stemMixSpec(spec, { name: "late", file: "2 late.wav", lane: 0 })));+ expect(lane).toHaveLength(1);+ near(lane[0], 1);+ });+});diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex 34dcabd6..576056de 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -32,10 +32,12 @@ import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paimport { renderElementFrames, renderElementPng } from "./textRender";import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";import { liveReader } from "./liveReader";+import { specSound } from "./soundSource";import type {Aspect,AudioClip,ClipAnim,+ ClipSpan,MediaAsset,Overlay,Selection,@@ -785,6 +787,19 @@ function renderClipBorderPng(return rasterCanvasToPng(canvas);}+/** A clip's sound as the spec carries it: the recording bound to its video,+ * and a split edit's reach past its picture with the ramp at the far end. */+function specSpanSound(sp: ClipSpan) {+ return {+ ...(sp.sound+ ? { soundFrom: { file: sp.sound.asset.fileName, offset: sp.sound.offset, duration: sp.sound.asset.duration } }+ : {}),+ ...(sp.soundLead || sp.soundTail+ ? { soundLead: sp.soundLead ?? 0, soundTail: sp.soundTail ?? 0, splitFade: sp.splitFade ?? 0 }+ : {}),+ };+}+/** What a source's code values mean, for the spec: the profile the header* settled (with the person's override), and the matrix and range the file* decodes with. A still is sRGB, decoded to full-range RGB, and a block is@@ -880,6 +895,7 @@ export async function buildExportPayload(: undefined;const clipEntries = spans.map((sp) => ({+ ...specSpanSound(sp),file: sp.asset.fileName,in: sp.clip.in,out: sp.clip.out,@@ -1094,10 +1110,11 @@ export async function buildExportPayload(r.soundBack = sp.soundBack;});return trackSpans- .map((sp, i) => ({ c: sp.clip, ramp: ramps[i] }))+ .map((sp, i) => ({ c: sp.clip, sp, ramp: ramps[i] })).filter(({ c }) => !c.hidden && c.start < duration)- .map(({ c, ramp }) => {+ .map(({ c, sp, ramp }) => {const entry = {+ ...specSpanSound(sp),file: assetById.get(c.assetId)!.fileName,in: c.in,out: c.out,@@ -1226,6 +1243,7 @@ export async function buildExportPayload(const audio = doc.audioClips.filter((a) => !a.hidden && a.start < duration && assetById.has(a.assetId)).map((a) => ({+ soundFrom: specSound(assetById.get(a.assetId)!, assetById, (x) => x.fileName),file: assetById.get(a.assetId)!.fileName,in: a.in,out: a.out,diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.tsindex 2a81ca14..382e517b 100644--- a/site/src/cut/lib/exportDelivery.ts+++ b/site/src/cut/lib/exportDelivery.ts@@ -161,9 +161,9 @@ export function videoBitrateFor(s: {/** Every media file a render spec plays — clips, picture-in-picture video,* sound — by name, once each. */export function specMediaFiles(spec: {- clips?: { file?: string; staged?: boolean }[];- overlayVideos?: { file?: string }[];- audio?: { file?: string }[];+ clips?: { file?: string; staged?: boolean; soundFrom?: { file?: string } }[];+ overlayVideos?: { file?: string; soundFrom?: { file?: string } }[];+ audio?: { file?: string; soundFrom?: { file?: string } }[];}): string[] {const names = new Set<string>();for (const c of [...(spec.clips ?? []), ...(spec.overlayVideos ?? []), ...(spec.audio ?? [])]) {@@ -172,6 +172,8 @@ export function specMediaFiles(spec: {// project's folder finds nothing, and staging it there puts it where the// render does not read.if (c?.file && !("staged" in c && c.staged)) names.add(c.file);+ // A recording bound to the video is read beside it.+ if (c?.soundFrom?.file) names.add(c.soundFrom.file);}return [...names];}diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex f3eb1108..d80f91d4 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -859,6 +859,14 @@ function trackZeroFades(spans: ClipSpan[], i: number): { fadeIn: number; fadeOut* while the picture keeps them, and everything after the first gap plays early.*/export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => string): MixSpec {+ // A clip's sound past its picture (a split edit) and the recording bound to+ // its video, as the fold reads them.+ const spanSound = (sp: ClipSpan) => ({+ ...(sp.sound ? { soundFrom: { file: resolve(sp.sound.asset), offset: sp.sound.offset, duration: sp.sound.asset.duration } } : {}),+ ...(sp.soundLead || sp.soundTail+ ? { soundLead: sp.soundLead, soundTail: sp.soundTail, splitFade: sp.splitFade }+ : {}),+ });const duration = projectDuration(doc);const spans = getClipSpans(doc.clips, doc.assets, 0);const byId = new Map(doc.assets.map((a) => [a.id, a]));@@ -878,20 +886,24 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinspeedCurve: sp.clip.speedCurve,reverse: sp.clip.reverse,sound: sp.clip.sound,- muted: sp.clip.muted || assetIsSilent(sp.asset),+ muted: sp.clip.muted || (!sp.sound && assetIsSilent(sp.asset)),fadeIn: ramps[i].head,fadeOut: ramps[i].tail,crossIn: ramps[i].crossIn,crossOut: ramps[i].crossOut,soundBack: sp.soundBack,soundAhead: sp.soundAhead,+ ...spanSound(sp),});});}for (const a of doc.audioClips) {const asset = byId.get(a.assetId);- if (!asset || a.hidden || a.start >= duration || assetIsSilent(asset)) continue;+ if (!asset || a.hidden || a.start >= duration) continue;+ const soundFrom = specSound(asset, byId, resolve);+ if (!soundFrom && assetIsSilent(asset)) continue;items.push({+ ...(soundFrom ? { soundFrom } : {}),file: resolve(asset),in: a.in,out: a.out,diff --git a/site/src/cut/lib/framePlan.ts b/site/src/cut/lib/framePlan.tsindex 322cc7f6..803bc7cd 100644--- a/site/src/cut/lib/framePlan.ts+++ b/site/src/cut/lib/framePlan.ts@@ -266,11 +266,32 @@ export function crossHandles(spans: ClipSpan[], t: number): { span: ClipSpan; gaconst from = sp.start - sp.soundBack;const to = sp.start + sp.len + sp.soundAhead;if (t < from || t >= to) continue;- out.push({ span: sp, gain: soundCrossGain(spans, sp, t) });+ out.push({ span: sp, gain: soundCrossGain(spans, sp, t) * splitEditGain(sp, t) });}return out;}+/**+ * The gain a split edit puts on a clip's sound at `t`: full inside the+ * picture, a short ramp up from silence where a J-cut's lead opens and down to+ * silence where an L-cut's tail closes. The offline fold and the ffmpeg graph+ * write the same ramps.+ */+export function splitEditGain(sp: ClipSpan, t: number): number {+ const lead = sp.soundLead ?? 0;+ const tail = sp.soundTail ?? 0;+ const end = sp.start + sp.len;+ if (lead > 0 && t < sp.start) {+ const f = Math.min(sp.splitFade ?? 0, lead);+ return f > 0 ? Math.max(0, Math.min(1, (t - (sp.start - lead)) / f)) : 1;+ }+ if (tail > 0 && t >= end) {+ const f = Math.min(sp.splitFade ?? 0, tail);+ return f > 0 ? Math.max(0, Math.min(1, (end + tail - t) / f)) : 1;+ }+ return 1;+}+/** How much of a clip's animation window is still running at `rel`, for the* sides that show a neighbour's frame behind them. */function animLive(a: ClipAnim | undefined, side: "in" | "out", rel: number, len: number) {diff --git a/site/src/cut/lib/itemKinds.ts b/site/src/cut/lib/itemKinds.tsindex 35e433ce..b7661794 100644--- a/site/src/cut/lib/itemKinds.ts+++ b/site/src/cut/lib/itemKinds.ts@@ -96,11 +96,16 @@ export const ITEM_KINDS: { [K in ItemKind]: ItemKindDef<K> } = {at: (c, start) => ({ ...c, start }),split: (c, at) => {const [left, right] = splitMedia(c, at);- return [{ ...left, transition: undefined, transitionStyle: undefined, animOut: undefined }, { ...right, animIn: undefined }];+ // A split edit stays on the outer edges: the new cut between the halves+ // carries none.+ return [+ { ...left, transition: undefined, transitionStyle: undefined, animOut: undefined, audioTail: undefined },+ { ...right, animIn: undefined, audioLead: undefined },+ ];},clone: deep,assetIds: (c) =>- [c.assetId, c.removal?.matte?.assetId, c.removal?.backdrop?.assetId].filter(+ [c.assetId, c.removal?.matte?.assetId, c.removal?.backdrop?.assetId, c.card?.matte?.assetId].filter((id): id is string => typeof id === "string" && id.length > 0),remapAssets: (c, to) => ({diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 11535b91..64580ff0 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1,6 +1,8 @@"use client";import { scanBeats, scanLevel, scanSilence, scanSpeech, type BeatScan, type LevelScan, type PcmChunk, type SpeechScan } from "./audioScan";+import { syncSound, type SyncAnswer } from "./soundSync";+import { LoudnessMeter, type LoudnessMeasure } from "./loudness";import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";import { registerBlobFile } from "./backend/browser/registry";import { cloudBackend } from "./backend/cloud";diff --git a/site/src/cut/lib/projectReference.ts b/site/src/cut/lib/projectReference.tsindex feadfb3b..3445feb7 100644--- a/site/src/cut/lib/projectReference.ts+++ b/site/src/cut/lib/projectReference.ts@@ -160,7 +160,14 @@ export async function landReferenceAssets(): Promise<LandedAsset[]> {const out: LandedAsset[] = [];const toCopy: StoredAsset[] = [];+ // A video bound to a separate recording brings the recording with it.+ const refAssets = new Map(ref.doc.assets.map((a) => [a.id, a]));+ const wanted = [...sources];for (const source of sources) {+ const rec = source.soundFrom && refAssets.get(source.soundFrom.assetId);+ if (rec && !wanted.some((w) => w.id === rec.id)) wanted.push(rec);+ }+ for (const source of wanted) {const have = useEditor.getState().assets.find((a) => a.copiedFrom?.projectId === ref.projectId && a.copiedFrom.assetId === source.id);@@ -194,7 +201,15 @@ export async function landReferenceAssets(else if (!isLinkedAssetType(asset.type)) void enrichAsset(asset).catch(() => {});out.push({ sourceId: source.id, asset, reused: false });}- // In the order asked for, so a caller's index arithmetic holds.+ // A copied video's binding points at the copy of its recording.const bySource = new Map(out.map((l) => [l.sourceId, l]));+ for (const l of out) {+ const from = l.asset.soundFrom;+ if (l.reused || !from) continue;+ const rec = bySource.get(from.assetId)?.asset.id;+ useEditor.getState().updateAsset(l.asset.id, { soundFrom: rec ? { ...from, assetId: rec } : undefined });+ l.asset = useEditor.getState().assets.find((a) => a.id === l.asset.id) ?? l.asset;+ }+ // In the order asked for, so a caller's index arithmetic holds.return sources.map((s) => bySource.get(s.id)!);}diff --git a/site/src/cut/lib/projectTemplate.test.ts b/site/src/cut/lib/projectTemplate.test.tsindex ea4e9d1b..9bcf4b61 100644--- a/site/src/cut/lib/projectTemplate.test.ts+++ b/site/src/cut/lib/projectTemplate.test.ts@@ -132,6 +132,25 @@ describe("insertTemplate", () => {const MAP: Record<string, string> = { v0: "b0", v1: "b1", m0: "bm", s0: "bs", f0: "" };const idsFor = (assetByMedia: string[], map = MAP) => assetByMedia.map((id) => map[id] ?? "");+ test("a video's bound recording travels with it and binds to the landed copy", () => {+ const doc = richDoc();+ const withRec = {+ ...doc,+ assets: [+ ...(doc.assets ?? []).map((a) => (a.id === "v0" ? { ...a, soundFrom: { assetId: "r0", offset: 0.25 } } : a)),+ { id: "r0", fileName: "lav.wav", name: "lav", type: "audio" as const, duration: 60 },+ ],+ };+ const { template, assetByMedia } = templateFromDoc(withRec);+ const v0 = assetByMedia.indexOf("v0");+ const r0 = assetByMedia.indexOf("r0");+ expect(r0).toBeGreaterThanOrEqual(0);+ expect(template.media[v0].soundFrom).toEqual({ media: r0, offset: 0.25 });+ useEditor.setState({ assets: [...useEditor.getState().assets, asset("br", "audio", 60)] });+ useEditor.getState().insertTemplate({ ...template, id: "t", addedAt: 0 }, idsFor(assetByMedia, { ...MAP, r0: "br" }), 0);+ expect(useEditor.getState().assets.find((a) => a.id === "b0")?.soundFrom).toEqual({ assetId: "br", offset: 0.25 });+ });+test("stands the whole edit up on other footage, treatment and all", () => {const { template, assetByMedia } = templateFromDoc(richDoc());useEditor.getState().insertTemplate({ ...template, id: "t", addedAt: 0 }, idsFor(assetByMedia), 0);diff --git a/site/src/cut/lib/projectTemplate.ts b/site/src/cut/lib/projectTemplate.tsindex 20594dfb..9181f182 100644--- a/site/src/cut/lib/projectTemplate.ts+++ b/site/src/cut/lib/projectTemplate.ts@@ -89,6 +89,11 @@ export function templateFromDoc(doc: ProjectDoc, opts: { items?: string[] } = {}assetByMedia.push(a.id);sourceAssets.push(a);indexByAsset.set(assetId, i);+ // The recording a video plays rides along with it.+ if (a.soundFrom) {+ const rec = mediaFor(a.soundFrom.assetId);+ if (rec != null) media[i] = { ...media[i], soundFrom: { media: rec, offset: a.soundFrom.offset } };+ }return i;};diff --git a/site/src/cut/lib/soundBind.ts b/site/src/cut/lib/soundBind.tsnew file mode 100644index 00000000..aec9727a--- /dev/null+++ b/site/src/cut/lib/soundBind.ts@@ -0,0 +1,88 @@+/**+ * Binding a separately recorded sound to a video: find where the recording+ * lines up with the camera's own track, and only when one alignment clearly+ * wins, write it onto the video asset (`soundFrom`). The Audio tab's Sound+ * picker and the assistant's sync_audio both land here, so the person and the+ * assistant hold the recording to the same bar.+ */++import { syncSourceSound } from "./media";+import { cutSound } from "./soundSettings";+import { useEditor } from "./store";+import { assetIsSilent, type MediaAsset } from "./types";+import { queueWatchSweep } from "./watch/sweep";++export interface BoundSound {+ assetId: string;+ audioAssetId: string;+ /** Recording seconds = video seconds + offset. */+ offset: number;+ /** The best alignment over the runner-up a second or more away. */+ confidence: number;+ /** Whether the offset sits on the sample (else on 20 ms). */+ refined: boolean;+ /** Clips of the video on the timeline, all now sounding from the recording. */+ clips: number;+}++/**+ * Line `audioId` up with `videoId` and bind it. `near` is the source span of+ * the clip the person is working on, so the camera's stretch that is matched+ * is the one they care about. Throws a readable Error when the files cannot+ * be matched or no alignment stands out; nothing is bound then.+ */+export async function bindRecording(+ videoId: string,+ audioId: string,+ near?: { in: number; out: number }+): Promise<BoundSound> {+ const s = useEditor.getState();+ const video = s.assets.find((a) => a.id === videoId);+ const rec = s.assets.find((a) => a.id === audioId);+ if (!video || video.type !== "video") throw new Error("That video is no longer in the project.");+ if (!rec) throw new Error("That recording is no longer in the project.");+ if (rec.type !== "audio") throw new Error(`"${rec.name}" is not an audio file.`);+ if (assetIsSilent(video))+ throw new Error(`"${video.name}" has no sound of its own to line the recording up against.`);+ if (!(video.duration > 0) || !(rec.duration > 0)) throw new Error("Both files must finish importing first.");+ const { syncProbeSeconds, syncMaxSeconds, syncMinConfidence } = cutSound();+ const [cameraFrom, cameraTo] = probeWindow(video, syncProbeSeconds, near);+ const answer = await syncSourceSound(video.url, rec.url, {+ cameraFrom,+ cameraTo,+ recordingTo: Math.min(rec.duration, syncMaxSeconds),+ });+ if (!answer || answer.confidence < syncMinConfidence) {+ const read = answer ? ` (confidence ${answer.confidence.toFixed(2)}, needs ${syncMinConfidence})` : "";+ throw new Error(+ `No clear alignment between "${rec.name}" and "${video.name}"${read}. They may be different takes, or share too little sound.`+ );+ }+ const offset = Math.round(answer.offset * 1e5) / 1e5;+ useEditor.getState().setAssetSoundFrom(video.id, { assetId: rec.id, offset });+ // Its transcript is heard again, off the recording.+ queueWatchSweep(video.id);+ return {+ assetId: video.id,+ audioAssetId: rec.id,+ offset,+ confidence: Math.round(answer.confidence * 100) / 100,+ refined: answer.refined,+ clips: useEditor.getState().clips.filter((c) => c.assetId === video.id).length,+ };+}++/** Return a video to its own sound. */+export function unbindRecording(videoId: string): void {+ useEditor.getState().setAssetSoundFrom(videoId, undefined);+ queueWatchSweep(videoId);+}++/** The camera's stretch to match: `seconds` long, centred on the clip being+ * worked on, else on the middle of the video, inside the file. */+function probeWindow(video: MediaAsset, seconds: number, near?: { in: number; out: number }): [number, number] {+ const len = Math.min(seconds, video.duration);+ const mid = near ? (near.in + near.out) / 2 : video.duration / 2;+ const from = Math.max(0, Math.min(video.duration - len, mid - len / 2));+ return [from, from + len];+}diff --git a/site/src/cut/lib/soundSettings.ts b/site/src/cut/lib/soundSettings.tsnew file mode 100644index 00000000..0b85cd45--- /dev/null+++ b/site/src/cut/lib/soundSettings.ts@@ -0,0 +1,17 @@+import { SETTINGS, type Settings } from "@/lib/config/registry";++// The sound editing tunables — the split edit's ramp and how a separate+// recording is lined up — bound once at session start from account config (the+// tab) or the global setting (a worker job), and read where a clip's sound is+// laid out.++let sound: Settings["cutSound"] = SETTINGS.cutSound.default;++export function bindCutSound(value: unknown): void {+ const parsed = SETTINGS.cutSound.schema.safeParse(value);+ if (parsed.success) sound = parsed.data;+}++export function cutSound(): Settings["cutSound"] {+ return sound;+}diff --git a/site/src/cut/lib/soundSource.ts b/site/src/cut/lib/soundSource.tsnew file mode 100644index 00000000..ee7945b5--- /dev/null+++ b/site/src/cut/lib/soundSource.ts@@ -0,0 +1,143 @@+/**+ * Where a clip's sound comes from.+ *+ * A camera records the picture and a scratch track beside it; a recorder on+ * the speaker records the sound worth keeping. Binding the recording to the+ * video asset (`soundFrom`) puts every clip of that video on the recording:+ * the clip keeps its trim, rate and direction, and each source second of the+ * video is read off the recording `offset` seconds later. The binding sits on+ * the asset, so a split, a trim, a retime, a reverse+ * or a copy cannot leave the sound behind. Every reader of a clip's sound —+ * the preview, the fold the tab exports and transcribes through, the spec the+ * engine and the worker render, the tools that listen — resolves it here.+ */++import { retimeOf, speedCurveOf, srcSpan, type Retimable, type SpeedNode } from "@donkeycut/effects-kit";+import { assetIsSilent, type StoredAsset } from "./types";++type Sourced = Pick<StoredAsset, "id" | "type" | "duration" | "soundFrom">;++/** The longest a split edit carries a clip's sound past its picture, either+ * side, in seconds. */+export const SPLIT_EDIT_MAX_S = 5;++/** The sound a video plays: its own file's track, or the recording bound to+ * it, with the seconds added to its source time to land on that file. */+export interface SoundSource<A> {+ asset: A;+ offset: number;+ /** The last second the sound file holds, for a bound recording; the+ * asset's own track is read as far as it goes. */+ limit: number;+ bound: boolean;+}++/** `asset`'s sound. A binding to a recording that is gone, silent, or the+ * video itself plays the camera's own track. One hop: a recording's own+ * binding is never followed. */+export function soundSourceOf<A extends Sourced>(+ asset: A,+ assets: readonly A[] | ReadonlyMap<string, A>+): SoundSource<A> {+ const from = asset.soundFrom;+ if (from && from.assetId !== asset.id && Number.isFinite(from.offset)) {+ const rec = isMap(assets) ? assets.get(from.assetId) : assets.find((a) => a.id === from.assetId);+ if (rec && !assetIsSilent(rec))+ return { asset: rec, offset: from.offset, limit: rec.duration > 0 ? rec.duration : Infinity, bound: true };+ }+ return { asset, offset: 0, limit: Infinity, bound: false };+}++const isMap = <A>(v: readonly A[] | ReadonlyMap<string, A>): v is ReadonlyMap<string, A> => v instanceof Map;++/** Source seconds of the sound file on either side of a clip's trim: what a+ * handle, a split edit or a crossing can reach into before its head and past+ * its tail. A reversed clip's head faces the source's end. */+export function soundRoom(+ c: { in: number; out: number; reverse?: boolean },+ src: { asset: { duration: number }; offset: number }+): { head: number; tail: number } {+ const lo = c.in + src.offset;+ const hi = c.out + src.offset;+ const end = src.asset.duration;+ return c.reverse ? { head: end - hi, tail: lo } : { head: lo, tail: end - hi };+}++/** A span moved onto its sound's clock: the same map from the timeline,+ * `offset` seconds later in the source. Speed curve nodes sit in source+ * seconds, so they move with it. */+export function shiftSpan<T extends Retimable>(c: T, offset: number): T {+ if (!offset) return c;+ const nodes = speedCurveOf(c);+ return {+ ...c,+ in: c.in + offset,+ out: c.out + offset,+ ...(nodes ? { speedCurve: nodes.map(([at, rate]): SpeedNode => [at + offset, rate]) } : {}),+ };+}++/** What one stretch of a span's sound reads from its file. */+export interface SoundWindow {+ /** Seconds on the sound file, low end first. */+ lo: number;+ hi: number;+ /** Timeline seconds into the window the sound begins: a recording that+ * starts after the window does leaves silence ahead of it. */+ at: number;+ /** Timeline seconds the sound runs. */+ len: number;+ /** The stretch as a span of its own on the sound file — the map a renderer+ * lays it through. */+ span: Retimable;+}++/**+ * The sound of span `c` over the window `[fromT, toT]`, timeline seconds from+ * its head (negative reaches before it, past its length beyond it), read from+ * the source `src` describes. The window narrows to what the file holds: a+ * recording rolled after the camera did, or stopped before it, leaves the+ * rest of the window silent. Null when nothing of the window is on the file.+ */+export function soundWindow(+ c: Retimable,+ src: { offset: number; limit: number },+ fromT: number,+ toT: number+): SoundWindow | null {+ const moved = shiftSpan(c, src.offset);+ const rt = retimeOf(moved);+ const want = srcSpan(rt, fromT, toT);+ const lo = Math.max(0, want.lo);+ const hi = Math.min(src.limit, want.hi);+ if (!(hi - lo > 1e-4)) return null;+ const t0 = rt.tAt(rt.reverse ? hi : lo);+ const t1 = rt.tAt(rt.reverse ? lo : hi);+ return {+ lo,+ hi,+ at: Math.max(0, t0 - fromT),+ len: Math.max(0, t1 - t0),+ span: { in: lo, out: hi, speed: moved.speed, speedCurve: moved.speedCurve, reverse: moved.reverse },+ };+}++/** A bound recording as a render spec carries it: the file the sound reads,+ * its clock's offset from the video's, and its length. */+export interface SpecSound {+ file: string;+ offset: number;+ duration: number;+}++/** The spec entry for `asset`'s bound recording, named the way the spec+ * names files; absent when the video plays its own track. */+export function specSound<A extends Sourced>(+ asset: A,+ assets: readonly A[] | ReadonlyMap<string, A>,+ fileOf: (a: A) => string+): SpecSound | undefined {+ const src = soundSourceOf(asset, assets);+ if (!src.bound) return undefined;+ return { file: fileOf(src.asset), offset: src.offset, duration: src.asset.duration };+}diff --git a/site/src/cut/lib/soundSync.test.ts b/site/src/cut/lib/soundSync.test.tsnew file mode 100644index 00000000..bc91e804--- /dev/null+++ b/site/src/cut/lib/soundSync.test.ts@@ -0,0 +1,197 @@+import { describe, expect, test } from "bun:test";+import {+ coarseSync,+ ENV_HOP_S,+ EnvelopeBuilder,+ envelopeOf,+ MonoResampler,+ SYNC_RATE,+ syncSignals,+ syncSound,+ type SyncChunk,+} from "./soundSync";++/** A seeded generator, so a failure replays. */+function rng(seed: number) {+ let s = seed >>> 0;+ return () => {+ s = (s * 1664525 + 1013904223) >>> 0;+ return s / 2 ** 32;+ };+}++/** Something with the shape of speech: bursts of tone and noise at random+ * levels, separated by pauses of random length. */+function speechLike(seconds: number, seed: number): Float32Array {+ const r = rng(seed);+ const out = new Float32Array(Math.round(seconds * SYNC_RATE));+ let i = 0;+ while (i < out.length) {+ const burst = Math.round((0.08 + r() * 0.3) * SYNC_RATE);+ const level = 0.1 + r() * 0.8;+ const pitch = 90 + r() * 200;+ for (let k = 0; k < burst && i < out.length; k++, i++) {+ const env = Math.sin((Math.PI * k) / burst);+ out[i] = level * env * (0.6 * Math.sin((2 * Math.PI * pitch * i) / SYNC_RATE) + 0.4 * (r() * 2 - 1));+ }+ i += Math.round((0.03 + r() * 0.4) * SYNC_RATE);+ }+ return out;+}++/** The same room heard by another device: its own gain, its own hiss, and a+ * polarity of its choosing. */+function heardBy(signal: Float32Array, gain: number, noise: number, seed: number, flip = false): Float32Array {+ const r = rng(seed);+ const out = new Float32Array(signal.length);+ for (let i = 0; i < signal.length; i++) out[i] = (flip ? -1 : 1) * gain * signal[i] + noise * (r() * 2 - 1);+ return out;+}++describe("syncSignals", () => {+ const room = speechLike(70, 7);++ test("finds where the camera's sound sits in the recording, to the sample", () => {+ // The recorder rolled 12.345 s before the camera.+ const startAt = Math.round(12.345 * SYNC_RATE);+ const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.01, 1);+ const recording = heardBy(room, 1, 0.002, 2);+ const answer = syncSignals(camera, 0, recording, 0)!;+ expect(answer).not.toBeNull();+ expect(Math.abs(answer.offset - 12.345)).toBeLessThan(1.5 / SYNC_RATE);+ expect(answer.refined).toBe(true);+ expect(answer.confidence).toBeGreaterThan(2);+ });++ test("a recording that started after the camera gives a negative offset", () => {+ // Camera time 0 is recording time -4.5: the recorder rolled late.+ const camera = heardBy(room.subarray(0, 40 * SYNC_RATE), 0.5, 0.01, 3);+ const late = Math.round(4.5 * SYNC_RATE);+ const recording = heardBy(room.subarray(late), 1, 0.002, 4);+ const answer = syncSignals(camera, 0, recording, 0)!;+ expect(Math.abs(answer.offset - -4.5)).toBeLessThan(1.5 / SYNC_RATE);+ });++ test("source times carry through: a camera window late in its file, a recording window late in its", () => {+ const startAt = Math.round(20 * SYNC_RATE);+ const camera = heardBy(room.subarray(startAt, startAt + 20 * SYNC_RATE), 0.4, 0.01, 5);+ const recording = heardBy(room.subarray(5 * SYNC_RATE), 1, 0.002, 6);+ // The camera window starts at camera second 100; the recording window at+ // recording second 5 of its file, which is room second 5. Camera second+ // 100 is room second 20, so recording = camera − 80.+ const answer = syncSignals(camera, 100, recording, 5)!;+ expect(Math.abs(answer.offset - -80)).toBeLessThan(1.5 / SYNC_RATE);+ });++ test("an inverted microphone still lines up", () => {+ const startAt = Math.round(7.25 * SYNC_RATE);+ const camera = heardBy(room.subarray(startAt, startAt + 25 * SYNC_RATE), 0.2, 0.005, 7, true);+ const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 8), 0)!;+ expect(Math.abs(answer.offset - 7.25)).toBeLessThan(1.5 / SYNC_RATE);+ });++ test("survives a noisy camera track", () => {+ const startAt = Math.round(30.01 * SYNC_RATE);+ const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.08, 9);+ const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 10), 0)!;+ // Within a frame at 60 fps even if the fine pass gives up.+ expect(Math.abs(answer.offset - 30.01)).toBeLessThan(0.017);+ expect(answer.confidence).toBeGreaterThan(1.4);+ });++ test("two unrelated recordings have no clear peak", () => {+ const camera = speechLike(30, 11);+ const recording = speechLike(70, 12);+ const answer = syncSignals(camera, 0, recording, 0);+ expect(answer === null || answer.confidence < 1.4).toBe(true);+ });++ test("digital silence on either side has nothing to agree on", () => {+ const answer = syncSignals(new Float32Array(20 * SYNC_RATE), 0, room, 0);+ expect(answer === null || answer.confidence < 1.4).toBe(true);+ });+});++describe("coarseSync", () => {+ test("the lag is in envelope hops", () => {+ const room = speechLike(40, 21);+ const startAt = Math.round(10 * SYNC_RATE);+ const env = envelopeOf(room);+ const cam = envelopeOf(room.subarray(startAt, startAt + 15 * SYNC_RATE));+ const c = coarseSync(cam, env)!;+ expect(c.lag * ENV_HOP_S).toBeCloseTo(10, 1);+ });+});++describe("streaming pieces", () => {+ test("the resampler folds 48 kHz stereo to 16 kHz mono at the right length", () => {+ const r = new MonoResampler(1);+ const left = new Float32Array(48000).fill(0.5);+ const right = new Float32Array(48000).fill(-0.1);+ r.push([left.subarray(0, 20000), right.subarray(0, 20000)], 48000);+ r.push([left.subarray(20000), right.subarray(20000)], 48000);+ expect(Math.abs(r.length - 16000)).toBeLessThanOrEqual(1);+ expect(r.samples()[100]).toBeCloseTo(0.2, 5);+ });++ test("the resampler keeps 44.1 kHz timing", () => {+ const r = new MonoResampler(2);+ r.push([new Float32Array(88200)], 44100);+ expect(Math.abs(r.length - 32000)).toBeLessThanOrEqual(1);+ });++ test("an envelope built in pieces equals one built whole", () => {+ const room = speechLike(5, 31);+ const b = new EnvelopeBuilder(1);+ for (let i = 0; i < room.length; i += 777) b.push(room.subarray(i, i + 777));+ expect([...b.envelope()]).toEqual([...envelopeOf(room)]);+ });+});++describe("syncSound", () => {+ /** A file read the way the decoders hand it over: 48 kHz stereo, in chunks+ * that start wherever the read was asked to. */+ const fileOf = (signal: Float32Array) => {+ let reads = 0;+ async function* read(from: number, to: number): AsyncGenerator<SyncChunk> {+ reads++;+ const lo = Math.max(0, Math.round(from * SYNC_RATE));+ const hi = Math.min(signal.length, Math.round(to * SYNC_RATE));+ for (let i = lo; i < hi; i += 1500) {+ const n = Math.min(1500, hi - i);+ const ch = new Float32Array(n * 3);+ for (let k = 0; k < n * 3; k++) ch[k] = signal[i + Math.floor(k / 3)];+ yield { channels: [ch, ch], timestamp: i / SYNC_RATE, sampleRate: SYNC_RATE * 3 };+ }+ }+ return { read, reads: () => reads };+ };++ test("streams the recording and lands the offset on the sample", async () => {+ const room = speechLike(80, 41);+ // The recorder rolled 7.25 s before the camera; the camera's file is the+ // room from 7.25 on.+ const startAt = Math.round(7.25 * SYNC_RATE);+ const camera = fileOf(heardBy(room.subarray(startAt), 0.4, 0.01, 42));+ const recording = fileOf(heardBy(room, 1, 0.002, 43));+ const answer = (await syncSound(+ { camera: camera.read, recording: recording.read },+ { cameraFrom: 10, cameraTo: 40, recordingTo: 80 }+ ))!;+ expect(answer).not.toBeNull();+ expect(Math.abs(answer.offset - 7.25)).toBeLessThan(1.5 / SYNC_RATE);+ expect(answer.refined).toBe(true);+ // The whole recording once for its envelope, then the fine window.+ expect(recording.reads()).toBe(2);+ });++ test("an unrelated recording is not confident", async () => {+ const camera = fileOf(speechLike(40, 51));+ const recording = fileOf(speechLike(80, 52));+ const answer = await syncSound(+ { camera: camera.read, recording: recording.read },+ { cameraFrom: 0, cameraTo: 30, recordingTo: 80 }+ );+ expect(answer === null || answer.confidence < 1.4).toBe(true);+ });+});diff --git a/site/src/cut/lib/soundSync.ts b/site/src/cut/lib/soundSync.tsnew file mode 100644index 00000000..a9b07a39--- /dev/null+++ b/site/src/cut/lib/soundSync.ts@@ -0,0 +1,452 @@+/**+ * Lining up a separately recorded sound with a camera's own.+ *+ * Both devices heard the same room, so the camera's scratch track and the+ * clean recording carry the same events — words, claps, a door — at times+ * that differ by one constant: when each device started rolling. That+ * constant is where the two signals agree best, which is a cross-correlation.+ *+ * Correlating raw waveforms over a long recording costs too much and trips on+ * what the two microphones disagree about (distance, polarity, the room), so+ * the search runs in two passes. The coarse pass correlates loudness+ * envelopes — log energy per 20 ms, with the slow level drift taken out —+ * which survive any difference in gain or tone and turn an hour of sound into+ * a few hundred thousand numbers. The fine pass correlates the waveforms+ * themselves at 16 kHz, a few seconds of each, within a hair of where the+ * coarse pass landed, to put the offset on the sample.+ *+ * How sure the answer is reads off the coarse pass: the best agreement over+ * the runner-up a second or more away from it. A recording of a different+ * take, or two tracks with nothing loud in common, has no clear winner, and+ * the caller refuses it.+ *+ * Pure: arrays in, numbers out. The decoding happens where the file lives.+ */++/** The rate both signals are compared at. */+export const SYNC_RATE = 16000;+/** Samples per envelope hop: 20 ms. */+export const ENV_HOP = 320;+export const ENV_HOP_S = ENV_HOP / SYNC_RATE;+/** Hops the slow level drift is measured over, taken out of the envelope. */+const DETREND_HOPS = 50;+/** A runner-up closer than this to the best lag is the same peak's shoulder. */+const PEAK_EXCLUDE_S = 1;+/** The least of either signal that has to overlap at a lag for it to count. */+const MIN_OVERLAP_SHARE = 0.5;+/** How far the fine pass searches either side of the coarse answer. */+export const FINE_REACH_S = 0.06;+/** How much of the camera's sound the fine pass compares. */+export const FINE_SECONDS = 4;+/** Confidence reported when the runner-up does not agree at all. */+const CONFIDENCE_CAP = 99;++/**+ * Folds decoded chunks of any rate and channel count into one mono signal at+ * `SYNC_RATE`: channels averaged, then each output sample the mean of the+ * input samples it covers — a box filter, enough to keep the fold from+ * aliasing what the correlation compares.+ */+export class MonoResampler {+ private out: Float32Array;+ private n = 0;+ private sum = 0;+ private count = 0;+ /** Input position, in output samples. */+ private pos = 0;++ constructor(expectedSeconds = 60) {+ this.out = new Float32Array(Math.max(1024, Math.ceil(expectedSeconds * SYNC_RATE)));+ }++ push(channels: Float32Array[], sampleRate: number): void {+ const step = SYNC_RATE / sampleRate;+ const len = channels[0]?.length ?? 0;+ const k = channels.length;+ for (let i = 0; i < len; i++) {+ let v = 0;+ for (let c = 0; c < k; c++) v += channels[c][i];+ this.sum += v / k;+ this.count++;+ const next = this.pos + step;+ if (Math.floor(next) > Math.floor(this.pos)) {+ this.emit(this.sum / this.count);+ this.sum = 0;+ this.count = 0;+ }+ this.pos = next;+ }+ }++ private emit(v: number): void {+ if (this.n === this.out.length) {+ const grown = new Float32Array(this.out.length * 2);+ grown.set(this.out);+ this.out = grown;+ }+ this.out[this.n++] = v;+ }++ /** The signal so far. */+ samples(): Float32Array {+ return this.out.subarray(0, this.n);+ }++ /** Hand the samples gathered since the last drain to `into` and let them+ * go, so a long signal streams through in one chunk's worth of memory. */+ drain(into: (mono: Float32Array) => void): void {+ if (this.n > 0) into(this.out.subarray(0, this.n));+ this.n = 0;+ }++ get length(): number {+ return this.n;+ }+}++/**+ * The loudness envelope the coarse pass compares, built from mono samples at+ * `SYNC_RATE` as they arrive: log RMS per hop. Kept separate from the+ * samples, so an hour of recording costs an hour of envelope.+ */+export class EnvelopeBuilder {+ private out: Float32Array;+ private n = 0;+ private acc = 0;+ private fill = 0;++ constructor(expectedSeconds = 60) {+ this.out = new Float32Array(Math.max(256, Math.ceil(expectedSeconds / ENV_HOP_S) + 1));+ }++ push(mono: Float32Array): void {+ for (let i = 0; i < mono.length; i++) {+ this.acc += mono[i] * mono[i];+ if (++this.fill === ENV_HOP) {+ if (this.n === this.out.length) {+ const grown = new Float32Array(this.out.length * 2);+ grown.set(this.out);+ this.out = grown;+ }+ this.out[this.n++] = Math.log(1e-8 + this.acc / ENV_HOP);+ this.acc = 0;+ this.fill = 0;+ }+ }+ }++ envelope(): Float32Array {+ return this.out.subarray(0, this.n);+ }+}++/** The envelope of a whole mono signal at `SYNC_RATE`. */+export function envelopeOf(mono: Float32Array): Float32Array {+ const b = new EnvelopeBuilder(mono.length / SYNC_RATE);+ b.push(mono);+ return b.envelope();+}++/** The envelope with its slow drift taken out and scaled to unit variance,+ * so the correlation weighs where loudness moves, whatever level a device+ * recorded. */+export function normalizeEnvelope(env: Float32Array): Float64Array {+ const n = env.length;+ const out = new Float64Array(n);+ if (n === 0) return out;+ // Running mean over a centered window, by prefix sums.+ const prefix = new Float64Array(n + 1);+ for (let i = 0; i < n; i++) prefix[i + 1] = prefix[i] + env[i];+ const half = Math.floor(DETREND_HOPS / 2);+ let sq = 0;+ for (let i = 0; i < n; i++) {+ const a = Math.max(0, i - half);+ const b = Math.min(n, i + half + 1);+ const v = env[i] - (prefix[b] - prefix[a]) / (b - a);+ out[i] = v;+ sq += v * v;+ }+ const sd = Math.sqrt(sq / n);+ if (sd > 1e-9) for (let i = 0; i < n; i++) out[i] /= sd;+ return out;+}++/** In-place radix-2 FFT; `inverse` runs it backward, unscaled. */+function fft(re: Float64Array, im: Float64Array, inverse: boolean): void {+ const n = re.length;+ for (let i = 1, j = 0; i < n; i++) {+ let bit = n >> 1;+ for (; j & bit; bit >>= 1) j ^= bit;+ j ^= bit;+ if (i < j) {+ let t = re[i];+ re[i] = re[j];+ re[j] = t;+ t = im[i];+ im[i] = im[j];+ im[j] = t;+ }+ }+ for (let len = 2; len <= n; len <<= 1) {+ const ang = ((inverse ? 2 : -2) * Math.PI) / len;+ const wr = Math.cos(ang);+ const wi = Math.sin(ang);+ const half = len >> 1;+ for (let i = 0; i < n; i += len) {+ let cr = 1;+ let ci = 0;+ for (let k = 0; k < half; k++) {+ const ar = re[i + k + half] * cr - im[i + k + half] * ci;+ const ai = re[i + k + half] * ci + im[i + k + half] * cr;+ re[i + k + half] = re[i + k] - ar;+ im[i + k + half] = im[i + k] - ai;+ re[i + k] += ar;+ im[i + k] += ai;+ const nr = cr * wr - ci * wi;+ ci = cr * wi + ci * wr;+ cr = nr;+ }+ }+ }+}++/**+ * Cross-correlation of `a` against `b` at every lag where they overlap:+ * `at(k)` is Σ a[i]·b[i + k], for k from −(a.length − 1) to b.length − 1.+ */+export function crossCorrelate(+ a: ArrayLike<number>,+ b: ArrayLike<number>+): { at: (k: number) => number; min: number; max: number } {+ const need = a.length + b.length - 1;+ let n = 1;+ while (n < need) n <<= 1;+ const ar = new Float64Array(n);+ const ai = new Float64Array(n);+ const br = new Float64Array(n);+ const bi = new Float64Array(n);+ for (let i = 0; i < a.length; i++) ar[i] = a[i];+ for (let i = 0; i < b.length; i++) br[i] = b[i];+ fft(ar, ai, false);+ fft(br, bi, false);+ // conj(A)·B+ for (let i = 0; i < n; i++) {+ const r = ar[i] * br[i] + ai[i] * bi[i];+ const im = ar[i] * bi[i] - ai[i] * br[i];+ ar[i] = r;+ ai[i] = im;+ }+ fft(ar, ai, true);+ const scale = 1 / n;+ return {+ at: (k) => ar[(k + n) % n] * scale,+ min: -(a.length - 1),+ max: b.length - 1,+ };+}++/** The coarse answer: the lag, in hops, at which the camera's envelope sits+ * best inside the recording's, and how clearly it beats the runner-up. */+export interface CoarseSync {+ lag: number;+ confidence: number;+}++/**+ * Where `camera` (an envelope) sits inside `recording` (another): camera hop+ * i lines up with recording hop i + lag. Lags where less than half of the+ * shorter signal overlaps are never considered — a sliver of overlap agrees+ * with anything.+ */+export function coarseSync(camera: Float32Array, recording: Float32Array): CoarseSync | null {+ const a = normalizeEnvelope(camera);+ const b = normalizeEnvelope(recording);+ if (a.length < 4 || b.length < 4) return null;+ const xc = crossCorrelate(a, b);+ const need = Math.ceil(Math.min(a.length, b.length) * MIN_OVERLAP_SHARE);+ // Overlap at lag k: the hops of a that land inside b.+ const overlap = (k: number) => Math.min(a.length, b.length - k) - Math.max(0, -k);+ let best = -Infinity;+ let lag = 0;+ for (let k = xc.min; k <= xc.max; k++) {+ if (overlap(k) < need) continue;+ const v = xc.at(k);+ if (v > best) {+ best = v;+ lag = k;+ }+ }+ if (!Number.isFinite(best)) return null;+ const exclude = Math.round(PEAK_EXCLUDE_S / ENV_HOP_S);+ let second = -Infinity;+ for (let k = xc.min; k <= xc.max; k++) {+ if (Math.abs(k - lag) <= exclude || overlap(k) < need) continue;+ second = Math.max(second, xc.at(k));+ }+ const confidence =+ best <= 0 ? 0 : second <= 0 ? CONFIDENCE_CAP : Math.min(CONFIDENCE_CAP, best / second);+ return { lag, confidence };+}++/**+ * The fine pass: `camera` and `recording` are waveforms at `SYNC_RATE`, the+ * recording starting `reach` samples earlier than where the coarse pass put+ * the camera's first sample and running `reach` samples longer. Returns the+ * shift, in samples, that lines them up best (negative = the recording sits+ * earlier than the coarse answer), or null when no shift stands out. Polarity+ * is ignored: two microphones can face the source from opposite sides.+ */+export function fineSync(camera: Float32Array, recording: Float32Array, reach: number): number | null {+ if (camera.length < 64 || recording.length < camera.length + 2 * reach) return null;+ const xc = crossCorrelate(camera, recording);+ let best = -1;+ let at = 0;+ for (let k = 0; k <= 2 * reach; k++) {+ const v = Math.abs(xc.at(k));+ if (v > best) {+ best = v;+ at = k;+ }+ }+ // The runner-up outside the peak's own cycle, a millisecond either side.+ const exclude = Math.round(SYNC_RATE / 1000);+ let second = 0;+ for (let k = 0; k <= 2 * reach; k++) {+ if (Math.abs(k - at) <= exclude) continue;+ second = Math.max(second, Math.abs(xc.at(k)));+ }+ if (!(best > 0) || best < second * 1.05) return null;+ return at - reach;+}++/** The loudest `seconds` of a signal at `SYNC_RATE`, by hop energy: where the+ * fine pass has the most to compare. Returns the first sample. */+export function loudestStretch(mono: Float32Array, seconds: number): number {+ const len = Math.min(mono.length, Math.round(seconds * SYNC_RATE));+ if (len >= mono.length) return 0;+ let sum = 0;+ for (let i = 0; i < len; i++) sum += mono[i] * mono[i];+ let best = sum;+ let at = 0;+ // Slide a hop at a time; exact enough and cheap.+ for (let s = ENV_HOP; s + len <= mono.length; s += ENV_HOP) {+ for (let i = s - ENV_HOP; i < s; i++) sum -= mono[i] * mono[i];+ for (let i = s + len - ENV_HOP; i < s + len; i++) sum += mono[i] * mono[i];+ if (sum > best) {+ best = sum;+ at = s;+ }+ }+ return at;+}++export interface SyncAnswer {+ /** Recording seconds = camera seconds + offset. */+ offset: number;+ confidence: number;+ /** Whether the fine pass put the offset on the sample; false leaves it at+ * the coarse pass's 20 ms grain. */+ refined: boolean;+}++/**+ * Both passes over two whole signals in hand, each at `SYNC_RATE` and each+ * with the source second its first sample sits at. The tools stream the+ * recording through `EnvelopeBuilder` instead, since a long one cannot be+ * held whole, and read the fine window on its own; this is the same+ * arithmetic for signals already decoded.+ */+export function syncSignals(+ camera: Float32Array,+ cameraStart: number,+ recording: Float32Array,+ recordingStart: number+): SyncAnswer | null {+ const coarse = coarseSync(envelopeOf(camera), envelopeOf(recording));+ if (!coarse) return null;+ const offset = recordingStart - cameraStart + coarse.lag * ENV_HOP_S;+ const reach = Math.round(FINE_REACH_S * SYNC_RATE);+ const from = loudestStretch(camera, FINE_SECONDS);+ const cam = camera.subarray(from, from + Math.round(FINE_SECONDS * SYNC_RATE));+ // Where the camera's stretch starts in the recording, by the coarse answer.+ const recAt = Math.round((cameraStart + from / SYNC_RATE + offset - recordingStart) * SYNC_RATE) - reach;+ const shift =+ recAt >= 0 && recAt + cam.length + 2 * reach <= recording.length+ ? fineSync(cam, recording.subarray(recAt, recAt + cam.length + 2 * reach), reach)+ : null;+ return {+ offset: shift === null ? offset : offset + shift / SYNC_RATE,+ confidence: coarse.confidence,+ refined: shift !== null,+ };+}++/** Decoded audio of any rate and channel count, chunk by chunk, starting at+ * `timestamp` source seconds. */+export interface SyncChunk {+ channels: Float32Array[];+ timestamp: number;+ sampleRate: number;+}++/** Reads a span of each file's sound, wherever it lives. */+export interface SyncReader {+ camera: (from: number, to: number) => AsyncIterable<SyncChunk>;+ recording: (from: number, to: number) => AsyncIterable<SyncChunk>;+}++/** A span folded to mono at `SYNC_RATE`, with the source second its first+ * sample sits at. */+async function monoOf(chunks: AsyncIterable<SyncChunk>, seconds: number) {+ const fold = new MonoResampler(seconds);+ let start: number | null = null;+ for await (const c of chunks) {+ start ??= c.timestamp;+ fold.push(c.channels, c.sampleRate);+ }+ return { mono: fold.samples(), start: start ?? 0 };+}++/**+ * Both passes over two files read as they stream: `cameraFrom..cameraTo` of+ * the camera (the stretch whose sound is matched), the recording's first+ * `recordingTo` seconds as an envelope only, and then the few seconds of the+ * recording the fine pass needs. Memory holds the camera's stretch, the+ * recording's envelope, and one chunk.+ */+export async function syncSound(+ read: SyncReader,+ opts: { cameraFrom: number; cameraTo: number; recordingTo: number }+): Promise<SyncAnswer | null> {+ const cam = await monoOf(read.camera(opts.cameraFrom, opts.cameraTo), opts.cameraTo - opts.cameraFrom);+ if (cam.mono.length < SYNC_RATE) return null;+ const fold = new MonoResampler(10);+ const env = new EnvelopeBuilder(opts.recordingTo);+ let recStart: number | null = null;+ for await (const c of read.recording(0, opts.recordingTo)) {+ recStart ??= c.timestamp;+ fold.push(c.channels, c.sampleRate);+ fold.drain((m) => env.push(m));+ }+ const coarse = coarseSync(envelopeOf(cam.mono), env.envelope());+ if (!coarse) return null;+ const offset = (recStart ?? 0) - cam.start + coarse.lag * ENV_HOP_S;+ // The fine pass: the camera's loudest few seconds against the recording+ // around where the coarse pass put them.+ const reach = Math.round(FINE_REACH_S * SYNC_RATE);+ const from = loudestStretch(cam.mono, FINE_SECONDS);+ const piece = cam.mono.subarray(from, from + Math.round(FINE_SECONDS * SYNC_RATE));+ const at = cam.start + from / SYNC_RATE + offset - reach / SYNC_RATE;+ const window = Math.max(0, at - 0.25);+ const rec = await monoOf(read.recording(window, at + (piece.length + 2 * reach) / SYNC_RATE + 0.25), FINE_SECONDS + 1);+ const recAt = Math.round((at - rec.start) * SYNC_RATE);+ const shift =+ recAt >= 0 && recAt + piece.length + 2 * reach <= rec.mono.length+ ? fineSync(piece, rec.mono.subarray(recAt, recAt + piece.length + 2 * reach), reach)+ : null;+ return {+ offset: shift === null ? offset : offset + shift / SYNC_RATE,+ confidence: coarse.confidence,+ refined: shift !== null,+ };+}diff --git a/site/src/cut/lib/sourceExportPlan.ts b/site/src/cut/lib/sourceExportPlan.tsindex cb48699b..440c0016 100644--- a/site/src/cut/lib/sourceExportPlan.ts+++ b/site/src/cut/lib/sourceExportPlan.ts@@ -15,7 +15,7 @@ export function sourceSequence(doc: ExportDoc) {const neutral: Partial<Record<keyof VideoClip, unknown>> = {track: 0, muted: false, volume: 1, fit: "fit", zoom: 1,panX: 0, panY: 0, flipH: false, flipV: false, rotation: 0, opacity: 1,- speed: 1, reverse: false, smoothSlow: false, transition: 0, hidden: false,+ speed: 1, reverse: false, smoothSlow: false, transition: 0, hidden: false, audioLead: 0, audioTail: 0,};const metadata = new Set(["id", "assetId", "name", "groupId", "in", "out", "start"]);for (const clip of clips) {@@ -25,7 +25,7 @@ export function sourceSequence(doc: ExportDoc) {if (!(key in neutral) || value !== neutral[key as keyof VideoClip]) return null;}const asset = doc.assets.find((a) => a.id === clip.assetId);- if (!asset || asset.type !== "video" || asset.block || !asset.width || !asset.height ||+ if (!asset || asset.type !== "video" || asset.block || asset.soundFrom || !asset.width || !asset.height ||!Number.isFinite(clip.in) || !Number.isFinite(clip.out) || clip.in < 0 || clip.out <= clip.in || clip.out > asset.duration + 0.000001) return null;// A file whose code values need converting (a log source, or an HDR file// into another delivery) has to render; an ungraded file already in thediff --git a/site/src/cut/lib/splitEdit.test.ts b/site/src/cut/lib/splitEdit.test.tsnew file mode 100644index 00000000..23a0f101--- /dev/null+++ b/site/src/cut/lib/splitEdit.test.ts@@ -0,0 +1,166 @@+import { beforeEach, describe, expect, test } from "bun:test";++import { runAiTool } from "./aiTools";+import { soundWindow } from "./soundSource";+import { getClipSpans, useEditor } from "./store";+import type { MediaAsset, VideoClip } from "./types";++/** Split edits and bound recordings as the document holds them: what the+ * spans every renderer reads make of a clip's lead, tail and recording, and+ * how the edits that reshape clips carry them. */++const video = (over: Partial<MediaAsset> = {}): MediaAsset => ({+ id: "cam",+ fileName: "cam.mp4",+ name: "cam.mp4",+ type: "video",+ duration: 60,+ url: "",+ ...over,+});+const recording: MediaAsset = { id: "rec", fileName: "rec.wav", name: "rec.wav", type: "audio", duration: 50, url: "" };++const clip = (id: string, start: number, inS: number, outS: number, over: Partial<VideoClip> = {}): VideoClip => ({+ id,+ assetId: "cam",+ start,+ in: inS,+ out: outS,+ track: 0,+ muted: false,+ ...over,+});++const spanOf = (id: string) => {+ const s = useEditor.getState();+ return getClipSpans(s.clips, s.assets).find((sp) => sp.clip.id === id)!;+};++beforeEach(() => {+ useEditor.setState({+ projectId: "p1",+ loaded: true,+ readOnly: false,+ clips: [clip("a", 0, 10, 14), clip("b", 4, 20, 24)],+ transitions: [],+ audioClips: [],+ overlays: [],+ assets: [video(), recording],+ selection: null,+ multiSelection: [],+ });+});++describe("set_split_edit", () => {+ test("a lead and a tail reach past the picture and nothing moves", async () => {+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4 });+ await runAiTool("set_split_edit", { clipId: "a", tail: 0.6 });+ const b = spanOf("b");+ const a = spanOf("a");+ expect(b.soundLead).toBeCloseTo(0.4, 5);+ expect(b.soundBack).toBeCloseTo(0.4, 5);+ expect(a.soundTail).toBeCloseTo(0.6, 5);+ expect(a.soundAhead).toBeCloseTo(0.6, 5);+ expect(b.start).toBe(4);+ expect(b.len).toBe(4);+ });++ test("a lead is capped by the source before the trim and by the timeline's start", async () => {+ useEditor.setState({ clips: [clip("a", 1, 0.3, 4)] });+ const res = (await runAiTool("set_split_edit", { clipId: "a", lead: 2 })) as { playing: { lead: number }; note?: string };+ expect(spanOf("a").soundLead).toBeCloseTo(0.3, 5);+ expect(res.playing.lead).toBeCloseTo(0.3, 2);+ expect(res.note).toBeDefined();+ });++ test("a cut carrying a transition keeps no split edit on that side", async () => {+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4 });+ expect(spanOf("b").soundLead).toBeCloseTo(0.4, 5);+ await runAiTool("set_transition", { clipId: "a", seconds: 0.5 });+ expect(spanOf("a").transitionOut).toBeGreaterThan(0);+ expect(spanOf("b").soundLead ?? 0).toBe(0);+ // The setting stays on the clip, and plays again once the cut is plain.+ await runAiTool("set_transition", { clipId: "a", seconds: 0 });+ expect(spanOf("b").soundLead).toBeCloseTo(0.4, 5);+ });++ test("0 clears a side and a side left out keeps its value", async () => {+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4, tail: 0.3 });+ await runAiTool("set_split_edit", { clipId: "b", lead: 0 });+ const b = useEditor.getState().clips.find((c) => c.id === "b")!;+ expect(b.audioLead).toBeUndefined();+ expect(b.audioTail).toBeCloseTo(0.3, 5);+ });++ test("a split keeps the lead on the left half and the tail on the right", async () => {+ await runAiTool("set_split_edit", { clipId: "b", lead: 0.4, tail: 0.3 });+ await runAiTool("split_at", { t: 6 });+ const halves = useEditor+ .getState()+ .clips.filter((c) => c.start >= 4)+ .sort((x, y) => x.start - y.start);+ expect(halves).toHaveLength(2);+ expect(halves[0].audioLead).toBeCloseTo(0.4, 5);+ expect(halves[0].audioTail).toBeUndefined();+ expect(halves[1].audioLead).toBeUndefined();+ expect(halves[1].audioTail).toBeCloseTo(0.3, 5);+ });++ test("ids lands one split edit on several clips", async () => {+ await runAiTool("set_split_edit", { ids: ["a", "b"], tail: 0.2 });+ const s = useEditor.getState();+ expect(s.clips.every((c) => c.audioTail === 0.2)).toBe(true);+ });+});++describe("a recording bound to the video", () => {+ test("every clip of the video reads it, a split's halves included", async () => {+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: -5 });+ await runAiTool("split_at", { t: 2 });+ const s = useEditor.getState();+ const spans = getClipSpans(s.clips, s.assets);+ expect(spans).toHaveLength(3);+ for (const sp of spans) {+ expect(sp.sound?.asset.id).toBe("rec");+ expect(sp.sound?.offset).toBe(-5);+ }+ });++ test("the recording's own room bounds the handles", () => {+ // The recording starts at video second 5; clip a starts on video 10, so+ // five seconds of it sit before the trim.+ useEditor.setState({ clips: [clip("a", 8, 10, 14, { audioLead: 9 })] });+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: -5 });+ expect(spanOf("a").soundLead).toBeCloseTo(5, 5);+ });++ test("a window the recording does not reach narrows and starts late", () => {+ // Recording second = video second − 12: video 10..12 is before it rolled.+ const w = soundWindow({ in: 10, out: 14 }, { offset: -12, limit: 50 }, 0, 4)!;+ expect(w.lo).toBe(0);+ expect(w.hi).toBeCloseTo(2, 5);+ expect(w.at).toBeCloseTo(2, 5);+ });++ test("binding is one undo step, and removing the recording unbinds", async () => {+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: 1.5 });+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toEqual({ assetId: "rec", offset: 1.5 });+ useEditor.getState().undo();+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();+ useEditor.getState().redo();+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom?.assetId).toBe("rec");+ useEditor.getState().removeAsset("rec");+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();+ });++ test("sync_audio clear returns the video to its own sound", async () => {+ useEditor.getState().setAssetSoundFrom("cam", { assetId: "rec", offset: 1 });+ await runAiTool("sync_audio", { clipId: "a", clear: true });+ expect(useEditor.getState().assets.find((a) => a.id === "cam")!.soundFrom).toBeUndefined();+ expect(spanOf("a").sound).toBeUndefined();+ });++ test("sync_audio refuses a file that is not audio", async () => {+ await expect(runAiTool("sync_audio", { clipId: "a", audio_asset_id: "cam" })).rejects.toThrow();+ });+});diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 7659b0c8..bf090f40 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -394,6 +394,9 @@ interface DocSnapshot {* chat; it lives on the asset like the beat grid, so the checkpoint* carries it the same way. */sourceColor: { id: string; profile: SourceProfile | undefined }[];+ /** Each video's bound recording, set from the Audio tab and the chat; on+ * the asset like the source-colour override, carried the same way. */+ soundFrom: { id: string; from: SoundFrom | undefined }[];}export type SubtitleStatus = "idle" | "running" | "ready" | "empty" | "error";@@ -1015,6 +1018,7 @@ const sharedFields = (s: Pick<EditorState, "guideLines" | "colorSpace" | "assetscolorSpace: s.colorSpace,beats: new Map(s.assets.map((a) => [a.id, a.beats])),profiles: new Map(s.assets.map((a) => [a.id, a.colorProfile])),+ sounds: new Map(s.assets.map((a) => [a.id, a.soundFrom])),});/** A project-wide setting changed on another timeline while this one was@@ -1033,6 +1037,8 @@ function rebaseShared(snaps: DocSnapshot[], then: SharedFields, now: SharedFieldsnap.sourceColor = snap.sourceColor.map((c) =>profiles.has(c.id) ? { id: c.id, profile: now.profiles.get(c.id) } : c);+ if (sounds.size)+ snap.soundFrom = snap.soundFrom.map((c) => (sounds.has(c.id) ? { id: c.id, from: now.sounds.get(c.id) } : c));}}@@ -1495,47 +1501,70 @@ export function cutTranscribeSpec(const spans = getClipSpans(s.clips, s.assets);const duration = projectDuration(s);const assetById = new Map(s.assets.map((a) => [a.id, a]));- const audio = s.audioClips- .filter(- (a) =>- !a.hidden &&- !a.reverse &&- a.start < duration &&- assetById.has(a.assetId) &&- !assetIsSilent(assetById.get(a.assetId)!)- )- .map((a) => ({- file: assetById.get(a.assetId)!.fileName,- in: a.in,- out: a.out,- start: a.start,- volume: a.volume,- speed: a.speed,- speedCurve: a.speedCurve,- reverse: a.reverse,- }))- .concat(- overlayLayers(s.clips)- .filter(- (c) =>- !c.hidden &&- !c.muted &&- !c.reverse &&- c.start < duration &&- assetById.has(c.assetId) &&- !assetIsSilent(assetById.get(c.assetId)!)- )- .map((c) => ({- file: assetById.get(c.assetId)!.fileName,- in: c.in,- out: c.out,- start: c.start,- volume: 1,- speed: c.speed,- speedCurve: c.speedCurve,- reverse: c.reverse,- }))- );+ type Item = CloudTranscribeSpec["audio"][number];+ /** The stretch `[fromT, toT]` of a span's sound (timeline seconds from its+ * head at `start`) as a placed item, read off its sound file: the video's+ * own track or the recording bound to it. Null when that file holds none+ * of it. */+ const placed = (+ c: Parameters<typeof soundWindow>[0],+ src: { asset: MediaAsset; offset: number; limit: number },+ start: number,+ fromT: number,+ toT: number,+ volume: number+ ): Item | null => {+ const w = soundWindow(c, src, fromT, toT);+ return w+ ? {+ file: src.asset.fileName,+ in: w.lo,+ out: w.hi,+ start: start + fromT + w.at,+ volume,+ speed: w.span.speed,+ speedCurve: w.span.speedCurve,+ reverse: w.span.reverse,+ }+ : null;+ };+ const own = (asset: MediaAsset) => ({ asset, offset: 0, limit: Infinity });+ const audio: Item[] = [];+ for (const a of s.audioClips) {+ const asset = assetById.get(a.assetId);+ if (!asset || a.hidden || a.reverse || a.start >= duration) continue;+ const src = soundSourceOf(asset, assetById);+ if (!src.bound && assetIsSilent(asset)) continue;+ const item = placed(a, src, a.start, 0, retimeOf(a).len, a.volume);+ if (item) audio.push(item);+ }+ // Every upper-track clip's sound, split edit included; on track 0 the fold+ // carries the clip itself, so only what plays off its own footprint — a+ // bound recording, a split edit's lead and tail — rides here.+ const audible = (sp: ClipSpan) =>+ !sp.clip.hidden && !sp.clip.muted && !sp.clip.reverse && (!!sp.sound || !assetIsSilent(sp.asset));+ for (const track of new Set(overlayLayers(s.clips).map((c) => c.track))) {+ for (const sp of getClipSpans(s.clips, s.assets, track)) {+ if (!audible(sp) || sp.start >= duration) continue;+ const item = placed(sp.clip, sp.sound ?? own(sp.asset), sp.start, -(sp.soundLead ?? 0), sp.len + (sp.soundTail ?? 0), 1);+ if (item) audio.push(item);+ }+ }+ for (const sp of spans) {+ if (!audible(sp)) continue;+ const src = sp.sound ?? own(sp.asset);+ const pieces: [number, number][] = sp.sound+ ? [[-(sp.soundLead ?? 0), sp.len + (sp.soundTail ?? 0)]]+ : [+ [-(sp.soundLead ?? 0), 0],+ [sp.len, sp.len + (sp.soundTail ?? 0)],+ ];+ for (const [from, to] of pieces) {+ if (to - from <= 1e-3) continue;+ const item = placed(sp.clip, src, sp.start, from, to, 1);+ if (item) audio.push(item);+ }+ }if (spans.length === 0 && audio.length === 0) return null;const silentSpacer = (len: number) => ({ file: "", in: 0, out: len, muted: true, speed: 1, transition: 0 });return {@@ -1554,8 +1583,9 @@ export function cutTranscribeSpec(in: sp.clip.in,out: sp.clip.out,// Sound played backward is not speech; the clip keeps its- // time in the mix as silence.- muted: sp.clip.muted || assetIsSilent(sp.asset) || !!sp.clip.reverse,+ // time in the mix as silence. A bound recording plays from+ // the items above, so the slot holds silence too.+ muted: sp.clip.muted || assetIsSilent(sp.asset) || !!sp.clip.reverse || !!sp.sound,speed: clipSpeed(sp.clip),speedCurve: sp.clip.speedCurve,// The clamped cross-dissolve overlap, so the mix overlaps clip@@ -1611,6 +1641,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {return {beats: assets.map((a) => ({ id: a.id, grid: a.beats })),sourceColor: assets.map((a) => ({ id: a.id, profile: a.colorProfile })),+ soundFrom: assets.map((a) => ({ id: a.id, from: a.soundFrom })),guideLines: { v: [...guideLines.v], h: [...guideLines.h] },colorSpace,// Render-owned clips are excluded — history captures the user's timeline,@@ -1633,7 +1664,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {* doc — an undo mid-session (a brush stroke, a panel edit) keeps the panel* and its stage gizmos on the clip being worked. */const restoreDoc = (snap: DocSnapshot) => {- const { beats, sourceColor, ...doc } = snap;+ const { beats, sourceColor, soundFrom, ...doc } = snap;const { clips, audioClips, assets, selection, multiSelection } = get();const genClips = clips.filter((c) => genClipIds.has(c.id));const genAudio = audioClips.filter((c) => genAudioIds.has(c.id));@@ -1641,11 +1672,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// since the checkpoint was taken is not in it and keeps the grid it has.const grids = new Map(beats.map((b) => [b.id, b.grid]));const profiles = new Map(sourceColor.map((c) => [c.id, c.profile]));+ const sounds = new Map(soundFrom.map((c) => [c.id, c.from]));const withBeats = assets.map((a) => {let next = a;if (grids.has(a.id) && grids.get(a.id) !== a.beats) next = { ...next, beats: grids.get(a.id) };if (profiles.has(a.id) && profiles.get(a.id) !== a.colorProfile)next = withColorProfile(next, profiles.get(a.id));+ if (sounds.has(a.id) && sounds.get(a.id) !== a.soundFrom) next = withSoundFrom(next, sounds.get(a.id));return next;});const beatsMoved = withBeats.some((a, i) => a !== assets[i]);@@ -2556,6 +2589,18 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {}));},+ setAssetSoundFrom: (id, from) => {+ const asset = get().assets.find((a) => a.id === id);+ if (!asset || get().readOnly) return;+ if (asset.soundFrom?.assetId === from?.assetId && asset.soundFrom?.offset === from?.offset) return;+ push();+ // The source's transcript was heard off the sound it had; the sweep+ // hears the new one.+ set((s) => ({+ assets: s.assets.map((a) => (a.id === id ? { ...withSoundFrom(a, from), speech: undefined } : a)),+ }));+ },+setAssetProxy: (id, proxy, proxyUrl) => {if (get().readOnly) return;set((s) => ({@@ -2697,7 +2742,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {const scrub = (snap: DocSnapshot) => {snap.clips = snap.clips.filter((c) => c.assetId !== id);snap.audioClips = snap.audioClips.filter((c) => c.assetId !== id);+ snap.soundFrom = snap.soundFrom.map((c) => (c.from?.assetId === id ? { id: c.id, from: undefined } : c));};+ // A video bound to this recording goes back to its own track.+ const remaining = (assets: MediaAsset[]) =>+ assets+ .filter((a) => a.id !== id)+ .map((a) => (a.soundFrom?.assetId === id ? withSoundFrom(a, undefined) : a));for (const snap of history) scrub(snap);for (const snap of future) scrub(snap);if (pending) scrub(pending.snap);@@ -2842,7 +2893,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// A video's own track plays on the soundtrack the same way a sound file// does — it is what detaching a clip's audio leaves behind — so a cut// laid out as empty shots can still carry the reference's sound.- if (!asset || asset.type === "image" || assetIsSilent(asset)) return;+ if (!asset || asset.type === "image" || (assetIsSilent(asset) && !soundSourceOf(asset, get().assets).bound)) return;push();// Within its lane the clip slides to the next free slot at or after the// target so it never lands on top of an existing sound.@@ -3704,7 +3755,8 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {const clip = clips.find((c) => c.id === selection.id);if (!clip || clip.muted) return; // no sound to detachconst span = clipWindow(clips, assets, clip.id);- if (!span || assetIsSilent(span.asset)) return; // a still has none either+ // A still has none either; a bound recording counts as the clip's sound.+ if (!span || (assetIsSilent(span.asset) && !soundSourceOf(span.asset, assets).bound)) return;push();const audio: AudioClip = {id: uid(),@@ -4105,7 +4157,15 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {style: t.style,...(t.hidden ? { hidden: true } : {}),}));+ // A video's separate recording binds to the landed copy of it.+ const bound = new Map<string, SoundFrom>();+ template.media.forEach((m, i) => {+ const video = assetIds[i];+ const rec = m.soundFrom ? assetIds[m.soundFrom.media] : undefined;+ if (video && rec && m.soundFrom) bound.set(video, { assetId: rec, offset: m.soundFrom.offset });+ });set((s) => ({+ ...(bound.size ? { assets: s.assets.map((a) => (bound.has(a.id) ? { ...a, soundFrom: bound.get(a.id) } : a)) } : {}),clips: [...s.clips, ...newClips, ...newLayers].sort((a, b) => a.start - b.start),audioClips: [...s.audioClips, ...newAudio],overlays: [...s.overlays, ...newTexts],@@ -4280,27 +4340,46 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {const projectId = s.projectId;const sp = clipWindow(s.clips, s.assets, clipId);if (!sp) throw new Error("The clip is no longer on the timeline.");- if (assetIsSilent(sp.asset)) throw new Error("This clip is a still — it has no sound.");+ const src = soundSourceOf(sp.asset, s.assets);+ if (assetIsSilent(sp.asset) && !src.bound) throw new Error("This clip is a still — it has no sound.");if (sp.clip.reverse) throw new Error("This clip plays backward — its sound is not speech.");+ // A recording bound to the video is what the clip says; it is read on+ // the video's clock, starting late where it rolled late.+ const heard = src.bound ? soundWindow(sp.clip, src, 0, sp.len) : null;+ if (src.bound && !heard) throw new Error("The recording bound to this video holds none of this clip.");const lane = s.subtitleLane;const epoch = laneEpoch;// The clip's own sound, deliberately unmuted: this transcribes what the// clip says even when its timeline audio is muted.- const spec = {+ const spec: CloudTranscribeSpec = {duration: sp.len,locale: trackLocale(s.subtitles, lane),clips: [- {- file: sp.asset.fileName,- in: sp.clip.in,- out: sp.clip.out,- muted: false,- speed: clipSpeed(sp.clip),- speedCurve: sp.clip.speedCurve,- transition: 0,- },+ heard+ ? { file: "", in: 0, out: sp.len, muted: true, speed: 1, transition: 0 }+ : {+ file: sp.asset.fileName,+ in: sp.clip.in,+ out: sp.clip.out,+ muted: false,+ speed: clipSpeed(sp.clip),+ speedCurve: sp.clip.speedCurve,+ transition: 0,+ },],- audio: [],+ audio: heard+ ? [+ {+ file: src.asset.fileName,+ in: heard.lo,+ out: heard.hi,+ start: heard.at,+ volume: 1,+ speed: heard.span.speed,+ speedCurve: heard.span.speedCurve,+ },+ ]+ : [],};set({ subtitleStatus: "running", subtitleError: null, subtitleStartedAt: Date.now() });try {@@ -5043,7 +5122,7 @@ function beatsEdited(prev: MediaAsset[], next: MediaAsset[]): boolean {const before = new Map(prev.map((a) => [a.id, a]));return next.some((a) => {const was = before.get(a.id);- return !!was && (was.beats !== a.beats || was.colorProfile !== a.colorProfile);+ return !!was && (was.beats !== a.beats || was.colorProfile !== a.colorProfile || was.soundFrom !== a.soundFrom);});}@@ -5055,6 +5134,13 @@ function withColorProfile<A extends StoredAsset>(asset: A, profile: SourceProfilreturn (profile ? { ...rest, colorProfile: profile } : rest) as A;}+/** The asset with its bound recording set or cleared. */+function withSoundFrom<A extends StoredAsset>(asset: A, from: SoundFrom | undefined): A {+ const { soundFrom: _was, ...rest } = asset;+ void _was;+ return (from ? { ...rest, soundFrom: from } : rest) as A;+}+/** A stored asset in the current shape. Projects saved while the override* rode inside the header record (`color.profile`) carry it out to* `colorProfile` once, on load. */@@ -5174,7 +5260,7 @@ export const docTimelines = (() => {export function storedAssets(assets: MediaAsset[]): StoredAsset[] {return assets.filter((a) => !tabOnlyUpload(a))- .map(({ id, fileName, name, type, duration, sizeBytes, width, height, origin, chatId, folderId, language, watch, speech, beats, sceneCuts, copiedFrom, block, color, colorProfile, proxy }) => ({+ .map(({ id, fileName, name, type, duration, sizeBytes, width, height, origin, chatId, folderId, language, watch, speech, beats, sceneCuts, copiedFrom, block, color, colorProfile, proxy, soundFrom }) => ({id,fileName,name,@@ -5196,6 +5282,7 @@ export function storedAssets(assets: MediaAsset[]): StoredAsset[] {...(color !== undefined ? { color } : {}),...(colorProfile !== undefined ? { colorProfile } : {}),...(proxy !== undefined ? { proxy } : {}),+ ...(soundFrom !== undefined ? { soundFrom } : {}),}));}@@ -5772,6 +5859,18 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)// and they are what lets the two clips be audible at once over a cut// their pictures hard-join.const prevSound = spans[spans.length - 1]?.soundOut ?? 0;+ // The handles are measured on the file the sound reads: a recording+ // bound to the video holds its own stretch on either side of the trim.+ const src = soundSourceOf(asset, byId);+ const room = soundRoom(clip, src);+ // A split edit carries the sound past an edge the picture cuts on. A+ // transition at that cut owns the handover, so the edge keeps none; the+ // lead never reaches before the timeline's start.+ const prevJoin = present[i - 1] ? transitionOverlap(present[i - 1].clip, clip) > 0 : false;+ const lead = prevJoin+ ? 0+ : Math.min(clip.start, handleSeconds(Math.min(SPLIT_EDIT_MAX_S, clip.audioLead ?? 0), room.head, clip, "head"));+ const tail = overlap > 0 ? 0 : handleSeconds(Math.min(SPLIT_EDIT_MAX_S, clip.audioTail ?? 0), room.tail, clip, "tail");spans.push({clip,asset,@@ -5780,8 +5879,12 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)transitionOut: onSound ? 0 : overlap,soundOut,// A reversed clip's head faces the source's end and its tail its start.- soundAhead: handleSeconds(soundOut, clip.reverse ? clip.in : asset.duration - clip.out, clip, "tail"),- soundBack: handleSeconds(prevSound, clip.reverse ? asset.duration - clip.out : clip.in, clip, "head"),+ soundAhead: Math.max(tail, handleSeconds(soundOut, room.tail, clip, "tail")),+ soundBack: Math.max(lead, handleSeconds(prevSound, room.head, clip, "head")),+ ...(lead > 0 || tail > 0+ ? { soundLead: lead, soundTail: tail, splitFade: cutSound().splitFadeSeconds }+ : {}),+ ...(src.bound ? { sound: { asset: src.asset, offset: src.offset, limit: src.limit } } : {}),});}return spans;@@ -6216,9 +6319,17 @@ export function assetIdsOnAnyTimeline(s: {transitions: TimelineTransition[];subtitles: SubtitlesBlock;timelines: Partial<Record<TimelineId, TimelineBody>>;+ assets?: readonly StoredAsset[];}): Set<string> {const used = assetIdsInUse(s);for (const body of Object.values(s.timelines)) for (const id of assetIdsInUse(body)) used.add(id);+ return withBoundSound(used, s.assets);+}++/** `used` plus the recordings bound to the videos in it: a bound recording+ * plays wherever its video does. */+function withBoundSound(used: Set<string>, assets: readonly StoredAsset[] | undefined): Set<string> {+ for (const a of assets ?? []) if (a.soundFrom && used.has(a.id)) used.add(a.soundFrom.assetId);return used;}@@ -6240,7 +6351,7 @@ export function assetIdsInUse(s: {});const captionFont = s.subtitles?.font ? fontAssetId(s.subtitles.font) : null;if (captionFont) used.add(captionFont);- return used;+ return withBoundSound(used, s.assets);}export function projectDuration(s: {diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex ca70bf75..0509d3f1 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -279,6 +279,18 @@ export interface StoredAsset {* file beside the master in the project's media. The preview reads it; an* export reads the master. */proxy?: { fileName: string; sizeBytes: number };+ /** A video's sound recorded on a separate device: every clip of this video+ * plays that recording, and the camera's own track stays out (see+ * soundSource.ts). Absent = the camera's sound. */+ soundFrom?: SoundFrom;+}++/** Where a video's sound comes from when it was recorded apart from the+ * picture: the recording, and the seconds its clock runs ahead of the+ * video's. Recording time = video source time + `offset`. */+export interface SoundFrom {+ assetId: string;+ offset: number;}/** A folder in the Media panel's Project Files view — a flat, project-local@@ -1122,6 +1134,9 @@ export interface TemplateMedia {duration: number;width?: number;height?: number;+ /** A video's separately recorded sound: the recording's index in `media`+ * and its clock offset (see SoundFrom). */+ soundFrom?: { media: number; offset: number };}export interface TemplateLayer {/** Explicit timeline group; absent means independent. */@@ -1546,6 +1561,19 @@ export interface ClipSpan {* the incoming half of the cross dissolve at the cut behind it. Zero when* the clip starts at its source's head. */soundBack: number;+ /** The split edit's share of `soundBack`: seconds this clip's sound plays+ * ahead of its picture, ramping up over `splitFade` from silence. Zero or+ * absent when the cut behind it carries a transition, which owns that+ * handover. */+ soundLead?: number;+ /** The split edit's share of `soundAhead`: seconds the sound carries past+ * the picture's end, ramping down over `splitFade` at its far end. */+ soundTail?: number;+ /** The ramp at the outer end of a split edit's extension, seconds. */+ splitFade?: number;+ /** The recording bound to the clip's video, when it has one: the clip's+ * sound reads it, `offset` seconds past the video's source time. */+ sound?: { asset: MediaAsset; offset: number; limit: number };}/** The document persisted as project.json inside each project folder. */diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.tsindex 0baa0332..1fb37e7f 100644--- a/site/src/cut/lib/watch/sweep.ts+++ b/site/src/cut/lib/watch/sweep.ts@@ -12,6 +12,7 @@ import { engineTranscribeSamples } from "../localStt";import { sampleWatchFrames } from "../media";import { runTranscription, useEditor } from "../store";import { trackLocale } from "../subtitles";+import { soundSourceOf, soundWindow } from "../soundSource";import { speechOnsets } from "./fuse";import { mergeSpeech, mergeWatch, nextUncoveredSpan } from "./merge";@@ -129,12 +130,18 @@ async function transcribeChunk(if (!s.projectId || !asset) return;const locale = asset.language ?? trackLocale(s.subtitles, 0);const specFrom = Math.max(0, span.from - (span.from > 0 ? SPEECH_PAD_S : 0));- const cues = await transcribeFree(s.projectId, {- duration: span.to - specFrom,- locale,- clips: [],- audio: [{ file: asset.fileName, in: specFrom, out: span.to, start: 0, volume: 1 }],- });+ // A video bound to a recording is transcribed off the recording, laid on+ // the video's clock; a stretch the recording does not reach is silence.+ const src = soundSourceOf(asset, s.assets);+ const heard = soundWindow({ in: specFrom, out: span.to }, src, 0, span.to - specFrom);+ const cues = heard+ ? await transcribeFree(s.projectId, {+ duration: span.to - specFrom,+ locale,+ clips: [],+ audio: [{ file: src.asset.fileName, in: heard.lo, out: heard.hi, start: heard.at, volume: 1 }],+ })+ : [];if (cues === null) return; // another project took over mid-runconst segments = cues.map((c) => ({ start: c.start + specFrom, end: c.end + specFrom, text: c.text }))diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 940570cc..477a86f9 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -12,6 +12,7 @@import { EXPORT_RESOLUTIONS } from "@/cut/lib/exportPresets";import { UI_TOOLS } from "@/cut/lib/projectCommands";import { STOCK_SFX_CATEGORIES } from "@/cut/lib/stock";+import { SPLIT_EDIT_MAX_S } from "@/cut/lib/soundSource";import { builtinLutCatalogText } from "@/cut/lib/builtinLuts";import { AI_PANEL_TOOLS } from "@/cut/components/AiPanel.tools";import {diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.tsindex 62672cb3..1174c9a9 100644--- a/site/src/cut/server/ai/headlessTools.ts+++ b/site/src/cut/server/ai/headlessTools.ts@@ -35,6 +35,7 @@ const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set(["detect_silence","detect_beats","refine_speech_cuts",+ "sync_audio","freeze_frame","capture_frame","compare_to_source",diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.tsindex e66d92c6..78098cb8 100644--- a/site/src/cut/server/cloud/library.ts+++ b/site/src/cut/server/cloud/library.ts@@ -890,6 +890,7 @@ export const libraryCloud = {duration: m.duration,width: m.width,height: m.height,+ ...(m.soundFrom ? { soundFrom: m.soundFrom } : {}),});}} catch (e) {diff --git a/site/src/cut/server/cloud/sharedView.test.ts b/site/src/cut/server/cloud/sharedView.test.tsindex b450a53d..f8ad13f3 100644--- a/site/src/cut/server/cloud/sharedView.test.ts+++ b/site/src/cut/server/cloud/sharedView.test.ts@@ -83,6 +83,18 @@ describe("filterDocForShare", () => {]);});+ test("a placed video's bound recording travels with it", () => {+ const bound = {+ ...doc,+ assets: [+ ...doc.assets.map((a) => (a.id === "placed" ? { ...a, soundFrom: { assetId: "lav", offset: 0.4 } } : a)),+ { id: "lav", fileName: "lav.wav", type: "audio", duration: 1 },+ ],+ } as unknown as ProjectDoc;+ expect(ids(filterDocForShare(bound, features()))).toEqual(["lav", "placed"]);+ expect(sharedFileNames(bound, features()).has("lav.wav")).toBe(true);+ });+test("a caption font travels only when subtitles are shared", () => {const captioned = {...doc,diff --git a/site/src/cut/server/cloud/sharedView.ts b/site/src/cut/server/cloud/sharedView.tsindex 82527084..f8823f39 100644--- a/site/src/cut/server/cloud/sharedView.ts+++ b/site/src/cut/server/cloud/sharedView.ts@@ -95,7 +95,10 @@ function allowedAssets(doc: ProjectDoc, features: ShareFeatures): StoredAsset[]for (const c of clips) {if (c.removal?.matte) referenced.add(c.removal.matte.assetId);if (c.removal?.backdrop?.assetId) referenced.add(c.removal.backdrop.assetId);+ if (c.card?.matte) referenced.add(c.card.matte.assetId);}+ // A video bound to a separate recording plays that recording.+ for (const a of assets) if (referenced.has(a.id) && a.soundFrom) referenced.add(a.soundFrom.assetId);const fonts = new Set<string>();for (const o of Array.isArray(doc.overlays) ? doc.overlays : []) {if (o.kind === "sticker" && o.assetId) referenced.add(o.assetId);diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex eb714cd6..0c49db29 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -137,6 +137,15 @@ export interface ExportSpec {* a cut their pictures hard-join. */soundAhead?: number;soundBack?: number;+ /** A split edit: seconds of `soundBack` the sound plays ahead of the+ * picture (J-cut) and of `soundAhead` it carries past it (L-cut), each+ * ramping from silence over `splitFade` at its far end. */+ soundLead?: number;+ soundTail?: number;+ splitFade?: number;+ /** The recording bound to this clip's video, which its sound reads in+ * place of the file's own track (see soundAlign.ts). */+ soundFrom?: SpecSound;/** Transition look id, resolved to an xfade name through the* TRANSITION_XFADE allowlist (unknown ids render as a plain fade). Cross* zoom renders as the fade plus zoom ramps on both segments' overlap@@ -244,6 +253,11 @@ export interface ExportSpec {tailSound?: number;soundBack?: number;soundAhead?: number;+ /** A split edit and a bound recording, as on `clips`. */+ soundLead?: number;+ soundTail?: number;+ splitFade?: number;+ soundFrom?: SpecSound;/** A still image: looped for the clip's length instead of trimmed. */image?: boolean;/** Manual color adjustments, baked into this overlay's segment. */@@ -296,6 +310,10 @@ export interface ExportSpec {/** Voiceover ducking: while this clip plays, every other sound drops to* this gain (0..1). Ducking clips never duck each other. */duck?: number;+ /** Detached from a video bound to a recording: it plays the recording. */+ soundFrom?: SpecSound;+ /** The soundtrack lane it sits on, which picks its stem; absent = 0. */+ lane?: number;}[];/** Overlay elements. A static element is one full-frame PNG windowed with* `enable`. An animated element ships `frames` — a region-cropped slideshow@@ -882,13 +900,61 @@ export async function runExport(turned.set(file, { video, audio, mono });return { ...mirrorRetimable(c, pivot), file };};+ // A video bound to a recording sounds from the recording. Each entry that+ // reads one gets a sound twin: the recording laid on the video's clock over+ // the stretch the entry reaches (soundAlign.ts), with the entry's trim and+ // map moved onto that file. Picture reads keep the entry; every sound read+ // below goes through `sounding`. A twin turns with its entry, and a+ // recording that holds none of the stretch leaves the entry silent.+ const twinFiles = new Set<string>();+ const twinOf = async <T extends Parameters<typeof turn>[0] & { soundFrom?: SpecSound; muted?: boolean }>(+ c: T,+ tag: string+ ): Promise<T | null | undefined> => {+ if (!c.soundFrom || !c.file || c.image || c.hidden || c.muted) return undefined;+ const rec = await resolveMedia(io.stat, mediaPathFor, c.soundFrom.file);+ const rt = retimeOf(c);+ const reach = srcSpan(rt, -(c.soundBack ?? 0), rt.len + (c.soundAhead ?? 0));+ const file = path.join(job.tmpDir, `bound_${tag}.wav`);+ const mono = (await io.audioChannels(rec)) === 1;+ const laid = await alignRecording((args) => io.runFfmpeg(job, args), rec, c.soundFrom, reach.lo, reach.hi, file, {+ mono,+ });+ if (!laid) return null;+ twinFiles.add(file);+ const { soundFrom: _bound, ...rest } = shiftSpan(c, -reach.lo);+ void _bound;+ return { ...rest, file } as T;+ };+ /** Each entry's sound twin, null for one whose recording is silent there. */+ const soundTwins = new Map<object, Sounding | null>();const turnedClips: ExportSpec["clips"] = [];- for (const [j, c] of spec.clips.entries()) turnedClips.push(await turn(c, `clip_${j}`));+ for (const [j, c] of spec.clips.entries()) {+ const twin = await twinOf(c, `clip_${j}`);+ const tc = await turn(c, `clip_${j}`);+ turnedClips.push(tc);+ if (twin !== undefined) soundTwins.set(tc, twin && (await turn(twin, `clipsnd_${j}`, true)));+ }const turnedOverlays: NonNullable<ExportSpec["overlayVideos"]> = [];- for (const [k, oc] of (spec.overlayVideos ?? []).entries()) turnedOverlays.push(await turn(oc, `ovl_${k}`));+ for (const [k, oc] of (spec.overlayVideos ?? []).entries()) {+ const twin = await twinOf(oc, `ovl_${k}`);+ const to = await turn(oc, `ovl_${k}`);+ turnedOverlays.push(to);+ if (twin !== undefined) soundTwins.set(to, twin && (await turn(twin, `ovlsnd_${k}`, true)));+ }const turnedAudio: ExportSpec["audio"] = [];- for (const [k, a] of spec.audio.entries()) turnedAudio.push(await turn(a, `snd_${k}`, true));+ for (const [k, a] of spec.audio.entries()) {+ const twin = await twinOf(a, `snd_${k}`);+ if (twin === null) continue;+ turnedAudio.push(await turn(twin ?? a, `snd_${k}`, true));+ }spec = { ...spec, clips: turnedClips, overlayVideos: turnedOverlays, audio: turnedAudio };+ /** What an entry's sound reads: its sound twin, or the entry itself. Null+ * when a bound recording holds nothing over it. */+ const sounding = <T extends Sounding>(c: T): T | Sounding | null => (soundTwins.has(c) ? soundTwins.get(c)! : c);+ /** The path an entry's file reads from: a twin is on disk already. */+ const sourcePath = (file: string) =>+ twinFiles.has(file) ? Promise.resolve(file) : resolveMedia(io.stat, mediaPathFor, file);// Tracks number 0..N bottom-up: track 0 folds sequentially into the base// picture, the rest overlay it in track order (highest last = frontmost).diff --git a/site/src/cut/server/library.ts b/site/src/cut/server/library.tsindex bf33c06c..639590f6 100644--- a/site/src/cut/server/library.ts+++ b/site/src/cut/server/library.ts@@ -81,6 +81,8 @@ export interface TemplateMedia {duration: number;width?: number;height?: number;+ /** A video's separately recorded sound: the recording's index in `media`. */+ soundFrom?: { media: number; offset: number };}export interface TemplateLayer {media: number; // index into template.media@@ -544,6 +546,7 @@ export interface TemplateInput {duration: number;width?: number;height?: number;+ soundFrom?: { media: number; offset: number };}[];layers: TemplateLayer[];audio: TemplateAudio[];@@ -631,6 +634,7 @@ export async function saveTemplate(duration: m.duration,width: m.width,height: m.height,+ ...(m.soundFrom ? { soundFrom: m.soundFrom } : {}),});}const template: LibraryTemplate = {diff --git a/site/src/cut/server/soundAlign.ts b/site/src/cut/server/soundAlign.tsnew file mode 100644index 00000000..16df43e3--- /dev/null+++ b/site/src/cut/server/soundAlign.ts@@ -0,0 +1,68 @@+/**+ * A recording laid on its video's clock, for the ffmpeg graphs.+ *+ * A video bound to a separately recorded sound (`soundFrom`) plays the+ * recording `offset` seconds past its own source time. The graphs read a+ * clip's sound through the clip's own trim and retime map, so the recording+ * is first written out over the stretch of video source seconds the clip can+ * reach, starting at that stretch's first second: second `τ` of the file is+ * video second `lo + τ`. Where the recording had not started yet, or had+ * already stopped, the file holds silence, so every read stays on time. The+ * clip then sounds from this file like any other, through the same map.+ */++import { BAKE_RATE } from "./retimeAudio";++/** Six decimals: a seek and a length to the microsecond, well under a sample. */+const exact = (n: number) => n.toFixed(6);++/**+ * Write video source seconds `[lo, hi]` of the recording `rec` (bound with+ * `offset`, `duration` seconds long) to `outFile`, a stereo float WAV at the+ * bake rate, silence-padded to the whole window. False when the recording+ * holds none of the window; nothing is written then.+ */+export async function alignRecording(+ ffmpeg: (args: string[]) => Promise<void>,+ rec: string,+ bound: { offset: number; duration: number },+ lo: number,+ hi: number,+ outFile: string,+ /** A mono recording is laid into both channels at full level, the way the+ * browser's mixer upmixes it. */+ opts: { mono?: boolean } = {}+): Promise<boolean> {+ const len = hi - lo;+ const start = lo + bound.offset;+ const from = Math.max(0, start);+ const to = Math.min(bound.duration, hi + bound.offset);+ if (!(len > 0.001) || !(to - from > 0.001)) return false;+ const pad = Math.round((from - start) * BAKE_RATE);+ await ffmpeg([+ "-y",+ "-hide_banner",+ "-loglevel",+ "error",+ "-ss",+ exact(from),+ "-i",+ rec,+ "-t",+ exact(to - from),+ "-vn",+ "-af",+ `aresample=${BAKE_RATE},` ++ (opts.mono ? "pan=stereo|c0=c0|c1=c0," : "aformat=channel_layouts=stereo,") ++ (pad > 0 ? `adelay=delays=${pad}S:all=1,` : "") ++ `apad=whole_len=${Math.round(len * BAKE_RATE)}`,+ "-t",+ exact(len),+ "-ar",+ String(BAKE_RATE),+ "-c:a",+ "pcm_f32le",+ outFile,+ ]);+ return true;+}diff --git a/site/src/cut/server/soundBound.e2e.test.ts b/site/src/cut/server/soundBound.e2e.test.tsnew file mode 100644index 00000000..46e15f29--- /dev/null+++ b/site/src/cut/server/soundBound.e2e.test.ts@@ -0,0 +1,267 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { spawnSync } from "node:child_process";+import { mkdtemp, rm } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";++// A clip's sound away from its own track and its own picture, through a real+// ffmpeg: a recording bound to the video (laid on the video's clock, then+// read through the clip's map) and a split edit's lead and tail. Each source+// is silence with one click at a known second; where the click lands in the+// exported file says which source second played at which timeline second.++const available = (cmd: string) => spawnSync(cmd, ["-version"], { stdio: "ignore" }).status === 0;+const tools = available("ffmpeg");+const RATE = 44100;++const ff = (args: string[]) => {+ const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { encoding: "utf8" });+ if (r.status !== 0) throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);+};++/** A click: 2 ms of a loud square at `at` seconds, silence elsewhere. */+const click = (at: number, secs: number) =>+ `aevalsrc=exprs='if(between(t\\,${at}\\,${at + 0.002})\\,0.9\\,0)':s=${RATE}:d=${secs}`;++let media = "";+let tmp = "";++beforeAll(async () => {+ if (!tools) return;+ media = await mkdtemp(path.join(os.tmpdir(), "cut-sound-media-"));+ tmp = await mkdtemp(path.join(os.tmpdir(), "cut-sound-job-"));+ const video = (name: string, audio: string, secs: number) =>+ ff([+ "-f", "lavfi", "-i", `testsrc2=size=160x120:rate=24:duration=${secs}`,+ "-f", "lavfi", "-i", audio,+ "-shortest", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "pcm_s16le",+ path.join(media, name),+ ]);+ // The camera's own scratch track clicks at 2.5; the recording at 3.0 of+ // its own clock.+ video("cam.mov", click(2.5, 8), 8);+ ff(["-f", "lavfi", "-i", click(3, 8), "-c:a", "pcm_s16le", path.join(media, "rec.wav")]);+ video("quiet.mov", `anullsrc=r=${RATE}:cl=stereo:d=8`, 8);+ video("lead.mov", click(0.8, 8), 8);+ video("tail.mov", click(2.3, 8), 8);+});++afterAll(async () => {+ if (media) await rm(media, { recursive: true, force: true });+ if (tmp) await rm(tmp, { recursive: true, force: true });+});++/** Render `over` and return the timeline second of the first click, or null. */+const clickIn = async (over: Partial<ExportSpec> & Pick<ExportSpec, "clips" | "duration">, name: string) => {+ const spec: ExportSpec = {+ projectId: "sound",+ width: 160,+ height: 120,+ fps: 24,+ crf: 35,+ preset: "ultrafast",+ container: "mov",+ audioCodec: "pcm",+ audio: [],+ overlays: [],+ ...over,+ };+ const job: RenderHandle = { tmpDir: tmp, outPath: path.join(tmp, `${name}.mov`), progress: 0, log: [] };+ await runExport(job, spec, (file) => path.join(media, file));+ const r = spawnSync("ffmpeg", ["-v", "error", "-i", job.outPath, "-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"], {+ maxBuffer: 64 * 1024 * 1024,+ });+ if (r.status !== 0) throw new Error(`decode failed: ${r.stderr}`);+ const buf = r.stdout as Buffer;+ const data = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));+ for (let i = 0; i < data.length; i++) if (Math.abs(data[i]) > 0.3) return i / RATE;+ return null;+};++/** Every click in a rendered file, as the timeline second it starts on. */+const clicksOf = (file: string): number[] => {+ const r = spawnSync("ffmpeg", ["-v", "error", "-i", file, "-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"], {+ maxBuffer: 64 * 1024 * 1024,+ });+ if (r.status !== 0) throw new Error(`decode failed: ${r.stderr}`);+ const buf = r.stdout as Buffer;+ const data = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));+ const at: number[] = [];+ let quietSince = 0;+ for (let i = 0; i < data.length; i++) {+ if (Math.abs(data[i]) <= 0.3) continue;+ if (i - quietSince > RATE * 0.1) at.push(i / RATE);+ quietSince = i;+ }+ return at;+};++const near = (got: number | null, want: number) => {+ expect(got).not.toBeNull();+ expect(Math.abs(got! - want)).toBeLessThan(0.005);+};++const bound = { file: "rec.wav", offset: -1, duration: 8 };++describe("a recording bound to the video, through ffmpeg", () => {+ test.skipIf(!tools)("plays the recording on the video's clock, and the camera's track stays out", async () => {+ // Recording 3.0 is video 4.0, which a clip from video 1 plays at 3.0.+ near(await clickIn({ duration: 4, clips: [{ file: "cam.mov", in: 1, out: 5, muted: false, soundFrom: bound }] }, "bound"), 3);+ });++ test.skipIf(!tools)("a reversed clip turns the recording with it", async () => {+ // Reversed over video 1..5, timeline t shows video 5 − t: video 4 at 1.+ near(+ await clickIn(+ { duration: 4, clips: [{ file: "cam.mov", in: 1, out: 5, muted: false, reverse: true, soundFrom: bound }] },+ "bound-reverse"+ ),+ 1+ );+ });++ test.skipIf(!tools)("a soundtrack entry detached from the video plays the recording", async () => {+ near(+ await clickIn(+ {+ duration: 4,+ clips: [{ file: "quiet.mov", in: 0, out: 4, muted: true }],+ audio: [{ file: "cam.mov", in: 2, out: 5, start: 1, volume: 1, soundFrom: bound }],+ },+ "bound-audio"+ ),+ 3+ );+ });++ test.skipIf(!tools)("a recording that holds nothing of the clip leaves it silent", async () => {+ expect(+ await clickIn(+ { duration: 1, clips: [{ file: "cam.mov", in: 0, out: 1, muted: false, soundFrom: { ...bound, offset: 20 } }] },+ "bound-gone"+ )+ ).toBeNull();+ });+});++describe("a mastered export with stems, through ffmpeg", () => {+ test.skipIf(!tools)("the dialogue stem carries the bound recording and the split edit, and the mix masters them", async () => {+ // Recording 3.0 is video 4.0, which the first clip (video 3..5) plays at+ // 1.0. The second clip starts at 2 on source 1; its lead plays source+ // 0.8 at 1.8. The soundtrack's click at 3.0 belongs to its own lane.+ const spec: ExportSpec = {+ projectId: "sound",+ width: 160,+ height: 120,+ fps: 24,+ crf: 35,+ preset: "ultrafast",+ container: "mov",+ audioCodec: "pcm",+ overlays: [],+ duration: 4,+ loudness: -16,+ truePeakCeiling: -1,+ clips: [+ { file: "cam.mov", in: 3, out: 5, muted: false, soundFrom: bound },+ { file: "lead.mov", in: 1, out: 3, muted: false, soundBack: 0.5, soundLead: 0.5, splitFade: 0.03 },+ ],+ audio: [{ file: "rec.wav", in: 0, out: 4, start: 0, volume: 1, lane: 0 }],+ stemPlan: [+ { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },+ { name: "rec", lane: 0, file: "2 rec.wav" },+ ],+ };+ const job: RenderHandle = {+ tmpDir: tmp,+ outPath: path.join(tmp, "stems.mov"),+ stemsPath: path.join(tmp, "stems.zip"),+ progress: 0,+ log: [],+ };+ await runExport(job, spec, (file) => path.join(media, file));+ const unzipped = path.join(tmp, "stems");+ expect(spawnSync("unzip", ["-o", "-q", job.stemsPath!, "-d", unzipped]).status).toBe(0);++ const dialogue = clicksOf(path.join(unzipped, "1 Dialogue.wav"));+ expect(dialogue).toHaveLength(2);+ near(dialogue[0], 1);+ near(dialogue[1], 1.8);+ const lane = clicksOf(path.join(unzipped, "2 rec.wav"));+ expect(lane).toHaveLength(1);+ near(lane[0], 3);+ const mix = clicksOf(job.outPath);+ expect(mix).toHaveLength(3);+ [1, 1.8, 3].forEach((want, i) => near(mix[i], want));+ });+});++describe("split edits, through ffmpeg", () => {+ test.skipIf(!tools)("a J-cut plays the incoming clip's sound ahead of its picture", async () => {+ // The second clip starts at 2 on source 1; its lead plays source 0.8 at 1.8.+ near(+ await clickIn(+ {+ duration: 4,+ clips: [+ { file: "quiet.mov", in: 0, out: 2, muted: false },+ { file: "lead.mov", in: 1, out: 3, muted: false, soundBack: 0.5, soundLead: 0.5, splitFade: 0.03 },+ ],+ },+ "jcut"+ ),+ 1.8+ );+ });++ test.skipIf(!tools)("an L-cut carries the outgoing clip's sound past its picture", async () => {+ near(+ await clickIn(+ {+ duration: 4,+ clips: [+ { file: "tail.mov", in: 0, out: 2, muted: false, soundAhead: 0.5, soundTail: 0.5, splitFade: 0.03 },+ { file: "quiet.mov", in: 0, out: 2, muted: false },+ ],+ },+ "lcut"+ ),+ 2.3+ );+ });++ test.skipIf(!tools)("an upper-track clip's lead lands ahead of its start", async () => {+ near(+ await clickIn(+ {+ duration: 4,+ clips: [{ file: "quiet.mov", in: 0, out: 4, muted: true }],+ overlayVideos: [+ {+ file: "lead.mov", in: 1, out: 2, start: 2, track: 1, muted: false,+ soundBack: 0.5, soundLead: 0.5, splitFade: 0.03,+ },+ ],+ },+ "overlay-jcut"+ ),+ 1.8+ );+ });++ test.skipIf(!tools)("without a split edit the trimmed-away sound stays out", async () => {+ expect(+ await clickIn(+ {+ duration: 4,+ clips: [+ { file: "tail.mov", in: 0, out: 2, muted: false },+ { file: "lead.mov", in: 1, out: 3, muted: false },+ ],+ },+ "plain"+ )+ ).toBeNull();+ });+});diff --git a/site/src/cut/worker/turnJob.ts b/site/src/cut/worker/turnJob.tsindex 790cc495..70249a5b 100644--- a/site/src/cut/worker/turnJob.ts+++ b/site/src/cut/worker/turnJob.ts@@ -2,6 +2,8 @@ import type { UIMessage } from "ai";import { getGlobalSetting } from "@/lib/config/effective";import { bindCutClip, bindCutJudge } from "@/cut/lib/chatRuntime";import { bindCutColor } from "@/cut/lib/colorSettings";+import { bindCutSound } from "@/cut/lib/soundSettings";+import { bindCutMotion } from "@/cut/lib/motionSettings";import { geminiModelRoleNames } from "@/lib/inference/gemini-models";import { openCloudProject, pushCloudProject } from "../lib/headless/docSession";import { dropPiSession, streamCutChat } from "../lib/pi/cutAgent";@@ -111,6 +113,8 @@ export async function runTurnJob(bindCutJudge(judgeSettings);bindCutClip(await getGlobalSetting("cutClip"));bindCutColor(await getGlobalSetting("cutColor"));+ bindCutSound(await getGlobalSetting("cutSound"));+ bindCutMotion(await getGlobalSetting("cutMotion"));deps.judgeSettings = judgeSettings;const toolCalls: string[] = [];const exec = deps.execTool;