Open source CapCut alternative.
TypeScript
/**
* Binding a separately recorded sound to a video: find where the recording
* lines up with the camera's own track, and only when one alignment clearly
* wins, write it onto the video asset (`soundFrom`). The Audio tab's Sound
* picker and the assistant's sync_audio both land here, so the person and the
* assistant hold the recording to the same bar.
*/
import { syncSourceSound } from "./media";
import { CUT_SOUND } from "./soundSettings";
import { useEditor } from "./store";
import { assetIsSilent, type MediaAsset } from "./types";
import { UserError } from "./userError";
import { queueWatchSweep } from "./watch/sweep";
interface BoundSound {
assetId: string;
audioAssetId: string;
/** Recording seconds = video seconds + offset. */
offset: number;
/** The best alignment over the runner-up a second or more away. */
confidence: number;
/** Whether the offset sits on the sample (else on 20 ms). */
refined: boolean;
/** Clips of the video on the timeline, all now sounding from the recording. */
clips: number;
}
/**
* Line `audioId` up with `videoId` and bind it. `near` is the source span of
* the clip the person is working on, so the camera's stretch that is matched
* is the one they care about. Throws a readable Error when the files cannot
* be matched or no alignment stands out; nothing is bound then.
*/
export async function bindRecording(
videoId: string,
audioId: string,
near?: { in: number; out: number }
): Promise<BoundSound> {
const s = useEditor.getState();
const video = s.assets.find((a) => a.id === videoId);
const rec = s.assets.find((a) => a.id === audioId);
if (!video || video.type !== "video") {
throw new UserError("sync_video_missing", "That video is no longer in the project.");
}
if (!rec) {
throw new UserError("sync_recording_missing", "That recording is no longer in the project.");
}
if (rec.type !== "audio") {
throw new UserError("sync_recording_not_audio", `"${rec.name}" is not an audio file.`, { name: rec.name });
}
if (assetIsSilent(video)) {
throw new UserError(
"sync_video_silent",
`"${video.name}" has no sound of its own to line the recording up against.`,
{ name: video.name },
);
}
if (!(video.duration > 0) || !(rec.duration > 0)) {
throw new UserError("sync_files_importing", "Both files must finish importing first.");
}
const { syncProbeSeconds, syncMaxSeconds, syncMinConfidence } = CUT_SOUND;
const [cameraFrom, cameraTo] = probeWindow(video, syncProbeSeconds, near);
const answer = await syncSourceSound(video.url, rec.url, {
cameraFrom,
cameraTo,
recordingTo: Math.min(rec.duration, syncMaxSeconds),
});
if (!answer || answer.confidence < syncMinConfidence) {
const read = answer ? ` (confidence ${answer.confidence.toFixed(2)}, needs ${syncMinConfidence})` : "";
const message = `No clear alignment between "${rec.name}" and "${video.name}"${read}. They may be different takes, or share too little sound.`;
// The weak-match line names the score it got and the score it needs.
if (answer) {
throw new UserError("sync_alignment_weak", message, {
recording: rec.name,
video: video.name,
confidence: answer.confidence.toFixed(2),
needs: syncMinConfidence,
});
}
throw new UserError("sync_alignment_none", message, { recording: rec.name, video: video.name });
}
const offset = Math.round(answer.offset * 1e5) / 1e5;
useEditor.getState().setAssetSoundFrom(video.id, { assetId: rec.id, offset });
// Its transcript is heard again, off the recording.
queueWatchSweep(video.id);
return {
assetId: video.id,
audioAssetId: rec.id,
offset,
confidence: Math.round(answer.confidence * 100) / 100,
refined: answer.refined,
clips: useEditor.getState().clips.filter((c) => c.assetId === video.id).length,
};
}
/** Return a video to its own sound. */
export function unbindRecording(videoId: string): void {
useEditor.getState().setAssetSoundFrom(videoId, undefined);
queueWatchSweep(videoId);
}
/** The camera's stretch to match: `seconds` long, centred on the clip being
* worked on, else on the middle of the video, inside the file. */
function probeWindow(video: MediaAsset, seconds: number, near?: { in: number; out: number }): [number, number] {
const len = Math.min(seconds, video.duration);
const mid = near ? (near.in + near.out) / 2 : video.duration / 2;
const from = Math.max(0, Math.min(video.duration - len, mid - len / 2));
return [from, from + len];
}