Skip to content

Open source CapCut alternative.

TypeScript

fix(site): harden separate mic sync [rebuild]

David Lef9e800fff764
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index bba8da49..bda2d5cc 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -14,7 +14,7 @@ import { blendInto, SYNTH_EDGE, synthWeight } from "@/cut/lib/frameSynth";
import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";
import { assetIsSilent, clipCovers, rectOf } from "@/cut/lib/types";
import type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
-import { soundSourceOf, soundWindow } from "@/cut/lib/soundSource";
+import { OWN_SOUND, soundSourceOf, soundWindow } from "@/cut/lib/soundSource";
import { SubjectMaskCompositor } from "@/cut/lib/behindPass";
import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";
import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";
@@ -169,15 +169,13 @@ export function engineLog(msg: string): void {
*/
const SOUND_LOOKAHEAD_S = 3;
-const OWN_TRACK = { offset: 0, limit: Infinity };
-
const voiceSpan = (sp: ClipSpan) => {
// The handles a crossing or a split edit reaches into play at the pace of
// the footage beside them, so the voice's span widens through the clip's
// own map and its retime is rebuilt over the wider span. A recording bound
// to the video is read through the same map, on its own clock, and starts
// late where it rolled after the camera did.
- const w = soundWindow(sp.clip, sp.sound ?? OWN_TRACK, -sp.soundBack, sp.len + sp.soundAhead);
+ const w = soundWindow(sp.clip, sp.sound ?? OWN_SOUND, -sp.soundBack, sp.len + sp.soundAhead);
if (!w) return null;
return {
url: (sp.sound?.asset ?? sp.asset).url,
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 88a3fd4d..25c7aa1e 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -3942,7 +3942,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
}
const chatId = chatOwner();
const toCopy = built.sourceAssets.filter((a) => !mapping.has(a.id));
- const landed = await landReferenceAssets(ref, toCopy, projectId, chatId);
+ const landed = await landReferenceAssets(ref, toCopy, projectId, chatId, new Set(mapping.keys()));
const landedBySource = new Map(landed.map((l) => [l.sourceId, l.asset]));
const assetIds = built.assetByMedia.map(
(sid) => mapping.get(sid) ?? landedBySource.get(sid)?.id ?? ""
@@ -5880,10 +5880,11 @@ async function landReferenceAssets(
ref: ReferenceProject,
sources: StoredAsset[],
projectId: string,
- chatId: string | null
+ chatId: string | null,
+ mapped?: ReadonlySet<string>
): Promise<LandedAsset[]> {
if (sources.length === 0) return [];
- return landReferenceAssetsInto(ref, sources, projectId, { chatId }).catch((e) => {
+ return landReferenceAssetsInto(ref, sources, projectId, { chatId, mapped }).catch((e) => {
throw new ToolError(e instanceof Error ? e.message : "Could not copy the reference's media.");
});
}
diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.ts
index 1b984e10..0380dbfb 100644
--- a/site/src/cut/lib/audioMix.ts
+++ b/site/src/cut/lib/audioMix.ts
@@ -28,7 +28,7 @@ import {
} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { fitSpan, retimeFits } from "./retimeFit";
-import { shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";
+import { OWN_SOUND, shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";
/** One track-0 clip's audio in the sequential fold. A spacer (no file) only
* shapes time. */
@@ -222,7 +222,7 @@ type Sounded = {
const soundRead = (c: Sounded, back = 0, ahead = 0): (SoundWindow & { file: string }) | null => {
const len = retimeOf(c).len;
const from = c.soundFrom;
- const w = soundWindow(c, from ? { offset: from.offset, limit: from.duration } : { offset: 0, limit: Infinity }, -back, len + ahead);
+ const w = soundWindow(c, from ? { offset: from.offset, limit: from.duration } : OWN_SOUND, -back, len + ahead);
return w && { ...w, file: from ? from.file : c.file };
};
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 8e05b852..40286be4 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -35,12 +35,11 @@ import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, pa
import { renderElementFrames, renderElementPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";
import { liveReader } from "./liveReader";
-import { specSound } from "./soundSource";
+import { soundSourceOf, spanSoundSpec, spanUsesSoundFeature, specSound } from "./soundSource";
import type {
Aspect,
AudioClip,
ClipAnim,
- ClipSpan,
MediaAsset,
Overlay,
Selection,
@@ -800,18 +799,8 @@ function renderClipBorderPng(
return rasterCanvasToPng(canvas);
}
-/** A clip's sound as the spec carries it: the recording bound to its video,
- * and a split edit's reach past its picture with the ramp at the far end. */
-function specSpanSound(sp: ClipSpan) {
- return {
- ...(sp.sound
- ? { soundFrom: { file: sp.sound.asset.fileName, offset: sp.sound.offset, duration: sp.sound.asset.duration } }
- : {}),
- ...(sp.soundLead || sp.soundTail
- ? { soundLead: sp.soundLead ?? 0, soundTail: sp.soundTail ?? 0, splitFade: sp.splitFade ?? 0 }
- : {}),
- };
-}
+/** The spec names a file by its name in the project's media folder. */
+const specFile = (a: MediaAsset) => a.fileName;
/** What a source's code values mean, for the spec: the profile the header
* settled (with the person's override), and the matrix and range the file
@@ -929,7 +918,7 @@ export async function buildExportPayload(
: undefined;
const clipEntries = spans.map((sp) => ({
- ...specSpanSound(sp),
+ ...spanSoundSpec(sp, specFile),
file: sp.asset.fileName,
in: sp.clip.in,
out: sp.clip.out,
@@ -1163,7 +1152,7 @@ export async function buildExportPayload(
.filter(({ c }) => !c.hidden && c.start < duration)
.map(({ c, sp, ramp }) => {
const entry = {
- ...specSpanSound(sp),
+ ...spanSoundSpec(sp, specFile),
file: assetById.get(c.assetId)!.fileName,
in: c.in,
out: c.out,
@@ -1637,10 +1626,17 @@ export function downloadExport(jobId: string, outName: string, backend: CutBacke
* not know — a range export would come back as the whole cut, a typed name
* as the project's — so what the settings ask of the engine is checked
* against what it says it carries, and the export refuses with the fix. */
-/** Whether the cut plays a bound recording or a split edit anywhere. */
+/** Whether the cut plays a bound recording or a split edit anywhere: on a
+ * clip of any track that sounds, or on the soundtrack. */
function cutUsesSoundFeatures(doc: ExportDoc): boolean {
- const bound = new Set(doc.assets.filter((a) => a.soundFrom).map((a) => a.id));
- return doc.clips.some((c) => bound.has(c.assetId) || !!c.audioLead || !!c.audioTail);
+ const byId = new Map(doc.assets.map((a) => [a.id, a]));
+ const soundtrack = doc.audioClips.some((a) => {
+ const asset = byId.get(a.assetId);
+ return !!asset && !a.hidden && soundSourceOf(asset, byId).bound;
+ });
+ if (soundtrack) return true;
+ const tracks = new Set(doc.clips.map((c) => c.track));
+ return [...tracks].some((t) => getClipSpans(doc.clips, doc.assets, t).some(spanUsesSoundFeature));
}
async function assertEngineCarries(
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index 4a9e3873..67ea0cca 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -34,7 +34,7 @@ import { audioFxSpans } from "./audioEffects";
import { duckWindows, renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix";
import { masterInPlace } from "./loudness";
import { STEM_CHANNELS, STEM_RATE, wavBytes, wavChunks, writeStoredZip, type StemDef } from "./stems";
-import { specSound } from "./soundSource";
+import { spanSoundSpec, specSound } from "./soundSource";
import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";
import { ensureClipLuts, sourceLookup } from "./lutBuild";
import { colorRead, type ColorRead, type ReadFile } from "./sourceColor";
@@ -889,14 +889,6 @@ function trackZeroFades(spans: ClipSpan[], i: number): { fadeIn: number; fadeOut
* while the picture keeps them, and everything after the first gap plays early.
*/
export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => string): MixSpec {
- // A clip's sound past its picture (a split edit) and the recording bound to
- // its video, as the fold reads them.
- const spanSound = (sp: ClipSpan) => ({
- ...(sp.sound ? { soundFrom: { file: resolve(sp.sound.asset), offset: sp.sound.offset, duration: sp.sound.asset.duration } } : {}),
- ...(sp.soundLead || sp.soundTail
- ? { soundLead: sp.soundLead, soundTail: sp.soundTail, splitFade: sp.splitFade }
- : {}),
- });
const duration = projectDuration(doc);
const spans = getClipSpans(doc.clips, doc.assets, 0);
const byId = new Map(doc.assets.map((a) => [a.id, a]));
@@ -923,7 +915,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
crossOut: ramps[i].crossOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
- ...spanSound(sp),
+ ...spanSoundSpec(sp, resolve),
});
});
}
@@ -971,7 +963,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
soundCross: sp.soundOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
- ...spanSound(sp),
+ ...spanSoundSpec(sp, resolve),
},
]);
diff --git a/site/src/cut/lib/framePlan.ts b/site/src/cut/lib/framePlan.ts
index 803bc7cd..8e9edb3e 100644
--- a/site/src/cut/lib/framePlan.ts
+++ b/site/src/cut/lib/framePlan.ts
@@ -277,7 +277,7 @@ export function crossHandles(spans: ClipSpan[], t: number): { span: ClipSpan; ga
* silence where an L-cut's tail closes. The offline fold and the ffmpeg graph
* write the same ramps.
*/
-export function splitEditGain(sp: ClipSpan, t: number): number {
+function splitEditGain(sp: ClipSpan, t: number): number {
const lead = sp.soundLead ?? 0;
const tail = sp.soundTail ?? 0;
const end = sp.start + sp.len;
diff --git a/site/src/cut/lib/library.ts b/site/src/cut/lib/library.ts
index fc82466c..34160f1f 100644
--- a/site/src/cut/lib/library.ts
+++ b/site/src/cut/lib/library.ts
@@ -48,6 +48,7 @@ import type {
TemplateMedia,
TemplateSaveInput,
} from "./types";
+import { landedSoundFrom } from "./projectTemplate";
import { IMAGE_CLIP_SECONDS, isLinkedAssetType, mediaUrl } from "./types";
@@ -1167,10 +1168,13 @@ export async function addTemplateToProject(
const s = useEditor.getState();
// Each copied media file (in template.media order) becomes a project asset;
// the layer/audio media indices resolve against this array. Enrichment gives
- // the new clips their filmstrip thumbnails and waveform peaks.
- const assetIds = media.map((m, i) => {
+ // the new clips their filmstrip thumbnails and waveform peaks. A video's
+ // separate recording binds to the copy of it.
+ const assetIds = media.map(() => crypto.randomUUID().slice(0, 8));
+ media.forEach((m, i) => {
+ const soundFrom = landedSoundFrom(m, assetIds);
const asset: MediaAsset = {
- id: crypto.randomUUID().slice(0, 8),
+ id: assetIds[i],
fileName: m.fileName,
name: m.name,
type: m.type,
@@ -1178,10 +1182,10 @@ export async function addTemplateToProject(
width: m.width,
height: m.height,
url: urls[i],
+ ...(soundFrom ? { soundFrom } : {}),
};
s.addAsset(asset);
void enrichAsset(asset);
- return asset.id;
});
s.insertTemplate(template, assetIds, at ?? playheadAt());
}
@@ -1225,11 +1229,15 @@ export async function addProjectTemplateToTimeline(
)
);
const s = useEditor.getState();
- const assetIds = template.media.map((m, i) => {
- const existing = s.assets.find((a) => a.fileName === m.fileName);
- if (existing) return existing.id;
+ // A media file still in the project plays as it stands; a re-registered
+ // one binds to its separate recording again.
+ const existing = template.media.map((m) => s.assets.find((a) => a.fileName === m.fileName));
+ const assetIds = existing.map((a) => a?.id ?? crypto.randomUUID().slice(0, 8));
+ template.media.forEach((m, i) => {
+ if (existing[i]) return;
+ const soundFrom = landedSoundFrom(m, assetIds);
const asset: MediaAsset = {
- id: crypto.randomUUID().slice(0, 8),
+ id: assetIds[i],
fileName: m.fileName,
name: m.name,
type: m.type,
@@ -1237,10 +1245,10 @@ export async function addProjectTemplateToTimeline(
width: m.width,
height: m.height,
url: urls[i] ?? mediaUrl(projectId, m.fileName),
+ ...(soundFrom ? { soundFrom } : {}),
};
s.addAsset(asset);
void enrichAsset(asset);
- return asset.id;
});
useEditor.getState().insertTemplate(template, assetIds, at ?? playheadAt());
}
diff --git a/site/src/cut/lib/projectReference.test.ts b/site/src/cut/lib/projectReference.test.ts
new file mode 100644
index 00000000..50cfa0de
--- /dev/null
+++ b/site/src/cut/lib/projectReference.test.ts
@@ -0,0 +1,59 @@
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+
+import { landReferenceAssets, setHostDocStore, type ReferenceProject } from "./projectReference";
+import { useEditor } from "./store";
+import type { StoredAsset } from "./types";
+
+/**
+ * A reference's media landing in the open project: a video bound to a
+ * separate recording brings the recording along, unless the caller plays
+ * that recording's part from an asset of its own.
+ */
+
+const copies: string[] = [];
+setHostDocStore({
+ readDoc: async () => null,
+ copyMedia: async (_src, file) => {
+ copies.push(file);
+ return `copy-${file}`;
+ },
+});
+afterAll(() => setHostDocStore(null));
+
+const video: StoredAsset = {
+ id: "v0",
+ fileName: "cam.mp4",
+ name: "cam",
+ type: "video",
+ duration: 30,
+ soundFrom: { assetId: "r0", offset: 0.25 },
+};
+const rec: StoredAsset = { id: "r0", fileName: "lav.wav", name: "lav", type: "audio", duration: 60 };
+const ref: ReferenceProject = {
+ link: { kind: "project", id: "ref" },
+ projectId: "ref",
+ doc: { assets: [video, rec] } as ReferenceProject["doc"],
+ residency: "local",
+ backend: null,
+ partial: [],
+};
+
+describe("landReferenceAssets", () => {
+ beforeEach(() => {
+ copies.length = 0;
+ useEditor.setState({ projectId: "p", assets: [] });
+ });
+
+ test("a bound video brings its recording and binds to the copy", async () => {
+ const [landed] = await landReferenceAssets(ref, [video], "p");
+ expect(copies).toEqual(["cam.mp4", "lav.wav"]);
+ const copy = useEditor.getState().assets.find((a) => a.fileName === "copy-lav.wav")!;
+ expect(landed.asset.soundFrom).toEqual({ assetId: copy.id, offset: 0.25 });
+ });
+
+ test("a recording the caller maps to its own asset stays behind", async () => {
+ const [landed] = await landReferenceAssets(ref, [video], "p", { mapped: new Set(["r0"]) });
+ expect(copies).toEqual(["cam.mp4"]);
+ expect(landed.asset.soundFrom).toBeUndefined();
+ });
+});
diff --git a/site/src/cut/lib/projectReference.ts b/site/src/cut/lib/projectReference.ts
index 3445feb7..330564d5 100644
--- a/site/src/cut/lib/projectReference.ts
+++ b/site/src/cut/lib/projectReference.ts
@@ -150,22 +150,26 @@ export interface LandedAsset {
* a font installs so titles set in it draw, and each copy remembers where it
* came from, so the same source pasted or replicated twice lands once. A
* chat's copies preview on its cards; a keyboard paste is the user's own
- * import.
+ * import. `mapped` names the reference sources the caller plays from assets
+ * of its own, which never copy across.
*/
export async function landReferenceAssets(
ref: ReferenceProject,
sources: StoredAsset[],
projectId: string,
- opts: { chatId?: string | null } = {}
+ opts: { chatId?: string | null; mapped?: ReadonlySet<string> } = {}
): Promise<LandedAsset[]> {
const out: LandedAsset[] = [];
const toCopy: StoredAsset[] = [];
- // A video bound to a separate recording brings the recording with it.
+ // A video bound to a separate recording brings the recording with it,
+ // unless the caller plays that recording's part from its own asset; the
+ // copy then plays its own track.
const refAssets = new Map(ref.doc.assets.map((a) => [a.id, a]));
const wanted = [...sources];
for (const source of sources) {
const rec = source.soundFrom && refAssets.get(source.soundFrom.assetId);
- if (rec && !wanted.some((w) => w.id === rec.id)) wanted.push(rec);
+ if (!rec || opts.mapped?.has(rec.id) || wanted.some((w) => w.id === rec.id)) continue;
+ wanted.push(rec);
}
for (const source of wanted) {
const have = useEditor
diff --git a/site/src/cut/lib/projectTemplate.test.ts b/site/src/cut/lib/projectTemplate.test.ts
index 9bcf4b61..2b6f0e08 100644
--- a/site/src/cut/lib/projectTemplate.test.ts
+++ b/site/src/cut/lib/projectTemplate.test.ts
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, test } from "bun:test";
import { richDoc, RICH_CLIP_S } from "./fixtures/richDoc";
-import { clampLayersToAssets, templateFromDoc } from "./projectTemplate";
+import { clampLayersToAssets, landedSoundFrom, templateFromDoc } from "./projectTemplate";
import { resolveTransitions, useEditor } from "./store";
import { emptySubtitles, isStickerOverlay, isTextOverlay, type MediaAsset, type TemplateSaveInput } from "./types";
@@ -132,7 +132,7 @@ describe("insertTemplate", () => {
const MAP: Record<string, string> = { v0: "b0", v1: "b1", m0: "bm", s0: "bs", f0: "" };
const idsFor = (assetByMedia: string[], map = MAP) => assetByMedia.map((id) => map[id] ?? "");
- test("a video's bound recording travels with it and binds to the landed copy", () => {
+ test("a video's bound recording travels with it, and lands onto the copies only", () => {
const doc = richDoc();
const withRec = {
...doc,
@@ -146,9 +146,15 @@ describe("insertTemplate", () => {
const r0 = assetByMedia.indexOf("r0");
expect(r0).toBeGreaterThanOrEqual(0);
expect(template.media[v0].soundFrom).toEqual({ media: r0, offset: 0.25 });
+ // The copies bind at landing: v0's entry onto whatever r0 landed as.
+ const landed = assetByMedia.map((id) => `new-${id}`);
+ expect(landedSoundFrom(template.media[v0], landed)).toEqual({ assetId: "new-r0", offset: 0.25 });
+ expect(landedSoundFrom(template.media[v0], landed.map((id, i) => (i === r0 ? "" : id)))).toBeUndefined();
+ // Placing the template on the project's own footage leaves that footage's
+ // sound as it was: b0 plays the reference's part, not its take.
useEditor.setState({ assets: [...useEditor.getState().assets, asset("br", "audio", 60)] });
useEditor.getState().insertTemplate({ ...template, id: "t", addedAt: 0 }, idsFor(assetByMedia, { ...MAP, r0: "br" }), 0);
- expect(useEditor.getState().assets.find((a) => a.id === "b0")?.soundFrom).toEqual({ assetId: "br", offset: 0.25 });
+ expect(useEditor.getState().assets.find((a) => a.id === "b0")?.soundFrom).toBeUndefined();
});
test("stands the whole edit up on other footage, treatment and all", () => {
diff --git a/site/src/cut/lib/projectTemplate.ts b/site/src/cut/lib/projectTemplate.ts
index 9181f182..ff18d6fc 100644
--- a/site/src/cut/lib/projectTemplate.ts
+++ b/site/src/cut/lib/projectTemplate.ts
@@ -15,6 +15,7 @@ import {
type AssetType,
type MediaAsset,
type ProjectDoc,
+ type SoundFrom,
type StoredAsset,
type TemplateAudio,
type TemplateLayer,
@@ -265,3 +266,11 @@ export const mediaTypeFits = (source: AssetType, replacement: AssetType): boolea
source === "audio" || isLinkedAssetType(source)
? replacement === source
: replacement === "video" || replacement === "image";
+
+/** The binding a template media entry lands with, onto the project assets
+ * `ids` the entries landed as: a video's recording binds to whatever the
+ * recording's entry landed as. Absent when the recording did not land. */
+export function landedSoundFrom(m: TemplateMedia, ids: readonly string[]): SoundFrom | undefined {
+ const rec = m.soundFrom ? ids[m.soundFrom.media] : undefined;
+ return rec && m.soundFrom ? { assetId: rec, offset: m.soundFrom.offset } : undefined;
+}
diff --git a/site/src/cut/lib/soundBind.ts b/site/src/cut/lib/soundBind.ts
index aec9727a..22a005a5 100644
--- a/site/src/cut/lib/soundBind.ts
+++ b/site/src/cut/lib/soundBind.ts
@@ -12,7 +12,7 @@ import { useEditor } from "./store";
import { assetIsSilent, type MediaAsset } from "./types";
import { queueWatchSweep } from "./watch/sweep";
-export interface BoundSound {
+interface BoundSound {
assetId: string;
audioAssetId: string;
/** Recording seconds = video seconds + offset. */
diff --git a/site/src/cut/lib/soundSource.test.ts b/site/src/cut/lib/soundSource.test.ts
new file mode 100644
index 00000000..f15ed1d3
--- /dev/null
+++ b/site/src/cut/lib/soundSource.test.ts
@@ -0,0 +1,61 @@
+import { describe, expect, test } from "bun:test";
+
+import { spanSoundSpec, spanUsesSoundFeature } from "./soundSource";
+import type { ClipSpan, MediaAsset, VideoClip } from "./types";
+
+/**
+ * What a clip's sound asks of a renderer: the spec entry that carries a bound
+ * recording and a split edit, and whether the clip needs a renderer that
+ * plays either at all.
+ */
+
+const asset = (id: string, type: MediaAsset["type"]): MediaAsset => ({
+ id,
+ fileName: `${id}.bin`,
+ name: id,
+ type,
+ duration: 30,
+ url: "",
+});
+const span = (a: MediaAsset, extra: Partial<ClipSpan> = {}, clip: Partial<VideoClip> = {}): ClipSpan => ({
+ clip: { id: "c", assetId: a.id, in: 0, out: 5, start: 0, track: 0, ...clip } as VideoClip,
+ asset: a,
+ start: 0,
+ len: 5,
+ transitionOut: 0,
+ soundOut: 0,
+ soundAhead: 0,
+ soundBack: 0,
+ ...extra,
+});
+
+describe("spanSoundSpec", () => {
+ test("carries the bound recording by the spec's file name and fills a split edit's missing side", () => {
+ const rec = asset("r", "audio");
+ const spec = spanSoundSpec(span(asset("v", "video"), { sound: { asset: rec, offset: 0.5, limit: 30 }, soundLead: 1 }), (a) => `f-${a.id}`);
+ expect(spec).toEqual({
+ soundFrom: { file: "f-r", offset: 0.5, duration: 30 },
+ soundLead: 1,
+ soundTail: 0,
+ splitFade: 0,
+ });
+ });
+
+ test("a clip on its own track with no split edit adds nothing", () => {
+ expect(spanSoundSpec(span(asset("v", "video")), (a) => a.fileName)).toEqual({});
+ });
+});
+
+describe("spanUsesSoundFeature", () => {
+ test("a split edit or a bound recording on a clip that sounds", () => {
+ expect(spanUsesSoundFeature(span(asset("v", "video"), { soundLead: 1 }))).toBe(true);
+ expect(spanUsesSoundFeature(span(asset("v", "video"), { sound: { asset: asset("r", "audio"), offset: 0, limit: 30 } }))).toBe(true);
+ expect(spanUsesSoundFeature(span(asset("v", "video")))).toBe(false);
+ });
+
+ test("a leftover split edit on a clip that plays nothing asks nothing", () => {
+ expect(spanUsesSoundFeature(span(asset("i", "image"), { soundLead: 1 }))).toBe(false);
+ expect(spanUsesSoundFeature(span(asset("v", "video"), { soundTail: 1 }, { muted: true }))).toBe(false);
+ expect(spanUsesSoundFeature(span(asset("v", "video"), { soundTail: 1 }, { hidden: true }))).toBe(false);
+ });
+});
diff --git a/site/src/cut/lib/soundSource.ts b/site/src/cut/lib/soundSource.ts
index ee7945b5..a0d31f0c 100644
--- a/site/src/cut/lib/soundSource.ts
+++ b/site/src/cut/lib/soundSource.ts
@@ -13,7 +13,7 @@
*/
import { retimeOf, speedCurveOf, srcSpan, type Retimable, type SpeedNode } from "@donkeycut/effects-kit";
-import { assetIsSilent, type StoredAsset } from "./types";
+import { assetIsSilent, type ClipSpan, type MediaAsset, type StoredAsset } from "./types";
type Sourced = Pick<StoredAsset, "id" | "type" | "duration" | "soundFrom">;
@@ -21,6 +21,9 @@ type Sourced = Pick<StoredAsset, "id" | "type" | "duration" | "soundFrom">;
* side, in seconds. */
export const SPLIT_EDIT_MAX_S = 5;
+/** A file's own track: read from its own clock, as far as it goes. */
+export const OWN_SOUND = { offset: 0, limit: Infinity } as const;
+
/** The sound a video plays: its own file's track, or the recording bound to
* it, with the seconds added to its source time to land on that file. */
export interface SoundSource<A> {
@@ -45,7 +48,7 @@ export function soundSourceOf<A extends Sourced>(
if (rec && !assetIsSilent(rec))
return { asset: rec, offset: from.offset, limit: rec.duration > 0 ? rec.duration : Infinity, bound: true };
}
- return { asset, offset: 0, limit: Infinity, bound: false };
+ return { asset, ...OWN_SOUND, bound: false };
}
const isMap = <A>(v: readonly A[] | ReadonlyMap<string, A>): v is ReadonlyMap<string, A> => v instanceof Map;
@@ -141,3 +144,25 @@ export function specSound<A extends Sourced>(
if (!src.bound) return undefined;
return { file: fileOf(src.asset), offset: src.offset, duration: src.asset.duration };
}
+
+/** A clip's sound as a spec entry carries it: the recording bound to its
+ * video, and a split edit's reach past its picture with the ramp at the far
+ * end. `fileOf` names files the way the spec does. */
+export function spanSoundSpec(sp: ClipSpan, fileOf: (a: MediaAsset) => string) {
+ return {
+ ...(sp.sound
+ ? { soundFrom: { file: fileOf(sp.sound.asset), offset: sp.sound.offset, duration: sp.sound.asset.duration } }
+ : {}),
+ ...(sp.soundLead || sp.soundTail
+ ? { soundLead: sp.soundLead ?? 0, soundTail: sp.soundTail ?? 0, splitFade: sp.splitFade ?? 0 }
+ : {}),
+ };
+}
+
+/** Whether a span plays sound off a bound recording or past its picture —
+ * what a renderer without either would get wrong. A clip that plays nothing
+ * (hidden, muted, a still, a silent file with no recording) never does. */
+export function spanUsesSoundFeature(sp: ClipSpan): boolean {
+ const sounds = !sp.clip.hidden && !sp.clip.muted && (!!sp.sound || !assetIsSilent(sp.asset));
+ return sounds && (!!sp.sound || !!sp.soundLead || !!sp.soundTail);
+}
diff --git a/site/src/cut/lib/soundSync.test.ts b/site/src/cut/lib/soundSync.test.ts
index bc91e804..e4f1b9f3 100644
--- a/site/src/cut/lib/soundSync.test.ts
+++ b/site/src/cut/lib/soundSync.test.ts
@@ -6,7 +6,6 @@ import {
envelopeOf,
MonoResampler,
SYNC_RATE,
- syncSignals,
syncSound,
type SyncChunk,
} from "./soundSync";
@@ -48,66 +47,97 @@ function heardBy(signal: Float32Array, gain: number, noise: number, seed: number
return out;
}
-describe("syncSignals", () => {
+/** A file read the way the decoders hand it over: 48 kHz stereo, in chunks
+ * that start wherever the read was asked to. `origin` is the file second the
+ * signal's first sample sits at. */
+const fileOf = (signal: Float32Array, origin = 0) => {
+ let reads = 0;
+ async function* read(from: number, to: number): AsyncGenerator<SyncChunk> {
+ reads++;
+ const lo = Math.max(0, Math.round((from - origin) * SYNC_RATE));
+ const hi = Math.min(signal.length, Math.round((to - origin) * SYNC_RATE));
+ for (let i = lo; i < hi; i += 1500) {
+ const n = Math.min(1500, hi - i);
+ const ch = new Float32Array(n * 3);
+ for (let k = 0; k < n * 3; k++) ch[k] = signal[i + Math.floor(k / 3)];
+ yield { channels: [ch, ch], timestamp: origin + i / SYNC_RATE, sampleRate: SYNC_RATE * 3 };
+ }
+ }
+ return { read, reads: () => reads };
+};
+
+/** Both passes over two whole signals, each read from a file whose first
+ * sample sits at the given second. */
+const syncWhole = (camera: Float32Array, cameraStart: number, recording: Float32Array, recordingStart: number) =>
+ syncSound(
+ { camera: fileOf(camera, cameraStart).read, recording: fileOf(recording, recordingStart).read },
+ {
+ cameraFrom: cameraStart,
+ cameraTo: cameraStart + camera.length / SYNC_RATE,
+ recordingTo: recordingStart + recording.length / SYNC_RATE,
+ }
+ );
+
+describe("syncSound over whole signals", () => {
const room = speechLike(70, 7);
- test("finds where the camera's sound sits in the recording, to the sample", () => {
+ test("finds where the camera's sound sits in the recording, to the sample", async () => {
// The recorder rolled 12.345 s before the camera.
const startAt = Math.round(12.345 * SYNC_RATE);
const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.01, 1);
const recording = heardBy(room, 1, 0.002, 2);
- const answer = syncSignals(camera, 0, recording, 0)!;
+ const answer = (await syncWhole(camera, 0, recording, 0))!;
expect(answer).not.toBeNull();
expect(Math.abs(answer.offset - 12.345)).toBeLessThan(1.5 / SYNC_RATE);
expect(answer.refined).toBe(true);
expect(answer.confidence).toBeGreaterThan(2);
});
- test("a recording that started after the camera gives a negative offset", () => {
+ test("a recording that started after the camera gives a negative offset", async () => {
// Camera time 0 is recording time -4.5: the recorder rolled late.
const camera = heardBy(room.subarray(0, 40 * SYNC_RATE), 0.5, 0.01, 3);
const late = Math.round(4.5 * SYNC_RATE);
const recording = heardBy(room.subarray(late), 1, 0.002, 4);
- const answer = syncSignals(camera, 0, recording, 0)!;
+ const answer = (await syncWhole(camera, 0, recording, 0))!;
expect(Math.abs(answer.offset - -4.5)).toBeLessThan(1.5 / SYNC_RATE);
});
- test("source times carry through: a camera window late in its file, a recording window late in its", () => {
+ test("source times carry through: a camera window late in its file, a recording window late in its", async () => {
const startAt = Math.round(20 * SYNC_RATE);
const camera = heardBy(room.subarray(startAt, startAt + 20 * SYNC_RATE), 0.4, 0.01, 5);
const recording = heardBy(room.subarray(5 * SYNC_RATE), 1, 0.002, 6);
// The camera window starts at camera second 100; the recording window at
// recording second 5 of its file, which is room second 5. Camera second
// 100 is room second 20, so recording = camera − 80.
- const answer = syncSignals(camera, 100, recording, 5)!;
+ const answer = (await syncWhole(camera, 100, recording, 5))!;
expect(Math.abs(answer.offset - -80)).toBeLessThan(1.5 / SYNC_RATE);
});
- test("an inverted microphone still lines up", () => {
+ test("an inverted microphone still lines up", async () => {
const startAt = Math.round(7.25 * SYNC_RATE);
const camera = heardBy(room.subarray(startAt, startAt + 25 * SYNC_RATE), 0.2, 0.005, 7, true);
- const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 8), 0)!;
+ const answer = (await syncWhole(camera, 0, heardBy(room, 1, 0.002, 8), 0))!;
expect(Math.abs(answer.offset - 7.25)).toBeLessThan(1.5 / SYNC_RATE);
});
- test("survives a noisy camera track", () => {
+ test("survives a noisy camera track", async () => {
const startAt = Math.round(30.01 * SYNC_RATE);
const camera = heardBy(room.subarray(startAt, startAt + 30 * SYNC_RATE), 0.3, 0.08, 9);
- const answer = syncSignals(camera, 0, heardBy(room, 1, 0.002, 10), 0)!;
+ const answer = (await syncWhole(camera, 0, heardBy(room, 1, 0.002, 10), 0))!;
// Within a frame at 60 fps even if the fine pass gives up.
expect(Math.abs(answer.offset - 30.01)).toBeLessThan(0.017);
expect(answer.confidence).toBeGreaterThan(1.4);
});
- test("two unrelated recordings have no clear peak", () => {
+ test("two unrelated recordings have no clear peak", async () => {
const camera = speechLike(30, 11);
const recording = speechLike(70, 12);
- const answer = syncSignals(camera, 0, recording, 0);
+ const answer = await syncWhole(camera, 0, recording, 0);
expect(answer === null || answer.confidence < 1.4).toBe(true);
});
- test("digital silence on either side has nothing to agree on", () => {
- const answer = syncSignals(new Float32Array(20 * SYNC_RATE), 0, room, 0);
+ test("digital silence on either side has nothing to agree on", async () => {
+ const answer = await syncWhole(new Float32Array(20 * SYNC_RATE), 0, room, 0);
expect(answer === null || answer.confidence < 1.4).toBe(true);
});
});
@@ -130,14 +160,25 @@ describe("streaming pieces", () => {
const right = new Float32Array(48000).fill(-0.1);
r.push([left.subarray(0, 20000), right.subarray(0, 20000)], 48000);
r.push([left.subarray(20000), right.subarray(20000)], 48000);
- expect(Math.abs(r.length - 16000)).toBeLessThanOrEqual(1);
+ expect(Math.abs(r.samples().length - 16000)).toBeLessThanOrEqual(1);
expect(r.samples()[100]).toBeCloseTo(0.2, 5);
});
test("the resampler keeps 44.1 kHz timing", () => {
const r = new MonoResampler(2);
r.push([new Float32Array(88200)], 44100);
- expect(Math.abs(r.length - 32000)).toBeLessThanOrEqual(1);
+ expect(Math.abs(r.samples().length - 32000)).toBeLessThanOrEqual(1);
+ });
+
+ test("the resampler raises a voice recorder's 8 kHz and 11.025 kHz to 16 kHz", () => {
+ for (const rate of [8000, 11025]) {
+ const r = new MonoResampler(1);
+ // Two pushes, so a chunk boundary sits inside the stretch.
+ r.push([new Float32Array(Math.floor(rate / 2)).fill(0.25)], rate);
+ r.push([new Float32Array(Math.ceil(rate / 2)).fill(0.25)], rate);
+ expect(Math.abs(r.samples().length - 16000)).toBeLessThanOrEqual(1);
+ expect(r.samples()[15000]).toBeCloseTo(0.25, 5);
+ }
});
test("an envelope built in pieces equals one built whole", () => {
@@ -149,24 +190,6 @@ describe("streaming pieces", () => {
});
describe("syncSound", () => {
- /** A file read the way the decoders hand it over: 48 kHz stereo, in chunks
- * that start wherever the read was asked to. */
- const fileOf = (signal: Float32Array) => {
- let reads = 0;
- async function* read(from: number, to: number): AsyncGenerator<SyncChunk> {
- reads++;
- const lo = Math.max(0, Math.round(from * SYNC_RATE));
- const hi = Math.min(signal.length, Math.round(to * SYNC_RATE));
- for (let i = lo; i < hi; i += 1500) {
- const n = Math.min(1500, hi - i);
- const ch = new Float32Array(n * 3);
- for (let k = 0; k < n * 3; k++) ch[k] = signal[i + Math.floor(k / 3)];
- yield { channels: [ch, ch], timestamp: i / SYNC_RATE, sampleRate: SYNC_RATE * 3 };
- }
- }
- return { read, reads: () => reads };
- };
-
test("streams the recording and lands the offset on the sample", async () => {
const room = speechLike(80, 41);
// The recorder rolled 7.25 s before the camera; the camera's file is the
diff --git a/site/src/cut/lib/soundSync.ts b/site/src/cut/lib/soundSync.ts
index a9b07a39..e39e7073 100644
--- a/site/src/cut/lib/soundSync.ts
+++ b/site/src/cut/lib/soundSync.ts
@@ -26,7 +26,7 @@
/** The rate both signals are compared at. */
export const SYNC_RATE = 16000;
/** Samples per envelope hop: 20 ms. */
-export const ENV_HOP = 320;
+const ENV_HOP = 320;
export const ENV_HOP_S = ENV_HOP / SYNC_RATE;
/** Hops the slow level drift is measured over, taken out of the envelope. */
const DETREND_HOPS = 50;
@@ -35,9 +35,9 @@ const PEAK_EXCLUDE_S = 1;
/** The least of either signal that has to overlap at a lag for it to count. */
const MIN_OVERLAP_SHARE = 0.5;
/** How far the fine pass searches either side of the coarse answer. */
-export const FINE_REACH_S = 0.06;
+const FINE_REACH_S = 0.06;
/** How much of the camera's sound the fine pass compares. */
-export const FINE_SECONDS = 4;
+const FINE_SECONDS = 4;
/** Confidence reported when the runner-up does not agree at all. */
const CONFIDENCE_CAP = 99;
@@ -45,7 +45,8 @@ const CONFIDENCE_CAP = 99;
* Folds decoded chunks of any rate and channel count into one mono signal at
* `SYNC_RATE`: channels averaged, then each output sample the mean of the
* input samples it covers — a box filter, enough to keep the fold from
- * aliasing what the correlation compares.
+ * aliasing what the correlation compares. A rate below `SYNC_RATE` (a voice
+ * recorder's 8 kHz) holds each input sample for the outputs it spans.
*/
export class MonoResampler {
private out: Float32Array;
@@ -69,8 +70,10 @@ export class MonoResampler {
this.sum += v / k;
this.count++;
const next = this.pos + step;
- if (Math.floor(next) > Math.floor(this.pos)) {
- this.emit(this.sum / this.count);
+ const outputs = Math.floor(next) - Math.floor(this.pos);
+ if (outputs > 0) {
+ const v = this.sum / this.count;
+ for (let o = 0; o < outputs; o++) this.emit(v);
this.sum = 0;
this.count = 0;
}
@@ -98,10 +101,6 @@ export class MonoResampler {
if (this.n > 0) into(this.out.subarray(0, this.n));
this.n = 0;
}
-
- get length(): number {
- return this.n;
- }
}
/**
@@ -150,7 +149,7 @@ export function envelopeOf(mono: Float32Array): Float32Array {
/** The envelope with its slow drift taken out and scaled to unit variance,
* so the correlation weighs where loudness moves, whatever level a device
* recorded. */
-export function normalizeEnvelope(env: Float32Array): Float64Array {
+function normalizeEnvelope(env: Float32Array): Float64Array {
const n = env.length;
const out = new Float64Array(n);
if (n === 0) return out;
@@ -214,7 +213,7 @@ function fft(re: Float64Array, im: Float64Array, inverse: boolean): void {
* Cross-correlation of `a` against `b` at every lag where they overlap:
* `at(k)` is Σ a[i]·b[i + k], for k from −(a.length − 1) to b.length − 1.
*/
-export function crossCorrelate(
+function crossCorrelate(
a: ArrayLike<number>,
b: ArrayLike<number>
): { at: (k: number) => number; min: number; max: number } {
@@ -247,7 +246,7 @@ export function crossCorrelate(
/** The coarse answer: the lag, in hops, at which the camera's envelope sits
* best inside the recording's, and how clearly it beats the runner-up. */
-export interface CoarseSync {
+interface CoarseSync {
lag: number;
confidence: number;
}
@@ -296,7 +295,7 @@ export function coarseSync(camera: Float32Array, recording: Float32Array): Coars
* earlier than the coarse answer), or null when no shift stands out. Polarity
* is ignored: two microphones can face the source from opposite sides.
*/
-export function fineSync(camera: Float32Array, recording: Float32Array, reach: number): number | null {
+function fineSync(camera: Float32Array, recording: Float32Array, reach: number): number | null {
if (camera.length < 64 || recording.length < camera.length + 2 * reach) return null;
const xc = crossCorrelate(camera, recording);
let best = -1;
@@ -321,7 +320,7 @@ export function fineSync(camera: Float32Array, recording: Float32Array, reach: n
/** The loudest `seconds` of a signal at `SYNC_RATE`, by hop energy: where the
* fine pass has the most to compare. Returns the first sample. */
-export function loudestStretch(mono: Float32Array, seconds: number): number {
+function loudestStretch(mono: Float32Array, seconds: number): number {
const len = Math.min(mono.length, Math.round(seconds * SYNC_RATE));
if (len >= mono.length) return 0;
let sum = 0;
@@ -349,38 +348,6 @@ export interface SyncAnswer {
refined: boolean;
}
-/**
- * Both passes over two whole signals in hand, each at `SYNC_RATE` and each
- * with the source second its first sample sits at. The tools stream the
- * recording through `EnvelopeBuilder` instead, since a long one cannot be
- * held whole, and read the fine window on its own; this is the same
- * arithmetic for signals already decoded.
- */
-export function syncSignals(
- camera: Float32Array,
- cameraStart: number,
- recording: Float32Array,
- recordingStart: number
-): SyncAnswer | null {
- const coarse = coarseSync(envelopeOf(camera), envelopeOf(recording));
- if (!coarse) return null;
- const offset = recordingStart - cameraStart + coarse.lag * ENV_HOP_S;
- const reach = Math.round(FINE_REACH_S * SYNC_RATE);
- const from = loudestStretch(camera, FINE_SECONDS);
- const cam = camera.subarray(from, from + Math.round(FINE_SECONDS * SYNC_RATE));
- // Where the camera's stretch starts in the recording, by the coarse answer.
- const recAt = Math.round((cameraStart + from / SYNC_RATE + offset - recordingStart) * SYNC_RATE) - reach;
- const shift =
- recAt >= 0 && recAt + cam.length + 2 * reach <= recording.length
- ? fineSync(cam, recording.subarray(recAt, recAt + cam.length + 2 * reach), reach)
- : null;
- return {
- offset: shift === null ? offset : offset + shift / SYNC_RATE,
- confidence: coarse.confidence,
- refined: shift !== null,
- };
-}
-
/** Decoded audio of any rate and channel count, chunk by chunk, starting at
* `timestamp` source seconds. */
export interface SyncChunk {
@@ -390,7 +357,7 @@ export interface SyncChunk {
}
/** Reads a span of each file's sound, wherever it lives. */
-export interface SyncReader {
+interface SyncReader {
camera: (from: number, to: number) => AsyncIterable<SyncChunk>;
recording: (from: number, to: number) => AsyncIterable<SyncChunk>;
}
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index e6a45c53..563579ec 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -95,7 +95,7 @@ import { cueEmphasis, emphasisField, joinEmphasis, remapEmphasis, sliceEmphasis,
import { clipboardItemAssetIds, clipboardItemFor, listedAssetIds, type TimelineClipboardItem } from "./itemKinds";
import { ANIM_STYLE_IDS, animStyleOfTransition, assetIsSilent, clipPoseAt, DEFAULT_BACKGROUND, emptySubtitles, frameOf, IMAGE_CLIP_SECONDS, isAudioTransition, fontAssetId, isEffectOverlay, isStickerOverlay, MAX_SUBTITLE_LANES, mediaUrl, migrateBehindSubject, migrateLegacyTransitions, normalizeAspect, overlayAnimStyle, projectBackground, SPEED_FLOOR, SPEED_MIN, stampOverlayKinds, stripDefaultOverlayKinds, isTimelineId, sanitizeTimelines, timelineAspect, TRANSITION_MAX, TRANSITION_STYLE_IDS, transitionBarAt, transitionBarStart, transitionStyleOfAnim, type TransitionBoundaryKind } from "./types";
import { liftMoveTracks } from "./textMotion";
-import { soundRoom, soundSourceOf, soundWindow, SPLIT_EDIT_MAX_S } from "./soundSource";
+import { OWN_SOUND, soundRoom, soundSourceOf, soundWindow, SPLIT_EDIT_MAX_S } from "./soundSource";
import { cutSound } from "./soundSettings";
import { readTextStyle } from "./textStyle";
import { loadUiState, saveUiState, type ProjectUiState } from "./uiState";
@@ -1562,7 +1562,7 @@ export function cutTranscribeSpec(
}
: null;
};
- const own = (asset: MediaAsset) => ({ asset, offset: 0, limit: Infinity });
+ const own = (asset: MediaAsset) => ({ asset, ...OWN_SOUND });
const audio: Item[] = [];
for (const a of s.audioClips) {
const asset = assetById.get(a.assetId);
@@ -1711,8 +1711,8 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
let next = a;
if (grids.has(a.id) && grids.get(a.id) !== a.beats) next = { ...next, beats: grids.get(a.id) };
if (profiles.has(a.id) && profiles.get(a.id) !== a.colorProfile)
- next = withColorProfile(next, profiles.get(a.id));
- if (sounds.has(a.id) && sounds.get(a.id) !== a.soundFrom) next = withSoundFrom(next, sounds.get(a.id));
+ next = withAssetField(next, "colorProfile", profiles.get(a.id));
+ if (sounds.has(a.id) && sounds.get(a.id) !== a.soundFrom) next = withAssetField(next, "soundFrom", sounds.get(a.id));
return next;
});
const beatsMoved = withBeats.some((a, i) => a !== assets[i]);
@@ -2619,7 +2619,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (!asset || get().readOnly || asset.colorProfile === profile) return;
push();
set((s) => ({
- assets: s.assets.map((a) => (a.id === id ? withColorProfile(a, profile) : a)),
+ assets: s.assets.map((a) => (a.id === id ? withAssetField(a, "colorProfile", profile) : a)),
}));
},
@@ -2631,7 +2631,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// The source's transcript was heard off the sound it had; the sweep
// hears the new one.
set((s) => ({
- assets: s.assets.map((a) => (a.id === id ? { ...withSoundFrom(a, from), speech: undefined } : a)),
+ assets: s.assets.map((a) => (a.id === id ? { ...withAssetField(a, "soundFrom", from), speech: undefined } : a)),
}));
},
@@ -2782,7 +2782,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const remaining = (assets: MediaAsset[]) =>
assets
.filter((a) => a.id !== id)
- .map((a) => (a.soundFrom?.assetId === id ? withSoundFrom(a, undefined) : a));
+ .map((a) => (a.soundFrom?.assetId === id ? withAssetField(a, "soundFrom", undefined) : a));
for (const snap of history) scrub(snap);
for (const snap of future) scrub(snap);
if (pending) scrub(pending.snap);
@@ -4194,15 +4194,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
style: t.style,
...(t.hidden ? { hidden: true } : {}),
}));
- // A video's separate recording binds to the landed copy of it.
- const bound = new Map<string, SoundFrom>();
- template.media.forEach((m, i) => {
- const video = assetIds[i];
- const rec = m.soundFrom ? assetIds[m.soundFrom.media] : undefined;
- if (video && rec && m.soundFrom) bound.set(video, { assetId: rec, offset: m.soundFrom.offset });
- });
set((s) => ({
- ...(bound.size ? { assets: s.assets.map((a) => (bound.has(a.id) ? { ...a, soundFrom: bound.get(a.id) } : a)) } : {}),
clips: [...s.clips, ...newClips, ...newLayers].sort((a, b) => a.start - b.start),
audioClips: [...s.audioClips, ...newAudio],
overlays: [...s.overlays, ...newTexts],
@@ -5251,19 +5243,17 @@ function beatsEdited(prev: MediaAsset[], next: MediaAsset[]): boolean {
});
}
-/** The asset with the person's source profile override set or cleared. The
- * header's record in `color` is left as it is. */
-function withColorProfile<A extends StoredAsset>(asset: A, profile: SourceProfile | undefined): A {
- const { colorProfile: _was, ...rest } = asset;
- void _was;
- return (profile ? { ...rest, colorProfile: profile } : rest) as A;
-}
-
-/** The asset with its bound recording set or cleared. */
-function withSoundFrom<A extends StoredAsset>(asset: A, from: SoundFrom | undefined): A {
- const { soundFrom: _was, ...rest } = asset;
+/** The asset with one optional setting set, or cleared off it entirely: the
+ * person's source profile override (the header's record in `color` is left
+ * as it is) or its bound recording. */
+function withAssetField<A extends StoredAsset, K extends "colorProfile" | "soundFrom">(
+ asset: A,
+ key: K,
+ value: StoredAsset[K] | undefined
+): A {
+ const { [key]: _was, ...rest } = asset;
void _was;
- return (from ? { ...rest, soundFrom: from } : rest) as A;
+ return (value ? { ...rest, [key]: value } : rest) as A;
}
/** A stored asset in the current shape. Projects saved while the override