The VS Code of video editing, built for agents.
TypeScript
feat: implement remove silence command and related functionality
konstantin-paulusb8f06155ede6
diff --git a/apps/web/src/assets/icons/remove-silence.svg b/apps/web/src/assets/icons/remove-silence.svgnew file mode 100644index 00000000..ac5c1a0a--- /dev/null+++ b/apps/web/src/assets/icons/remove-silence.svg@@ -0,0 +1,3 @@+<svg viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">+<path d="M6.5 8.5V10.5M8.5 6.5V12.5M10.5 7.5V11.5M14.5 7.5V11.5M16.5 6.5V12.5M18.5 8.5V10.5M6.5 16.5H10.5M8.5 14.5L10.5 16.5L8.5 18.5M18.5 16.5H14.5M16.5 14.5L14.5 16.5L16.5 18.5" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"/>+</svg>diff --git a/apps/web/src/components/commands/index.tsx b/apps/web/src/components/commands/index.tsxindex 15974f69..125ee217 100644--- a/apps/web/src/components/commands/index.tsx+++ b/apps/web/src/components/commands/index.tsx@@ -11,6 +11,7 @@ import { GenerateCommand } from "./generate";import { MakeVideoCommand } from "./make-video";import { ProjectFolderCommands } from "./project-folder";import { RemoveBackgroundCommand } from "./remove-background";+import { RemoveSilenceCommand } from "./remove-silence";import { RerunCommand } from "./rerun";import { ReuseSettingsCommand } from "./reuse-settings";import { ShortcutCommands } from "./shortcuts";@@ -40,6 +41,7 @@ export function CanvasCommands() {<UpscaleCommand /><RemoveBackgroundCommand /><AutoCaptionsCommand />+ <RemoveSilenceCommand /><ExportVideoCommand /><ExportCurrentFrameCommand /><ToolCommand tool={ToolType.MOVE} />diff --git a/apps/web/src/components/commands/remove-silence.tsx b/apps/web/src/components/commands/remove-silence.tsxnew file mode 100644index 00000000..731d0a58--- /dev/null+++ b/apps/web/src/components/commands/remove-silence.tsx@@ -0,0 +1,85 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { createMemo } from "solid-js";+import { toast } from "somoto";+import { useWorld } from "@diffusionstudio/koota-solid";+import { Separator } from "@/components/ui/separator";+import { useCommand, useCommands } from "@/context/commands";+import { useSelection } from "@/engine/hooks";+import { detectClipSilences, removeSilences, silenceTargets } from "@/engine/silence/remove";+import { ActionBar, ActionBarCancel, ActionBarConfirm, ActionBarTarget } from "@/components/canvas/action-bar";++import type { Entity } from "koota";+import type { ClipSilences } from "@/engine/silence/remove";++/** Cuts the silent parts out of the selected video and audio clips and closes the gaps. */+export function RemoveSilenceCommand() {+ const { targets, run } = useRemoveSilence();++ useCommand({+ id: "remove-silence",+ title: "Remove silence",+ group: "Media",+ icon: "remove-silence",+ keywords: ["silent parts", "pauses", "gaps", "jump cut", "trim", "dead air", "audio"],+ ready: () => targets().length > 0,+ run: () => void run(),+ bar: RemoveSilenceBar,+ });++ return null;+}++/** The selected clips with sound, cut on Confirm. */+function RemoveSilenceBar() {+ const { close } = useCommands();+ const { targets, run } = useRemoveSilence();++ const confirm = () => {+ void run();+ close();+ };++ return (+ <ActionBar>+ <ActionBarTarget nodes={targets()} verb="Remove silence from" plural="clips" empty="Select a video or audio clip" />+ <Separator orientation="vertical" class="min-h-5" />+ <ActionBarCancel onCancel={close} />+ <ActionBarConfirm disabled={targets().length === 0} onConfirm={confirm} hint="Remove" />+ </ActionBar>+ );+}++function useRemoveSilence() {+ const world = useWorld();+ const { nodes } = useSelection();++ // Tracks the selection; which of it has sound is read off the world.+ const targets = createMemo(() => (nodes(), silenceTargets(world)), [], { equals: sameEntities });++ const run = async () => {+ const clips = targets();+ if (clips.length === 0) return;++ const found: ClipSilences[] = [];+ try {+ for (const clip of clips) {+ found.push({ clip, silences: await detectClipSilences(world, clip) });+ }+ } catch (e) {+ console.error("Silence detection failed:", e);+ toast.error("Failed to remove silence", { description: (e as Error).message });+ return;+ }++ removeSilences(world, found);+ };++ return { targets, run };+}++function sameEntities(a: Entity[], b: Entity[]): boolean {+ return a.length === b.length && a.every((entity, i) => entity === b[i]);+}diff --git a/apps/web/src/components/sidebar-left/asset-item.tsx b/apps/web/src/components/sidebar-left/asset-item.tsxindex 50a98011..c49f557e 100644--- a/apps/web/src/components/sidebar-left/asset-item.tsx+++ b/apps/web/src/components/sidebar-left/asset-item.tsx@@ -127,7 +127,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {<MiddleEllipsistext={name()}data-selected={props.selected}- class="text-xs text-center text-muted-foreground select-none data-[selected=true]:text-foreground"+ class="text-xs text-muted-foreground select-none data-[selected=true]:text-foreground"title={props.asset.source}/>}@@ -143,7 +143,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {onClick={(e) => e.stopPropagation()}onContextMenu={(e) => e.stopPropagation()}onPointerDown={(e) => e.stopPropagation()}- class="text-xs text-center text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"+ class="text-xs text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"/></Show></ContextMenuTrigger>diff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsxindex 52c412ca..de723c89 100644--- a/apps/web/src/components/sidebar-left/folder-item.tsx+++ b/apps/web/src/components/sidebar-left/folder-item.tsx@@ -155,7 +155,7 @@ export function FolderItem(props: FolderItemProps) {<MiddleEllipsistext={name()}data-selected={props.selected}- class="text-xs text-center text-muted-foreground select-none data-[selected=true]:text-foreground"+ class="text-xs text-muted-foreground select-none data-[selected=true]:text-foreground"/>}>@@ -170,7 +170,7 @@ export function FolderItem(props: FolderItemProps) {onClick={(e) => e.stopPropagation()}onContextMenu={(e) => e.stopPropagation()}onPointerDown={(e) => e.stopPropagation()}- class="text-xs text-center text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"+ class="text-xs text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"/></Show></ContextMenuTrigger>diff --git a/apps/web/src/engine/silence/detect.ts b/apps/web/src/engine/silence/detect.tsnew file mode 100644index 00000000..5a05397a--- /dev/null+++ b/apps/web/src/engine/silence/detect.ts@@ -0,0 +1,269 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Finding the silent stretches of a recording. The audio is reduced, as it+ * streams past, to a loudness envelope — one RMS level in dBFS per hop of a+ * few milliseconds, taken after a high-pass that drops rumble and DC — and+ * the decisions are made on that envelope, which is small enough to keep+ * whole: an hour of it is a megabyte and a half.+ *+ * What counts as silent is worked out from the recording itself rather than+ * a fixed level: the noise floor (a low percentile of the envelope) and the+ * level of the sound (a high one) are measured, and the threshold sits a+ * fraction of the way up from the one to the other, clamped to a sane range.+ * A quiet room and a noisy one are then cut the same way, and a recording+ * with no contrast between the two — music, constant noise — is left alone.+ *+ * The envelope is then gated with hysteresis, so a level hovering around the+ * threshold does not chatter; bursts of sound too short to be speech (a+ * click, a knock) are folded into the silence around them; silences too short+ * to be anything but a pause are kept; and what is cut is pulled back from+ * the sound either side, so words keep their breath and their decay.+ */++export type SilenceOptions = {+ /** The envelope's hop, in seconds. */+ hop: number;+ /** Below this, in Hz, the signal is filtered out before it is measured. */+ highpass: number;+ /** Where the threshold sits between the noise floor (0) and the sound (1). */+ sensitivity: number;+ /** The quietest the threshold may be, in dBFS. */+ minThreshold: number;+ /** The loudest the threshold may be, in dBFS: nothing above it is ever silence. */+ maxThreshold: number;+ /**+ * How far, in dB, the sound must stand above the noise floor for there to+ * be a threshold between them at all.+ */+ minContrast: number;+ /** How far, in dB, the level has to fall below the threshold to close the gate again. */+ hysteresis: number;+ /** Silence shorter than this, in seconds, is a pause, and stays. */+ minSilence: number;+ /** Sound shorter than this, in seconds, between two silences is cut with them. */+ minSound: number;+ /** Kept either side of the sound, in seconds, out of the silence around it. */+ padding: number;+};++/**+ * Tuned on speech: a pause has to last half a second to be cut, and a quarter+ * of a second of it (an eighth either side) is left in, which is about the+ * gap a speaker leaves between sentences.+ */+export const DEFAULT_SILENCE_OPTIONS: SilenceOptions = {+ hop: 0.01,+ highpass: 80,+ sensitivity: 0.3,+ minThreshold: -60,+ maxThreshold: -30,+ minContrast: 10,+ hysteresis: 4,+ minSilence: 0.5,+ minSound: 0.1,+ padding: 0.12,+};++/** A stretch of time, in seconds. */+export type Span = [start: number, end: number];++/** What the envelope reads where there is nothing: digital silence, or no audio at all. */+const FLOOR_DB = -100;++/** The resolution, in dB, the levels are measured at. */+const BIN_DB = 0.5;++/**+ * Reduces audio to the loudness envelope, a block at a time and in the order+ * it plays. Placed by timestamp rather than appended, so a track with a gap+ * in it, or one that starts before `start`, still lines up with the time it+ * is measured against; hops nothing reached read as silent.+ */+export class Envelope {+ readonly levels: Float32Array;+ readonly start: number;+ readonly hop: number;++ private readonly highpass: number;+ /** The filter's coefficients, for the sample rate they were made for. */+ private coefficients: { rate: number; b0: number; b1: number; b2: number; a1: number; a2: number } | null = null;+ /** The filter's state: the last two inputs and outputs. */+ private x1 = 0;+ private x2 = 0;+ private y1 = 0;+ private y2 = 0;+ /** Where the next sample is expected, in seconds, to tell a gap from a continuation. */+ private expected = -Infinity;+ private index = -1;+ private sum = 0;+ private count = 0;++ constructor(start: number, end: number, options: Pick<SilenceOptions, 'hop' | 'highpass'>) {+ this.start = start;+ this.hop = options.hop;+ this.highpass = options.highpass;+ this.levels = new Float32Array(Math.max(0, Math.ceil((end - start) / options.hop))).fill(FLOOR_DB);+ }++ /**+ * Takes in one block of interleaved samples starting at `timestamp`+ * seconds. Channels are averaged, which is how a listener would hear a+ * mono fold of them.+ */+ push(samples: Float32Array, channels: number, rate: number, timestamp: number): void {+ const filter = this.filter(rate);+ const frames = Math.floor(samples.length / channels);++ // A block that does not pick up where the last one stopped follows a+ // gap (or a seek): the filter's memory is of audio that isn't adjacent.+ if (Math.abs(timestamp - this.expected) > 1 / rate * 2) {+ this.x1 = this.x2 = this.y1 = this.y2 = 0;+ }+ this.expected = timestamp + frames / rate;++ for (let frame = 0; frame < frames; frame++) {+ let x = 0;+ for (let channel = 0; channel < channels; channel++) x += samples[frame * channels + channel]!;+ x /= channels;++ const y = filter.b0 * x + filter.b1 * this.x1 + filter.b2 * this.x2 - filter.a1 * this.y1 - filter.a2 * this.y2;+ this.x2 = this.x1;+ this.x1 = x;+ this.y2 = this.y1;+ this.y1 = y;++ const index = Math.floor((timestamp + frame / rate - this.start) / this.hop);+ if (index !== this.index) {+ this.flush();+ this.index = index;+ }+ this.sum += y * y;+ this.count++;+ }+ }++ /** Writes out the hop still being summed. Call once the last block is in. */+ flush(): void {+ if (this.count > 0 && this.index >= 0 && this.index < this.levels.length) {+ this.levels[this.index] = Math.max(FLOOR_DB, 10 * Math.log10(this.sum / this.count));+ }+ this.sum = 0;+ this.count = 0;+ }++ /** A second-order Butterworth high-pass at the cutoff, for `rate`. */+ private filter(rate: number) {+ if (this.coefficients?.rate === rate) return this.coefficients;++ const w = (2 * Math.PI * Math.min(this.highpass, rate / 4)) / rate;+ const alpha = Math.sin(w) / Math.SQRT2;+ const cos = Math.cos(w);+ const a0 = 1 + alpha;++ this.coefficients = {+ rate,+ b0: (1 + cos) / 2 / a0,+ b1: -(1 + cos) / a0,+ b2: (1 + cos) / 2 / a0,+ a1: (-2 * cos) / a0,+ a2: (1 - alpha) / a0,+ };+ return this.coefficients;+ }+}++/** The level `fraction` of the envelope's hops are at or below, in dB. */+function percentile(levels: Float32Array, fraction: number): number {+ const bins = new Uint32Array(Math.ceil(-FLOOR_DB / BIN_DB) + 1);+ for (const level of levels) {+ bins[Math.min(bins.length - 1, Math.max(0, Math.round((level - FLOOR_DB) / BIN_DB)))]!++;+ }++ const target = fraction * levels.length;+ let seen = 0;+ for (let bin = 0; bin < bins.length; bin++) {+ seen += bins[bin]!;+ if (seen >= target) return FLOOR_DB + bin * BIN_DB;+ }+ return 0;+}++/**+ * The level below which the envelope is silent, or null when it has no+ * silence to tell apart: sound and floor are too close together for a line+ * between them to mean anything. A recording that is quiet throughout is+ * silent throughout, and gets the loudest threshold allowed.+ */+export function silenceThreshold(levels: Float32Array, options: SilenceOptions): number | null {+ if (levels.length === 0) return null;++ const floor = percentile(levels, 0.1);+ const sound = percentile(levels, 0.95);++ if (sound <= options.minThreshold) return options.maxThreshold;+ if (sound - floor < options.minContrast) return null;++ const threshold = floor + (sound - floor) * options.sensitivity;+ return Math.min(options.maxThreshold, Math.max(options.minThreshold, threshold));+}++/** Runs of equal values in `gate`, as [value, from, to) in hops. */+function runs(gate: Uint8Array): [value: number, from: number, to: number][] {+ const result: [number, number, number][] = [];+ let from = 0;+ for (let i = 1; i <= gate.length; i++) {+ if (i === gate.length || gate[i] !== gate[from]) {+ result.push([gate[from]!, from, i]);+ from = i;+ }+ }+ return result;+}++/**+ * The silent stretches of the envelope worth cutting, in seconds of the+ * source, already pulled back from the sound either side by the padding. A+ * silence at either end of the envelope keeps no padding on that side: there+ * is no sound beyond it to lead into.+ */+export function findSilences(envelope: Envelope, options: SilenceOptions = DEFAULT_SILENCE_OPTIONS): Span[] {+ const { levels, hop, start } = envelope;+ const threshold = silenceThreshold(levels, options);+ if (threshold === null) return [];++ // 1 is sound. Opens above the threshold and closes only once the level+ // has fallen clearly below it.+ const gate = new Uint8Array(levels.length);+ let open = false;+ for (let i = 0; i < levels.length; i++) {+ const level = levels[i]!;+ open = open ? level >= threshold - options.hysteresis : level > threshold;+ gate[i] = open ? 1 : 0;+ }++ // A burst of sound too short to be speech, with silence either side, is+ // part of that silence. One at the very edge has silence on one side only+ // and is still a click.+ const minSound = Math.round(options.minSound / hop);+ for (const [value, from, to] of runs(gate)) {+ if (value === 1 && to - from < minSound) gate.fill(0, from, to);+ }++ const minSilence = Math.round(options.minSilence / hop);+ const silences: Span[] = [];++ for (const [value, from, to] of runs(gate)) {+ if (value === 1 || to - from < minSilence) continue;++ const head = from === 0 ? 0 : options.padding;+ const tail = to === levels.length ? 0 : options.padding;+ const cutFrom = start + from * hop + head;+ const cutTo = start + to * hop - tail;+ if (cutTo - cutFrom >= hop) silences.push([cutFrom, cutTo]);+ }++ return silences;+}diff --git a/apps/web/src/engine/silence/remove.tsx b/apps/web/src/engine/silence/remove.tsxnew file mode 100644index 00000000..eb776164--- /dev/null+++ b/apps/web/src/engine/silence/remove.tsx@@ -0,0 +1,219 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Taking the silent parts out of a clip. The silences are found in the+ * stretch of the source the clip plays (see `./detect`), then the clip is cut+ * the way a split cuts it — copied where it stands, each copy trimmed to one+ * of the stretches of sound — and the pieces are pulled together so the gaps+ * close. Clips after it in the same sequence move up by as much as was taken+ * out, so the cut leaves no hole behind.+ *+ * Finding is asynchronous and cutting is not: everything the cut does lands+ * in one go, so it is one step to undo.+ */++import { Sequence } from '@diffusionstudio/reconciler';+import {+ AssetId,+ Computed,+ FrameRate,+ Host,+ Selected,+ getAsset,+ getAssetFile,+ getEntityChildren,+ getNextName,+ getParentEntity,+ getSourceWindow,+ getTimelineOrigin,+ isGroup,+ isSequence,+} from '@diffusionstudio/runtime';++import { getDocumentEditor } from '../editor';+import { cloneFramesForSplit, clonePeaksForSplit } from '../timeline';+import { authoredTime, moveEntityTo, trimIn, trimOut } from '../timing';+import { DEFAULT_SILENCE_OPTIONS } from './detect';++import type { Entity, World } from 'koota';+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';+import type { SilenceOptions, Span } from './detect';+import type { SilenceRequest, SilenceResponse } from './worker';++/** The silences found in one clip, in seconds of its source. */+export type ClipSilences = { clip: Entity; silences: Span[] };++/** The media `clip` plays sound from, or null when it plays none. */+function soundAsset(world: World, clip: Entity): AudioAsset | VideoAsset | null {+ const id = clip.get(AssetId)?.value;+ const asset = id ? getAsset(world, id) : undefined;+ if (asset?.type === 'AUDIO') return asset;+ if (asset?.type === 'VIDEO' && asset.channels) return asset;+ return null;+}++/** The selected clips that have sound to take silence out of. */+export function silenceTargets(world: World): Entity[] {+ return [...world.query(Selected)].filter((entity) => soundAsset(world, entity) !== null);+}++/**+ * The silences in the stretch of its source `clip` plays, in seconds of the+ * source. Decoded in a worker of its own, gone when it answers.+ */+export async function detectClipSilences(+ world: World,+ clip: Entity,+ options: SilenceOptions = DEFAULT_SILENCE_OPTIONS,+): Promise<Span[]> {+ const asset = soundAsset(world, clip);+ if (!asset) return [];++ const fps = world.get(FrameRate)?.value ?? 30;+ const window = getSourceWindow(clip);+ const file = await getAssetFile(asset);++ const worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' });+ try {+ const silences = await new Promise<Span[] | null>((resolve, reject) => {+ worker.onmessage = ({ data }: MessageEvent<SilenceResponse>) => {+ if ('error' in data) reject(new Error(data.error));+ else resolve(data.silences);+ };+ worker.onerror = (event) => reject(new Error(event.message || 'The silence worker failed'));+ worker.postMessage({+ file,+ start: window.in / fps,+ end: window.out / fps,+ options,+ } satisfies SilenceRequest);+ });+ return silences ?? [];+ } finally {+ worker.terminate();+ }+}++/**+ * The stretches of `clip` to keep, in scene frames: the clip's span with the+ * silences taken out. Worked out against the clip as it is now, so an edit+ * made while the silences were being found is respected.+ */+function keptSpans(world: World, clip: Entity, silences: Span[]): [number, number][] {+ const computed = clip.get(Computed);+ if (!computed) return [];++ const fps = world.get(FrameRate)?.value ?? 30;+ const rate = computed.playbackRate || 1;+ const { start, end, origin } = computed;+ const toFrame = (seconds: number) => Math.min(end, Math.max(start, Math.round(origin + (seconds * fps) / rate)));++ const kept: [number, number][] = [];+ let cursor = start;+ for (const [from, to] of silences) {+ const cutFrom = toFrame(from);+ const cutTo = toFrame(to);+ if (cutTo <= cutFrom) continue;+ if (cutFrom > cursor) kept.push([cursor, cutFrom]);+ cursor = Math.max(cursor, cutTo);+ }+ if (cursor < end) kept.push([cursor, end]);++ return kept;+}++/**+ * The clips after `clip` in the sequence it is in, which move up when it+ * gets shorter. None outside a sequence: layers there don't follow one+ * another, so nothing is out of place when one of them gets shorter.+ */+function followers(world: World, clip: Entity): Entity[] {+ const parent = getParentEntity(clip);+ if (parent === null || !isSequence(parent)) return [];++ const end = (clip.get(Computed)?.end ?? 0) - getTimelineOrigin(clip);+ return getEntityChildren(world, parent).filter(+ (sibling) => sibling !== clip && (authoredTime(world, sibling, 'start') ?? 0) >= end,+ );+}++/** Moves `entity` along its timeline by `delta` frames, by what the file says rather than what was last computed. */+function shift(world: World, entity: Entity, delta: number): void {+ const start = authoredTime(world, entity, 'start') ?? 0;+ moveEntityTo(world, entity, getTimelineOrigin(entity) + start + delta);+}++/**+ * Cuts the silences out of each clip and closes the gaps they leave. A clip+ * that would have nothing left is left alone. Returns the pieces the clips+ * were cut into, which are what the selection is left on.+ */+export function removeSilences(world: World, found: ClipSilences[]): Entity[] {+ const editor = getDocumentEditor(world);+ const cut: Entity[] = [];++ // Latest first: a clip earlier in a sequence moves the ones after it up,+ // and those have to have been cut already, so their own pieces move too.+ const ordered = found+ .filter(({ clip }) => clip.isAlive())+ .sort((a, b) => (b.clip.get(Computed)?.start ?? 0) - (a.clip.get(Computed)?.start ?? 0));++ for (const { clip, silences } of ordered) {+ const start = clip.get(Computed)?.start ?? 0;+ const end = clip.get(Computed)?.end ?? 0;+ const kept = keptSpans(world, clip, silences);+ const length = kept.reduce((sum, [from, to]) => sum + to - from, 0);+ const removed = end - start - length;+ if (kept.length === 0 || removed <= 0) continue;++ // Read before anything moves: the cut changes where the clip ends.+ const after = followers(world, clip);+ // Only the last piece leads into the next clip; the cuts between the+ // pieces are hard ones.+ const transition = clip.get(Host)?.props.transition !== undefined;++ // Each copy is of the one before it, made before anything is trimmed,+ // so every piece is spelled from the whole clip and in playing order.+ const pieces = [clip];+ for (let i = 1; i < kept.length; i++) {+ const [pair] = editor.duplicateInPlace([pieces[i - 1]!]);+ if (!pair) break;+ pieces.push(pair.copy);+ clonePeaksForSplit(clip.id(), pair.copy.id());+ cloneFramesForSplit(clip.id(), pair.copy.id());+ }+ if (pieces.length < kept.length) {+ editor.remove(pieces.slice(1));+ continue;+ }++ // Out before in: the out point is read against where the clip starts+ // now, which the in point moves.+ let at = start;+ for (const [index, piece] of pieces.entries()) {+ const [from, to] = kept[index]!;+ if (to < end) trimOut(world, piece, to);+ if (from > start) trimIn(world, piece, from);+ moveEntityTo(world, piece, at);+ if (transition && index < pieces.length - 1) editor.editProperty(piece, 'transition', false);+ at += to - from;+ }++ for (const follower of after) shift(world, follower, -removed);++ // Pieces side by side under a scene would be layers; wrapped, they+ // still read as the one clip they were (as a split does it).+ const parent = getParentEntity(clip);+ if (pieces.length > 1 && parent !== null && !isGroup(parent)) {+ editor.wrap(pieces, () => <Sequence name={getNextName(world, 'Sequence')} />);+ }++ cut.push(...pieces);+ }++ if (cut.length > 0) editor.select(cut);++ return cut;+}diff --git a/apps/web/src/engine/silence/worker.ts b/apps/web/src/engine/silence/worker.tsnew file mode 100644index 00000000..a346a546--- /dev/null+++ b/apps/web/src/engine/silence/worker.ts@@ -0,0 +1,69 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The silences of a media file's audio, found off the main thread: the+// stretch asked about is decoded block by block and each block is folded+// into the loudness envelope before the next is decoded, so nothing more than+// one block of samples is ever held. PCM formats are unpacked in JS, which+// would stall the UI for a long file. One worker per request.++import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';++import { Envelope, findSilences } from './detect';++import type { SilenceOptions, Span } from './detect';++export interface SilenceRequest {+ file: Blob;+ /** The stretch of the file to look in, in seconds. */+ start: number;+ end: number;+ options: SilenceOptions;+}++export type SilenceResponse = { silences: Span[] | null } | { error: string };++// The app's lib is the DOM's, so the worker scope is narrowed by hand.+const scope = self as unknown as {+ onmessage: ((event: MessageEvent<SilenceRequest>) => void) | null;+ postMessage(message: SilenceResponse): void;+};++scope.onmessage = async ({ data }) => {+ try {+ scope.postMessage({ silences: await detect(data) });+ } catch (error) {+ scope.postMessage({ error: error instanceof Error ? error.message : String(error) });+ }+};++/** The silences between `start` and `end` of the file's primary audio track, or null when it has none. */+async function detect({ file, start, end, options }: SilenceRequest): Promise<Span[] | null> {+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });+ try {+ const track = await input.getPrimaryAudioTrack();+ if (!track) return null;++ const envelope = new Envelope(start, end, options);+ const sink = new AudioSampleSink(track);++ for await (const sample of sink.samples(start, end)) {+ // Everything the sample holds is taken before it is closed, and it+ // is closed however that goes: they are decoder resources.+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });+ envelope.push(floats, sample.numberOfChannels, sample.sampleRate, sample.timestamp);+ } finally {+ sample.close();+ }+ }+ envelope.flush();++ return findSilences(envelope, options);+ } finally {+ input.dispose();+ }+}