Skip to content

The VS Code of video editing, built for agents.

TypeScript

feat: implement remove silence command and related functionality

konstantin-paulusb8f06155ede6
diff --git a/apps/web/src/assets/icons/remove-silence.svg b/apps/web/src/assets/icons/remove-silence.svg
new file mode 100644
index 00000000..ac5c1a0a
--- /dev/null
+++ b/apps/web/src/assets/icons/remove-silence.svg
@@ -0,0 +1,3 @@
+<svg viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.5 8.5V10.5M8.5 6.5V12.5M10.5 7.5V11.5M14.5 7.5V11.5M16.5 6.5V12.5M18.5 8.5V10.5M6.5 16.5H10.5M8.5 14.5L10.5 16.5L8.5 18.5M18.5 16.5H14.5M16.5 14.5L14.5 16.5L16.5 18.5" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>
diff --git a/apps/web/src/components/commands/index.tsx b/apps/web/src/components/commands/index.tsx
index 15974f69..125ee217 100644
--- a/apps/web/src/components/commands/index.tsx
+++ b/apps/web/src/components/commands/index.tsx
@@ -11,6 +11,7 @@ import { GenerateCommand } from "./generate";
import { MakeVideoCommand } from "./make-video";
import { ProjectFolderCommands } from "./project-folder";
import { RemoveBackgroundCommand } from "./remove-background";
+import { RemoveSilenceCommand } from "./remove-silence";
import { RerunCommand } from "./rerun";
import { ReuseSettingsCommand } from "./reuse-settings";
import { ShortcutCommands } from "./shortcuts";
@@ -40,6 +41,7 @@ export function CanvasCommands() {
<UpscaleCommand />
<RemoveBackgroundCommand />
<AutoCaptionsCommand />
+ <RemoveSilenceCommand />
<ExportVideoCommand />
<ExportCurrentFrameCommand />
<ToolCommand tool={ToolType.MOVE} />
diff --git a/apps/web/src/components/commands/remove-silence.tsx b/apps/web/src/components/commands/remove-silence.tsx
new file mode 100644
index 00000000..731d0a58
--- /dev/null
+++ b/apps/web/src/components/commands/remove-silence.tsx
@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { createMemo } from "solid-js";
+import { toast } from "somoto";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { Separator } from "@/components/ui/separator";
+import { useCommand, useCommands } from "@/context/commands";
+import { useSelection } from "@/engine/hooks";
+import { detectClipSilences, removeSilences, silenceTargets } from "@/engine/silence/remove";
+import { ActionBar, ActionBarCancel, ActionBarConfirm, ActionBarTarget } from "@/components/canvas/action-bar";
+
+import type { Entity } from "koota";
+import type { ClipSilences } from "@/engine/silence/remove";
+
+/** Cuts the silent parts out of the selected video and audio clips and closes the gaps. */
+export function RemoveSilenceCommand() {
+ const { targets, run } = useRemoveSilence();
+
+ useCommand({
+ id: "remove-silence",
+ title: "Remove silence",
+ group: "Media",
+ icon: "remove-silence",
+ keywords: ["silent parts", "pauses", "gaps", "jump cut", "trim", "dead air", "audio"],
+ ready: () => targets().length > 0,
+ run: () => void run(),
+ bar: RemoveSilenceBar,
+ });
+
+ return null;
+}
+
+/** The selected clips with sound, cut on Confirm. */
+function RemoveSilenceBar() {
+ const { close } = useCommands();
+ const { targets, run } = useRemoveSilence();
+
+ const confirm = () => {
+ void run();
+ close();
+ };
+
+ return (
+ <ActionBar>
+ <ActionBarTarget nodes={targets()} verb="Remove silence from" plural="clips" empty="Select a video or audio clip" />
+ <Separator orientation="vertical" class="min-h-5" />
+ <ActionBarCancel onCancel={close} />
+ <ActionBarConfirm disabled={targets().length === 0} onConfirm={confirm} hint="Remove" />
+ </ActionBar>
+ );
+}
+
+function useRemoveSilence() {
+ const world = useWorld();
+ const { nodes } = useSelection();
+
+ // Tracks the selection; which of it has sound is read off the world.
+ const targets = createMemo(() => (nodes(), silenceTargets(world)), [], { equals: sameEntities });
+
+ const run = async () => {
+ const clips = targets();
+ if (clips.length === 0) return;
+
+ const found: ClipSilences[] = [];
+ try {
+ for (const clip of clips) {
+ found.push({ clip, silences: await detectClipSilences(world, clip) });
+ }
+ } catch (e) {
+ console.error("Silence detection failed:", e);
+ toast.error("Failed to remove silence", { description: (e as Error).message });
+ return;
+ }
+
+ removeSilences(world, found);
+ };
+
+ return { targets, run };
+}
+
+function sameEntities(a: Entity[], b: Entity[]): boolean {
+ return a.length === b.length && a.every((entity, i) => entity === b[i]);
+}
diff --git a/apps/web/src/components/sidebar-left/asset-item.tsx b/apps/web/src/components/sidebar-left/asset-item.tsx
index 50a98011..c49f557e 100644
--- a/apps/web/src/components/sidebar-left/asset-item.tsx
+++ b/apps/web/src/components/sidebar-left/asset-item.tsx
@@ -127,7 +127,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
<MiddleEllipsis
text={name()}
data-selected={props.selected}
- class="text-xs text-center text-muted-foreground select-none data-[selected=true]:text-foreground"
+ class="text-xs text-muted-foreground select-none data-[selected=true]:text-foreground"
title={props.asset.source}
/>
}
@@ -143,7 +143,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
onClick={(e) => e.stopPropagation()}
onContextMenu={(e) => e.stopPropagation()}
onPointerDown={(e) => e.stopPropagation()}
- class="text-xs text-center text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"
+ class="text-xs text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"
/>
</Show>
</ContextMenuTrigger>
diff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsx
index 52c412ca..de723c89 100644
--- a/apps/web/src/components/sidebar-left/folder-item.tsx
+++ b/apps/web/src/components/sidebar-left/folder-item.tsx
@@ -155,7 +155,7 @@ export function FolderItem(props: FolderItemProps) {
<MiddleEllipsis
text={name()}
data-selected={props.selected}
- class="text-xs text-center text-muted-foreground select-none data-[selected=true]:text-foreground"
+ class="text-xs text-muted-foreground select-none data-[selected=true]:text-foreground"
/>
}
>
@@ -170,7 +170,7 @@ export function FolderItem(props: FolderItemProps) {
onClick={(e) => e.stopPropagation()}
onContextMenu={(e) => e.stopPropagation()}
onPointerDown={(e) => e.stopPropagation()}
- class="text-xs text-center text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"
+ class="text-xs text-foreground truncate bg-transparent rounded-sm outline-none ring-1 ring-primary px-0.5"
/>
</Show>
</ContextMenuTrigger>
diff --git a/apps/web/src/engine/silence/detect.ts b/apps/web/src/engine/silence/detect.ts
new file mode 100644
index 00000000..5a05397a
--- /dev/null
+++ b/apps/web/src/engine/silence/detect.ts
@@ -0,0 +1,269 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Finding the silent stretches of a recording. The audio is reduced, as it
+ * streams past, to a loudness envelope — one RMS level in dBFS per hop of a
+ * few milliseconds, taken after a high-pass that drops rumble and DC — and
+ * the decisions are made on that envelope, which is small enough to keep
+ * whole: an hour of it is a megabyte and a half.
+ *
+ * What counts as silent is worked out from the recording itself rather than
+ * a fixed level: the noise floor (a low percentile of the envelope) and the
+ * level of the sound (a high one) are measured, and the threshold sits a
+ * fraction of the way up from the one to the other, clamped to a sane range.
+ * A quiet room and a noisy one are then cut the same way, and a recording
+ * with no contrast between the two — music, constant noise — is left alone.
+ *
+ * The envelope is then gated with hysteresis, so a level hovering around the
+ * threshold does not chatter; bursts of sound too short to be speech (a
+ * click, a knock) are folded into the silence around them; silences too short
+ * to be anything but a pause are kept; and what is cut is pulled back from
+ * the sound either side, so words keep their breath and their decay.
+ */
+
+export type SilenceOptions = {
+ /** The envelope's hop, in seconds. */
+ hop: number;
+ /** Below this, in Hz, the signal is filtered out before it is measured. */
+ highpass: number;
+ /** Where the threshold sits between the noise floor (0) and the sound (1). */
+ sensitivity: number;
+ /** The quietest the threshold may be, in dBFS. */
+ minThreshold: number;
+ /** The loudest the threshold may be, in dBFS: nothing above it is ever silence. */
+ maxThreshold: number;
+ /**
+ * How far, in dB, the sound must stand above the noise floor for there to
+ * be a threshold between them at all.
+ */
+ minContrast: number;
+ /** How far, in dB, the level has to fall below the threshold to close the gate again. */
+ hysteresis: number;
+ /** Silence shorter than this, in seconds, is a pause, and stays. */
+ minSilence: number;
+ /** Sound shorter than this, in seconds, between two silences is cut with them. */
+ minSound: number;
+ /** Kept either side of the sound, in seconds, out of the silence around it. */
+ padding: number;
+};
+
+/**
+ * Tuned on speech: a pause has to last half a second to be cut, and a quarter
+ * of a second of it (an eighth either side) is left in, which is about the
+ * gap a speaker leaves between sentences.
+ */
+export const DEFAULT_SILENCE_OPTIONS: SilenceOptions = {
+ hop: 0.01,
+ highpass: 80,
+ sensitivity: 0.3,
+ minThreshold: -60,
+ maxThreshold: -30,
+ minContrast: 10,
+ hysteresis: 4,
+ minSilence: 0.5,
+ minSound: 0.1,
+ padding: 0.12,
+};
+
+/** A stretch of time, in seconds. */
+export type Span = [start: number, end: number];
+
+/** What the envelope reads where there is nothing: digital silence, or no audio at all. */
+const FLOOR_DB = -100;
+
+/** The resolution, in dB, the levels are measured at. */
+const BIN_DB = 0.5;
+
+/**
+ * Reduces audio to the loudness envelope, a block at a time and in the order
+ * it plays. Placed by timestamp rather than appended, so a track with a gap
+ * in it, or one that starts before `start`, still lines up with the time it
+ * is measured against; hops nothing reached read as silent.
+ */
+export class Envelope {
+ readonly levels: Float32Array;
+ readonly start: number;
+ readonly hop: number;
+
+ private readonly highpass: number;
+ /** The filter's coefficients, for the sample rate they were made for. */
+ private coefficients: { rate: number; b0: number; b1: number; b2: number; a1: number; a2: number } | null = null;
+ /** The filter's state: the last two inputs and outputs. */
+ private x1 = 0;
+ private x2 = 0;
+ private y1 = 0;
+ private y2 = 0;
+ /** Where the next sample is expected, in seconds, to tell a gap from a continuation. */
+ private expected = -Infinity;
+ private index = -1;
+ private sum = 0;
+ private count = 0;
+
+ constructor(start: number, end: number, options: Pick<SilenceOptions, 'hop' | 'highpass'>) {
+ this.start = start;
+ this.hop = options.hop;
+ this.highpass = options.highpass;
+ this.levels = new Float32Array(Math.max(0, Math.ceil((end - start) / options.hop))).fill(FLOOR_DB);
+ }
+
+ /**
+ * Takes in one block of interleaved samples starting at `timestamp`
+ * seconds. Channels are averaged, which is how a listener would hear a
+ * mono fold of them.
+ */
+ push(samples: Float32Array, channels: number, rate: number, timestamp: number): void {
+ const filter = this.filter(rate);
+ const frames = Math.floor(samples.length / channels);
+
+ // A block that does not pick up where the last one stopped follows a
+ // gap (or a seek): the filter's memory is of audio that isn't adjacent.
+ if (Math.abs(timestamp - this.expected) > 1 / rate * 2) {
+ this.x1 = this.x2 = this.y1 = this.y2 = 0;
+ }
+ this.expected = timestamp + frames / rate;
+
+ for (let frame = 0; frame < frames; frame++) {
+ let x = 0;
+ for (let channel = 0; channel < channels; channel++) x += samples[frame * channels + channel]!;
+ x /= channels;
+
+ const y = filter.b0 * x + filter.b1 * this.x1 + filter.b2 * this.x2 - filter.a1 * this.y1 - filter.a2 * this.y2;
+ this.x2 = this.x1;
+ this.x1 = x;
+ this.y2 = this.y1;
+ this.y1 = y;
+
+ const index = Math.floor((timestamp + frame / rate - this.start) / this.hop);
+ if (index !== this.index) {
+ this.flush();
+ this.index = index;
+ }
+ this.sum += y * y;
+ this.count++;
+ }
+ }
+
+ /** Writes out the hop still being summed. Call once the last block is in. */
+ flush(): void {
+ if (this.count > 0 && this.index >= 0 && this.index < this.levels.length) {
+ this.levels[this.index] = Math.max(FLOOR_DB, 10 * Math.log10(this.sum / this.count));
+ }
+ this.sum = 0;
+ this.count = 0;
+ }
+
+ /** A second-order Butterworth high-pass at the cutoff, for `rate`. */
+ private filter(rate: number) {
+ if (this.coefficients?.rate === rate) return this.coefficients;
+
+ const w = (2 * Math.PI * Math.min(this.highpass, rate / 4)) / rate;
+ const alpha = Math.sin(w) / Math.SQRT2;
+ const cos = Math.cos(w);
+ const a0 = 1 + alpha;
+
+ this.coefficients = {
+ rate,
+ b0: (1 + cos) / 2 / a0,
+ b1: -(1 + cos) / a0,
+ b2: (1 + cos) / 2 / a0,
+ a1: (-2 * cos) / a0,
+ a2: (1 - alpha) / a0,
+ };
+ return this.coefficients;
+ }
+}
+
+/** The level `fraction` of the envelope's hops are at or below, in dB. */
+function percentile(levels: Float32Array, fraction: number): number {
+ const bins = new Uint32Array(Math.ceil(-FLOOR_DB / BIN_DB) + 1);
+ for (const level of levels) {
+ bins[Math.min(bins.length - 1, Math.max(0, Math.round((level - FLOOR_DB) / BIN_DB)))]!++;
+ }
+
+ const target = fraction * levels.length;
+ let seen = 0;
+ for (let bin = 0; bin < bins.length; bin++) {
+ seen += bins[bin]!;
+ if (seen >= target) return FLOOR_DB + bin * BIN_DB;
+ }
+ return 0;
+}
+
+/**
+ * The level below which the envelope is silent, or null when it has no
+ * silence to tell apart: sound and floor are too close together for a line
+ * between them to mean anything. A recording that is quiet throughout is
+ * silent throughout, and gets the loudest threshold allowed.
+ */
+export function silenceThreshold(levels: Float32Array, options: SilenceOptions): number | null {
+ if (levels.length === 0) return null;
+
+ const floor = percentile(levels, 0.1);
+ const sound = percentile(levels, 0.95);
+
+ if (sound <= options.minThreshold) return options.maxThreshold;
+ if (sound - floor < options.minContrast) return null;
+
+ const threshold = floor + (sound - floor) * options.sensitivity;
+ return Math.min(options.maxThreshold, Math.max(options.minThreshold, threshold));
+}
+
+/** Runs of equal values in `gate`, as [value, from, to) in hops. */
+function runs(gate: Uint8Array): [value: number, from: number, to: number][] {
+ const result: [number, number, number][] = [];
+ let from = 0;
+ for (let i = 1; i <= gate.length; i++) {
+ if (i === gate.length || gate[i] !== gate[from]) {
+ result.push([gate[from]!, from, i]);
+ from = i;
+ }
+ }
+ return result;
+}
+
+/**
+ * The silent stretches of the envelope worth cutting, in seconds of the
+ * source, already pulled back from the sound either side by the padding. A
+ * silence at either end of the envelope keeps no padding on that side: there
+ * is no sound beyond it to lead into.
+ */
+export function findSilences(envelope: Envelope, options: SilenceOptions = DEFAULT_SILENCE_OPTIONS): Span[] {
+ const { levels, hop, start } = envelope;
+ const threshold = silenceThreshold(levels, options);
+ if (threshold === null) return [];
+
+ // 1 is sound. Opens above the threshold and closes only once the level
+ // has fallen clearly below it.
+ const gate = new Uint8Array(levels.length);
+ let open = false;
+ for (let i = 0; i < levels.length; i++) {
+ const level = levels[i]!;
+ open = open ? level >= threshold - options.hysteresis : level > threshold;
+ gate[i] = open ? 1 : 0;
+ }
+
+ // A burst of sound too short to be speech, with silence either side, is
+ // part of that silence. One at the very edge has silence on one side only
+ // and is still a click.
+ const minSound = Math.round(options.minSound / hop);
+ for (const [value, from, to] of runs(gate)) {
+ if (value === 1 && to - from < minSound) gate.fill(0, from, to);
+ }
+
+ const minSilence = Math.round(options.minSilence / hop);
+ const silences: Span[] = [];
+
+ for (const [value, from, to] of runs(gate)) {
+ if (value === 1 || to - from < minSilence) continue;
+
+ const head = from === 0 ? 0 : options.padding;
+ const tail = to === levels.length ? 0 : options.padding;
+ const cutFrom = start + from * hop + head;
+ const cutTo = start + to * hop - tail;
+ if (cutTo - cutFrom >= hop) silences.push([cutFrom, cutTo]);
+ }
+
+ return silences;
+}
diff --git a/apps/web/src/engine/silence/remove.tsx b/apps/web/src/engine/silence/remove.tsx
new file mode 100644
index 00000000..eb776164
--- /dev/null
+++ b/apps/web/src/engine/silence/remove.tsx
@@ -0,0 +1,219 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Taking the silent parts out of a clip. The silences are found in the
+ * stretch of the source the clip plays (see `./detect`), then the clip is cut
+ * the way a split cuts it — copied where it stands, each copy trimmed to one
+ * of the stretches of sound — and the pieces are pulled together so the gaps
+ * close. Clips after it in the same sequence move up by as much as was taken
+ * out, so the cut leaves no hole behind.
+ *
+ * Finding is asynchronous and cutting is not: everything the cut does lands
+ * in one go, so it is one step to undo.
+ */
+
+import { Sequence } from '@diffusionstudio/reconciler';
+import {
+ AssetId,
+ Computed,
+ FrameRate,
+ Host,
+ Selected,
+ getAsset,
+ getAssetFile,
+ getEntityChildren,
+ getNextName,
+ getParentEntity,
+ getSourceWindow,
+ getTimelineOrigin,
+ isGroup,
+ isSequence,
+} from '@diffusionstudio/runtime';
+
+import { getDocumentEditor } from '../editor';
+import { cloneFramesForSplit, clonePeaksForSplit } from '../timeline';
+import { authoredTime, moveEntityTo, trimIn, trimOut } from '../timing';
+import { DEFAULT_SILENCE_OPTIONS } from './detect';
+
+import type { Entity, World } from 'koota';
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
+import type { SilenceOptions, Span } from './detect';
+import type { SilenceRequest, SilenceResponse } from './worker';
+
+/** The silences found in one clip, in seconds of its source. */
+export type ClipSilences = { clip: Entity; silences: Span[] };
+
+/** The media `clip` plays sound from, or null when it plays none. */
+function soundAsset(world: World, clip: Entity): AudioAsset | VideoAsset | null {
+ const id = clip.get(AssetId)?.value;
+ const asset = id ? getAsset(world, id) : undefined;
+ if (asset?.type === 'AUDIO') return asset;
+ if (asset?.type === 'VIDEO' && asset.channels) return asset;
+ return null;
+}
+
+/** The selected clips that have sound to take silence out of. */
+export function silenceTargets(world: World): Entity[] {
+ return [...world.query(Selected)].filter((entity) => soundAsset(world, entity) !== null);
+}
+
+/**
+ * The silences in the stretch of its source `clip` plays, in seconds of the
+ * source. Decoded in a worker of its own, gone when it answers.
+ */
+export async function detectClipSilences(
+ world: World,
+ clip: Entity,
+ options: SilenceOptions = DEFAULT_SILENCE_OPTIONS,
+): Promise<Span[]> {
+ const asset = soundAsset(world, clip);
+ if (!asset) return [];
+
+ const fps = world.get(FrameRate)?.value ?? 30;
+ const window = getSourceWindow(clip);
+ const file = await getAssetFile(asset);
+
+ const worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' });
+ try {
+ const silences = await new Promise<Span[] | null>((resolve, reject) => {
+ worker.onmessage = ({ data }: MessageEvent<SilenceResponse>) => {
+ if ('error' in data) reject(new Error(data.error));
+ else resolve(data.silences);
+ };
+ worker.onerror = (event) => reject(new Error(event.message || 'The silence worker failed'));
+ worker.postMessage({
+ file,
+ start: window.in / fps,
+ end: window.out / fps,
+ options,
+ } satisfies SilenceRequest);
+ });
+ return silences ?? [];
+ } finally {
+ worker.terminate();
+ }
+}
+
+/**
+ * The stretches of `clip` to keep, in scene frames: the clip's span with the
+ * silences taken out. Worked out against the clip as it is now, so an edit
+ * made while the silences were being found is respected.
+ */
+function keptSpans(world: World, clip: Entity, silences: Span[]): [number, number][] {
+ const computed = clip.get(Computed);
+ if (!computed) return [];
+
+ const fps = world.get(FrameRate)?.value ?? 30;
+ const rate = computed.playbackRate || 1;
+ const { start, end, origin } = computed;
+ const toFrame = (seconds: number) => Math.min(end, Math.max(start, Math.round(origin + (seconds * fps) / rate)));
+
+ const kept: [number, number][] = [];
+ let cursor = start;
+ for (const [from, to] of silences) {
+ const cutFrom = toFrame(from);
+ const cutTo = toFrame(to);
+ if (cutTo <= cutFrom) continue;
+ if (cutFrom > cursor) kept.push([cursor, cutFrom]);
+ cursor = Math.max(cursor, cutTo);
+ }
+ if (cursor < end) kept.push([cursor, end]);
+
+ return kept;
+}
+
+/**
+ * The clips after `clip` in the sequence it is in, which move up when it
+ * gets shorter. None outside a sequence: layers there don't follow one
+ * another, so nothing is out of place when one of them gets shorter.
+ */
+function followers(world: World, clip: Entity): Entity[] {
+ const parent = getParentEntity(clip);
+ if (parent === null || !isSequence(parent)) return [];
+
+ const end = (clip.get(Computed)?.end ?? 0) - getTimelineOrigin(clip);
+ return getEntityChildren(world, parent).filter(
+ (sibling) => sibling !== clip && (authoredTime(world, sibling, 'start') ?? 0) >= end,
+ );
+}
+
+/** Moves `entity` along its timeline by `delta` frames, by what the file says rather than what was last computed. */
+function shift(world: World, entity: Entity, delta: number): void {
+ const start = authoredTime(world, entity, 'start') ?? 0;
+ moveEntityTo(world, entity, getTimelineOrigin(entity) + start + delta);
+}
+
+/**
+ * Cuts the silences out of each clip and closes the gaps they leave. A clip
+ * that would have nothing left is left alone. Returns the pieces the clips
+ * were cut into, which are what the selection is left on.
+ */
+export function removeSilences(world: World, found: ClipSilences[]): Entity[] {
+ const editor = getDocumentEditor(world);
+ const cut: Entity[] = [];
+
+ // Latest first: a clip earlier in a sequence moves the ones after it up,
+ // and those have to have been cut already, so their own pieces move too.
+ const ordered = found
+ .filter(({ clip }) => clip.isAlive())
+ .sort((a, b) => (b.clip.get(Computed)?.start ?? 0) - (a.clip.get(Computed)?.start ?? 0));
+
+ for (const { clip, silences } of ordered) {
+ const start = clip.get(Computed)?.start ?? 0;
+ const end = clip.get(Computed)?.end ?? 0;
+ const kept = keptSpans(world, clip, silences);
+ const length = kept.reduce((sum, [from, to]) => sum + to - from, 0);
+ const removed = end - start - length;
+ if (kept.length === 0 || removed <= 0) continue;
+
+ // Read before anything moves: the cut changes where the clip ends.
+ const after = followers(world, clip);
+ // Only the last piece leads into the next clip; the cuts between the
+ // pieces are hard ones.
+ const transition = clip.get(Host)?.props.transition !== undefined;
+
+ // Each copy is of the one before it, made before anything is trimmed,
+ // so every piece is spelled from the whole clip and in playing order.
+ const pieces = [clip];
+ for (let i = 1; i < kept.length; i++) {
+ const [pair] = editor.duplicateInPlace([pieces[i - 1]!]);
+ if (!pair) break;
+ pieces.push(pair.copy);
+ clonePeaksForSplit(clip.id(), pair.copy.id());
+ cloneFramesForSplit(clip.id(), pair.copy.id());
+ }
+ if (pieces.length < kept.length) {
+ editor.remove(pieces.slice(1));
+ continue;
+ }
+
+ // Out before in: the out point is read against where the clip starts
+ // now, which the in point moves.
+ let at = start;
+ for (const [index, piece] of pieces.entries()) {
+ const [from, to] = kept[index]!;
+ if (to < end) trimOut(world, piece, to);
+ if (from > start) trimIn(world, piece, from);
+ moveEntityTo(world, piece, at);
+ if (transition && index < pieces.length - 1) editor.editProperty(piece, 'transition', false);
+ at += to - from;
+ }
+
+ for (const follower of after) shift(world, follower, -removed);
+
+ // Pieces side by side under a scene would be layers; wrapped, they
+ // still read as the one clip they were (as a split does it).
+ const parent = getParentEntity(clip);
+ if (pieces.length > 1 && parent !== null && !isGroup(parent)) {
+ editor.wrap(pieces, () => <Sequence name={getNextName(world, 'Sequence')} />);
+ }
+
+ cut.push(...pieces);
+ }
+
+ if (cut.length > 0) editor.select(cut);
+
+ return cut;
+}
diff --git a/apps/web/src/engine/silence/worker.ts b/apps/web/src/engine/silence/worker.ts
new file mode 100644
index 00000000..a346a546
--- /dev/null
+++ b/apps/web/src/engine/silence/worker.ts
@@ -0,0 +1,69 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The silences of a media file's audio, found off the main thread: the
+// stretch asked about is decoded block by block and each block is folded
+// into the loudness envelope before the next is decoded, so nothing more than
+// one block of samples is ever held. PCM formats are unpacked in JS, which
+// would stall the UI for a long file. One worker per request.
+
+import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
+
+import { Envelope, findSilences } from './detect';
+
+import type { SilenceOptions, Span } from './detect';
+
+export interface SilenceRequest {
+ file: Blob;
+ /** The stretch of the file to look in, in seconds. */
+ start: number;
+ end: number;
+ options: SilenceOptions;
+}
+
+export type SilenceResponse = { silences: Span[] | null } | { error: string };
+
+// The app's lib is the DOM's, so the worker scope is narrowed by hand.
+const scope = self as unknown as {
+ onmessage: ((event: MessageEvent<SilenceRequest>) => void) | null;
+ postMessage(message: SilenceResponse): void;
+};
+
+scope.onmessage = async ({ data }) => {
+ try {
+ scope.postMessage({ silences: await detect(data) });
+ } catch (error) {
+ scope.postMessage({ error: error instanceof Error ? error.message : String(error) });
+ }
+};
+
+/** The silences between `start` and `end` of the file's primary audio track, or null when it has none. */
+async function detect({ file, start, end, options }: SilenceRequest): Promise<Span[] | null> {
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ try {
+ const track = await input.getPrimaryAudioTrack();
+ if (!track) return null;
+
+ const envelope = new Envelope(start, end, options);
+ const sink = new AudioSampleSink(track);
+
+ for await (const sample of sink.samples(start, end)) {
+ // Everything the sample holds is taken before it is closed, and it
+ // is closed however that goes: they are decoder resources.
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+ envelope.push(floats, sample.numberOfChannels, sample.sampleRate, sample.timestamp);
+ } finally {
+ sample.close();
+ }
+ }
+ envelope.flush();
+
+ return findSilences(envelope, options);
+ } finally {
+ input.dispose();
+ }
+}