Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

updates

Jareer9ecba9621a2e
diff --git a/desktop/src-tauri/src/export.rs b/desktop/src-tauri/src/export.rs
index c0fb9f4f..11385d3c 100644
--- a/desktop/src-tauri/src/export.rs
+++ b/desktop/src-tauri/src/export.rs
@@ -48,6 +48,9 @@ pub struct ExportClip {
pub fade_in: f64,
#[serde(default)]
pub fade_out: f64,
+ /// FFmpeg filter chain from the Filters tab, or empty.
+ #[serde(default)]
+ pub filter_chain: String,
}
fn unity() -> f64 {
@@ -305,6 +308,14 @@ fn mix_audio(clips: &[&ExportClip], duration: f64, destination: &Path) -> Result
clip.source_start, clip.duration
);
+ // Filters first: they are what the clip *is* now, and gain and fades
+ // are adjustments applied to that result. Reversing the order would
+ // let a compressor inside a filter chain undo the fade.
+ if !clip.filter_chain.is_empty() {
+ stage.push(',');
+ stage.push_str(&clip.filter_chain);
+ }
+
if (clip.volume - 1.0).abs() > f64::EPSILON {
stage.push_str(&format!(",volume={:.4}", clip.volume.max(0.0)));
}
diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs
index 9180b523..a414c702 100644
--- a/desktop/src-tauri/src/lib.rs
+++ b/desktop/src-tauri/src/lib.rs
@@ -195,6 +195,63 @@ fn filmstrip(path: &str, count: u32, height: u32) -> Result<Vec<u8>, String> {
Ok(output.stdout)
}
+/// Renders one clip's audio through a filter chain and returns it as WAV.
+///
+/// This is what lets the preview play filtered audio *exactly* as the export
+/// will render it. The alternative - reimplementing pitch shifting, companding
+/// and de-essing in Web Audio - would be a second implementation to keep in
+/// step with the first, and it would be wrong in ways nobody would notice
+/// until the exported file sounded different from the edit.
+///
+/// WAV because it decodes instantly in the browser and needs no container
+/// seeking; the result is short-lived and never written to disk.
+#[tauri::command]
+async fn render_filtered_audio(
+ path: String,
+ source_start: f64,
+ duration: f64,
+ chain: String,
+) -> Result<tauri::ipc::Response, String> {
+ let bytes = tauri::async_runtime::spawn_blocking(move || {
+ // A filter chain arrives as text and goes straight to FFmpeg, so it
+ // must not be able to smuggle in another argument.
+ if chain.contains('\n') || chain.contains('\r') {
+ return Err("filter chain contains a line break".to_owned());
+ }
+
+ let mut command = std::process::Command::new(relay_media::ffmpeg());
+ command
+ .args(["-hide_banner", "-nostdin", "-loglevel", "error"])
+ .args(["-ss", &format!("{source_start:.6}")])
+ .args(["-t", &format!("{duration:.6}")])
+ .args(["-i", &path])
+ .args(["-vn", "-af", &chain])
+ // 16-bit stereo at 48k: what the filters run at anyway, and small
+ // enough to move over the bridge without a second thought.
+ .args(["-ac", "2", "-ar", "48000", "-c:a", "pcm_s16le"])
+ .args(["-f", "wav", "-"]);
+
+ let output = command
+ .output()
+ .map_err(|error| format!("could not run ffmpeg: {error}"))?;
+
+ if !output.status.success() {
+ let detail = String::from_utf8_lossy(&output.stderr);
+ let tail = detail.lines().rev().take(2).collect::<Vec<_>>().join(" / ");
+ return Err(format!("filter failed: {tail}"));
+ }
+ if output.stdout.is_empty() {
+ return Err("filter produced no audio".to_owned());
+ }
+
+ Ok(output.stdout)
+ })
+ .await
+ .map_err(|error| format!("filter task failed: {error}"))??;
+
+ Ok(tauri::ipc::Response::new(bytes))
+}
+
/// Creates a project folder, writes its manifest and records it as recent.
#[tauri::command]
async fn create_project(
@@ -319,6 +376,7 @@ pub fn run() {
probe_media,
engine_version,
read_media_bytes,
+ render_filtered_audio,
extract_filmstrip,
create_project,
open_project,
diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx
index 54ce1786..3e610442 100644
--- a/desktop/src/App.tsx
+++ b/desktop/src/App.tsx
@@ -42,9 +42,11 @@ import {
updateClip,
trimClip,
whyNotMerge,
+ type Clip,
type MediaItem,
type Project,
} from "./lib/project";
+import { buildChain, chainKey } from "./lib/filters";
import { useTheme, type Theme } from "./lib/theme";
import { useTransport } from "./lib/transport";
@@ -205,8 +207,13 @@ function Editor({
const audio = useRef<AudioPreview | null>(null);
if (audio.current === null) audio.current = new AudioPreview();
+ const [renderingClips, setRenderingClips] = useState<ReadonlySet<string>>(new Set());
+
useEffect(() => {
const preview = audio.current;
+ if (preview) {
+ preview.onRenderChange = () => setRenderingClips(new Set(preview.pending()));
+ }
return () => preview?.dispose();
}, []);
@@ -221,6 +228,9 @@ function Editor({
clipId: clip.id,
path: media.path,
time: clip.sourceStart + (playhead - clip.start),
+ // A filtered clip plays a render of itself, produced by the same
+ // ffmpeg chain the exporter will use.
+ filter: chainFor(clip),
// Uncapped: the preview routes through a gain node, so what you
// hear matches what the exporter will render.
volume: clipGainAt(clip, playhead),
@@ -439,6 +449,19 @@ function Editor({
return () => window.removeEventListener("keydown", onKey);
}, [deleteSelected, duration, frameRate, mergeSelected, splitAtPlayhead, transport]);
+ // Filters are rendered per clip, keyed so that returning a slider to a value
+ // you already heard reuses the render instead of producing it again.
+ const chainFor = useCallback((clip: Clip) => {
+ const chain = buildChain(clip.filters);
+ if (!chain) return undefined;
+ return {
+ key: chainKey(clip.mediaId, clip.sourceStart, clip.duration, clip.filters),
+ chain,
+ sourceStart: clip.sourceStart,
+ duration: clip.duration,
+ };
+ }, []);
+
// The exporter works from a flat list: the engine rebuilds a real timeline
// from it, so track *order* has to survive the trip even though track
// identity does not.
@@ -462,6 +485,7 @@ function Editor({
volume: clip.volume,
fadeIn: clip.fadeIn,
fadeOut: clip.fadeOut,
+ filterChain: buildChain(clip.filters) ?? "",
},
];
}),
@@ -646,6 +670,7 @@ function Editor({
clip={selectedClip}
media={inspectorMedia}
frameRate={frameRate}
+ rendering={selectedClipIds.some((id) => renderingClips.has(id))}
onChangeClip={(patch) => {
if (selectedClipIds.length !== 1) return;
setProject((current) => updateClip(current, selectedClipIds[0], patch));
diff --git a/desktop/src/components/FiltersPanel.tsx b/desktop/src/components/FiltersPanel.tsx
new file mode 100644
index 00000000..afd2ab68
--- /dev/null
+++ b/desktop/src/components/FiltersPanel.tsx
@@ -0,0 +1,174 @@
+import { useState } from "react";
+
+import {
+ CATEGORIES,
+ FILTERS,
+ findFilter,
+ resolveParams,
+ type ClipFilter,
+ type FilterCategory,
+} from "../lib/filters";
+import type { Clip } from "../lib/project";
+import { Slider } from "./controls";
+import { Icon, IconButton } from "./Icon";
+import { Empty } from "./Panel";
+
+/**
+ * The Filters tab.
+ *
+ * Filters are a *list* on the clip, not a set of toggles, because they apply
+ * in order and order is audible. Adding the same filter twice is allowed for
+ * the same reason - two gentle EQ lifts are a real thing to want.
+ *
+ * Everything here writes plain numbers onto the clip. The FFmpeg chain is
+ * built from those in one place (`lib/filters.ts`) and used by both the
+ * preview and the exporter, so what you hear is what gets rendered.
+ */
+export function FiltersPanel({
+ clip,
+ rendering,
+ onChange,
+}: {
+ clip: Clip | null;
+ /** True while the preview is re-rendering this clip's audio. */
+ rendering: boolean;
+ onChange: (filters: ClipFilter[]) => void;
+}) {
+ const [category, setCategory] = useState<FilterCategory>("voice");
+
+ if (!clip) {
+ return (
+ <Empty icon={<Icon name="settings" size={26} strokeWidth={1.5} />}>
+ Select a single clip to add filters.
+ </Empty>
+ );
+ }
+
+ if (clip.kind === "image") {
+ return (
+ <Empty icon={<Icon name="image" size={26} strokeWidth={1.5} />}>
+ A still has no audio to filter.
+ </Empty>
+ );
+ }
+
+ const applied = clip.filters;
+
+ const add = (id: string) => onChange([...applied, { id, params: {} }]);
+ const remove = (index: number) => onChange(applied.filter((_, at) => at !== index));
+ const setParam = (index: number, key: string, value: number) =>
+ onChange(
+ applied.map((filter, at) =>
+ at === index ? { ...filter, params: { ...filter.params, [key]: value } } : filter,
+ ),
+ );
+ const move = (index: number, by: number) => {
+ const target = index + by;
+ if (target < 0 || target >= applied.length) return;
+ const next = [...applied];
+ [next[index], next[target]] = [next[target], next[index]];
+ onChange(next);
+ };
+
+ return (
+ <div className="px-3 py-3">
+ {applied.length > 0 && (
+ <section className="mb-5">
+ <h3 className="mb-2 flex items-center gap-2 text-[11px] font-semibold uppercase tracking-wider text-tertiary">
+ Applied
+ {rendering && (
+ <span className="font-normal normal-case tracking-normal text-accent">
+ rendering...
+ </span>
+ )}
+ </h3>
+
+ <ul className="flex flex-col gap-2">
+ {applied.map((entry, index) => {
+ const definition = findFilter(entry.id);
+ if (!definition) return null;
+ const values = resolveParams(definition, entry.params);
+
+ return (
+ <li key={`${entry.id}-${index}`} className="rounded-lg bg-sunken p-2.5">
+ <div className="mb-2 flex items-center gap-1">
+ <span className="min-w-0 flex-1 truncate text-xs text-primary">
+ {definition.label}
+ </span>
+ <IconButton
+ icon="chevronDown"
+ label="Move later in the chain"
+ size={7}
+ disabled={index === applied.length - 1}
+ onClick={() => move(index, 1)}
+ />
+ <IconButton
+ icon="close"
+ label={`Remove ${definition.label}`}
+ size={7}
+ tone="danger"
+ onClick={() => remove(index)}
+ />
+ </div>
+
+ {definition.params.map((param) => (
+ <Slider
+ key={param.key}
+ label={param.label}
+ value={values[param.key]}
+ min={param.min}
+ max={param.max}
+ step={param.step}
+ format={param.format}
+ onReset={() => setParam(index, param.key, param.default)}
+ onChange={(value) => setParam(index, param.key, value)}
+ />
+ ))}
+ </li>
+ );
+ })}
+ </ul>
+ </section>
+ )}
+
+ <div className="mb-3 flex rounded-lg bg-sunken p-0.5">
+ {CATEGORIES.map((entry) => (
+ <button
+ key={entry.id}
+ type="button"
+ aria-pressed={category === entry.id}
+ onClick={() => setCategory(entry.id)}
+ className={`flex-1 cursor-pointer rounded-[6px] px-2 py-1 text-[12px] transition-colors ${
+ category === entry.id
+ ? "bg-panel text-primary shadow-[0_1px_2px_rgba(0,0,0,0.14)]"
+ : "text-secondary hover:text-primary"
+ }`}
+ >
+ {entry.label}
+ </button>
+ ))}
+ </div>
+
+ <ul className="flex flex-col gap-1">
+ {FILTERS.filter((filter) => filter.category === category).map((filter) => (
+ <li key={filter.id}>
+ <button
+ type="button"
+ onClick={() => add(filter.id)}
+ className="w-full cursor-pointer rounded-lg px-2 py-2 text-left transition-colors
+ hover:bg-hover"
+ >
+ <span className="flex items-center gap-2">
+ <Icon name="plus" size={12} className="shrink-0 text-tertiary" />
+ <span className="text-xs text-primary">{filter.label}</span>
+ </span>
+ <span className="mt-0.5 block pl-5 text-[11px] leading-snug text-tertiary">
+ {filter.blurb}
+ </span>
+ </button>
+ </li>
+ ))}
+ </ul>
+ </div>
+ );
+}
diff --git a/desktop/src/components/RightPanel.tsx b/desktop/src/components/RightPanel.tsx
index e8dd5096..2f9bd21e 100644
--- a/desktop/src/components/RightPanel.tsx
+++ b/desktop/src/components/RightPanel.tsx
@@ -1,8 +1,9 @@
import type { Clip, MediaItem } from "../lib/project";
+import type { ClipFilter } from "../lib/filters";
import { AdjustPanel } from "./AdjustPanel";
-import { Icon } from "./Icon";
+import { FiltersPanel } from "./FiltersPanel";
import { Inspector } from "./Inspector";
-import { Empty, Panel } from "./Panel";
+import { Panel } from "./Panel";
export type RightTab = "details" | "adjust" | "filters";
@@ -27,6 +28,7 @@ export function RightPanel({
media,
frameRate,
onChangeClip,
+ rendering,
}: {
tab: RightTab;
onTab: (tab: RightTab) => void;
@@ -34,6 +36,8 @@ export function RightPanel({
media: MediaItem | null;
frameRate: number;
onChangeClip: (patch: Partial<Clip>) => void;
+ /** True while the selected clip's filtered audio is being rendered. */
+ rendering: boolean;
}) {
return (
<Panel>
@@ -60,13 +64,11 @@ export function RightPanel({
{tab === "details" && <Inspector clip={clip} media={media} frameRate={frameRate} />}
{tab === "adjust" && <AdjustPanel clip={clip} onChange={onChangeClip} />}
{tab === "filters" && (
- // Said plainly rather than dressed up with a disabled list of effects
- // that do not exist. Audio filters need the preview to run through Web
- // Audio so that what you hear matches what ffmpeg will render.
- <Empty icon={<Icon name="settings" size={26} strokeWidth={1.5} />}>
- No filters yet. Audio effects arrive once the preview runs through a real
- mixer rather than a media element.
- </Empty>
+ <FiltersPanel
+ clip={clip}
+ rendering={rendering}
+ onChange={(filters: ClipFilter[]) => onChangeClip({ filters })}
+ />
)}
</Panel>
);
diff --git a/desktop/src/lib/audio.ts b/desktop/src/lib/audio.ts
index f6e60399..e6a32419 100644
--- a/desktop/src/lib/audio.ts
+++ b/desktop/src/lib/audio.ts
@@ -23,7 +23,7 @@
* See docs/decisions/0004-audio-preview-in-the-webview.md.
*/
-import { readMediaBytes } from "./engine";
+import { readMediaBytes, renderFilteredAudio } from "./engine";
/** One audio clip that should be sounding right now. */
export interface Voice {
@@ -33,6 +33,19 @@ export interface Voice {
time: number;
/** Linear gain, fades already applied. May exceed 1. */
volume: number;
+ /**
+ * A filtered render of this clip, when it has filters.
+ *
+ * The rendered audio begins at the clip's in-point rather than the file's,
+ * so a position inside it is offset by `sourceStart`.
+ */
+ filter?: {
+ /** Identity of this exact render. A change means re-render. */
+ key: string;
+ chain: string;
+ sourceStart: number;
+ duration: number;
+ };
}
/**
@@ -45,6 +58,15 @@ export interface Voice {
const TOLERANCE_PLAYING = 0.3;
const TOLERANCE_PAUSED = 0.02;
+/**
+ * How long the filter settings must hold still before re-rendering.
+ *
+ * Without this, every step of a slider drag spawned an FFmpeg process and tore
+ * down the audio element mid-playback - dozens of renders for one gesture, and
+ * audio that cut out while you were trying to hear what you were adjusting.
+ */
+const RENDER_DEBOUNCE_MS = 400;
+
let sharedContext: AudioContext | null = null;
/** The one AudioContext. Browsers cap how many a page may create. */
@@ -89,10 +111,26 @@ interface Loaded {
gain: GainNode;
objectUrl?: string;
loading: boolean;
+ /** Which render this element holds; undefined means the unfiltered file. */
+ key?: string;
+ /** How far into the file this element's audio begins. */
+ offset: number;
}
export class AudioPreview {
private voices = new Map<string, Loaded>();
+ private rendering = new Set<string>();
+ /** Pending re-renders, keyed by clip. */
+ private timers = new Map<string, number>();
+ /** Which key each pending timer is waiting to render. */
+ private scheduled = new Map<string, string | undefined>();
+ /** Notified when a render finishes, so the UI can drop its spinner. */
+ onRenderChange: (() => void) | null = null;
+
+ /** Clips whose filtered audio is still rendering. */
+ pending(): ReadonlySet<string> {
+ return this.rendering;
+ }
/**
* Brings the audio output in line with the transport.
@@ -111,17 +149,22 @@ export class AudioPreview {
const tolerance = playing ? TOLERANCE_PLAYING : TOLERANCE_PAUSED;
for (const voice of active) {
- const loaded = this.load(voice.clipId, voice.path);
+ const loaded = this.load(voice);
const element = loaded.element;
// No clamp. A boost is meant to be audible.
loaded.gain.gain.value = Math.max(0, voice.volume);
+ // A filtered render starts at the clip's in-point, so the same instant
+ // sits at a different offset inside it than in the original file.
+ const target = Math.max(0, voice.time - loaded.offset);
+
// readyState 0 means metadata has not arrived; setting currentTime then
// throws in some engines and is discarded in others.
- if (element.readyState > 0 && Math.abs(element.currentTime - voice.time) > tolerance) {
+
+ if (element.readyState > 0 && Math.abs(element.currentTime - target) > tolerance) {
try {
- element.currentTime = Math.max(0, voice.time);
+ element.currentTime = target;
} catch {
// Still opening; the next sync lands it.
}
@@ -141,43 +184,127 @@ export class AudioPreview {
/** Drops a voice, for instance when its clip is deleted. */
release(clipId: string): void {
+ const timer = this.timers.get(clipId);
+ if (timer !== undefined) {
+ clearTimeout(timer);
+ this.timers.delete(clipId);
+ this.scheduled.delete(clipId);
+ }
+
const loaded = this.voices.get(clipId);
if (!loaded) return;
+ this.voices.delete(clipId);
+ this.destroy(loaded);
+
+ if (this.rendering.delete(clipId)) this.onRenderChange?.();
+ }
+ /** Tears down one element and its graph node. */
+ private destroy(loaded: Loaded): void {
loaded.element.pause();
loaded.element.removeAttribute("src");
loaded.element.load();
loaded.gain.disconnect();
if (loaded.objectUrl) URL.revokeObjectURL(loaded.objectUrl);
- this.voices.delete(clipId);
}
dispose(): void {
+ for (const timer of this.timers.values()) clearTimeout(timer);
+ this.timers.clear();
+ this.scheduled.clear();
for (const clipId of [...this.voices.keys()]) this.release(clipId);
}
- private load(clipId: string, path: string): Loaded {
- const existing = this.voices.get(clipId);
- if (existing) return existing;
+ private load(voice: Voice): Loaded {
+ const key = voice.filter?.key;
+ const existing = this.voices.get(voice.clipId);
+ if (existing && existing.key === key) return existing;
+
+ // The settings changed. Do *not* tear down what is currently audible:
+ // schedule the swap and keep playing the old render until the new one is
+ // ready, so adjusting a filter does not silence the thing you are judging.
+ if (existing) {
+ this.schedule(voice);
+ return existing;
+ }
+
+ return this.create(voice);
+ }
+
+ /** Queues a re-render, restarting the clock on every further change. */
+ private schedule(voice: Voice): void {
+ const key = voice.filter?.key;
+ if (this.scheduled.get(voice.clipId) === key) return;
+
+ const timer = this.timers.get(voice.clipId);
+ if (timer !== undefined) clearTimeout(timer);
+
+ this.scheduled.set(voice.clipId, key);
+ this.timers.set(
+ voice.clipId,
+ window.setTimeout(() => {
+ this.timers.delete(voice.clipId);
+ this.scheduled.delete(voice.clipId);
+
+ const previous = this.voices.get(voice.clipId);
+ this.voices.delete(voice.clipId);
+ this.create(voice);
+ if (previous) this.destroy(previous);
+ }, RENDER_DEBOUNCE_MS),
+ );
+ }
+
+ private create(voice: Voice): Loaded {
+ const key = voice.filter?.key;
const element = new Audio();
element.preload = "auto";
- const loaded: Loaded = { element, gain: connectElement(element), loading: true };
- this.voices.set(clipId, loaded);
+ const loaded: Loaded = {
+ element,
+ gain: connectElement(element),
+ loading: true,
+ key,
+ offset: voice.filter ? voice.filter.sourceStart : 0,
+ };
+ this.voices.set(voice.clipId, loaded);
+
+ const source = voice.filter
+ ? renderFilteredAudio({
+ path: voice.path,
+ sourceStart: voice.filter.sourceStart,
+ duration: voice.filter.duration,
+ chain: voice.filter.chain,
+ })
+ : readMediaBytes(voice.path);
+
+ if (voice.filter) {
+ this.rendering.add(voice.clipId);
+ this.onRenderChange?.();
+ }
// Source arrives asynchronously; until then the element is silent and the
- // sync pass above simply has nothing to play, which is correct.
- void readMediaBytes(path)
+ // sync pass simply has nothing to play, which is correct.
+ void source
.then((bytes) => {
const url = URL.createObjectURL(new Blob([bytes]));
+
+ // A render that finishes after its clip was replaced or removed has
+ // nothing to play into; hand the URL straight back rather than
+ // leaking it into a detached element.
+ if (this.voices.get(voice.clipId) !== loaded) {
+ URL.revokeObjectURL(url);
+ return;
+ }
+
loaded.objectUrl = url;
loaded.element.src = url;
loaded.element.load();
})
- .catch((cause) => console.error(`Relay: could not load audio for ${path}`, cause))
+ .catch((cause) => console.error(`Relay: could not load audio for ${voice.path}`, cause))
.finally(() => {
loaded.loading = false;
+ if (this.rendering.delete(voice.clipId)) this.onRenderChange?.();
});
return loaded;
diff --git a/desktop/src/lib/engine.ts b/desktop/src/lib/engine.ts
index e41ece88..e50bdd85 100644
--- a/desktop/src/lib/engine.ts
+++ b/desktop/src/lib/engine.ts
@@ -60,6 +60,21 @@ export async function readMediaBytes(path: string): Promise<ArrayBuffer> {
return invoke<ArrayBuffer>("read_media_bytes", { path });
}
+/**
+ * Renders a clip's audio through an FFmpeg filter chain, returning WAV bytes.
+ *
+ * The same chain string the exporter uses, so the preview cannot drift from
+ * the rendered file.
+ */
+export async function renderFilteredAudio(request: {
+ path: string;
+ sourceStart: number;
+ duration: number;
+ chain: string;
+}): Promise<ArrayBuffer> {
+ return invoke<ArrayBuffer>("render_filtered_audio", request);
+}
+
/** A project on disk, as the launch screen sees it. */
export interface ProjectInfo {
path: string;
@@ -119,6 +134,8 @@ export interface ExportClip {
volume: number;
fadeIn: number;
fadeOut: number;
+ /** FFmpeg filter chain, or empty for none. */
+ filterChain: string;
}
export interface ExportRequest {
diff --git a/desktop/src/lib/filters.ts b/desktop/src/lib/filters.ts
new file mode 100644
index 00000000..a887dcdd
--- /dev/null
+++ b/desktop/src/lib/filters.ts
@@ -0,0 +1,301 @@
+/**
+ * The audio filter catalogue.
+ *
+ * Every filter is a function from parameters to an FFmpeg filter fragment.
+ * That is the whole design: there is one description of what a filter does,
+ * it produces a string, and both the exporter and the preview run that same
+ * string through FFmpeg. They cannot drift, because there is nothing to drift
+ * between - no second implementation in Web Audio approximating the first.
+ *
+ * The cost is that a filtered clip has to be rendered before it can be heard.
+ * See `lib/audio.ts`; the alternative was a preview that lies.
+ */
+
+export type FilterCategory = "voice" | "tone" | "space";
+
+export interface FilterParam {
+ key: string;
+ label: string;
+ min: number;
+ max: number;
+ step: number;
+ default: number;
+ /** Turns the raw number into what the reader sees. */
+ format: (value: number) => string;
+}
+
+export interface FilterDefinition {
+ id: string;
+ label: string;
+ category: FilterCategory;
+ /** One line on what it does, shown under the name. */
+ blurb: string;
+ params: FilterParam[];
+ /** Builds the FFmpeg fragment for these parameter values. */
+ chain: (params: Record<string, number>) => string;
+}
+
+/** A filter applied to a clip, with whatever parameters were set. */
+export interface ClipFilter {
+ id: string;
+ params: Record<string, number>;
+}
+
+export const CATEGORIES: { id: FilterCategory; label: string }[] = [
+ { id: "voice", label: "Voice" },
+ { id: "tone", label: "Tone" },
+ { id: "space", label: "Space" },
+];
+
+const percent = (value: number) => `${Math.round(value)}%`;
+const semitones = (value: number) =>
+ `${value > 0 ? "+" : ""}${value.toFixed(1)} st`;
+const decibels = (value: number) => `${value > 0 ? "+" : ""}${value.toFixed(1)} dB`;
+const seconds = (value: number) => `${value.toFixed(2)}s`;
+
+/**
+ * Pitch shift that moves the formants with the pitch.
+ *
+ * Raising the sample rate and then resampling back shifts everything - which
+ * is why it sounds like a different voice rather than the same voice
+ * transposed. `atempo` then puts the duration back, because `asetrate` alone
+ * would also make the clip shorter.
+ *
+ * This is the technique the whole Voice category is built on.
+ */
+function pitchShift(semitoneShift: number): string[] {
+ const ratio = 2 ** (semitoneShift / 12);
+ return [
+ "aresample=48000",
+ `asetrate=${(48000 * ratio).toFixed(6)}`,
+ "aresample=48000",
+ `atempo=${(1 / ratio).toFixed(8)}`,
+ ];
+}
+
+export const FILTERS: FilterDefinition[] = [
+ {
+ id: "sweet",
+ label: "Sweet",
+ category: "voice",
+ blurb: "Brighter, softer and a little higher. Ported from the reference chain.",
+ params: [
+ {
+ key: "amount",
+ label: "Amount",
+ min: 0,
+ max: 100,
+ step: 1,
+ default: 65,
+ format: percent,
+ },
+ {
+ key: "pitch",
+ label: "Pitch",
+ min: 0,
+ max: 8,
+ step: 0.1,
+ // 0 means "follow amount", which is what the original did when no
+ // explicit pitch was passed.
+ default: 0,
+ format: (value) => (value === 0 ? "auto" : semitones(value)),
+ },
+ ],
+ chain: ({ amount = 65, pitch = 0 }) => {
+ const strength = Math.max(0, Math.min(100, amount)) / 100;
+
+ /*
+ * The filter chain below is the reference implementation, unchanged.
+ * These *coefficients* are not.
+ *
+ * The original mapped the whole 0-100 slider onto 1.2-2.7 semitones and
+ * about 2 dB of EQ, so dragging it end to end barely changed anything -
+ * a faithful port of numbers that made a poor control. The shape is the
+ * same and the low end still matches; the top now goes somewhere.
+ *
+ * Original, if you want the subtler feel back:
+ * shift = 1.2 + 1.5 * strength
+ * presence = 0.8 + 1.5 * strength
+ * air = 1.0 + 2.0 * strength
+ * deess = 0.18 + 0.22 * strength
+ * echo = 0.012 + 0.018 * strength
+ */
+ const shift = pitch > 0 ? pitch : 1.2 + 3.8 * strength;
+ const presence = 0.8 + 5.2 * strength;
+ const air = 1.0 + 7.0 * strength;
+ const deess = 0.18 + 0.32 * strength;
+ const echo = 0.012 + 0.048 * strength;
+
+ return [
+ ...pitchShift(shift),
+ "highpass=f=85",
+ "equalizer=f=300:t=q:w=1.1:g=-1.4",
+ `equalizer=f=4200:t=q:w=0.8:g=${presence.toFixed(3)}`,
+ `equalizer=f=10500:t=q:w=0.7:g=${air.toFixed(3)}`,
+ "compand=attacks=0.010:decays=0.180:points=-80/-80|-30/-24|-18/-13|-8/-5|0/-1.5:soft-knee=5:gain=1",
+ `deesser=i=${deess.toFixed(3)}:m=0.45:f=0.55`,
+ `aecho=0.8:0.82:38:${echo.toFixed(4)}`,
+ "alimiter=limit=0.94:attack=5:release=60",
+ ].join(",");
+ },
+ },
+ {
+ id: "deep",
+ label: "Deep",
+ category: "voice",
+ blurb: "Lower and heavier, with the low end lifted to match.",
+ params: [
+ { key: "pitch", label: "Pitch", min: -8, max: -1, step: 0.1, default: -3, format: semitones },
+ { key: "body", label: "Body", min: 0, max: 8, step: 0.5, default: 3, format: decibels },
+ ],
+ chain: ({ pitch = -3, body = 3 }) =>
+ [
+ ...pitchShift(pitch),
+ "highpass=f=55",
+ `equalizer=f=140:t=q:w=1.0:g=${body.toFixed(2)}`,
+ "equalizer=f=2600:t=q:w=0.9:g=-1.2",
+ "alimiter=limit=0.94:attack=5:release=60",
+ ].join(","),
+ },
+ {
+ id: "chipmunk",
+ label: "Chipmunk",
+ category: "voice",
+ blurb: "Sharply higher. Formants move too, so it is a different voice.",
+ params: [
+ { key: "pitch", label: "Pitch", min: 3, max: 12, step: 0.5, default: 7, format: semitones },
+ ],
+ chain: ({ pitch = 7 }) =>
+ [...pitchShift(pitch), "highpass=f=120", "alimiter=limit=0.94"].join(","),
+ },
+ {
+ id: "robot",
+ label: "Robot",
+ category: "voice",
+ blurb: "Flattened and metallic, from a very short modulated delay.",
+ params: [
+ { key: "depth", label: "Depth", min: 1, max: 10, step: 0.5, default: 5, format: percent },
+ ],
+ chain: ({ depth = 5 }) =>
+ [
+ // aphaser caps speed at 2, so depth maps into 0.4-2.0 rather than
+ // being halved - which put it out of range and made the filter refuse.
+ `aphaser=type=t:speed=${(0.4 + (depth / 10) * 1.6).toFixed(2)}:decay=0.6`,
+ "flanger=delay=2:depth=4:speed=0.8",
+ "equalizer=f=1800:t=q:w=1.2:g=3",
+ "alimiter=limit=0.94",
+ ].join(","),
+ },
+
+ {
+ id: "bass",
+ label: "Bass boost",
+ category: "tone",
+ blurb: "Lifts the low end without touching anything above it.",
+ params: [
+ { key: "gain", label: "Gain", min: 0, max: 12, step: 0.5, default: 5, format: decibels },
+ ],
+ chain: ({ gain = 5 }) => `bass=g=${gain.toFixed(2)}:f=110:w=0.6,alimiter=limit=0.95`,
+ },
+ {
+ id: "treble",
+ label: "Treble boost",
+ category: "tone",
+ blurb: "Adds air at the top. Useful on dull recordings.",
+ params: [
+ { key: "gain", label: "Gain", min: 0, max: 12, step: 0.5, default: 4, format: decibels },
+ ],
+ chain: ({ gain = 4 }) => `treble=g=${gain.toFixed(2)}:f=9000:w=0.7,alimiter=limit=0.95`,
+ },
+ {
+ id: "telephone",
+ label: "Telephone",
+ category: "tone",
+ blurb: "Band-limited to a phone line, everything else discarded.",
+ params: [
+ { key: "drive", label: "Drive", min: 0, max: 10, step: 0.5, default: 3, format: percent },
+ ],
+ chain: ({ drive = 3 }) =>
+ [
+ "highpass=f=400",
+ "lowpass=f=3400",
+ `equalizer=f=1600:t=q:w=1.4:g=${(3 + drive / 2).toFixed(2)}`,
+ "alimiter=limit=0.92",
+ ].join(","),
+ },
+
+ {
+ id: "echo",
+ label: "Echo",
+ category: "space",
+ blurb: "A single distinct repeat.",
+ params: [
+ { key: "delay", label: "Delay", min: 0.05, max: 1, step: 0.01, default: 0.25, format: seconds },
+ { key: "decay", label: "Decay", min: 0.1, max: 0.9, step: 0.05, default: 0.4, format: percent },
+ ],
+ chain: ({ delay = 0.25, decay = 0.4 }) =>
+ `aecho=0.8:0.85:${Math.round(delay * 1000)}:${decay.toFixed(2)}`,
+ },
+ {
+ id: "room",
+ label: "Room",
+ category: "space",
+ blurb: "Several short reflections, so it sits in a space rather than repeating.",
+ params: [
+ { key: "size", label: "Size", min: 0, max: 100, step: 1, default: 40, format: percent },
+ ],
+ chain: ({ size = 40 }) => {
+ const scale = 0.4 + (size / 100) * 1.6;
+ const taps = [23, 41, 67, 97].map((ms) => Math.round(ms * scale));
+ const gains = [0.32, 0.24, 0.17, 0.11].map((g) => g.toFixed(3));
+ return `aecho=0.8:0.88:${taps.join("|")}:${gains.join("|")}`;
+ },
+ },
+];
+
+export function findFilter(id: string): FilterDefinition | null {
+ return FILTERS.find((filter) => filter.id === id) ?? null;
+}
+
+/** Fills in any parameter the clip did not set. */
+export function resolveParams(
+ definition: FilterDefinition,
+ params: Record<string, number>,
+): Record<string, number> {
+ const resolved: Record<string, number> = {};
+ for (const param of definition.params) {
+ resolved[param.key] = params[param.key] ?? param.default;
+ }
+ return resolved;
+}
+
+/**
+ * The complete FFmpeg filter string for a clip, or null if it has none.
+ *
+ * Filters apply in the order they were added, which is why `filters` is an
+ * array: EQ before a limiter is a different sound from the reverse.
+ */
+export function buildChain(filters: readonly ClipFilter[]): string | null {
+ const fragments = filters.flatMap((applied) => {
+ const definition = findFilter(applied.id);
+ if (!definition) return [];
+ return [definition.chain(resolveParams(definition, applied.params))];
+ });
+
+ return fragments.length > 0 ? fragments.join(",") : null;
+}
+
+/**
+ * A stable key for "this exact filtered audio".
+ *
+ * Used to cache rendered results, so dragging a slider back to a value you
+ * already heard plays instantly instead of re-rendering.
+ */
+export function chainKey(
+ mediaId: string,
+ sourceStart: number,
+ duration: number,
+ filters: readonly ClipFilter[],
+): string {
+ return `${mediaId}|${sourceStart.toFixed(3)}|${duration.toFixed(3)}|${buildChain(filters) ?? ""}`;
+}
diff --git a/desktop/src/lib/project.ts b/desktop/src/lib/project.ts
index 76efeffa..bacaa9f9 100644
--- a/desktop/src/lib/project.ts
+++ b/desktop/src/lib/project.ts
@@ -21,6 +21,7 @@
*/
import type { MediaSummary } from "./engine";
+import type { ClipFilter } from "./filters";
/**
* What a piece of media is.
@@ -95,6 +96,14 @@ export interface Clip {
fadeIn: number;
/** Fade-out length in seconds. Zero means no fade. */
fadeOut: number;
+
+ /**
+ * Audio filters, applied in order.
+ *
+ * An array rather than a set because order is audible: an EQ before a
+ * limiter is a different sound from the reverse.
+ */
+ filters: ClipFilter[];
}
export interface Project {
@@ -208,6 +217,7 @@ export function addClip(
volume: 1,
fadeIn: 0,
fadeOut: 0,
+ filters: [],
};
return { project: { ...project, clips: [...project.clips, clip] }, clipId: clip.id };
@@ -338,13 +348,14 @@ export function splitClip(project: Project, clipId: string, time: number): Proje
return project;
}
- const head: Clip = { ...clip, duration: offset };
+ const head: Clip = { ...clip, duration: offset, filters: [...clip.filters] };
const tail: Clip = {
...clip,
id: nextId("c"),
start: clip.start + offset,
duration: clip.duration - offset,
sourceStart: clip.sourceStart + offset,
+ filters: [...clip.filters],
};
return {