Skip to content

Open source CapCut alternative.

TypeScript

fix(site): a sound treatment is loud enough to hear [rebuild]

David Lea50555a7e101
diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.ts
index 1bea0df2..4732bbac 100644
--- a/site/packages/effects-kit/src/audioFx.ts
+++ b/site/packages/effects-kit/src/audioFx.ts
@@ -95,9 +95,11 @@ export const AUDIO_FX_RAMP = 0.04;
const clampAmount = (k: number | undefined) => Math.max(0.05, Math.min(1, k ?? 0.5));
const fmt = (n: number) => (Math.round(n * 1000) / 1000).toString();
-/** The delays a reverb's early reflections arrive at, in milliseconds. They
- * are mutually prime so the copies never stack into a single flutter. */
-const ROOM_TAPS = [23, 41, 59, 79, 101, 127, 151, 181];
+/** The delays a reverb's reflections arrive at, in milliseconds. They are
+ * mutually prime so the copies never stack into a single flutter, and they run
+ * out past two thirds of a second: a handful of early reflections alone reads
+ * as a thickening, and what makes a room audible is the tail behind them. */
+const ROOM_TAPS = [29, 43, 67, 89, 113, 149, 191, 239, 307, 383, 467, 571, 683];
/**
* The recipe for one effect at strength `amount` (0.05..1).
@@ -114,32 +116,37 @@ export function audioFxRecipe(effect: string, amount?: number): AudioFxRecipe |
return {
dry: 1,
taps: [
- { ms: 250, gain: 0.6 * k },
- { ms: 500, gain: 0.36 * k },
- { ms: 750, gain: 0.2 * k },
+ { ms: 250, gain: 0.75 * k },
+ { ms: 500, gain: 0.5 * k },
+ { ms: 750, gain: 0.3 * k },
],
bands: [],
};
case "reverb":
- // A room, built from its early reflections: dense taps inside a fifth of
- // a second, decaying, with the top rolled off so the tail sits behind
- // the voice rather than beside it.
+ // A room, built from its reflections: dense taps decaying across three
+ // quarters of a second, with the top rolled off so the tail sits behind
+ // the voice. The decay is slow enough that the tail is still there after
+ // the word that made it, which is the part that is heard as a room.
return {
dry: 1,
- taps: ROOM_TAPS.map((ms, i) => ({ ms, gain: 0.5 * k * Math.exp(-0.45 * i) })),
- bands: [{ type: "lowpass", hz: 6000 }],
- gain: 0.9,
+ taps: ROOM_TAPS.map((ms, i) => ({ ms, gain: 0.55 * k * Math.exp(-0.22 * i) })),
+ bands: [{ type: "lowpass", hz: 5000 }],
+ gain: 0.8,
};
case "muffle":
- // Heard through a wall: two passes, so the top goes rather than dims.
+ // Heard through a wall: two passes take the top off at twenty-four
+ // decibels an octave, and a shelf pulls down what leaks past them, so
+ // the consonants go and the body stays. Makeup gain keeps the window at
+ // the level of the mix around it.
return {
dry: 1,
taps: [],
bands: [
- { type: "lowpass", hz: 3200 - 2400 * k },
- { type: "lowpass", hz: 3200 - 2400 * k },
+ { type: "lowpass", hz: 2000 - 1500 * k },
+ { type: "lowpass", hz: 2000 - 1500 * k },
+ { type: "highshelf", hz: 3000, db: -(6 + 12 * k) },
],
- gain: 1 + 0.4 * k,
+ gain: 1 + 0.8 * k,
};
case "telephone":
// The band a phone line passes, with the middle pushed up so speech
@@ -148,36 +155,55 @@ export function audioFxRecipe(effect: string, amount?: number): AudioFxRecipe |
dry: 1,
taps: [],
bands: [
- { type: "highpass", hz: 400 + 200 * k },
- { type: "lowpass", hz: 3400 - 800 * k },
- { type: "peaking", hz: 1800, q: 1.2, db: 6 * k },
+ { type: "highpass", hz: 500 + 300 * k },
+ { type: "highpass", hz: 500 + 300 * k },
+ { type: "lowpass", hz: 3200 - 1000 * k },
+ { type: "peaking", hz: 1800, q: 1.2, db: 4 + 6 * k },
],
- gain: 1.2,
+ gain: 1.3,
};
case "deep":
+ // Bigger and lower. The lift sits at 180 Hz, inside what a laptop
+ // speaker and a phone can actually reproduce, and the top comes down
+ // with it — a voice reads as deep by what it loses as much as by what it
+ // gains, and the loss is what carries on small speakers.
return {
dry: 1,
taps: [],
- bands: [{ type: "lowshelf", hz: 110, db: 4 + 8 * k }],
- gain: 0.85,
+ bands: [
+ { type: "lowshelf", hz: 180, db: 6 + 10 * k },
+ { type: "highshelf", hz: 2600, db: -(5 + 9 * k) },
+ ],
+ gain: 0.9,
};
case "bright":
+ // Thin and forward: the bottom comes off, a presence peak pushes the
+ // consonants out, and the shelf above it opens the air.
return {
dry: 1,
taps: [],
- bands: [{ type: "highshelf", hz: 4000, db: 3 + 7 * k }],
- gain: 0.9,
+ bands: [
+ { type: "lowshelf", hz: 220, db: -(4 + 8 * k) },
+ { type: "peaking", hz: 4500, q: 0.9, db: 4 + 7 * k },
+ { type: "highshelf", hz: 8000, db: 4 + 8 * k },
+ ],
+ gain: 0.95,
};
case "wobble":
- return { dry: 1, taps: [], bands: [], tremolo: { hz: 5, depth: 0.85 * k } };
+ // Six swings a second, and the shallowest of them still drops the level
+ // by a third — a tremolo that only grazes the signal is heard as nothing
+ // at all.
+ return { dry: 1, taps: [], bands: [], tremolo: { hz: 6, depth: 0.35 + 0.65 * k } };
case "crush":
- // Down to four bits at the top of the slider, mixed in over the clean
- // signal so the words survive the grit.
+ // Down to three bits at the top of the slider, mixed in over the clean
+ // signal so the words survive the grit. Eight bits and up is a change a
+ // meter sees and an ear does not, so even the low end of the slider
+ // quantizes hard enough to hear.
return {
dry: 1,
taps: [],
bands: [],
- crush: { bits: Math.max(4, Math.round(12 - 8 * k)), mix: 0.4 + 0.6 * k },
+ crush: { bits: Math.max(3, Math.round(7 - 4 * k)), mix: 0.5 + 0.5 * k },
gain: 0.9,
};
default:
@@ -412,20 +438,36 @@ export function audioFxBars(
}
bars = withTaps;
}
+ // The passes are read as a pair of edges rather than one at a time: two
+ // sections at the same cutoff are one steeper filter, and blurring the
+ // figure twice for them leaves a bar with no shape in it at all.
+ let lowest = Infinity;
+ let highest = 0;
for (const b of recipe.bands) {
- if (b.type === "lowpass") bars = smooth(bars, Math.min(4, 2400 / Math.max(200, b.hz)));
- if (b.type === "highpass") {
- const body = smooth(bars, 3);
- bars = bars.map((v, i) => Math.max(0, v - body[i] * Math.min(0.9, b.hz / 1200)));
+ if (b.type === "lowpass") {
+ lowest = Math.min(lowest, b.hz);
+ continue;
}
- if (b.type === "lowshelf" || b.type === "highshelf" || b.type === "peaking") {
- const body = smooth(bars, 3);
- const lift = (b.db ?? 0) / 24;
- bars =
- b.type === "highshelf"
- ? bars.map((v, i) => v + Math.max(0, v - body[i]) * lift * 3)
- : bars.map((v, i) => v + body[i] * lift);
+ if (b.type === "highpass") {
+ highest = Math.max(highest, b.hz);
+ continue;
}
+ const body = smooth(bars, 3);
+ const lift = (b.db ?? 0) / 24;
+ bars =
+ b.type === "highshelf"
+ ? bars.map((v, i) => v + Math.max(0, v - body[i]) * lift * 3)
+ : bars.map((v, i) => v + body[i] * lift);
+ }
+ if (lowest < Infinity) {
+ // Rounded off and smaller: what the pass takes off the top is gone from
+ // the figure the same way it is gone from the sound.
+ const keep = Math.max(0.45, Math.min(1, lowest / 3500));
+ bars = smooth(bars, Math.min(3, 2400 / Math.max(200, lowest))).map((v) => v * keep);
+ }
+ if (highest > 0) {
+ const body = smooth(bars, 3);
+ bars = bars.map((v, i) => Math.max(0, v - body[i] * Math.min(0.9, highest / 1200)));
}
if (recipe.tremolo) {
const { hz, depth } = recipe.tremolo;
@@ -442,8 +484,12 @@ export function audioFxBars(
}
// Louder than the tile is tall: the figure is scaled back to fit rather than
// flattened against the top, so an effect that adds level keeps its shape.
+ // A figure a filter has taken most of the level out of is scaled the other
+ // way for the same reason — pressed against the floor it has no shape left
+ // to read either.
const peak = Math.max(...bars);
if (peak > 1) bars = bars.map((v) => v / peak);
+ else if (peak > 0.01 && peak < 0.55) bars = bars.map((v) => (v * 0.55) / peak);
// A floor keeps every bar visible: the figure shows the shape of the effect,
// and a bar at zero reads as a gap in the tile.
return bars.map((v) => Math.max(0.1, Math.min(1, v)));
diff --git a/site/packages/effects-kit/src/audioFxAudible.test.ts b/site/packages/effects-kit/src/audioFxAudible.test.ts
new file mode 100644
index 00000000..c7fd5682
--- /dev/null
+++ b/site/packages/effects-kit/src/audioFxAudible.test.ts
@@ -0,0 +1,187 @@
+import { describe, expect, test } from "bun:test";
+import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";
+import { AUDIO_EFFECT_IDS, audioFxRecipe, buildAudioFx } from "./audioFx";
+
+/**
+ * What the recipes have to be: audible.
+ *
+ * An effect whose numbers are right and whose sound is a small change nobody
+ * notices is a broken tile — it was placed, it renders, and it does nothing a
+ * listener can hear. These render each effect at the strength a drop places
+ * (no amount set) over a speech-shaped signal and hold it to a floor: the
+ * output has to differ from the untreated signal by more than a level trim,
+ * and each effect has to differ in its own direction.
+ */
+
+const globals = globalThis as Record<string, unknown>;
+globals.AudioBuffer ??= AudioBuffer;
+globals.OfflineAudioContext ??= OfflineAudioContext;
+
+const RATE = 44100;
+const SECONDS = 1.5;
+/** The harmonics the test voice is built from, low to high. */
+const HARMONICS = [150, 300, 600, 1200, 2400, 4800, 9000];
+
+/** A voice-shaped signal: a harmonic series under a syllable envelope, with a
+ * little noise on top so the filters have something to take off. */
+function voice(): AudioBuffer {
+ const buf = new AudioBuffer({
+ length: Math.round(SECONDS * RATE),
+ numberOfChannels: 1,
+ sampleRate: RATE,
+ });
+ const ch = buf.getChannelData(0);
+ let seed = 3;
+ const noise = () => ((seed = (seed * 1664525 + 1013904223) % 4294967296) / 2147483648 - 1);
+ for (let i = 0; i < ch.length; i++) {
+ const t = i / RATE;
+ const syllable = Math.max(0, Math.sin(Math.PI * ((t * 3) % 1)));
+ let v = 0;
+ let a = 1;
+ for (const hz of HARMONICS) {
+ v += a * Math.sin(2 * Math.PI * hz * t);
+ a *= 0.62;
+ }
+ ch[i] = (v / 2.3 + noise() * 0.05) * syllable * 0.5;
+ }
+ return buf;
+}
+
+const SOURCE = voice();
+
+async function render(effect: string | null): Promise<Float32Array> {
+ const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);
+ const node = ctx.createBufferSource();
+ node.buffer = SOURCE;
+ if (effect) {
+ const chain = buildAudioFx(ctx, audioFxRecipe(effect)!, 0);
+ node.connect(chain.input);
+ chain.output.connect(ctx.destination);
+ } else {
+ node.connect(ctx.destination);
+ }
+ node.start();
+ // Copied out: the rendering's channel data is a view the next render is free
+ // to write over, and the untreated take has to survive the treated ones.
+ return Float32Array.from((await ctx.startRendering()).getChannelData(0));
+}
+
+const rms = (d: Float32Array, from = 0, to = d.length) => {
+ let sum = 0;
+ for (let i = from; i < to; i++) sum += d[i] * d[i];
+ return Math.sqrt(sum / Math.max(1, to - from));
+};
+
+/** The level at one frequency, by the Goertzel sum — an FFT bin without the
+ * FFT, which is all a handful of harmonics needs. */
+const levelAt = (d: Float32Array, hz: number) => {
+ const w = (2 * Math.PI * hz) / RATE;
+ const c = 2 * Math.cos(w);
+ let s1 = 0;
+ let s2 = 0;
+ for (let i = 0; i < d.length; i++) {
+ const s = d[i] + c * s1 - s2;
+ s2 = s1;
+ s1 = s;
+ }
+ return Math.sqrt(Math.abs(s1 * s1 + s2 * s2 - c * s1 * s2)) / d.length;
+};
+
+const dB = (x: number) => 20 * Math.log10(Math.max(1e-12, x));
+
+/** How much of the treated signal is not the untreated one, in decibels,
+ * after matching the levels — so an effect cannot pass on makeup gain alone. */
+function change(wet: Float32Array, dry: Float32Array): number {
+ const k = rms(dry) / Math.max(1e-9, rms(wet));
+ let sum = 0;
+ for (let i = 0; i < wet.length; i++) sum += (wet[i] * k - dry[i]) ** 2;
+ return dB(Math.sqrt(sum / wet.length) / rms(dry));
+}
+
+/** The swing an amplitude modulation puts on the level, in decibels. */
+function swing(d: Float32Array): number {
+ const w = Math.round(RATE * 0.015);
+ const env: number[] = [];
+ for (let i = 0; i + w < d.length; i += w) env.push(rms(d, i, i + w));
+ const peak = Math.max(...env);
+ const loud = env.filter((v) => v > 0.02 * peak).sort((a, b) => a - b);
+ if (loud.length < 8) return 0;
+ return dB(loud[loud.length - 1]) - dB(loud[Math.floor(loud.length * 0.15)]);
+}
+
+describe("every audio effect is audible at the strength a drop places", () => {
+ test("each one changes the sound by more than a level trim", async () => {
+ const dry = await render(null);
+ for (const id of AUDIO_EFFECT_IDS) {
+ const wet = await render(id);
+ // −26 dB is around where a change stops being a detail and starts being
+ // the point; every effect clears it with room to spare.
+ expect({ id, change: change(wet, dry) > -26 }).toEqual({ id, change: true });
+ }
+ });
+
+ test("each one moves the sound in the direction its name promises", async () => {
+ const dry = await render(null);
+ const band = async (id: string) => {
+ const wet = await render(id);
+ const at = (hz: number) => dB(levelAt(wet, hz)) - dB(levelAt(dry, hz));
+ return { low: at(150), mid: at(1200), high: at(4800) };
+ };
+
+ // Muffle and telephone take the top off; the phone takes the bottom too.
+ const muffle = await band("muffle");
+ expect(muffle.high).toBeLessThan(-20);
+ expect(muffle.low).toBeGreaterThan(0);
+ const telephone = await band("telephone");
+ expect(telephone.low).toBeLessThan(-20);
+ expect(telephone.mid).toBeGreaterThan(3);
+
+ // Deep leans into the body and loses the detail; bright does the reverse.
+ const deep = await band("deep");
+ expect(deep.low).toBeGreaterThan(4);
+ expect(deep.high).toBeLessThan(-4);
+ const bright = await band("bright");
+ expect(bright.high).toBeGreaterThan(4);
+ expect(bright.low).toBeLessThan(-4);
+
+ // A wobble that only grazes the level is heard as nothing at all.
+ expect(swing(await render("wobble")) - swing(dry)).toBeGreaterThan(4);
+ });
+
+ test("the rooms are still sounding after the sound that made them stops", async () => {
+ // A burst, then silence: a reverb tail and an echo repeat both live in
+ // the silence, which is the part a spectrum of the whole take cannot show.
+ const burst = new AudioBuffer({
+ length: Math.round(SECONDS * RATE),
+ numberOfChannels: 1,
+ sampleRate: RATE,
+ });
+ const ch = burst.getChannelData(0);
+ for (let i = 0; i < 0.3 * RATE; i++) {
+ const t = i / RATE;
+ let v = 0;
+ let a = 1;
+ for (const hz of [180, 360, 720, 1440, 2880]) {
+ v += a * Math.sin(2 * Math.PI * hz * t);
+ a *= 0.6;
+ }
+ ch[i] = (v / 2.2) * 0.5 * Math.min(1, (0.3 - t) * 20);
+ }
+ const tail = async (id: string) => {
+ const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);
+ const node = ctx.createBufferSource();
+ node.buffer = burst;
+ const chain = buildAudioFx(ctx, audioFxRecipe(id)!, 0);
+ node.connect(chain.input);
+ chain.output.connect(ctx.destination);
+ node.start();
+ const out = Float32Array.from((await ctx.startRendering()).getChannelData(0));
+ const body = rms(out, 0, Math.round(0.3 * RATE));
+ return dB(rms(out, Math.round(0.35 * RATE), Math.round(0.9 * RATE)) / body);
+ };
+ // Half a second after the burst, a room is still ringing well above the
+ // floor, and an echo is repeating louder still.
+ expect(await tail("reverb")).toBeGreaterThan(-30);
+ expect(await tail("echo")).toBeGreaterThan(-18);
+ });
+});
diff --git a/site/src/cut/lib/audioMixFx.test.ts b/site/src/cut/lib/audioMixFx.test.ts
index 9651a7be..7edb75df 100644
--- a/site/src/cut/lib/audioMixFx.test.ts
+++ b/site/src/cut/lib/audioMixFx.test.ts
@@ -49,7 +49,9 @@ const fold = async (effects: { effect: string; amount?: number; start: number; e
{ sampleRate: RATE, channels: 1, resolve: (f) => f }
);
expect(buffer).not.toBeNull();
- return buffer!.getChannelData(0);
+ // Copied out: the fold's channel data is a view the next fold may write
+ // over, and a comparison between two folds has to hold both.
+ return Float32Array.from(buffer!.getChannelData(0));
};
describe("audio effects in the offline fold", () => {