Open source CapCut alternative.
TypeScript
fix(site): a sound treatment is loud enough to hear [rebuild]
David Lea50555a7e101
diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.tsindex 1bea0df2..4732bbac 100644--- a/site/packages/effects-kit/src/audioFx.ts+++ b/site/packages/effects-kit/src/audioFx.ts@@ -95,9 +95,11 @@ export const AUDIO_FX_RAMP = 0.04;const clampAmount = (k: number | undefined) => Math.max(0.05, Math.min(1, k ?? 0.5));const fmt = (n: number) => (Math.round(n * 1000) / 1000).toString();-/** The delays a reverb's early reflections arrive at, in milliseconds. They- * are mutually prime so the copies never stack into a single flutter. */-const ROOM_TAPS = [23, 41, 59, 79, 101, 127, 151, 181];+/** The delays a reverb's reflections arrive at, in milliseconds. They are+ * mutually prime so the copies never stack into a single flutter, and they run+ * out past two thirds of a second: a handful of early reflections alone reads+ * as a thickening, and what makes a room audible is the tail behind them. */+const ROOM_TAPS = [29, 43, 67, 89, 113, 149, 191, 239, 307, 383, 467, 571, 683];/*** The recipe for one effect at strength `amount` (0.05..1).@@ -114,32 +116,37 @@ export function audioFxRecipe(effect: string, amount?: number): AudioFxRecipe |return {dry: 1,taps: [- { ms: 250, gain: 0.6 * k },- { ms: 500, gain: 0.36 * k },- { ms: 750, gain: 0.2 * k },+ { ms: 250, gain: 0.75 * k },+ { ms: 500, gain: 0.5 * k },+ { ms: 750, gain: 0.3 * k },],bands: [],};case "reverb":- // A room, built from its early reflections: dense taps inside a fifth of- // a second, decaying, with the top rolled off so the tail sits behind- // the voice rather than beside it.+ // A room, built from its reflections: dense taps decaying across three+ // quarters of a second, with the top rolled off so the tail sits behind+ // the voice. The decay is slow enough that the tail is still there after+ // the word that made it, which is the part that is heard as a room.return {dry: 1,- taps: ROOM_TAPS.map((ms, i) => ({ ms, gain: 0.5 * k * Math.exp(-0.45 * i) })),- bands: [{ type: "lowpass", hz: 6000 }],- gain: 0.9,+ taps: ROOM_TAPS.map((ms, i) => ({ ms, gain: 0.55 * k * Math.exp(-0.22 * i) })),+ bands: [{ type: "lowpass", hz: 5000 }],+ gain: 0.8,};case "muffle":- // Heard through a wall: two passes, so the top goes rather than dims.+ // Heard through a wall: two passes take the top off at twenty-four+ // decibels an octave, and a shelf pulls down what leaks past them, so+ // the consonants go and the body stays. Makeup gain keeps the window at+ // the level of the mix around it.return {dry: 1,taps: [],bands: [- { type: "lowpass", hz: 3200 - 2400 * k },- { type: "lowpass", hz: 3200 - 2400 * k },+ { type: "lowpass", hz: 2000 - 1500 * k },+ { type: "lowpass", hz: 2000 - 1500 * k },+ { type: "highshelf", hz: 3000, db: -(6 + 12 * k) },],- gain: 1 + 0.4 * k,+ gain: 1 + 0.8 * k,};case "telephone":// The band a phone line passes, with the middle pushed up so speech@@ -148,36 +155,55 @@ export function audioFxRecipe(effect: string, amount?: number): AudioFxRecipe |dry: 1,taps: [],bands: [- { type: "highpass", hz: 400 + 200 * k },- { type: "lowpass", hz: 3400 - 800 * k },- { type: "peaking", hz: 1800, q: 1.2, db: 6 * k },+ { type: "highpass", hz: 500 + 300 * k },+ { type: "highpass", hz: 500 + 300 * k },+ { type: "lowpass", hz: 3200 - 1000 * k },+ { type: "peaking", hz: 1800, q: 1.2, db: 4 + 6 * k },],- gain: 1.2,+ gain: 1.3,};case "deep":+ // Bigger and lower. The lift sits at 180 Hz, inside what a laptop+ // speaker and a phone can actually reproduce, and the top comes down+ // with it — a voice reads as deep by what it loses as much as by what it+ // gains, and the loss is what carries on small speakers.return {dry: 1,taps: [],- bands: [{ type: "lowshelf", hz: 110, db: 4 + 8 * k }],- gain: 0.85,+ bands: [+ { type: "lowshelf", hz: 180, db: 6 + 10 * k },+ { type: "highshelf", hz: 2600, db: -(5 + 9 * k) },+ ],+ gain: 0.9,};case "bright":+ // Thin and forward: the bottom comes off, a presence peak pushes the+ // consonants out, and the shelf above it opens the air.return {dry: 1,taps: [],- bands: [{ type: "highshelf", hz: 4000, db: 3 + 7 * k }],- gain: 0.9,+ bands: [+ { type: "lowshelf", hz: 220, db: -(4 + 8 * k) },+ { type: "peaking", hz: 4500, q: 0.9, db: 4 + 7 * k },+ { type: "highshelf", hz: 8000, db: 4 + 8 * k },+ ],+ gain: 0.95,};case "wobble":- return { dry: 1, taps: [], bands: [], tremolo: { hz: 5, depth: 0.85 * k } };+ // Six swings a second, and the shallowest of them still drops the level+ // by a third — a tremolo that only grazes the signal is heard as nothing+ // at all.+ return { dry: 1, taps: [], bands: [], tremolo: { hz: 6, depth: 0.35 + 0.65 * k } };case "crush":- // Down to four bits at the top of the slider, mixed in over the clean- // signal so the words survive the grit.+ // Down to three bits at the top of the slider, mixed in over the clean+ // signal so the words survive the grit. Eight bits and up is a change a+ // meter sees and an ear does not, so even the low end of the slider+ // quantizes hard enough to hear.return {dry: 1,taps: [],bands: [],- crush: { bits: Math.max(4, Math.round(12 - 8 * k)), mix: 0.4 + 0.6 * k },+ crush: { bits: Math.max(3, Math.round(7 - 4 * k)), mix: 0.5 + 0.5 * k },gain: 0.9,};default:@@ -412,20 +438,36 @@ export function audioFxBars(}bars = withTaps;}+ // The passes are read as a pair of edges rather than one at a time: two+ // sections at the same cutoff are one steeper filter, and blurring the+ // figure twice for them leaves a bar with no shape in it at all.+ let lowest = Infinity;+ let highest = 0;for (const b of recipe.bands) {- if (b.type === "lowpass") bars = smooth(bars, Math.min(4, 2400 / Math.max(200, b.hz)));- if (b.type === "highpass") {- const body = smooth(bars, 3);- bars = bars.map((v, i) => Math.max(0, v - body[i] * Math.min(0.9, b.hz / 1200)));+ if (b.type === "lowpass") {+ lowest = Math.min(lowest, b.hz);+ continue;}- if (b.type === "lowshelf" || b.type === "highshelf" || b.type === "peaking") {- const body = smooth(bars, 3);- const lift = (b.db ?? 0) / 24;- bars =- b.type === "highshelf"- ? bars.map((v, i) => v + Math.max(0, v - body[i]) * lift * 3)- : bars.map((v, i) => v + body[i] * lift);+ if (b.type === "highpass") {+ highest = Math.max(highest, b.hz);+ continue;}+ const body = smooth(bars, 3);+ const lift = (b.db ?? 0) / 24;+ bars =+ b.type === "highshelf"+ ? bars.map((v, i) => v + Math.max(0, v - body[i]) * lift * 3)+ : bars.map((v, i) => v + body[i] * lift);+ }+ if (lowest < Infinity) {+ // Rounded off and smaller: what the pass takes off the top is gone from+ // the figure the same way it is gone from the sound.+ const keep = Math.max(0.45, Math.min(1, lowest / 3500));+ bars = smooth(bars, Math.min(3, 2400 / Math.max(200, lowest))).map((v) => v * keep);+ }+ if (highest > 0) {+ const body = smooth(bars, 3);+ bars = bars.map((v, i) => Math.max(0, v - body[i] * Math.min(0.9, highest / 1200)));}if (recipe.tremolo) {const { hz, depth } = recipe.tremolo;@@ -442,8 +484,12 @@ export function audioFxBars(}// Louder than the tile is tall: the figure is scaled back to fit rather than// flattened against the top, so an effect that adds level keeps its shape.+ // A figure a filter has taken most of the level out of is scaled the other+ // way for the same reason — pressed against the floor it has no shape left+ // to read either.const peak = Math.max(...bars);if (peak > 1) bars = bars.map((v) => v / peak);+ else if (peak > 0.01 && peak < 0.55) bars = bars.map((v) => (v * 0.55) / peak);// A floor keeps every bar visible: the figure shows the shape of the effect,// and a bar at zero reads as a gap in the tile.return bars.map((v) => Math.max(0.1, Math.min(1, v)));diff --git a/site/packages/effects-kit/src/audioFxAudible.test.ts b/site/packages/effects-kit/src/audioFxAudible.test.tsnew file mode 100644index 00000000..c7fd5682--- /dev/null+++ b/site/packages/effects-kit/src/audioFxAudible.test.ts@@ -0,0 +1,187 @@+import { describe, expect, test } from "bun:test";+import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";+import { AUDIO_EFFECT_IDS, audioFxRecipe, buildAudioFx } from "./audioFx";++/**+ * What the recipes have to be: audible.+ *+ * An effect whose numbers are right and whose sound is a small change nobody+ * notices is a broken tile — it was placed, it renders, and it does nothing a+ * listener can hear. These render each effect at the strength a drop places+ * (no amount set) over a speech-shaped signal and hold it to a floor: the+ * output has to differ from the untreated signal by more than a level trim,+ * and each effect has to differ in its own direction.+ */++const globals = globalThis as Record<string, unknown>;+globals.AudioBuffer ??= AudioBuffer;+globals.OfflineAudioContext ??= OfflineAudioContext;++const RATE = 44100;+const SECONDS = 1.5;+/** The harmonics the test voice is built from, low to high. */+const HARMONICS = [150, 300, 600, 1200, 2400, 4800, 9000];++/** A voice-shaped signal: a harmonic series under a syllable envelope, with a+ * little noise on top so the filters have something to take off. */+function voice(): AudioBuffer {+ const buf = new AudioBuffer({+ length: Math.round(SECONDS * RATE),+ numberOfChannels: 1,+ sampleRate: RATE,+ });+ const ch = buf.getChannelData(0);+ let seed = 3;+ const noise = () => ((seed = (seed * 1664525 + 1013904223) % 4294967296) / 2147483648 - 1);+ for (let i = 0; i < ch.length; i++) {+ const t = i / RATE;+ const syllable = Math.max(0, Math.sin(Math.PI * ((t * 3) % 1)));+ let v = 0;+ let a = 1;+ for (const hz of HARMONICS) {+ v += a * Math.sin(2 * Math.PI * hz * t);+ a *= 0.62;+ }+ ch[i] = (v / 2.3 + noise() * 0.05) * syllable * 0.5;+ }+ return buf;+}++const SOURCE = voice();++async function render(effect: string | null): Promise<Float32Array> {+ const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);+ const node = ctx.createBufferSource();+ node.buffer = SOURCE;+ if (effect) {+ const chain = buildAudioFx(ctx, audioFxRecipe(effect)!, 0);+ node.connect(chain.input);+ chain.output.connect(ctx.destination);+ } else {+ node.connect(ctx.destination);+ }+ node.start();+ // Copied out: the rendering's channel data is a view the next render is free+ // to write over, and the untreated take has to survive the treated ones.+ return Float32Array.from((await ctx.startRendering()).getChannelData(0));+}++const rms = (d: Float32Array, from = 0, to = d.length) => {+ let sum = 0;+ for (let i = from; i < to; i++) sum += d[i] * d[i];+ return Math.sqrt(sum / Math.max(1, to - from));+};++/** The level at one frequency, by the Goertzel sum — an FFT bin without the+ * FFT, which is all a handful of harmonics needs. */+const levelAt = (d: Float32Array, hz: number) => {+ const w = (2 * Math.PI * hz) / RATE;+ const c = 2 * Math.cos(w);+ let s1 = 0;+ let s2 = 0;+ for (let i = 0; i < d.length; i++) {+ const s = d[i] + c * s1 - s2;+ s2 = s1;+ s1 = s;+ }+ return Math.sqrt(Math.abs(s1 * s1 + s2 * s2 - c * s1 * s2)) / d.length;+};++const dB = (x: number) => 20 * Math.log10(Math.max(1e-12, x));++/** How much of the treated signal is not the untreated one, in decibels,+ * after matching the levels — so an effect cannot pass on makeup gain alone. */+function change(wet: Float32Array, dry: Float32Array): number {+ const k = rms(dry) / Math.max(1e-9, rms(wet));+ let sum = 0;+ for (let i = 0; i < wet.length; i++) sum += (wet[i] * k - dry[i]) ** 2;+ return dB(Math.sqrt(sum / wet.length) / rms(dry));+}++/** The swing an amplitude modulation puts on the level, in decibels. */+function swing(d: Float32Array): number {+ const w = Math.round(RATE * 0.015);+ const env: number[] = [];+ for (let i = 0; i + w < d.length; i += w) env.push(rms(d, i, i + w));+ const peak = Math.max(...env);+ const loud = env.filter((v) => v > 0.02 * peak).sort((a, b) => a - b);+ if (loud.length < 8) return 0;+ return dB(loud[loud.length - 1]) - dB(loud[Math.floor(loud.length * 0.15)]);+}++describe("every audio effect is audible at the strength a drop places", () => {+ test("each one changes the sound by more than a level trim", async () => {+ const dry = await render(null);+ for (const id of AUDIO_EFFECT_IDS) {+ const wet = await render(id);+ // −26 dB is around where a change stops being a detail and starts being+ // the point; every effect clears it with room to spare.+ expect({ id, change: change(wet, dry) > -26 }).toEqual({ id, change: true });+ }+ });++ test("each one moves the sound in the direction its name promises", async () => {+ const dry = await render(null);+ const band = async (id: string) => {+ const wet = await render(id);+ const at = (hz: number) => dB(levelAt(wet, hz)) - dB(levelAt(dry, hz));+ return { low: at(150), mid: at(1200), high: at(4800) };+ };++ // Muffle and telephone take the top off; the phone takes the bottom too.+ const muffle = await band("muffle");+ expect(muffle.high).toBeLessThan(-20);+ expect(muffle.low).toBeGreaterThan(0);+ const telephone = await band("telephone");+ expect(telephone.low).toBeLessThan(-20);+ expect(telephone.mid).toBeGreaterThan(3);++ // Deep leans into the body and loses the detail; bright does the reverse.+ const deep = await band("deep");+ expect(deep.low).toBeGreaterThan(4);+ expect(deep.high).toBeLessThan(-4);+ const bright = await band("bright");+ expect(bright.high).toBeGreaterThan(4);+ expect(bright.low).toBeLessThan(-4);++ // A wobble that only grazes the level is heard as nothing at all.+ expect(swing(await render("wobble")) - swing(dry)).toBeGreaterThan(4);+ });++ test("the rooms are still sounding after the sound that made them stops", async () => {+ // A burst, then silence: a reverb tail and an echo repeat both live in+ // the silence, which is the part a spectrum of the whole take cannot show.+ const burst = new AudioBuffer({+ length: Math.round(SECONDS * RATE),+ numberOfChannels: 1,+ sampleRate: RATE,+ });+ const ch = burst.getChannelData(0);+ for (let i = 0; i < 0.3 * RATE; i++) {+ const t = i / RATE;+ let v = 0;+ let a = 1;+ for (const hz of [180, 360, 720, 1440, 2880]) {+ v += a * Math.sin(2 * Math.PI * hz * t);+ a *= 0.6;+ }+ ch[i] = (v / 2.2) * 0.5 * Math.min(1, (0.3 - t) * 20);+ }+ const tail = async (id: string) => {+ const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);+ const node = ctx.createBufferSource();+ node.buffer = burst;+ const chain = buildAudioFx(ctx, audioFxRecipe(id)!, 0);+ node.connect(chain.input);+ chain.output.connect(ctx.destination);+ node.start();+ const out = Float32Array.from((await ctx.startRendering()).getChannelData(0));+ const body = rms(out, 0, Math.round(0.3 * RATE));+ return dB(rms(out, Math.round(0.35 * RATE), Math.round(0.9 * RATE)) / body);+ };+ // Half a second after the burst, a room is still ringing well above the+ // floor, and an echo is repeating louder still.+ expect(await tail("reverb")).toBeGreaterThan(-30);+ expect(await tail("echo")).toBeGreaterThan(-18);+ });+});diff --git a/site/src/cut/lib/audioMixFx.test.ts b/site/src/cut/lib/audioMixFx.test.tsindex 9651a7be..7edb75df 100644--- a/site/src/cut/lib/audioMixFx.test.ts+++ b/site/src/cut/lib/audioMixFx.test.ts@@ -49,7 +49,9 @@ const fold = async (effects: { effect: string; amount?: number; start: number; e{ sampleRate: RATE, channels: 1, resolve: (f) => f });expect(buffer).not.toBeNull();- return buffer!.getChannelData(0);+ // Copied out: the fold's channel data is a view the next fold may write+ // over, and a comparison between two folds has to hold both.+ return Float32Array.from(buffer!.getChannelData(0));};describe("audio effects in the offline fold", () => {