Skip to content

The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.

Studio: another speed is heard at once, its pitch kept by the browser (#146)

riverisbuilding29346ca43b42
diff --git a/.changeset/speed-native-pitch.md b/.changeset/speed-native-pitch.md
new file mode 100644
index 0000000..1cb50e3
--- /dev/null
+++ b/.changeset/speed-native-pitch.md
@@ -0,0 +1,7 @@
+---
+"openfilm": patch
+---
+
+Sound played faster or slower is heard the moment the speed changes, and sounds natural. The browser plays it at that speed with its pitch kept, in the same mix, kept in step with the picture; nothing is made on the server first. 0.2.3 re-encoded the whole sound for each speed, which left long videos silent for seconds after a change and made voices sound hollow.
+
+The viewer's playback button shows a speed gauge instead of the inspector's icon.
diff --git a/packages/openfilm/src/shared.d.mts b/packages/openfilm/src/shared.d.mts
index e32718f..416d63b 100644
--- a/packages/openfilm/src/shared.d.mts
+++ b/packages/openfilm/src/shared.d.mts
@@ -3,8 +3,6 @@ export interface MixFade { in: number; out: number; length: number; skip: number
export interface MixClip { file: string; at: number; from: number; to?: number; volume: number; speed?: number; fade?: MixFade }
export declare function audioClips(root: string, entry: string, clips: Array<{ src: string; at?: number; from?: number; to?: number; volume?: number; speed?: number; fade?: readonly [number, number] }> | undefined, from?: number): MixClip[];
export declare function fadeFilter(fade: MixFade | undefined): string;
-/** The atempo filter chain for a playback speed, the pitch kept (each stage takes 0.5–2); empty at 1. */
-export declare function tempo(speed: number): string;
export declare function mix(clips: MixClip[], duration: number, file: string, codec?: string[]): Promise<void>;
/** A mix's inputs and filter graph: a file used by several clips is one input, split. */
export declare function mixGraph(clips: MixClip[], duration: number): { inputs: string[]; filter: string };
diff --git a/packages/openfilm/src/shared.mjs b/packages/openfilm/src/shared.mjs
index d99e0d3..19a65e5 100644
--- a/packages/openfilm/src/shared.mjs
+++ b/packages/openfilm/src/shared.mjs
@@ -227,7 +227,7 @@ export function hasAudio(file) {
}
/** atempo chain for a playback rate (pitch kept); each stage takes 0.5–2. */
-export function tempo(speed) {
+function tempo(speed) {
const out = [];
let s = speed;
while (s > 2) { out.push('atempo=2'); s /= 2; }
diff --git a/packages/openfilm/studio/server/derived.mjs b/packages/openfilm/studio/server/derived.mjs
index 2d529ab..f58ea08 100644
--- a/packages/openfilm/studio/server/derived.mjs
+++ b/packages/openfilm/studio/server/derived.mjs
@@ -12,7 +12,6 @@ import { createHash } from 'node:crypto';
import { existsSync } from 'node:fs';
import { mkdir, readFile, rename, rm, stat } from 'node:fs/promises';
import { join } from 'node:path';
-import { tempo } from '../../src/shared.mjs';
import { readable } from './files.mjs';
import { TOOL_DIR } from './projects.mjs';
import { writeAtomic } from './atomic.mjs';
@@ -155,24 +154,18 @@ const COMPACT = /\.(mp3|m4a|aac|ogg|oga|opus)$/i;
const COMPACT_BYTES = 16 << 20;
/** @type {Map<string, Promise<string | null>>} copies being made, by where they go */
const makingSound = new Map();
-/** The speeds a sound is made at: a clip's (SPEED_MIN to SPEED_MAX) by the viewer's (0.25× to 2×), and some room. */
-export const SOUND_SPEED = { min: 1 / 16, max: 16 };
/**
* The sound of a media file as the editor plays it, small: a compressed sound file of up to COMPACT_BYTES is its
* own; any other (a video, a WAV, a long recording) gets a copy of its sound in `.film/cache/audio/`, AAC in .m4a:
- * the file's own AAC as it is, any other sound encoded at 128 kb/s, stereo at most. At another `speed` it is stretched
- * to it with its pitch kept, by the same atempo filter exports use, so the editor plays it as the export has it (a
- * sound sped up as a tape is would be heard higher). Made once per version of the file and speed; returns the path to
- * play, or null when there is no sound in it. Exports mix from the files themselves.
- * @param {string} root @param {string} rel @param {number} [speed]
+ * the file's own AAC as it is, any other sound encoded at 128 kb/s, stereo at most. Made once per version of the
+ * file; returns the path to play, or null when there is no sound in it. Exports mix from the files themselves.
+ * @param {string} root @param {string} rel
*/
-export async function soundOf(root, rel, speed = 1) {
- if (!(speed >= SOUND_SPEED.min && speed <= SOUND_SPEED.max)) throw new MediaError('speed: from 1/16 to 16', 400);
- const at = Math.round(speed * 1e4) / 1e4;
+export async function soundOf(root, rel) {
const { abs, key } = await source(root, rel);
- if (at === 1 && COMPACT.test(rel) && (await stat(abs)).size <= COMPACT_BYTES) return abs;
- const out = join(root, TOOL_DIR, 'cache', 'audio', at === 1 ? `${key}.m4a` : `${key}-x${at}.m4a`);
+ if (COMPACT.test(rel) && (await stat(abs)).size <= COMPACT_BYTES) return abs;
+ const out = join(root, TOOL_DIR, 'cache', 'audio', `${key}.m4a`);
if (existsSync(out)) return out;
let made = makingSound.get(out);
if (!made) {
@@ -182,9 +175,7 @@ export async function soundOf(root, rel, speed = 1) {
const stream = info.streams?.[0] ?? {};
const channels = Number(stream.channels) || 2;
/* AAC is what a browser decodes everywhere: kept as it is when it is that already (no loss, no time) */
- const codec = at === 1 && stream.codec_name === 'aac' && channels <= 2
- ? ['-c:a', 'copy']
- : [...(at === 1 ? [] : ['-af', tempo(at)]), '-c:a', 'aac', '-b:a', '128k', '-ac', String(Math.min(2, channels))];
+ const codec = stream.codec_name === 'aac' && channels <= 2 ? ['-c:a', 'copy'] : ['-c:a', 'aac', '-b:a', '128k', '-ac', String(Math.min(2, channels))];
await mkdir(join(root, TOOL_DIR, 'cache', 'audio'), { recursive: true });
const part = `${out}.part`;
try {
diff --git a/packages/openfilm/studio/server/derived.test.mjs b/packages/openfilm/studio/server/derived.test.mjs
index 4c0f28e..9cda345 100644
--- a/packages/openfilm/studio/server/derived.test.mjs
+++ b/packages/openfilm/studio/server/derived.test.mjs
@@ -125,29 +125,3 @@ test('a file\'s sound for the editor is small and in step: a video\'s AAC kept a
assert.deepEqual(readdirSync(join(root, '.film', 'cache', 'audio')).sort(), made, 'asked again: the copy made');
await assert.rejects(soundOf(root, 'assets/none.mp4'), (e) => e instanceof MediaError && e.status === 404);
});
-
-/** A tone's frequency (Hz) and length (s), from its zero crossings. @param {string} file */
-function toneOf(file) {
- const pcm = execFileSync('ffmpeg', ['-v', 'error', '-i', file, '-ac', '1', '-ar', '48000', '-f', 's16le', '-'], { maxBuffer: 1 << 28 });
- const samples = new Int16Array(pcm.buffer, pcm.byteOffset, Math.floor(pcm.length / 2));
- let ups = 0;
- for (let i = 1; i < samples.length; i++) if (samples[i - 1] < 0 && samples[i] >= 0) ups++;
- const seconds = samples.length / 48000;
- return { hz: ups / seconds, seconds };
-}
-
-test('a sound played at another speed is stretched to it with its pitch kept, once per speed', { skip: !hasFfmpeg && 'no ffmpeg' }, async () => {
- execFileSync('ffmpeg', ['-v', 'error', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=6', '-b:a', '128k', join(root, 'assets', 'tone.mp3')]);
- const own = String(await soundOf(root, 'assets/tone.mp3'));
- assert.equal(own, join(root, 'assets', 'tone.mp3'), 'at 1× a small sound is played as it is');
- for (const speed of [1.5, 0.5, 4]) {
- const copy = String(await soundOf(root, 'assets/tone.mp3', speed));
- assert.match(copy, /-x[\d.]+\.m4a$/, `${speed}×: a copy of its own`);
- const tone = toneOf(copy);
- assert.ok(Math.abs(tone.hz - 440) < 440 * 0.03, `${speed}×: still 440 Hz, not ${Math.round(tone.hz)}`);
- assert.ok(Math.abs(tone.seconds - 6 / speed) < 0.1, `${speed}×: ${6 / speed} s long, not ${tone.seconds}`);
- assert.equal(String(await soundOf(root, 'assets/tone.mp3', speed)), copy, 'asked again: the copy made');
- }
- await assert.rejects(soundOf(root, 'assets/tone.mp3', 0), (e) => e instanceof MediaError && e.status === 400);
- await assert.rejects(soundOf(root, 'assets/tone.mp3', 32), (e) => e instanceof MediaError && e.status === 400);
-});
diff --git a/packages/openfilm/studio/server/pages.mjs b/packages/openfilm/studio/server/pages.mjs
index 23b161d..9b1013a 100644
--- a/packages/openfilm/studio/server/pages.mjs
+++ b/packages/openfilm/studio/server/pages.mjs
@@ -16,8 +16,7 @@
* Pages get the host flag (the film's clock, see src/host.mjs) before any of their scripts, as `look` and `render`
* give it, so a film draws here exactly as it renders. Everything may be read by the editor's origin (its sound mixer
* fetches the media), which frames the pages. A media file asked for with `?film-sound` is its sound,
- * small (derived.mjs soundOf): what the mixer plays, never the whole of a video; 204 when it has none. With
- * `&speed=…` it is stretched to that speed, its pitch kept.
+ * small (derived.mjs soundOf): what the mixer plays, never the whole of a video; 204 when it has none.
*/
import { execFile } from 'node:child_process';
import { createHash, createHmac } from 'node:crypto';
@@ -32,7 +31,7 @@ import { readPrefs } from './prefs.mjs';
import { readyProxy } from './proxies.mjs';
import { madeCopy } from '../../src/seekable.mjs';
import { isPreviewKey, previewDir } from './preview-cache.mjs';
-import { SOUND_SPEED, soundOf } from './derived.mjs';
+import { soundOf } from './derived.mjs';
const BRIDGE = readFileSync(new URL('./bridge.js', import.meta.url), 'utf8');
@@ -179,10 +178,7 @@ export async function startPages({ port, editorOrigin, secret, projects }) {
if (!servable(entry.root, plain)) { res.writeHead(404, headers).end('not found'); return; }
if (url.searchParams.has('film-sound')) {
/* waited for while it is made (a few seconds for a long recording): the mixer plays the film without it meanwhile */
- /* `speed`: stretched to it, its pitch kept (the clip's speed by the viewer's) */
- const speed = Number(url.searchParams.get('speed') ?? 1);
- if (!(speed >= SOUND_SPEED.min && speed <= SOUND_SPEED.max)) { res.writeHead(400, headers).end('speed: from 1/16 to 16'); return; }
- const file = await soundOf(entry.root, plain.replace(/^\/+/, ''), speed).catch((/** @type {any} */ e) => (e?.status === 404 ? undefined : null));
+ const file = await soundOf(entry.root, plain.replace(/^\/+/, '')).catch((/** @type {any} */ e) => (e?.status === 404 ? undefined : null));
if (file === undefined) { res.writeHead(404, headers).end('not found'); return; }
if (!file) { res.writeHead(204, headers).end(); return; }
if (!folderFiles(dirname(file), { headers: policy() })(req, res, `/${basename(file)}`, url.searchParams)) res.writeHead(403, headers).end();
diff --git a/packages/openfilm/studio/ui/src/components/PlaybackMenu.tsx b/packages/openfilm/studio/ui/src/components/PlaybackMenu.tsx
index 9724cc0..97e5756 100644
--- a/packages/openfilm/studio/ui/src/components/PlaybackMenu.tsx
+++ b/packages/openfilm/studio/ui/src/components/PlaybackMenu.tsx
@@ -1,6 +1,6 @@
import React from 'react';
import { createPortal } from 'react-dom';
-import { Captions, Repeat, Settings2 } from 'lucide-react';
+import { Captions, Gauge, Repeat } from 'lucide-react';
import { useT } from '@/i18n';
import { useKeyHint } from '@/lib/keymap-store';
import { Tooltip } from './Tooltip';
@@ -88,7 +88,7 @@ export function PlaybackMenu({ className, disabled, settings }: {
}}>
{loop ? <Repeat size={14} className="text-[var(--accent)]" aria-hidden /> : null}
{speed !== 1 ? <span className="text-[12px] leading-none tabular-nums">{speedText(speed)}</span> : null}
- {!loop && speed === 1 ? <Settings2 size={14} aria-hidden /> : null}
+ {!loop && speed === 1 ? <Gauge size={14} aria-hidden /> : null}
</button>
</Tooltip>
{anchor && createPortal(<div ref={panel} id={id} role="dialog" aria-label={t('viewer.playback')}
diff --git a/packages/openfilm/studio/ui/src/editor/player.test.ts b/packages/openfilm/studio/ui/src/editor/player.test.ts
index 1ca19a1..304637c 100644
--- a/packages/openfilm/studio/ui/src/editor/player.test.ts
+++ b/packages/openfilm/studio/ui/src/editor/player.test.ts
@@ -1,6 +1,6 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
-import { anchorTime, EASE_STEP, easedLead, fadeSteps, JUMP, soundsDue, soundSpeed, stretchedSpan } from './player.ts';
+import { anchorTime, EASE_STEP, easedLead, fadeSteps, fileTime, JUMP, soundsDue, stretchRate } from './player.ts';
test('a sound\'s fades as ramps: where it is now, then each corner after it', () => {
/* 4 s long, 1 s in, 2 s out */
@@ -40,16 +40,24 @@ test('sounds are started a few seconds ahead of the film, each once, solo leavin
assert.deepEqual(soundsDue([{ ...s(0, 4, 'e'), speed: 2 }], 2.5, 3, new Set(), null), []);
});
-test('a sound at another speed is played from a copy stretched to it: where to start in it and for how long', () => {
- /* a clip at 2× of the file's seconds 10–20, at film second 4: 5 film seconds long */
+test('a sound heard at another speed is played by a media element at its rate, pitch kept; at 1× it is decoded', () => {
+ /* its clip's speed by the viewer's */
+ assert.equal(stretchRate(2, 1), 2);
+ assert.equal(stretchRate(2, 0.75), 1.5);
+ assert.equal(stretchRate(1, 0.25), 0.25);
+ assert.equal(stretchRate(1, 1 / 3), 0.3333);
+ /* at 1× (by any pair), the decoded sound plays */
+ assert.equal(stretchRate(1, 1), null);
+ assert.equal(stretchRate(2, 0.5), null);
+ /* within what a browser plays */
+ assert.equal(stretchRate(64, 2), 16);
+ assert.equal(stretchRate(0.1, 0.25), 1 / 16);
+});
+
+test('where in its file a sound is at a film second', () => {
+ /* a clip at 2× of the file's seconds 10–20, at film second 4 */
const s = { src: 'a.wav', at: 4, from: 10, to: 20, volume: 1, speed: 2 };
- assert.equal(soundSpeed(2, 1), 2);
- assert.equal(soundSpeed(2, 0.75), 1.5);
- assert.equal(soundSpeed(1, 1 / 3), 0.3333);
- /* from its start, the viewer at 1×: the 2× copy from 10 / 2, all 10 file seconds as 5 */
- assert.deepEqual(stretchedSpan(s, 4, 2), { offset: 5, duration: 5 });
- /* the film started 1 s into it, the viewer at 1.5× (the copy at 3×): file second 12, 8 file seconds left */
- assert.deepEqual(stretchedSpan(s, 5, 3), { offset: 4, duration: 8 / 3 });
- /* at 1× of a 1× clip it is the file itself */
- assert.deepEqual(stretchedSpan({ ...s, speed: 1, to: 15 }, 6, 1), { offset: 12, duration: 3 });
+ assert.equal(fileTime(s, 4), 10);
+ assert.equal(fileTime(s, 5), 12);
+ assert.equal(fileTime({ ...s, speed: 1 }, 6.5), 12.5);
});
diff --git a/packages/openfilm/studio/ui/src/editor/player.ts b/packages/openfilm/studio/ui/src/editor/player.ts
index 5d5ab04..0dc73e7 100644
--- a/packages/openfilm/studio/ui/src/editor/player.ts
+++ b/packages/openfilm/studio/ui/src/editor/player.ts
@@ -10,11 +10,12 @@
*
* `sounds` is the timeline's own list (src/timeline.mjs `__filmSounds`): each sound clip's sound and each video's own,
* already placed and cut to its clip, with its volume and speed. Each file's sound is fetched small (`?film-sound`, see
- * studio/server/pages.mjs): a video's is a copy of its sound alone, never the whole video. A sound played faster or
- * slower (its clip's speed by the viewer's) is fetched stretched to that speed with its pitch kept (`&speed=`, the
- * atempo filter exports use) and played as it is: never sped up as a tape is, which would be heard higher. A sound not
- * here yet (its copy still being made) is silent until it is, then joins where the film is. Sounds are started a few seconds ahead of
- * the film, not all at once: a film of hundreds of sounds starts as quickly as one of a few.
+ * studio/server/pages.mjs): a video's is a copy of its sound alone, never the whole video. A sound heard at its own
+ * speed (its clip's speed by the viewer's is 1) is decoded and played sample-accurate. One heard faster or slower is
+ * played by the browser's own media element with its pitch kept (`preservesPitch`), in the same mix: another speed is
+ * heard at once, with nothing made for it, and its place is checked against the film as it plays. A sound not here yet
+ * (its copy still being made) is silent until it is, then joins where the film is. Sounds are started a few seconds
+ * ahead of the film, not all at once: a film of hundreds of sounds starts as quickly as one of a few.
*
* Every sound goes through one mix, metered (`levels`: the audio meters read it while the film plays, nothing reads it
* paused) before the person's volume. Solo (`setHeard`) leaves the clips of the other tracks out of what plays here
@@ -29,6 +30,10 @@ const AHEAD = 8;
const CHECK_MS = 250;
/** How long after now a sound can be started at the soonest (s): sooner, its start is cut. */
const LEAD = 0.03;
+/** A sound played by a media element further than this from where the film is (s) is put back in its place. */
+const DRIFT = 0.12;
+/** After a sound is put back in its place, how long it is left to settle before it is checked again (ms). */
+const SETTLE_MS = 1000;
/** `fade`: its fades in and out, film seconds, already fitted to its length (src/timeline.mjs). */
export type Sound = { src: string; at: number; from: number; to: number; volume: number; speed: number; clip?: string; fade?: [number, number] };
@@ -84,25 +89,22 @@ export type MixSamples = [Float32Array<ArrayBuffer>, Float32Array<ArrayBuffer>];
/** A sound's length on the film (s). */
const lengthOf = (s: Sound) => (s.to - s.from) / s.speed;
-/** The speed a sound's file is played at: its clip's by the viewer's, to 4 places (the copy made for it, see soundOf). */
-export function soundSpeed(speed: number, rate: number): number {
- return Math.round(speed * rate * 1e4) / 1e4;
-}
-
/**
- * Where to start in a copy of the sound stretched to `speed` (soundSpeed) and how long to play it (its own seconds),
- * for the sound heard from film second `startFilm` to its end: the copy's second is `speed` of the file's.
+ * The rate a sound's media element plays its file at, its pitch kept: its clip's speed by the viewer's, within what a
+ * browser plays (1/16 to 16). Null when that is 1: the sound is decoded and played as it is.
*/
-export function stretchedSpan(s: Sound, startFilm: number, speed: number): { offset: number; duration: number } {
- const end = s.at + lengthOf(s);
- return {
- offset: (s.from + (startFilm - s.at) * s.speed) / speed,
- duration: ((end - startFilm) * s.speed) / speed,
- };
+export function stretchRate(speed: number, rate: number): number | null {
+ const v = Math.round(speed * rate * 1e4) / 1e4;
+ return v === 1 ? null : Math.max(1 / 16, Math.min(16, v));
+}
+
+/** Where in its file a sound is at film second `t` (s). */
+export function fileTime(s: Sound, t: number): number {
+ return s.from + (t - s.at) * s.speed;
}
-/** A decoded sound's place: its file, at a speed. */
-const bufferKey = (src: string, speed: number) => `${speed}|${src}`;
+/** A sound played by a media element: the element, what it goes through in the mix, and when it was last put back. */
+type Voice = { el: HTMLAudioElement; source: MediaElementAudioSourceNode; gain: GainNode; sound: Sound; timer: number; settled: number };
/** How far film time `t` is from a sound: 0 while it plays, else until it starts; one already past after all those. */
function awayFrom(s: Sound, t: number) {
@@ -125,13 +127,14 @@ export function soundsDue(sounds: readonly Sound[], from: number, until: number,
export class Player {
private ctx: AudioContext | null = null;
- /** each file's decoded sound at a speed (bufferKey), and the version of the file it was decoded from (undefined: not
- known then) */
+ /** each file's decoded sound, and the version of the file it was decoded from (undefined: not known then) */
private buffers = new Map<string, { version: number | undefined; buffer: Promise<AudioBuffer | null> }>();
private origin = '';
private versionOf: (src: string) => number | undefined = () => undefined;
private sounds: Sound[] = [];
private playingNodes: AudioScheduledSourceNode[] = [];
+ /** the sounds heard faster or slower, each played by a media element (see stretchRate) */
+ private voices: Voice[] = [];
/** where the film is while playing, on the page's clock (see Anchor) */
private line: Anchor = { t: 0, at: 0, rate: 1 };
/** the same line on the audio clock while the sound plays: film second `t` is heard at audio time `at` (s); null silent */
@@ -181,32 +184,24 @@ export class Player {
}
/**
- * Fetches the sounds not decoded yet at the speed they play at now, or decoded from an older version; says whether
- * there were any. Those at no speed played now or at 1× are let go (a long sound decoded is tens of MB).
+ * Fetches the sounds not decoded yet, or decoded from an older version; says whether there were any. A clip at another
+ * speed is never decoded: a media element plays it.
*/
private fetchSounds(): boolean {
if (!this.origin) return false;
const ctx = this.context();
let fetched = false;
const t = this.time();
- const kept = new Set(this.sounds.flatMap((s) => [bufferKey(s.src, soundSpeed(s.speed, this.rate)), bufferKey(s.src, soundSpeed(s.speed, 1))]));
- for (const key of [...this.buffers.keys()]) if (!kept.has(key)) this.buffers.delete(key);
- for (const s of [...this.sounds].sort((a, b) => awayFrom(a, t) - awayFrom(b, t))) {
+ for (const s of [...this.sounds].filter((x) => x.speed === 1).sort((a, b) => awayFrom(a, t) - awayFrom(b, t))) {
const version = this.versionOf(s.src);
- const speed = soundSpeed(s.speed, this.rate);
- const key = bufferKey(s.src, speed);
- const had = this.buffers.get(key);
+ const had = this.buffers.get(s.src);
/* a version not known on one side is taken as the same: the file was not seen to change */
if (had && (had.version === version || had.version === undefined || version === undefined)) {
had.version ??= version;
continue;
}
- const url = new URL(s.src, this.origin);
- url.searchParams.set('film-sound', '1');
- if (speed !== 1) url.searchParams.set('speed', String(speed));
- /* past the browser's cache, which keeps the old file under the same address */
- if (version !== undefined) url.searchParams.set('v', String(version));
- this.buffers.set(key, { version, buffer: this.slot(s.src)
+ const url = this.soundUrl(s.src);
+ this.buffers.set(s.src, { version, buffer: this.slot(s.src)
.then((done) => fetch(url)
/* 204: there is no sound in it */
.then((r) => (r.status === 204 ? null : r.ok ? r.arrayBuffer() : Promise.reject(new Error(String(r.status)))))
@@ -218,6 +213,15 @@ export class Player {
return fetched;
}
+ /** The address of a file's sound, small (`?film-sound`), past the browser's cache when the file was made again. */
+ private soundUrl(src: string): string {
+ const url = new URL(src, this.origin);
+ url.searchParams.set('film-sound', '1');
+ const version = this.versionOf(src);
+ if (version !== undefined) url.searchParams.set('v', String(version));
+ return url.href;
+ }
+
/** A turn to fetch `src`'s sound; resolves with what ends it. */
private slot(src: string): Promise<() => void> {
return new Promise((resolve) => {
@@ -315,13 +319,12 @@ export class Player {
if (this.master) this.master.gain.value = muted ? 0 : this.volume;
}
- /** Play faster or slower, the pitch kept (each sound at that speed is fetched); playing goes on from where it is. */
+ /** Play faster or slower, the pitch kept (see stretchRate); playing goes on from where it is, heard at once. */
setRate(rate: number) {
const t = this.time();
const next = Math.max(0.25, Math.min(2, rate));
if (next === this.rate) return;
this.rate = next;
- this.fetchSounds();
if (this.playing) this.restart(t);
}
@@ -405,6 +408,7 @@ export class Player {
}
}
this.startDue();
+ this.keepVoices();
}, CHECK_MS);
}
@@ -416,7 +420,8 @@ export class Player {
for (const i of soundsDue(this.sounds, now, now + AHEAD * this.rate, this.started, this.heard)) {
this.started.add(i);
const s = this.sounds[i];
- void this.buffers.get(bufferKey(s.src, soundSpeed(s.speed, this.rate)))?.buffer.then((buffer) => {
+ if (stretchRate(s.speed, this.rate) != null) { this.voice(s, heard); continue; }
+ void this.buffers.get(s.src)?.buffer.then((buffer) => {
if (!buffer || !this.playing || this.heardLine !== heard) return;
this.schedule(buffer, s, heard);
});
@@ -435,19 +440,11 @@ export class Player {
const startFilm = Math.max(s.at, heard.t + Math.max(0, soonest - heard.at) * r);
if (startFilm >= end) return;
const when = Math.max(soonest, ctxAt(startFilm));
- /* the buffer is the sound stretched to this speed already: played as it is, its pitch is the file's */
- const span = stretchedSpan(s, startFilm, soundSpeed(s.speed, r));
const source = ctx.createBufferSource();
source.buffer = buffer;
- const gain = ctx.createGain();
- if (s.fade) {
- /* its fades as ramps on the audio clock, sample-accurate: from where it starts, each corner at its own moment */
- const [[, first], ...rest] = fadeSteps(startFilm - s.at, end - s.at, s.fade);
- gain.gain.setValueAtTime(s.volume * first, when);
- for (const [u, g] of rest) gain.gain.linearRampToValueAtTime(s.volume * g, Math.max(when, ctxAt(s.at + u)));
- } else gain.gain.value = s.volume;
+ const gain = this.fades(s, startFilm, when, ctxAt);
source.connect(gain).connect(this.mix ?? ctx.destination);
- source.start(when, span.offset, span.duration);
+ source.start(when, fileTime(s, startFilm), end - startFilm);
source.onended = () => {
const i = this.playingNodes.indexOf(source);
if (i >= 0) this.playingNodes.splice(i, 1);
@@ -455,6 +452,78 @@ export class Player {
this.playingNodes.push(source);
}
+ /** A sound's gain in the mix, its fades as ramps on the audio clock from `when` (film second `startFilm`) on. */
+ private fades(s: Sound, startFilm: number, when: number, ctxAt: (filmT: number) => number): GainNode {
+ const gain = this.context().createGain();
+ if (s.fade) {
+ /* sample-accurate: from where it starts, each corner at its own moment */
+ const [[, first], ...rest] = fadeSteps(startFilm - s.at, lengthOf(s), s.fade);
+ gain.gain.setValueAtTime(s.volume * first, when);
+ for (const [u, g] of rest) gain.gain.linearRampToValueAtTime(s.volume * g, Math.max(when, ctxAt(s.at + u)));
+ } else gain.gain.value = s.volume;
+ return gain;
+ }
+
+ /**
+ * One sound heard faster or slower: its file played by a media element at its rate, the pitch kept, through the mix.
+ * It is put where the film will be when it starts and started then; keepVoices keeps it there.
+ */
+ private voice(s: Sound, heard: { t: number; at: number }) {
+ const ctx = this.context();
+ const r = this.rate;
+ const ctxAt = (filmT: number) => heard.at + (filmT - heard.t) / r;
+ const end = s.at + lengthOf(s);
+ const soonest = ctx.currentTime + LEAD;
+ const startFilm = Math.max(s.at, heard.t + Math.max(0, soonest - heard.at) * r);
+ if (startFilm >= end) return;
+ const when = Math.max(soonest, ctxAt(startFilm));
+ const el = new Audio();
+ /* the film's origin is not the editor's: read across, so the mix hears it (its server says the editor may) */
+ el.crossOrigin = 'anonymous';
+ el.preload = 'auto';
+ el.preservesPitch = true;
+ el.src = this.soundUrl(s.src);
+ el.defaultPlaybackRate = el.playbackRate = stretchRate(s.speed, r) ?? 1;
+ el.currentTime = fileTime(s, startFilm);
+ const source = ctx.createMediaElementSource(el);
+ const gain = this.fades(s, startFilm, when, ctxAt);
+ source.connect(gain).connect(this.mix ?? ctx.destination);
+ const voice: Voice = { el, source, gain, sound: s, settled: 0, timer: 0 };
+ voice.timer = window.setTimeout(() => {
+ if (!this.voices.includes(voice)) return;
+ el.playbackRate = stretchRate(s.speed, this.rate) ?? 1;
+ void el.play().catch(() => {});
+ }, Math.max(0, (when - ctx.currentTime) * 1000));
+ this.voices.push(voice);
+ }
+
+ /** Each sound a media element plays, kept where the film is: put back when it drifts, let go when it has ended. */
+ private keepVoices() {
+ const heard = this.heardLine;
+ const ctx = this.ctx;
+ if (!heard || !ctx) return;
+ /* the film second the mix plays now */
+ const film = heard.t + (ctx.currentTime - heard.at) * this.rate;
+ const now = performance.now();
+ for (const v of [...this.voices]) {
+ const s = v.sound;
+ if (film >= s.at + lengthOf(s)) { this.dropVoice(v); continue; }
+ if (film < s.at || v.el.paused || v.el.seeking || v.el.readyState < 2 || now - v.settled < SETTLE_MS) continue;
+ const want = fileTime(s, film);
+ if (Math.abs(v.el.currentTime - want) > DRIFT) { v.el.currentTime = want; v.settled = now; }
+ }
+ }
+
+ private dropVoice(v: Voice) {
+ window.clearTimeout(v.timer);
+ v.el.pause();
+ v.el.removeAttribute('src');
+ v.el.load();
+ try { v.source.disconnect(); v.gain.disconnect(); } catch { /* already apart */ }
+ const i = this.voices.indexOf(v);
+ if (i >= 0) this.voices.splice(i, 1);
+ }
+
pause() {
this.pausedT = this.time();
this.playing = false;
@@ -474,6 +543,7 @@ export class Player {
private stopNodes() {
for (const node of this.playingNodes) { try { node.onended = null; node.stop(); } catch { /* already stopped */ } }
this.playingNodes = [];
+ for (const v of [...this.voices]) this.dropVoice(v);
this.started.clear();
}