Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor peaks and waveform management in timeline

konstantin-paulusfcaa8b95ed72
diff --git a/apps/web/src/engine/timeline/peaks.ts b/apps/web/src/engine/timeline/peaks.ts
index 25a7693e..5980bac1 100644
--- a/apps/web/src/engine/timeline/peaks.ts
+++ b/apps/web/src/engine/timeline/peaks.ts
@@ -2,79 +2,59 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-/**
- * The peaks the timeline's waveforms are drawn from, in two layers.
- *
- * The asset layer decodes a file once, at a fixed high resolution, and hands
- * every couple of seconds of it to IndexedDB. Peaks that fine are the better
- * part of a kilobyte a second, so a few long files would be tens of megabytes
- * held for a whole session; the store is what keeps them off the JS heap. It
- * is emptied when it is opened, since the index of which chunks exist lives
- * in memory — nothing written in an earlier session could be found again.
- *
- * The clip layer takes whichever chunks the visible stretch of one clip
- * covers and downsamples them to one peak per pixel, which is the only
- * resolution a draw pass can use. Zoom changes what a pixel is worth, so this
- * layer is rebuilt as the timeline is zoomed; the asset layer never is.
- *
- * Both are asked for by the draw pass and filled in behind it: a request says
- * what a clip needs, and the next frame draws whatever has arrived. Nothing
- * here blocks a frame.
- */
-
-import { AudioSampleSink } from 'mediabunny';
-import { getAudioTrack } from '@diffusionstudio/runtime';
-
-import type { AudioSample } from 'mediabunny';
-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
-
-/** Peaks a second the asset layer decodes at: finer than any zoom reaches. */
-const SOURCE_PEAKS_PER_SECOND = 800;
+import { Library } from '@diffusionstudio/runtime';
+import { WAVEFORM_PEAKS_PER_SECOND, downsamplePeaks } from '@diffusionstudio/assets';
-/** How much audio one stored chunk covers, in seconds. */
-const CHUNK_DURATION = 2;
+import type { World } from 'koota';
+import type { AssetCache, AudioAsset, VideoAsset } from '@diffusionstudio/assets';
/** How wide one drawn sample is, in pixels. */
export const SAMPLE_WIDTH = 1;
-const IDB_NAME = 'peak-cache-db';
-const IDB_VERSION = 1;
-const IDB_STORE = 'data';
+/**
+ * How much source either side of the visible stretch is cut with it, as a
+ * fraction of that stretch: a scroll that stays inside the margin draws from
+ * what has already been cut instead of cutting again.
+ */
+const WINDOW_MARGIN = 0.5;
-/** Where a chunk of an asset's peaks sits in it, in seconds, and its key. */
-type PeakChunk = {
+/** The stretch of one clip's waveform it can draw, at the zoom it asked for. */
+export type ClipSamples = {
+ /** What it covers, in source seconds. */
start: number;
end: number;
- id: string;
-};
-
-/** A chunk of peaks, downsampled to what one clip draws it at. */
-export type ClipSamples = PeakChunk & {
- data: Uint8ClampedArray;
peaksPerSecond: number;
-};
-
-/** What the decode produced for one stretch of a file, before it is stored. */
-type DecodedPeaks = {
- start: number;
- end: number;
data: Uint8ClampedArray;
};
-/** The asset layer's record of one file: which chunks of it exist. */
+/** The asset layer's record of one file's waveform. */
type AssetPeaks = {
- decoding: boolean;
- /** True once the file has been read through, whatever came of it. */
- decoded: boolean;
- chunks: PeakChunk[];
+ /** The derived waveform, once there is one to read from. */
+ file: File | null;
+ /** How many peaks it holds, which is how many bytes it is. */
+ length: number;
+ loading: boolean;
+ /** True once the cache has answered, whatever it answered. */
+ loaded: boolean;
+};
+
+/** A stretch of a waveform to cut, and what it is being cut for. */
+type Window = {
+ /** The stretch to read, in peaks, margins and all. */
+ from: number;
+ to: number;
+ /** What of it is on screen, so a scroll inside the margins costs nothing. */
+ visibleFrom: number;
+ visibleTo: number;
+ peaksPerSecond: number;
};
-/** The clip layer's record of one clip: the chunks it can draw, at its zoom. */
+/** The clip layer's record of one clip: what it can draw, at its zoom. */
type ClipPeaks = {
assetId: string;
- samples: ClipSamples[];
- /** The range and zoom `samples` was built for; unchanged means no work. */
- hash: string;
+ samples: ClipSamples | null;
+ /** What `samples` was cut for; unchanged means there is nothing to do. */
+ cut: Window | null;
updating: boolean;
pending: PeakRequest | null;
};
@@ -94,26 +74,32 @@ const assetPeaks = new Map<string, AssetPeaks>();
const clipPeaks = new Map<number, ClipPeaks>();
/**
- * The chunks `clip` can draw right now, ordered by where they start. Empty
- * (or short) while the decode behind them is still running.
+ * The stretch `clip` can draw right now, or null while the read behind it is
+ * still running.
*/
-export function getClipSamples(clip: number): ClipSamples[] | undefined {
- return clipPeaks.get(clip)?.samples;
+export function getClipSamples(clip: number): ClipSamples | null {
+ return clipPeaks.get(clip)?.samples ?? null;
}
/**
- * Asks for the peaks a clip needs. Starts the file's decode the first time it
- * is asked about, and re-cuts the clip's own samples whenever the stretch it
- * wants or the zoom it wants them at has changed.
+ * Asks for the peaks a clip needs. Has the file's waveform derived the first
+ * time it is asked about, and re-cuts the clip's own samples whenever the
+ * stretch it wants has moved out of what was cut, or the zoom it wants them
+ * at has changed.
*/
-export function requestPeaks(request: PeakRequest): void {
+export function requestPeaks(world: World, request: PeakRequest): void {
+ // The waveform is the cache's to derive and to keep. Until a project has
+ // attached one there is nothing to draw from and nothing to do about it.
+ const cache = world.get(Library)?.cache;
+ if (!cache) return;
+
const record = assetPeaks.get(request.asset.id);
- if (!record || (!record.decoding && !record.decoded)) {
- void decodeAsset(request);
+ if (!record || (!record.loading && !record.loaded)) {
+ void loadAsset(cache, request);
}
- if (record && record.chunks.length > 0) {
+ if (record && record.length > 0) {
void updateClip(request, record);
}
}
@@ -121,8 +107,8 @@ export function requestPeaks(request: PeakRequest): void {
/**
* Gives a clip's samples to the half that was cut off it. Both halves play
* the same source at the same zoom, so the copy starts with everything the
- * original had — the peak buffers themselves are shared, only the records
- * that say what is being drawn are its own.
+ * original had — the peak buffer itself is shared, only the record that says
+ * what is being drawn is its own.
*/
export function clonePeaksForSplit(clip: number, copy: number): void {
const source = clipPeaks.get(clip);
@@ -130,8 +116,8 @@ export function clonePeaksForSplit(clip: number, copy: number): void {
clipPeaks.set(copy, {
assetId: source.assetId,
- samples: source.samples.map((sample) => ({ ...sample })),
- hash: source.hash,
+ samples: source.samples && { ...source.samples },
+ cut: source.cut && { ...source.cut },
updating: false,
pending: null,
});
@@ -143,7 +129,7 @@ export function clearClipPeaks(clip?: number): void {
else clipPeaks.delete(clip);
}
-/** Forgets everything: the clips' samples and the index of what was decoded. */
+/** Forgets everything: the clips' samples and the files they were read from. */
export function clearPeaks(): void {
clipPeaks.clear();
assetPeaks.clear();
@@ -161,101 +147,31 @@ export function forgetAssetPeaks(assetId: string): void {
// The asset layer
/**
- * Reads the file through once, writing its peaks to the store a couple of
- * seconds at a time. Each chunk is usable the moment it lands, so a long file
- * draws from the top down rather than all at once at the end.
+ * Takes the waveform the cache has for a file, waiting on the decode where it
+ * has to derive one. Marked loaded either way: a file with no audio in it, or
+ * one that would not decode, is not worth asking about on every frame.
*/
-async function decodeAsset(request: PeakRequest): Promise<void> {
+async function loadAsset(cache: AssetCache, request: PeakRequest): Promise<void> {
const { asset } = request;
- const record = assetPeaks.get(asset.id) ?? { decoding: false, decoded: false, chunks: [] };
+ const record = assetPeaks.get(asset.id) ?? { file: null, length: 0, loading: false, loaded: false };
assetPeaks.set(asset.id, record);
- if (record.decoding) return;
+ if (record.loading) return;
- record.decoding = true;
+ record.loading = true;
try {
- const track = await getAudioTrack(asset);
- if (!track) return;
-
- const sink = new AudioSampleSink(track);
- const pending: DecodedPeaks[] = [];
- let covered = 0;
-
- const flush = async (): Promise<void> => {
- const merged = mergePeaks(pending);
- const id = `${asset.id}-${merged.start}`;
- record.chunks.push({ start: merged.start, end: merged.end, id });
- pending.length = 0;
- covered = 0;
- await storePeaks(id, merged.data);
- };
-
- for await (const sample of sink.samples()) {
- covered += sample.duration;
- try {
- pending.push(peaksOfSample(sample));
- } finally {
- sample.close();
- }
-
- if (covered > CHUNK_DURATION) await flush();
- }
+ const file = await cache.waveform(asset);
+ if (!file) return;
- if (pending.length > 0) await flush();
-
- record.chunks.sort((a, b) => a.start - b.start);
+ record.file = file;
+ record.length = file.size;
} catch (error) {
- console.error('[timeline] could not decode peaks', error);
+ console.error('[timeline] could not load peaks', error);
} finally {
- record.decoding = false;
- record.decoded = true;
- }
-}
-
-/**
- * One decoded sample's peaks: the loudest frame of each slice, across every
- * channel, raised to a power so quiet passages still show as something.
- */
-function peaksOfSample(sample: AudioSample): DecodedPeaks {
- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
- const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
-
- const channels = sample.numberOfChannels;
- const frames = floats.length / channels;
- const target = Math.ceil(sample.duration * SOURCE_PEAKS_PER_SECOND);
- const peaks = new Uint8ClampedArray(target);
- const ratio = frames / target;
-
- for (let i = 0; i < target; i++) {
- const from = Math.floor(i * ratio);
- const to = Math.floor((i + 1) * ratio);
- let max = 0;
-
- for (let frame = from; frame < to && frame < frames; frame++) {
- for (let channel = 0; channel < channels; channel++) {
- max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));
- }
- }
-
- peaks[i] = Math.floor(255 * max);
- }
-
- return { start: sample.timestamp, end: sample.timestamp + sample.duration, data: peaks };
-}
-
-function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {
- const length = peaks.reduce((total, peak) => total + peak.data.length, 0);
- const merged = new Uint8ClampedArray(length);
-
- let offset = 0;
- for (const peak of peaks) {
- merged.set(peak.data, offset);
- offset += peak.data.length;
+ record.loading = false;
+ record.loaded = true;
}
-
- return { start: peaks[0]!.start, end: peaks[peaks.length - 1]!.end, data: merged };
}
// ---------------------------------------------------------------------------
@@ -264,13 +180,12 @@ function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {
/**
* Re-cuts a clip's samples for the stretch and zoom it is asking about. Only
* one runs at a time per clip — a zoom or a scroll that arrives while one is
- * reading the store replaces whatever was queued behind it, so the work that
- * finally runs is for where the timeline ended up rather than where it went
- * through.
+ * reading replaces whatever was queued behind it, so the work that finally
+ * runs is for where the timeline ended up rather than where it went through.
*/
async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<void> {
const peaks = clipPeaks.get(request.clip)
- ?? { assetId: request.asset.id, samples: [], hash: '', updating: false, pending: null };
+ ?? { assetId: request.asset.id, samples: null, cut: null, updating: false, pending: null };
clipPeaks.set(request.clip, peaks);
if (peaks.updating) {
@@ -279,30 +194,29 @@ async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<voi
}
try {
- const range = coveringRange(request, record);
- if (range.hash === peaks.hash) return;
+ const wanted = coveringWindow(request, record);
+ if (!wanted || !needsCut(peaks.cut, wanted)) return;
peaks.updating = true;
- peaks.hash = range.hash;
-
- for (const chunk of record.chunks) {
- if (!inRange(chunk.start, range.start, range.end)) continue;
+ peaks.cut = wanted;
- const data = await loadPeaks(chunk.id);
- if (!data) continue;
-
- peaks.samples = peaks.samples.filter((sample) => sample.id !== chunk.id);
- peaks.samples.push({
- ...chunk,
- data: downsample(data, Math.ceil((chunk.end - chunk.start) * request.peaksPerSecond)),
- peaksPerSecond: request.peaksPerSecond,
- });
- peaks.samples.sort((a, b) => a.start - b.start);
+ const data = await readPeaks(request.asset.id, record, wanted.from, wanted.to);
+ if (!data) {
+ peaks.cut = null;
+ return;
}
- peaks.samples = peaks.samples.filter((sample) => inRange(sample.start, range.start, range.end));
+ const seconds = data.length / WAVEFORM_PEAKS_PER_SECOND;
+
+ peaks.samples = {
+ start: wanted.from / WAVEFORM_PEAKS_PER_SECOND,
+ end: (wanted.from + data.length) / WAVEFORM_PEAKS_PER_SECOND,
+ peaksPerSecond: request.peaksPerSecond,
+ data: downsamplePeaks(data, Math.max(1, Math.round(seconds * request.peaksPerSecond))),
+ };
} catch (error) {
console.error('[timeline] could not cut peaks', error);
+ peaks.cut = null;
} finally {
peaks.updating = false;
}
@@ -315,102 +229,44 @@ async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<voi
}
/**
- * The stretch of the asset to cut, snapped out to whole chunks: what is on
- * screen, widened to the chunks its ends fall inside, so the waveform does
- * not stop short of the edge of the viewport.
+ * The stretch of the waveform to cut, in peaks: what is on screen, widened by
+ * a margin either side so a scroll draws from what has already been cut, and
+ * clamped to what there is to read. Null while there is nothing.
*/
-function coveringRange(request: PeakRequest, record: AssetPeaks) {
- const from = Math.max(0, request.start);
- const to = Math.min(request.asset.duration, request.end);
-
- const first = record.chunks.find((chunk) => inRange(from, chunk.start, chunk.end));
- const last = record.chunks.find((chunk) => inRange(to, chunk.start, chunk.end));
-
- const start = first?.start ?? record.chunks.at(0)?.start ?? 0;
- const end = last?.end ?? record.chunks.at(-1)?.end ?? 0;
-
- return { start, end, hash: `${request.peaksPerSecond}.${start}.${end}` };
-}
-
-/**
- * The peaks of `data` at `target` positions: the loudest of each slice, and
- * never fewer than one sample per position.
- *
- * Zoomed in past the resolution the peaks were stored at, a position's slice
- * of the source is narrower than one stored sample, and every slice whose
- * ends round to the same sample would read nothing at all — a fifth of them
- * at the far end of the zoom, drawn as a regular pattern of gaps through the
- * waveform. Holding the sample the slice starts in draws what there is to
- * draw instead: past that zoom the file has no more detail to show.
- */
-function downsample(data: Uint8ClampedArray, target: number): Uint8ClampedArray {
- const peaks = new Uint8ClampedArray(target);
- const ratio = data.length / target;
-
- for (let i = 0; i < target; i++) {
- const from = Math.floor(i * ratio);
- const to = Math.max(from + 1, Math.floor((i + 1) * ratio));
- let max = 0;
-
- for (let index = from; index < to && index < data.length; index++) {
- max = Math.max(max, data[index]!);
- }
-
- peaks[i] = max;
- }
-
- return peaks;
-}
+function coveringWindow(request: PeakRequest, record: AssetPeaks): Window | null {
+ const peaksOf = (seconds: number): number => seconds * WAVEFORM_PEAKS_PER_SECOND;
+ const clamp = (peak: number): number => Math.min(record.length, Math.max(0, peak));
-const inRange = (value: number, from: number, to: number): boolean => value >= from && value <= to;
-
-// ---------------------------------------------------------------------------
-// The store
+ const visibleFrom = clamp(Math.floor(peaksOf(request.start)));
+ const visibleTo = clamp(Math.ceil(peaksOf(request.end)));
-let database: Promise<IDBDatabase> | null = null;
+ const margin = peaksOf(Math.max(0, request.end - request.start)) * WINDOW_MARGIN;
+ const from = clamp(Math.floor(peaksOf(request.start) - margin));
+ const to = clamp(Math.ceil(peaksOf(request.end) + margin));
+ if (to <= from) return null;
-function getDatabase(): Promise<IDBDatabase> {
- database ??= new Promise((resolve, reject) => {
- const request = indexedDB.open(IDB_NAME, IDB_VERSION);
-
- request.onupgradeneeded = () => request.result.createObjectStore(IDB_STORE);
- request.onerror = () => reject(request.error);
-
- request.onsuccess = () => {
- // What an earlier session left behind is unreachable — the index of
- // which chunks exist was never written — so the session starts on
- // an empty store rather than one that only grows.
- const transaction = request.result.transaction(IDB_STORE, 'readwrite');
- transaction.objectStore(IDB_STORE).clear();
- transaction.oncomplete = () => resolve(request.result);
- transaction.onerror = () => reject(transaction.error);
- };
- });
-
- return database;
+ return { from, to, visibleFrom, visibleTo, peaksPerSecond: request.peaksPerSecond };
}
-async function storePeaks(key: string, peaks: Uint8ClampedArray): Promise<void> {
- const db = await getDatabase();
- const transaction = db.transaction(IDB_STORE, 'readwrite');
-
- transaction.objectStore(IDB_STORE).put(new Blob([new Uint8Array(peaks)]), key);
+/** Whether what is on screen is outside what the last cut covers. */
+function needsCut(cut: Window | null, wanted: Window): boolean {
+ if (!cut) return true;
+ if (cut.peaksPerSecond !== wanted.peaksPerSecond) return true;
- return new Promise((resolve, reject) => {
- transaction.oncomplete = () => resolve();
- transaction.onerror = () => reject(transaction.error);
- });
+ return wanted.visibleFrom < cut.from || wanted.visibleTo > cut.to;
}
-async function loadPeaks(key: string): Promise<Uint8ClampedArray | null> {
- const db = await getDatabase();
- const store = db.transaction(IDB_STORE, 'readonly').objectStore(IDB_STORE);
-
- const blob = await new Promise<Blob | null>((resolve, reject) => {
- const request = store.get(key);
- request.onsuccess = () => resolve(request.result ?? null);
- request.onerror = () => reject(request.error);
- });
+/** The peaks in `[from, to)`, off disk: only the bytes that stretch covers. */
+async function readPeaks(assetId: string, record: AssetPeaks, from: number, to: number): Promise<Uint8ClampedArray | null> {
+ if (!record.file) return null;
- return blob ? new Uint8ClampedArray(await blob.arrayBuffer()) : null;
+ try {
+ return new Uint8ClampedArray(await record.file.slice(from, to).arrayBuffer());
+ } catch (error) {
+ // The file went out from under us: forget it, and let the next frame
+ // that asks find it again or have it derived afresh.
+ console.warn('[timeline] could not read stored peaks', error);
+ assetPeaks.delete(assetId);
+ return null;
+ }
}
diff --git a/apps/web/src/engine/timeline/render/waveform.ts b/apps/web/src/engine/timeline/render/waveform.ts
index 6b18c948..725bbc9b 100644
--- a/apps/web/src/engine/timeline/render/waveform.ts
+++ b/apps/web/src/engine/timeline/render/waveform.ts
@@ -10,7 +10,6 @@ import { framesToPixels, getFrameRate, getResolution, getViewport } from '../vie
import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
import type { Entity, World } from 'koota';
-import type { ClipSamples } from '../peaks';
import type { RowCursor } from '../layout';
import type { TimelineSurfaceState } from '../surface';
@@ -32,8 +31,9 @@ export type WaveformOptions = {
* A column is a moment of the source, so what it is worth changes with the
* zoom: the peaks are asked for at one per pixel (see `../peaks`), and only
* for the stretch of the clip that is on screen. Drawing then reads them back
- * by time rather than by index, so a column drawn before its peaks have
- * arrived is simply left out and filled in on a later frame.
+ * by time rather than by index, so a column outside what has been cut — a
+ * waveform not derived yet, or the margin a scroll has just left — is simply
+ * left out and filled in on a later frame.
*
* Tall rows show the waveform about its middle, short ones stand it on the
* bottom of the clip and fade it, so the label above it stays readable.
@@ -92,7 +92,7 @@ export function renderWaveform(
const firstSample = Math.max(inPx, viewportLeft - VIEWPORT_MARGIN - originPx);
const lastSample = Math.min(outPx, viewportRight + VIEWPORT_MARGIN - originPx);
- requestPeaks({
+ requestPeaks(world, {
clip: eid,
asset: options.asset,
start: pixelsToSeconds(firstSample) * playbackRate,
@@ -114,19 +114,11 @@ export function renderWaveform(
const samples = getClipSamples(eid);
- // The chunks are ordered by where they start and the columns are drawn
- // left to right, so the search for the chunk a column falls in only ever
- // moves forwards.
- let cursor = 0;
-
- for (let x = firstSample; x < lastSample; x += SAMPLE_WIDTH) {
+ for (let x = firstSample; samples && x < lastSample; x += SAMPLE_WIDTH) {
const time = pixelsToSeconds(x) * playbackRate;
+ if (time < samples.start || time > samples.end) continue;
- cursor = findChunk(samples, time, cursor);
- const chunk = samples?.[cursor];
- if (!chunk || time < chunk.start || time > chunk.end) continue;
-
- const peak = chunk.data[Math.floor((time - chunk.start) * chunk.peaksPerSecond)];
+ const peak = samples.data[Math.floor((time - samples.start) * samples.peaksPerSecond)];
if (peak === undefined) continue;
const height = Math.max((peak / 255) * waveformHeight, minSampleHeight);
@@ -135,13 +127,3 @@ export function renderWaveform(
ctx.restore();
}
-
-/** The first chunk at or after `from` that has not already ended by `time`. */
-function findChunk(samples: ClipSamples[] | undefined, time: number, from: number): number {
- if (!samples) return 0;
-
- let index = from;
- while (index < samples.length && samples[index]!.end < time) index++;
-
- return index;
-}
diff --git a/packages/assets/src/cache.ts b/packages/assets/src/cache.ts
index df00a15b..81737049 100644
--- a/packages/assets/src/cache.ts
+++ b/packages/assets/src/cache.ts
@@ -3,8 +3,9 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { derivePeaks } from './derive/peaks';
+import { PEAK_BARS } from './derive/peaks';
import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH } from './derive/thumbnail';
+import { deriveWaveform, downsamplePeaks } from './derive/waveform';
import type { ProjectFS } from './fs';
import type { Asset } from './types';
@@ -41,6 +42,19 @@ export const PEAKS: CacheKind<Uint8ClampedArray> = {
decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),
};
+/**
+ * The full waveform: the whole file at `WAVEFORM_PEAKS_PER_SECOND`, one byte
+ * a peak and nothing else, so the peak at a moment of the file is the byte at
+ * that moment's index. The one entry read a stretch at a time rather than
+ * whole (see `ensure`) — an hour of audio is a few megabytes of it.
+ */
+export const WAVEFORM: CacheKind<Uint8ClampedArray> = {
+ dir: 'waveforms',
+ ext: 'u8',
+ encode: (peaks) => new Blob([peaks as Uint8ClampedArray<ArrayBuffer>]),
+ decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),
+};
+
/** What an entry is keyed by: an asset, or just its id. */
type Keyed = string | Pick<Asset, 'id'>;
@@ -50,6 +64,8 @@ export class AssetCache {
private readonly fs: ProjectFS;
/** Reads and derivations under way, by `kind.dir/id[@variant]`. */
private readonly inflight = new Map<string, Promise<unknown>>();
+ /** The same, for entries asked for as files rather than as values. */
+ private readonly files = new Map<string, Promise<File | null>>();
/** Every kind used, so `remove` and `prune` know which directories to look in. */
private readonly kinds = new Map<string, CacheKind<unknown>>();
@@ -57,6 +73,7 @@ export class AssetCache {
this.fs = fs;
this.register(THUMBNAIL);
this.register(PEAKS);
+ this.register(WAVEFORM);
}
// -----------------------------------------------------------------------
@@ -72,9 +89,27 @@ export class AssetCache {
return this.get(THUMBNAIL, asset, async () => deriveThumbnail(await asset.handle.getFile(), asset.mimeType, width), variant);
}
- /** The asset's rough waveform, or null when it has no audio. */
+ /**
+ * The asset's rough waveform, or null when it has no audio. Cut down from
+ * the full one, so a file is decoded once however many resolutions of it
+ * are asked for.
+ */
public peaks(asset: Asset): Promise<Uint8ClampedArray | null> {
- return this.get(PEAKS, asset, async () => derivePeaks(await asset.handle.getFile()));
+ return this.get(PEAKS, asset, async () => {
+ const waveform = await this.waveform(asset);
+ return waveform && downsamplePeaks(new Uint8ClampedArray(await waveform.arrayBuffer()), PEAK_BARS);
+ });
+ }
+
+ /**
+ * The file the asset's full waveform is stored in, derived first where
+ * there is none, or null when it has no audio. A file rather than its
+ * bytes: the timeline draws the stretch of a clip that is on screen by
+ * reading the bytes that stretch covers (`file.slice(from, to)`) and no
+ * more.
+ */
+ public waveform(asset: Asset): Promise<File | null> {
+ return this.ensure(WAVEFORM, asset, async () => deriveWaveform(await asset.handle.getFile()));
}
// -----------------------------------------------------------------------
@@ -96,6 +131,23 @@ export class AssetCache {
return promise;
}
+ /**
+ * The file (kind, asset[, variant]) is stored in, `produce()`d and written
+ * there first where there is none: for entries whose consumer reads a
+ * stretch of them at a time rather than all of them at once. Concurrent
+ * calls share one derivation; null when there was nothing to derive.
+ */
+ public ensure<T>(kind: CacheKind<T>, keyed: Keyed, produce: () => Promise<T | null>, variant?: string): Promise<File | null> {
+ this.register(kind);
+ const key = this.key(kind, keyed, variant);
+ const running = this.files.get(key);
+ if (running) return running;
+
+ const promise = this.loadFile(kind, key, produce).finally(() => this.files.delete(key));
+ this.files.set(key, promise);
+ return promise;
+ }
+
/** Drops every entry of an asset on disk. */
public async remove(keyed: Keyed): Promise<void> {
const id = idOf(keyed);
@@ -158,6 +210,38 @@ export class AssetCache {
return value;
}
+ /**
+ * The entry's file, derived and written where there is none. The write is
+ * awaited, unlike `load`'s: here the file is the value.
+ */
+ private async loadFile<T>(kind: CacheKind<T>, key: string, produce: () => Promise<T | null>): Promise<File | null> {
+ const path = this.pathOf(kind, key);
+ const stored = await this.fileAt(path);
+ if (stored && stored.size > 0) return stored;
+
+ let value: T | null;
+ try {
+ value = await produce();
+ } catch (error) {
+ console.warn(`[assets] could not derive ${key}:`, error);
+ value = null;
+ }
+ if (value === null) return null;
+
+ await this.write(kind, key, value);
+ return this.fileAt(path);
+ }
+
+ /** The file at a cache path, or null when it is missing or unreadable. */
+ private async fileAt(path: string): Promise<File | null> {
+ try {
+ if (!(await this.fs.stat(path))) return null;
+ return await this.fs.file(path);
+ } catch {
+ return null;
+ }
+ }
+
/** The entry on disk, or undefined when missing or unreadable. */
private async read<T>(kind: CacheKind<T>, key: string): Promise<T | undefined> {
const path = this.pathOf(kind, key);
diff --git a/packages/assets/src/derive/peaks-worker.ts b/packages/assets/src/derive/peaks-worker.ts
deleted file mode 100644
index f727db3c..00000000
--- a/packages/assets/src/derive/peaks-worker.ts
+++ /dev/null
@@ -1,94 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// The rough waveform of a media file, computed off the main thread: a fixed
-// number of bars, each the peak amplitude (across channels, gamma-lifted so
-// quiet passages still show) of its slice of the file, as a byte. WebCodecs
-// decodes elsewhere already, but PCM formats (WAV) are unpacked in JS and
-// would stall the UI. One worker per request: a file and a bar count in,
-// the peaks out.
-
-import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
-
-export interface PeaksRequest {
- file: Blob;
- bars: number;
-}
-
-export type PeaksResponse = { peaks: Uint8ClampedArray | null } | { error: string };
-
-// The package is typed against the DOM lib, so the worker scope is narrowed by hand.
-const scope = self as unknown as {
- onmessage: ((event: MessageEvent<PeaksRequest>) => void) | null;
- postMessage(message: PeaksResponse, transfer?: Transferable[]): void;
-};
-
-scope.onmessage = async ({ data: { file, bars } }) => {
- try {
- const peaks = await computePeaks(file, bars);
- scope.postMessage({ peaks }, peaks ? [peaks.buffer as ArrayBuffer] : []);
- } catch (error) {
- scope.postMessage({ error: error instanceof Error ? error.message : String(error) });
- }
-};
-
-/** The peaks of `file`'s primary audio track, or null when it has none. */
-async function computePeaks(file: Blob, bars: number): Promise<Uint8ClampedArray | null> {
- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
- try {
- const track = await input.getPrimaryAudioTrack();
- if (!track) return null;
-
- const sink = new AudioSampleSink(track);
- const chunks: Uint8ClampedArray[] = [];
-
- for await (const sample of sink.samples()) {
- let floats: Float32Array;
- let channels: number;
- let frames: number;
- try {
- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
- floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
- channels = sample.numberOfChannels;
- frames = floats.length / channels;
- } finally {
- sample.close();
- }
-
- const framePeaks = new Uint8ClampedArray(frames);
- for (let f = 0; f < frames; f++) {
- let max = 0;
- for (let ch = 0; ch < channels; ch++) {
- max = Math.max(max, Math.pow(Math.abs(floats[f * channels + ch]!), 0.8));
- }
- framePeaks[f] = Math.floor(max * 255);
- }
- chunks.push(framePeaks);
- }
-
- if (chunks.length === 0) return null;
-
- const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
- const merged = new Uint8ClampedArray(total);
- let offset = 0;
- for (const chunk of chunks) {
- merged.set(chunk, offset);
- offset += chunk.length;
- }
-
- const peaks = new Uint8ClampedArray(bars);
- const ratio = total / bars;
- for (let i = 0; i < bars; i++) {
- const start = Math.floor(i * ratio);
- const end = Math.min(total, Math.floor((i + 1) * ratio));
- let max = 0;
- for (let j = start; j < end; j++) max = Math.max(max, merged[j]!);
- peaks[i] = max;
- }
- return peaks;
- } finally {
- input.dispose();
- }
-}
diff --git a/packages/assets/src/derive/peaks.ts b/packages/assets/src/derive/peaks.ts
index 7a7455f9..4399654e 100644
--- a/packages/assets/src/derive/peaks.ts
+++ b/packages/assets/src/derive/peaks.ts
@@ -2,23 +2,17 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import type { PeaksRequest, PeaksResponse } from './peaks-worker';
+import { deriveWaveform, downsamplePeaks } from './waveform';
/** How many bars a rough waveform has. */
export const PEAK_BARS = 128;
/**
- * The peaks of `file`'s primary audio track, or null when it has none.
- * Decoded in a worker of its own, gone when it answers.
+ * The rough waveform of `file`: a fixed number of bars, each the peak
+ * amplitude of its slice of the file, or null when it has no audio. The full
+ * waveform cut down — a file is decoded at one resolution and read at any.
*/
-export function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {
- const worker = new Worker(new URL('./peaks-worker.ts', import.meta.url), { type: 'module' });
- return new Promise<Uint8ClampedArray | null>((resolve, reject) => {
- worker.onmessage = ({ data }: MessageEvent<PeaksResponse>) => {
- if ('error' in data) reject(new Error(data.error));
- else resolve(data.peaks);
- };
- worker.onerror = (event) => reject(new Error(event.message || 'The peaks worker failed'));
- worker.postMessage({ file, bars } satisfies PeaksRequest);
- }).finally(() => worker.terminate());
+export async function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {
+ const waveform = await deriveWaveform(file);
+ return waveform && downsamplePeaks(waveform, bars);
}
diff --git a/packages/assets/src/derive/waveform-worker.ts b/packages/assets/src/derive/waveform-worker.ts
new file mode 100644
index 00000000..1d16d490
--- /dev/null
+++ b/packages/assets/src/derive/waveform-worker.ts
@@ -0,0 +1,101 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The waveform of a media file, computed off the main thread: the peak
+// amplitude of each slice of its audio (across channels, gamma-lifted so
+// quiet passages still show), as a byte. WebCodecs decodes elsewhere already,
+// but PCM formats (WAV) are unpacked in JS and would stall the UI. One worker
+// per request: a file and a resolution in, the whole waveform out.
+
+import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
+
+export interface WaveformRequest {
+ file: Blob;
+ peaksPerSecond: number;
+}
+
+export type WaveformResponse = { peaks: Uint8ClampedArray | null } | { error: string };
+
+// The package is typed against the DOM lib, so the worker scope is narrowed by hand.
+const scope = self as unknown as {
+ onmessage: ((event: MessageEvent<WaveformRequest>) => void) | null;
+ postMessage(message: WaveformResponse, transfer?: Transferable[]): void;
+};
+
+scope.onmessage = async ({ data: { file, peaksPerSecond } }) => {
+ try {
+ const peaks = await computeWaveform(file, peaksPerSecond);
+ scope.postMessage({ peaks }, peaks ? [peaks.buffer as ArrayBuffer] : []);
+ } catch (error) {
+ scope.postMessage({ error: error instanceof Error ? error.message : String(error) });
+ }
+};
+
+/**
+ * The waveform of `file`'s primary audio track, or null when it has none.
+ *
+ * Every peak is written where it belongs rather than appended, so a track
+ * that starts at a timestamp of its own or has a gap in it still lines up
+ * with the time it is drawn against.
+ */
+async function computeWaveform(file: Blob, peaksPerSecond: number): Promise<Uint8ClampedArray | null> {
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ try {
+ const track = await input.getPrimaryAudioTrack();
+ if (!track) return null;
+
+ // The track's own end, so a file whose sound outlasts its picture (or
+ // starts after it) is as long here as it is on disk.
+ const peaks = new Uint8ClampedArray(Math.ceil((await track.computeDuration()) * peaksPerSecond));
+ const sink = new AudioSampleSink(track);
+ let covered = 0;
+
+ for await (const sample of sink.samples()) {
+ let floats: Float32Array;
+ let channels: number;
+ let timestamp: number;
+ let duration: number;
+
+ // Everything the sample holds is taken before it is closed, and it
+ // is closed however that goes: they are decoder resources.
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+ channels = sample.numberOfChannels;
+ timestamp = sample.timestamp;
+ duration = sample.duration;
+ } finally {
+ sample.close();
+ }
+
+ const frames = floats.length / channels;
+ const from = Math.floor(timestamp * peaksPerSecond);
+ const to = Math.min(peaks.length, from + Math.ceil(duration * peaksPerSecond));
+ const ratio = frames / Math.max(1, to - from);
+
+ for (let peak = from; peak < to; peak++) {
+ const start = Math.floor((peak - from) * ratio);
+ const end = Math.floor((peak - from + 1) * ratio);
+ let max = 0;
+
+ for (let frame = start; frame < end && frame < frames; frame++) {
+ for (let channel = 0; channel < channels; channel++) {
+ max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));
+ }
+ }
+
+ peaks[peak] = Math.floor(255 * max);
+ }
+
+ covered = Math.max(covered, to);
+ }
+
+ if (covered === 0) return null;
+
+ return covered === peaks.length ? peaks : peaks.slice(0, covered);
+ } finally {
+ input.dispose();
+ }
+}
diff --git a/packages/assets/src/derive/waveform.ts b/packages/assets/src/derive/waveform.ts
new file mode 100644
index 00000000..3f14f5cf
--- /dev/null
+++ b/packages/assets/src/derive/waveform.ts
@@ -0,0 +1,59 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { WaveformRequest, WaveformResponse } from './waveform-worker';
+
+/**
+ * Peaks a second a derived waveform holds: finer than any zoom the timeline
+ * reaches, and a byte each, so an hour of audio is a few megabytes of it.
+ */
+export const WAVEFORM_PEAKS_PER_SECOND = 800;
+
+/**
+ * The waveform of `file`'s primary audio track, or null when it has none:
+ * one byte a peak, `peaksPerSecond` of them a second, and the peak at a
+ * moment of the file at that moment's index. Decoded in a worker of its own,
+ * gone when it answers.
+ */
+export function deriveWaveform(file: Blob, peaksPerSecond = WAVEFORM_PEAKS_PER_SECOND): Promise<Uint8ClampedArray | null> {
+ const worker = new Worker(new URL('./waveform-worker.ts', import.meta.url), { type: 'module' });
+ return new Promise<Uint8ClampedArray | null>((resolve, reject) => {
+ worker.onmessage = ({ data }: MessageEvent<WaveformResponse>) => {
+ if ('error' in data) reject(new Error(data.error));
+ else resolve(data.peaks);
+ };
+ worker.onerror = (event) => reject(new Error(event.message || 'The waveform worker failed'));
+ worker.postMessage({ file, peaksPerSecond } satisfies WaveformRequest);
+ }).finally(() => worker.terminate());
+}
+
+/**
+ * The peaks of `data` at `target` positions: the loudest of each slice, and
+ * never fewer than one position per sample.
+ *
+ * Asked for more positions than there are samples — a waveform zoomed in past
+ * the resolution it was derived at — a position's slice is narrower than one
+ * sample, and every slice whose ends round to the same sample would read
+ * nothing at all: a fifth of them at the far end of the zoom, drawn as a
+ * regular pattern of gaps. Holding the sample the slice starts in draws what
+ * there is to draw instead; past that zoom there is no more detail to show.
+ */
+export function downsamplePeaks(data: Uint8ClampedArray, target: number): Uint8ClampedArray {
+ const peaks = new Uint8ClampedArray(target);
+ const ratio = data.length / target;
+
+ for (let i = 0; i < target; i++) {
+ const from = Math.floor(i * ratio);
+ const to = Math.max(from + 1, Math.floor((i + 1) * ratio));
+ let max = 0;
+
+ for (let index = from; index < to && index < data.length; index++) {
+ max = Math.max(max, data[index]!);
+ }
+
+ peaks[i] = max;
+ }
+
+ return peaks;
+}
diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.ts
index 0eeeb641..ef9f6294 100644
--- a/packages/assets/src/index.ts
+++ b/packages/assets/src/index.ts
@@ -11,4 +11,5 @@ export * from './library';
export * from './browser';
export * from './cache';
export * from './derive/peaks';
+export * from './derive/waveform';
export * from './derive/thumbnail';