The VS Code of video editing, built for agents.
TypeScript
Refactor peaks and waveform management in timeline
konstantin-paulusfcaa8b95ed72
diff --git a/apps/web/src/engine/timeline/peaks.ts b/apps/web/src/engine/timeline/peaks.tsindex 25a7693e..5980bac1 100644--- a/apps/web/src/engine/timeline/peaks.ts+++ b/apps/web/src/engine/timeline/peaks.ts@@ -2,79 +2,59 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-/**- * The peaks the timeline's waveforms are drawn from, in two layers.- *- * The asset layer decodes a file once, at a fixed high resolution, and hands- * every couple of seconds of it to IndexedDB. Peaks that fine are the better- * part of a kilobyte a second, so a few long files would be tens of megabytes- * held for a whole session; the store is what keeps them off the JS heap. It- * is emptied when it is opened, since the index of which chunks exist lives- * in memory — nothing written in an earlier session could be found again.- *- * The clip layer takes whichever chunks the visible stretch of one clip- * covers and downsamples them to one peak per pixel, which is the only- * resolution a draw pass can use. Zoom changes what a pixel is worth, so this- * layer is rebuilt as the timeline is zoomed; the asset layer never is.- *- * Both are asked for by the draw pass and filled in behind it: a request says- * what a clip needs, and the next frame draws whatever has arrived. Nothing- * here blocks a frame.- */--import { AudioSampleSink } from 'mediabunny';-import { getAudioTrack } from '@diffusionstudio/runtime';--import type { AudioSample } from 'mediabunny';-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';--/** Peaks a second the asset layer decodes at: finer than any zoom reaches. */-const SOURCE_PEAKS_PER_SECOND = 800;+import { Library } from '@diffusionstudio/runtime';+import { WAVEFORM_PEAKS_PER_SECOND, downsamplePeaks } from '@diffusionstudio/assets';-/** How much audio one stored chunk covers, in seconds. */-const CHUNK_DURATION = 2;+import type { World } from 'koota';+import type { AssetCache, AudioAsset, VideoAsset } from '@diffusionstudio/assets';/** How wide one drawn sample is, in pixels. */export const SAMPLE_WIDTH = 1;-const IDB_NAME = 'peak-cache-db';-const IDB_VERSION = 1;-const IDB_STORE = 'data';+/**+ * How much source either side of the visible stretch is cut with it, as a+ * fraction of that stretch: a scroll that stays inside the margin draws from+ * what has already been cut instead of cutting again.+ */+const WINDOW_MARGIN = 0.5;-/** Where a chunk of an asset's peaks sits in it, in seconds, and its key. */-type PeakChunk = {+/** The stretch of one clip's waveform it can draw, at the zoom it asked for. */+export type ClipSamples = {+ /** What it covers, in source seconds. */start: number;end: number;- id: string;-};--/** A chunk of peaks, downsampled to what one clip draws it at. */-export type ClipSamples = PeakChunk & {- data: Uint8ClampedArray;peaksPerSecond: number;-};--/** What the decode produced for one stretch of a file, before it is stored. */-type DecodedPeaks = {- start: number;- end: number;data: Uint8ClampedArray;};-/** The asset layer's record of one file: which chunks of it exist. */+/** The asset layer's record of one file's waveform. */type AssetPeaks = {- decoding: boolean;- /** True once the file has been read through, whatever came of it. */- decoded: boolean;- chunks: PeakChunk[];+ /** The derived waveform, once there is one to read from. */+ file: File | null;+ /** How many peaks it holds, which is how many bytes it is. */+ length: number;+ loading: boolean;+ /** True once the cache has answered, whatever it answered. */+ loaded: boolean;+};++/** A stretch of a waveform to cut, and what it is being cut for. */+type Window = {+ /** The stretch to read, in peaks, margins and all. */+ from: number;+ to: number;+ /** What of it is on screen, so a scroll inside the margins costs nothing. */+ visibleFrom: number;+ visibleTo: number;+ peaksPerSecond: number;};-/** The clip layer's record of one clip: the chunks it can draw, at its zoom. */+/** The clip layer's record of one clip: what it can draw, at its zoom. */type ClipPeaks = {assetId: string;- samples: ClipSamples[];- /** The range and zoom `samples` was built for; unchanged means no work. */- hash: string;+ samples: ClipSamples | null;+ /** What `samples` was cut for; unchanged means there is nothing to do. */+ cut: Window | null;updating: boolean;pending: PeakRequest | null;};@@ -94,26 +74,32 @@ const assetPeaks = new Map<string, AssetPeaks>();const clipPeaks = new Map<number, ClipPeaks>();/**- * The chunks `clip` can draw right now, ordered by where they start. Empty- * (or short) while the decode behind them is still running.+ * The stretch `clip` can draw right now, or null while the read behind it is+ * still running.*/-export function getClipSamples(clip: number): ClipSamples[] | undefined {- return clipPeaks.get(clip)?.samples;+export function getClipSamples(clip: number): ClipSamples | null {+ return clipPeaks.get(clip)?.samples ?? null;}/**- * Asks for the peaks a clip needs. Starts the file's decode the first time it- * is asked about, and re-cuts the clip's own samples whenever the stretch it- * wants or the zoom it wants them at has changed.+ * Asks for the peaks a clip needs. Has the file's waveform derived the first+ * time it is asked about, and re-cuts the clip's own samples whenever the+ * stretch it wants has moved out of what was cut, or the zoom it wants them+ * at has changed.*/-export function requestPeaks(request: PeakRequest): void {+export function requestPeaks(world: World, request: PeakRequest): void {+ // The waveform is the cache's to derive and to keep. Until a project has+ // attached one there is nothing to draw from and nothing to do about it.+ const cache = world.get(Library)?.cache;+ if (!cache) return;+const record = assetPeaks.get(request.asset.id);- if (!record || (!record.decoding && !record.decoded)) {- void decodeAsset(request);+ if (!record || (!record.loading && !record.loaded)) {+ void loadAsset(cache, request);}- if (record && record.chunks.length > 0) {+ if (record && record.length > 0) {void updateClip(request, record);}}@@ -121,8 +107,8 @@ export function requestPeaks(request: PeakRequest): void {/*** Gives a clip's samples to the half that was cut off it. Both halves play* the same source at the same zoom, so the copy starts with everything the- * original had — the peak buffers themselves are shared, only the records- * that say what is being drawn are its own.+ * original had — the peak buffer itself is shared, only the record that says+ * what is being drawn is its own.*/export function clonePeaksForSplit(clip: number, copy: number): void {const source = clipPeaks.get(clip);@@ -130,8 +116,8 @@ export function clonePeaksForSplit(clip: number, copy: number): void {clipPeaks.set(copy, {assetId: source.assetId,- samples: source.samples.map((sample) => ({ ...sample })),- hash: source.hash,+ samples: source.samples && { ...source.samples },+ cut: source.cut && { ...source.cut },updating: false,pending: null,});@@ -143,7 +129,7 @@ export function clearClipPeaks(clip?: number): void {else clipPeaks.delete(clip);}-/** Forgets everything: the clips' samples and the index of what was decoded. */+/** Forgets everything: the clips' samples and the files they were read from. */export function clearPeaks(): void {clipPeaks.clear();assetPeaks.clear();@@ -161,101 +147,31 @@ export function forgetAssetPeaks(assetId: string): void {// The asset layer/**- * Reads the file through once, writing its peaks to the store a couple of- * seconds at a time. Each chunk is usable the moment it lands, so a long file- * draws from the top down rather than all at once at the end.+ * Takes the waveform the cache has for a file, waiting on the decode where it+ * has to derive one. Marked loaded either way: a file with no audio in it, or+ * one that would not decode, is not worth asking about on every frame.*/-async function decodeAsset(request: PeakRequest): Promise<void> {+async function loadAsset(cache: AssetCache, request: PeakRequest): Promise<void> {const { asset } = request;- const record = assetPeaks.get(asset.id) ?? { decoding: false, decoded: false, chunks: [] };+ const record = assetPeaks.get(asset.id) ?? { file: null, length: 0, loading: false, loaded: false };assetPeaks.set(asset.id, record);- if (record.decoding) return;+ if (record.loading) return;- record.decoding = true;+ record.loading = true;try {- const track = await getAudioTrack(asset);- if (!track) return;-- const sink = new AudioSampleSink(track);- const pending: DecodedPeaks[] = [];- let covered = 0;-- const flush = async (): Promise<void> => {- const merged = mergePeaks(pending);- const id = `${asset.id}-${merged.start}`;- record.chunks.push({ start: merged.start, end: merged.end, id });- pending.length = 0;- covered = 0;- await storePeaks(id, merged.data);- };-- for await (const sample of sink.samples()) {- covered += sample.duration;- try {- pending.push(peaksOfSample(sample));- } finally {- sample.close();- }-- if (covered > CHUNK_DURATION) await flush();- }+ const file = await cache.waveform(asset);+ if (!file) return;- if (pending.length > 0) await flush();-- record.chunks.sort((a, b) => a.start - b.start);+ record.file = file;+ record.length = file.size;} catch (error) {- console.error('[timeline] could not decode peaks', error);+ console.error('[timeline] could not load peaks', error);} finally {- record.decoding = false;- record.decoded = true;- }-}--/**- * One decoded sample's peaks: the loudest frame of each slice, across every- * channel, raised to a power so quiet passages still show as something.- */-function peaksOfSample(sample: AudioSample): DecodedPeaks {- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });- const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });-- const channels = sample.numberOfChannels;- const frames = floats.length / channels;- const target = Math.ceil(sample.duration * SOURCE_PEAKS_PER_SECOND);- const peaks = new Uint8ClampedArray(target);- const ratio = frames / target;-- for (let i = 0; i < target; i++) {- const from = Math.floor(i * ratio);- const to = Math.floor((i + 1) * ratio);- let max = 0;-- for (let frame = from; frame < to && frame < frames; frame++) {- for (let channel = 0; channel < channels; channel++) {- max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));- }- }-- peaks[i] = Math.floor(255 * max);- }-- return { start: sample.timestamp, end: sample.timestamp + sample.duration, data: peaks };-}--function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {- const length = peaks.reduce((total, peak) => total + peak.data.length, 0);- const merged = new Uint8ClampedArray(length);-- let offset = 0;- for (const peak of peaks) {- merged.set(peak.data, offset);- offset += peak.data.length;+ record.loading = false;+ record.loaded = true;}-- return { start: peaks[0]!.start, end: peaks[peaks.length - 1]!.end, data: merged };}// ---------------------------------------------------------------------------@@ -264,13 +180,12 @@ function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {/*** Re-cuts a clip's samples for the stretch and zoom it is asking about. Only* one runs at a time per clip — a zoom or a scroll that arrives while one is- * reading the store replaces whatever was queued behind it, so the work that- * finally runs is for where the timeline ended up rather than where it went- * through.+ * reading replaces whatever was queued behind it, so the work that finally+ * runs is for where the timeline ended up rather than where it went through.*/async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<void> {const peaks = clipPeaks.get(request.clip)- ?? { assetId: request.asset.id, samples: [], hash: '', updating: false, pending: null };+ ?? { assetId: request.asset.id, samples: null, cut: null, updating: false, pending: null };clipPeaks.set(request.clip, peaks);if (peaks.updating) {@@ -279,30 +194,29 @@ async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<voi}try {- const range = coveringRange(request, record);- if (range.hash === peaks.hash) return;+ const wanted = coveringWindow(request, record);+ if (!wanted || !needsCut(peaks.cut, wanted)) return;peaks.updating = true;- peaks.hash = range.hash;-- for (const chunk of record.chunks) {- if (!inRange(chunk.start, range.start, range.end)) continue;+ peaks.cut = wanted;- const data = await loadPeaks(chunk.id);- if (!data) continue;-- peaks.samples = peaks.samples.filter((sample) => sample.id !== chunk.id);- peaks.samples.push({- ...chunk,- data: downsample(data, Math.ceil((chunk.end - chunk.start) * request.peaksPerSecond)),- peaksPerSecond: request.peaksPerSecond,- });- peaks.samples.sort((a, b) => a.start - b.start);+ const data = await readPeaks(request.asset.id, record, wanted.from, wanted.to);+ if (!data) {+ peaks.cut = null;+ return;}- peaks.samples = peaks.samples.filter((sample) => inRange(sample.start, range.start, range.end));+ const seconds = data.length / WAVEFORM_PEAKS_PER_SECOND;++ peaks.samples = {+ start: wanted.from / WAVEFORM_PEAKS_PER_SECOND,+ end: (wanted.from + data.length) / WAVEFORM_PEAKS_PER_SECOND,+ peaksPerSecond: request.peaksPerSecond,+ data: downsamplePeaks(data, Math.max(1, Math.round(seconds * request.peaksPerSecond))),+ };} catch (error) {console.error('[timeline] could not cut peaks', error);+ peaks.cut = null;} finally {peaks.updating = false;}@@ -315,102 +229,44 @@ async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<voi}/**- * The stretch of the asset to cut, snapped out to whole chunks: what is on- * screen, widened to the chunks its ends fall inside, so the waveform does- * not stop short of the edge of the viewport.+ * The stretch of the waveform to cut, in peaks: what is on screen, widened by+ * a margin either side so a scroll draws from what has already been cut, and+ * clamped to what there is to read. Null while there is nothing.*/-function coveringRange(request: PeakRequest, record: AssetPeaks) {- const from = Math.max(0, request.start);- const to = Math.min(request.asset.duration, request.end);-- const first = record.chunks.find((chunk) => inRange(from, chunk.start, chunk.end));- const last = record.chunks.find((chunk) => inRange(to, chunk.start, chunk.end));-- const start = first?.start ?? record.chunks.at(0)?.start ?? 0;- const end = last?.end ?? record.chunks.at(-1)?.end ?? 0;-- return { start, end, hash: `${request.peaksPerSecond}.${start}.${end}` };-}--/**- * The peaks of `data` at `target` positions: the loudest of each slice, and- * never fewer than one sample per position.- *- * Zoomed in past the resolution the peaks were stored at, a position's slice- * of the source is narrower than one stored sample, and every slice whose- * ends round to the same sample would read nothing at all — a fifth of them- * at the far end of the zoom, drawn as a regular pattern of gaps through the- * waveform. Holding the sample the slice starts in draws what there is to- * draw instead: past that zoom the file has no more detail to show.- */-function downsample(data: Uint8ClampedArray, target: number): Uint8ClampedArray {- const peaks = new Uint8ClampedArray(target);- const ratio = data.length / target;-- for (let i = 0; i < target; i++) {- const from = Math.floor(i * ratio);- const to = Math.max(from + 1, Math.floor((i + 1) * ratio));- let max = 0;-- for (let index = from; index < to && index < data.length; index++) {- max = Math.max(max, data[index]!);- }-- peaks[i] = max;- }-- return peaks;-}+function coveringWindow(request: PeakRequest, record: AssetPeaks): Window | null {+ const peaksOf = (seconds: number): number => seconds * WAVEFORM_PEAKS_PER_SECOND;+ const clamp = (peak: number): number => Math.min(record.length, Math.max(0, peak));-const inRange = (value: number, from: number, to: number): boolean => value >= from && value <= to;--// ----------------------------------------------------------------------------// The store+ const visibleFrom = clamp(Math.floor(peaksOf(request.start)));+ const visibleTo = clamp(Math.ceil(peaksOf(request.end)));-let database: Promise<IDBDatabase> | null = null;+ const margin = peaksOf(Math.max(0, request.end - request.start)) * WINDOW_MARGIN;+ const from = clamp(Math.floor(peaksOf(request.start) - margin));+ const to = clamp(Math.ceil(peaksOf(request.end) + margin));+ if (to <= from) return null;-function getDatabase(): Promise<IDBDatabase> {- database ??= new Promise((resolve, reject) => {- const request = indexedDB.open(IDB_NAME, IDB_VERSION);-- request.onupgradeneeded = () => request.result.createObjectStore(IDB_STORE);- request.onerror = () => reject(request.error);-- request.onsuccess = () => {- // What an earlier session left behind is unreachable — the index of- // which chunks exist was never written — so the session starts on- // an empty store rather than one that only grows.- const transaction = request.result.transaction(IDB_STORE, 'readwrite');- transaction.objectStore(IDB_STORE).clear();- transaction.oncomplete = () => resolve(request.result);- transaction.onerror = () => reject(transaction.error);- };- });-- return database;+ return { from, to, visibleFrom, visibleTo, peaksPerSecond: request.peaksPerSecond };}-async function storePeaks(key: string, peaks: Uint8ClampedArray): Promise<void> {- const db = await getDatabase();- const transaction = db.transaction(IDB_STORE, 'readwrite');-- transaction.objectStore(IDB_STORE).put(new Blob([new Uint8Array(peaks)]), key);+/** Whether what is on screen is outside what the last cut covers. */+function needsCut(cut: Window | null, wanted: Window): boolean {+ if (!cut) return true;+ if (cut.peaksPerSecond !== wanted.peaksPerSecond) return true;- return new Promise((resolve, reject) => {- transaction.oncomplete = () => resolve();- transaction.onerror = () => reject(transaction.error);- });+ return wanted.visibleFrom < cut.from || wanted.visibleTo > cut.to;}-async function loadPeaks(key: string): Promise<Uint8ClampedArray | null> {- const db = await getDatabase();- const store = db.transaction(IDB_STORE, 'readonly').objectStore(IDB_STORE);-- const blob = await new Promise<Blob | null>((resolve, reject) => {- const request = store.get(key);- request.onsuccess = () => resolve(request.result ?? null);- request.onerror = () => reject(request.error);- });+/** The peaks in `[from, to)`, off disk: only the bytes that stretch covers. */+async function readPeaks(assetId: string, record: AssetPeaks, from: number, to: number): Promise<Uint8ClampedArray | null> {+ if (!record.file) return null;- return blob ? new Uint8ClampedArray(await blob.arrayBuffer()) : null;+ try {+ return new Uint8ClampedArray(await record.file.slice(from, to).arrayBuffer());+ } catch (error) {+ // The file went out from under us: forget it, and let the next frame+ // that asks find it again or have it derived afresh.+ console.warn('[timeline] could not read stored peaks', error);+ assetPeaks.delete(assetId);+ return null;+ }}diff --git a/apps/web/src/engine/timeline/render/waveform.ts b/apps/web/src/engine/timeline/render/waveform.tsindex 6b18c948..725bbc9b 100644--- a/apps/web/src/engine/timeline/render/waveform.ts+++ b/apps/web/src/engine/timeline/render/waveform.ts@@ -10,7 +10,6 @@ import { framesToPixels, getFrameRate, getResolution, getViewport } from '../vieimport type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';import type { Entity, World } from 'koota';-import type { ClipSamples } from '../peaks';import type { RowCursor } from '../layout';import type { TimelineSurfaceState } from '../surface';@@ -32,8 +31,9 @@ export type WaveformOptions = {* A column is a moment of the source, so what it is worth changes with the* zoom: the peaks are asked for at one per pixel (see `../peaks`), and only* for the stretch of the clip that is on screen. Drawing then reads them back- * by time rather than by index, so a column drawn before its peaks have- * arrived is simply left out and filled in on a later frame.+ * by time rather than by index, so a column outside what has been cut — a+ * waveform not derived yet, or the margin a scroll has just left — is simply+ * left out and filled in on a later frame.** Tall rows show the waveform about its middle, short ones stand it on the* bottom of the clip and fade it, so the label above it stays readable.@@ -92,7 +92,7 @@ export function renderWaveform(const firstSample = Math.max(inPx, viewportLeft - VIEWPORT_MARGIN - originPx);const lastSample = Math.min(outPx, viewportRight + VIEWPORT_MARGIN - originPx);- requestPeaks({+ requestPeaks(world, {clip: eid,asset: options.asset,start: pixelsToSeconds(firstSample) * playbackRate,@@ -114,19 +114,11 @@ export function renderWaveform(const samples = getClipSamples(eid);- // The chunks are ordered by where they start and the columns are drawn- // left to right, so the search for the chunk a column falls in only ever- // moves forwards.- let cursor = 0;-- for (let x = firstSample; x < lastSample; x += SAMPLE_WIDTH) {+ for (let x = firstSample; samples && x < lastSample; x += SAMPLE_WIDTH) {const time = pixelsToSeconds(x) * playbackRate;+ if (time < samples.start || time > samples.end) continue;- cursor = findChunk(samples, time, cursor);- const chunk = samples?.[cursor];- if (!chunk || time < chunk.start || time > chunk.end) continue;-- const peak = chunk.data[Math.floor((time - chunk.start) * chunk.peaksPerSecond)];+ const peak = samples.data[Math.floor((time - samples.start) * samples.peaksPerSecond)];if (peak === undefined) continue;const height = Math.max((peak / 255) * waveformHeight, minSampleHeight);@@ -135,13 +127,3 @@ export function renderWaveform(ctx.restore();}--/** The first chunk at or after `from` that has not already ended by `time`. */-function findChunk(samples: ClipSamples[] | undefined, time: number, from: number): number {- if (!samples) return 0;-- let index = from;- while (index < samples.length && samples[index]!.end < time) index++;-- return index;-}diff --git a/packages/assets/src/cache.ts b/packages/assets/src/cache.tsindex df00a15b..81737049 100644--- a/packages/assets/src/cache.ts+++ b/packages/assets/src/cache.ts@@ -3,8 +3,9 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { derivePeaks } from './derive/peaks';+import { PEAK_BARS } from './derive/peaks';import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH } from './derive/thumbnail';+import { deriveWaveform, downsamplePeaks } from './derive/waveform';import type { ProjectFS } from './fs';import type { Asset } from './types';@@ -41,6 +42,19 @@ export const PEAKS: CacheKind<Uint8ClampedArray> = {decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),};+/**+ * The full waveform: the whole file at `WAVEFORM_PEAKS_PER_SECOND`, one byte+ * a peak and nothing else, so the peak at a moment of the file is the byte at+ * that moment's index. The one entry read a stretch at a time rather than+ * whole (see `ensure`) — an hour of audio is a few megabytes of it.+ */+export const WAVEFORM: CacheKind<Uint8ClampedArray> = {+ dir: 'waveforms',+ ext: 'u8',+ encode: (peaks) => new Blob([peaks as Uint8ClampedArray<ArrayBuffer>]),+ decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),+};+/** What an entry is keyed by: an asset, or just its id. */type Keyed = string | Pick<Asset, 'id'>;@@ -50,6 +64,8 @@ export class AssetCache {private readonly fs: ProjectFS;/** Reads and derivations under way, by `kind.dir/id[@variant]`. */private readonly inflight = new Map<string, Promise<unknown>>();+ /** The same, for entries asked for as files rather than as values. */+ private readonly files = new Map<string, Promise<File | null>>();/** Every kind used, so `remove` and `prune` know which directories to look in. */private readonly kinds = new Map<string, CacheKind<unknown>>();@@ -57,6 +73,7 @@ export class AssetCache {this.fs = fs;this.register(THUMBNAIL);this.register(PEAKS);+ this.register(WAVEFORM);}// -----------------------------------------------------------------------@@ -72,9 +89,27 @@ export class AssetCache {return this.get(THUMBNAIL, asset, async () => deriveThumbnail(await asset.handle.getFile(), asset.mimeType, width), variant);}- /** The asset's rough waveform, or null when it has no audio. */+ /**+ * The asset's rough waveform, or null when it has no audio. Cut down from+ * the full one, so a file is decoded once however many resolutions of it+ * are asked for.+ */public peaks(asset: Asset): Promise<Uint8ClampedArray | null> {- return this.get(PEAKS, asset, async () => derivePeaks(await asset.handle.getFile()));+ return this.get(PEAKS, asset, async () => {+ const waveform = await this.waveform(asset);+ return waveform && downsamplePeaks(new Uint8ClampedArray(await waveform.arrayBuffer()), PEAK_BARS);+ });+ }++ /**+ * The file the asset's full waveform is stored in, derived first where+ * there is none, or null when it has no audio. A file rather than its+ * bytes: the timeline draws the stretch of a clip that is on screen by+ * reading the bytes that stretch covers (`file.slice(from, to)`) and no+ * more.+ */+ public waveform(asset: Asset): Promise<File | null> {+ return this.ensure(WAVEFORM, asset, async () => deriveWaveform(await asset.handle.getFile()));}// -----------------------------------------------------------------------@@ -96,6 +131,23 @@ export class AssetCache {return promise;}+ /**+ * The file (kind, asset[, variant]) is stored in, `produce()`d and written+ * there first where there is none: for entries whose consumer reads a+ * stretch of them at a time rather than all of them at once. Concurrent+ * calls share one derivation; null when there was nothing to derive.+ */+ public ensure<T>(kind: CacheKind<T>, keyed: Keyed, produce: () => Promise<T | null>, variant?: string): Promise<File | null> {+ this.register(kind);+ const key = this.key(kind, keyed, variant);+ const running = this.files.get(key);+ if (running) return running;++ const promise = this.loadFile(kind, key, produce).finally(() => this.files.delete(key));+ this.files.set(key, promise);+ return promise;+ }+/** Drops every entry of an asset on disk. */public async remove(keyed: Keyed): Promise<void> {const id = idOf(keyed);@@ -158,6 +210,38 @@ export class AssetCache {return value;}+ /**+ * The entry's file, derived and written where there is none. The write is+ * awaited, unlike `load`'s: here the file is the value.+ */+ private async loadFile<T>(kind: CacheKind<T>, key: string, produce: () => Promise<T | null>): Promise<File | null> {+ const path = this.pathOf(kind, key);+ const stored = await this.fileAt(path);+ if (stored && stored.size > 0) return stored;++ let value: T | null;+ try {+ value = await produce();+ } catch (error) {+ console.warn(`[assets] could not derive ${key}:`, error);+ value = null;+ }+ if (value === null) return null;++ await this.write(kind, key, value);+ return this.fileAt(path);+ }++ /** The file at a cache path, or null when it is missing or unreadable. */+ private async fileAt(path: string): Promise<File | null> {+ try {+ if (!(await this.fs.stat(path))) return null;+ return await this.fs.file(path);+ } catch {+ return null;+ }+ }+/** The entry on disk, or undefined when missing or unreadable. */private async read<T>(kind: CacheKind<T>, key: string): Promise<T | undefined> {const path = this.pathOf(kind, key);diff --git a/packages/assets/src/derive/peaks-worker.ts b/packages/assets/src/derive/peaks-worker.tsdeleted file mode 100644index f727db3c..00000000--- a/packages/assets/src/derive/peaks-worker.ts+++ /dev/null@@ -1,94 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// The rough waveform of a media file, computed off the main thread: a fixed-// number of bars, each the peak amplitude (across channels, gamma-lifted so-// quiet passages still show) of its slice of the file, as a byte. WebCodecs-// decodes elsewhere already, but PCM formats (WAV) are unpacked in JS and-// would stall the UI. One worker per request: a file and a bar count in,-// the peaks out.--import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';--export interface PeaksRequest {- file: Blob;- bars: number;-}--export type PeaksResponse = { peaks: Uint8ClampedArray | null } | { error: string };--// The package is typed against the DOM lib, so the worker scope is narrowed by hand.-const scope = self as unknown as {- onmessage: ((event: MessageEvent<PeaksRequest>) => void) | null;- postMessage(message: PeaksResponse, transfer?: Transferable[]): void;-};--scope.onmessage = async ({ data: { file, bars } }) => {- try {- const peaks = await computePeaks(file, bars);- scope.postMessage({ peaks }, peaks ? [peaks.buffer as ArrayBuffer] : []);- } catch (error) {- scope.postMessage({ error: error instanceof Error ? error.message : String(error) });- }-};--/** The peaks of `file`'s primary audio track, or null when it has none. */-async function computePeaks(file: Blob, bars: number): Promise<Uint8ClampedArray | null> {- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });- try {- const track = await input.getPrimaryAudioTrack();- if (!track) return null;-- const sink = new AudioSampleSink(track);- const chunks: Uint8ClampedArray[] = [];-- for await (const sample of sink.samples()) {- let floats: Float32Array;- let channels: number;- let frames: number;- try {- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });- floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });- channels = sample.numberOfChannels;- frames = floats.length / channels;- } finally {- sample.close();- }-- const framePeaks = new Uint8ClampedArray(frames);- for (let f = 0; f < frames; f++) {- let max = 0;- for (let ch = 0; ch < channels; ch++) {- max = Math.max(max, Math.pow(Math.abs(floats[f * channels + ch]!), 0.8));- }- framePeaks[f] = Math.floor(max * 255);- }- chunks.push(framePeaks);- }-- if (chunks.length === 0) return null;-- const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);- const merged = new Uint8ClampedArray(total);- let offset = 0;- for (const chunk of chunks) {- merged.set(chunk, offset);- offset += chunk.length;- }-- const peaks = new Uint8ClampedArray(bars);- const ratio = total / bars;- for (let i = 0; i < bars; i++) {- const start = Math.floor(i * ratio);- const end = Math.min(total, Math.floor((i + 1) * ratio));- let max = 0;- for (let j = start; j < end; j++) max = Math.max(max, merged[j]!);- peaks[i] = max;- }- return peaks;- } finally {- input.dispose();- }-}diff --git a/packages/assets/src/derive/peaks.ts b/packages/assets/src/derive/peaks.tsindex 7a7455f9..4399654e 100644--- a/packages/assets/src/derive/peaks.ts+++ b/packages/assets/src/derive/peaks.ts@@ -2,23 +2,17 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import type { PeaksRequest, PeaksResponse } from './peaks-worker';+import { deriveWaveform, downsamplePeaks } from './waveform';/** How many bars a rough waveform has. */export const PEAK_BARS = 128;/**- * The peaks of `file`'s primary audio track, or null when it has none.- * Decoded in a worker of its own, gone when it answers.+ * The rough waveform of `file`: a fixed number of bars, each the peak+ * amplitude of its slice of the file, or null when it has no audio. The full+ * waveform cut down — a file is decoded at one resolution and read at any.*/-export function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {- const worker = new Worker(new URL('./peaks-worker.ts', import.meta.url), { type: 'module' });- return new Promise<Uint8ClampedArray | null>((resolve, reject) => {- worker.onmessage = ({ data }: MessageEvent<PeaksResponse>) => {- if ('error' in data) reject(new Error(data.error));- else resolve(data.peaks);- };- worker.onerror = (event) => reject(new Error(event.message || 'The peaks worker failed'));- worker.postMessage({ file, bars } satisfies PeaksRequest);- }).finally(() => worker.terminate());+export async function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {+ const waveform = await deriveWaveform(file);+ return waveform && downsamplePeaks(waveform, bars);}diff --git a/packages/assets/src/derive/waveform-worker.ts b/packages/assets/src/derive/waveform-worker.tsnew file mode 100644index 00000000..1d16d490--- /dev/null+++ b/packages/assets/src/derive/waveform-worker.ts@@ -0,0 +1,101 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The waveform of a media file, computed off the main thread: the peak+// amplitude of each slice of its audio (across channels, gamma-lifted so+// quiet passages still show), as a byte. WebCodecs decodes elsewhere already,+// but PCM formats (WAV) are unpacked in JS and would stall the UI. One worker+// per request: a file and a resolution in, the whole waveform out.++import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';++export interface WaveformRequest {+ file: Blob;+ peaksPerSecond: number;+}++export type WaveformResponse = { peaks: Uint8ClampedArray | null } | { error: string };++// The package is typed against the DOM lib, so the worker scope is narrowed by hand.+const scope = self as unknown as {+ onmessage: ((event: MessageEvent<WaveformRequest>) => void) | null;+ postMessage(message: WaveformResponse, transfer?: Transferable[]): void;+};++scope.onmessage = async ({ data: { file, peaksPerSecond } }) => {+ try {+ const peaks = await computeWaveform(file, peaksPerSecond);+ scope.postMessage({ peaks }, peaks ? [peaks.buffer as ArrayBuffer] : []);+ } catch (error) {+ scope.postMessage({ error: error instanceof Error ? error.message : String(error) });+ }+};++/**+ * The waveform of `file`'s primary audio track, or null when it has none.+ *+ * Every peak is written where it belongs rather than appended, so a track+ * that starts at a timestamp of its own or has a gap in it still lines up+ * with the time it is drawn against.+ */+async function computeWaveform(file: Blob, peaksPerSecond: number): Promise<Uint8ClampedArray | null> {+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });+ try {+ const track = await input.getPrimaryAudioTrack();+ if (!track) return null;++ // The track's own end, so a file whose sound outlasts its picture (or+ // starts after it) is as long here as it is on disk.+ const peaks = new Uint8ClampedArray(Math.ceil((await track.computeDuration()) * peaksPerSecond));+ const sink = new AudioSampleSink(track);+ let covered = 0;++ for await (const sample of sink.samples()) {+ let floats: Float32Array;+ let channels: number;+ let timestamp: number;+ let duration: number;++ // Everything the sample holds is taken before it is closed, and it+ // is closed however that goes: they are decoder resources.+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });+ channels = sample.numberOfChannels;+ timestamp = sample.timestamp;+ duration = sample.duration;+ } finally {+ sample.close();+ }++ const frames = floats.length / channels;+ const from = Math.floor(timestamp * peaksPerSecond);+ const to = Math.min(peaks.length, from + Math.ceil(duration * peaksPerSecond));+ const ratio = frames / Math.max(1, to - from);++ for (let peak = from; peak < to; peak++) {+ const start = Math.floor((peak - from) * ratio);+ const end = Math.floor((peak - from + 1) * ratio);+ let max = 0;++ for (let frame = start; frame < end && frame < frames; frame++) {+ for (let channel = 0; channel < channels; channel++) {+ max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));+ }+ }++ peaks[peak] = Math.floor(255 * max);+ }++ covered = Math.max(covered, to);+ }++ if (covered === 0) return null;++ return covered === peaks.length ? peaks : peaks.slice(0, covered);+ } finally {+ input.dispose();+ }+}diff --git a/packages/assets/src/derive/waveform.ts b/packages/assets/src/derive/waveform.tsnew file mode 100644index 00000000..3f14f5cf--- /dev/null+++ b/packages/assets/src/derive/waveform.ts@@ -0,0 +1,59 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { WaveformRequest, WaveformResponse } from './waveform-worker';++/**+ * Peaks a second a derived waveform holds: finer than any zoom the timeline+ * reaches, and a byte each, so an hour of audio is a few megabytes of it.+ */+export const WAVEFORM_PEAKS_PER_SECOND = 800;++/**+ * The waveform of `file`'s primary audio track, or null when it has none:+ * one byte a peak, `peaksPerSecond` of them a second, and the peak at a+ * moment of the file at that moment's index. Decoded in a worker of its own,+ * gone when it answers.+ */+export function deriveWaveform(file: Blob, peaksPerSecond = WAVEFORM_PEAKS_PER_SECOND): Promise<Uint8ClampedArray | null> {+ const worker = new Worker(new URL('./waveform-worker.ts', import.meta.url), { type: 'module' });+ return new Promise<Uint8ClampedArray | null>((resolve, reject) => {+ worker.onmessage = ({ data }: MessageEvent<WaveformResponse>) => {+ if ('error' in data) reject(new Error(data.error));+ else resolve(data.peaks);+ };+ worker.onerror = (event) => reject(new Error(event.message || 'The waveform worker failed'));+ worker.postMessage({ file, peaksPerSecond } satisfies WaveformRequest);+ }).finally(() => worker.terminate());+}++/**+ * The peaks of `data` at `target` positions: the loudest of each slice, and+ * never fewer than one position per sample.+ *+ * Asked for more positions than there are samples — a waveform zoomed in past+ * the resolution it was derived at — a position's slice is narrower than one+ * sample, and every slice whose ends round to the same sample would read+ * nothing at all: a fifth of them at the far end of the zoom, drawn as a+ * regular pattern of gaps. Holding the sample the slice starts in draws what+ * there is to draw instead; past that zoom there is no more detail to show.+ */+export function downsamplePeaks(data: Uint8ClampedArray, target: number): Uint8ClampedArray {+ const peaks = new Uint8ClampedArray(target);+ const ratio = data.length / target;++ for (let i = 0; i < target; i++) {+ const from = Math.floor(i * ratio);+ const to = Math.max(from + 1, Math.floor((i + 1) * ratio));+ let max = 0;++ for (let index = from; index < to && index < data.length; index++) {+ max = Math.max(max, data[index]!);+ }++ peaks[i] = max;+ }++ return peaks;+}diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.tsindex 0eeeb641..ef9f6294 100644--- a/packages/assets/src/index.ts+++ b/packages/assets/src/index.ts@@ -11,4 +11,5 @@ export * from './library';export * from './browser';export * from './cache';export * from './derive/peaks';+export * from './derive/waveform';export * from './derive/thumbnail';