Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor timeline media and peaks management

konstantin-paulus03f1078b3c52
diff --git a/apps/web/src/engine/asset-actions.ts b/apps/web/src/engine/asset-actions.ts
index 69b274c1..b79e238d 100644
--- a/apps/web/src/engine/asset-actions.ts
+++ b/apps/web/src/engine/asset-actions.ts
@@ -8,7 +8,8 @@
import { toast } from "somoto";
import { importFiles as importFilesInto, pickFiles, saveAssetAs as saveAs } from "@diffusionstudio/assets";
import { insertAsset } from "./insert-asset";
-import { forgetPoster } from "./timeline/media";
+import { forgetAssetMedia } from "./timeline/media";
+import { forgetAssetPeaks } from "./timeline/peaks";
import type { World } from "koota";
@@ -55,8 +56,10 @@ export async function replaceAssetSource(library: AssetLibrary, asset: Asset): P
if (!path) return null;
try {
const relinked = await library.relink(asset, path);
- // The picture the timeline is showing is of the file it used to be.
- forgetPoster(asset.id);
+ // The picture and the waveform the timeline is showing are of the file
+ // it used to be.
+ forgetAssetMedia(asset.id);
+ forgetAssetPeaks(asset.id);
return relinked;
} catch (error) {
toast.error("Failed to replace", { description: (error as Error).message });
diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.ts
index e03d7cc6..30abafe6 100644
--- a/apps/web/src/engine/create-engine.ts
+++ b/apps/web/src/engine/create-engine.ts
@@ -2,12 +2,12 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';
+import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Geometry, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';
import { hudSystem } from './hud';
import { createSignal, type Accessor, type Setter } from 'solid-js';
import { AssetSelection, Hud, Keys, Pointer, PointerEvents, ProjectConfig, SnapLines } from './traits';
import { inputSystem } from './input/input-system';
-import { timelineSystem, TimelineSurface } from './timeline';
+import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';
import { shortcutSystem } from './input/shortcuts';
import { sourceErrorSystem } from './source-errors';
@@ -54,6 +54,13 @@ class Engine {
this.unsubscribe.push(
this.world.onAdd(ChildOf('*'), () => (this.interactiveDirty = true)),
this.world.onRemove(ChildOf('*'), () => (this.interactiveDirty = true)),
+ // The timeline keeps a clip's waveform and thumbnails by entity id,
+ // and koota hands ids back out: a clip that is gone has to take its
+ // pictures with it, or the next clip to be given its id inherits them.
+ this.world.onRemove(Geometry, (entity) => {
+ clearClipPeaks(entity.id());
+ clearClipFrames(entity.id());
+ }),
);
[this.running, this.setRunning] = createSignal(false);
@@ -268,6 +275,11 @@ class Engine {
this.audioContext.close();
}
+ // The timeline's decoded pictures and peaks are of this project's
+ // files, and are held outside the world: they go with the engine.
+ clearPeaks();
+ clearMedia();
+
this.unsubscribeEventListeners();
this.unsubscribe.forEach(unsubscribe => unsubscribe());
this.unsubscribe.length = 0;
diff --git a/apps/web/src/engine/split.tsx b/apps/web/src/engine/split.tsx
index b2143667..dee88f1f 100644
--- a/apps/web/src/engine/split.tsx
+++ b/apps/web/src/engine/split.tsx
@@ -29,6 +29,7 @@ import {
import { Or } from 'koota';
import { getDocumentEditor } from './editor';
+import { cloneFramesForSplit, clonePeaksForSplit } from './timeline';
import { trimIn, trimOut } from './timing';
import type { Entity, World } from 'koota';
@@ -105,6 +106,11 @@ export function splitAtPlayhead(world: World): Entity[] {
trimOut(world, original, frame);
trimIn(world, copy, frame);
handOffDecoders(world, original, copy);
+ // The two halves play the same source at the same zoom, so the copy
+ // starts with the waveform and thumbnails the whole clip had rather
+ // than decoding them all over again.
+ clonePeaksForSplit(original.id(), copy.id());
+ cloneFramesForSplit(original.id(), copy.id());
}
// Two clips side by side under a scene are two layers of the timeline,
diff --git a/apps/web/src/engine/timeline/index.ts b/apps/web/src/engine/timeline/index.ts
index ad16eba3..f9d34035 100644
--- a/apps/web/src/engine/timeline/index.ts
+++ b/apps/web/src/engine/timeline/index.ts
@@ -8,6 +8,8 @@ export * from './controller';
export * from './drag';
export * from './snapping';
export * from './layout';
+export * from './media';
+export * from './peaks';
export * from './style';
export * from './surface';
export * from './timeline';
diff --git a/apps/web/src/engine/timeline/media.ts b/apps/web/src/engine/timeline/media.ts
index dba81144..24f4ec6e 100644
--- a/apps/web/src/engine/timeline/media.ts
+++ b/apps/web/src/engine/timeline/media.ts
@@ -3,60 +3,444 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
- * The pictures the timeline draws inside clips. The library derives and keeps
- * an asset's poster (see `AssetCache.thumbnail`); what is needed here is the
- * same picture decoded and ready to draw, which asking for it every frame
- * would not give — so it is asked for once and kept until the asset changes.
+ * The pictures the timeline draws inside clips, in the two kinds the two
+ * kinds of clip need.
*
- * Keyed by asset rather than by clip: two clips of the same footage show the
- * same picture, and a clip that is split into two does not need a second copy
- * of anything.
+ * A still — an image, or a frames directory shown by its first frame — is one
+ * picture tiled along the clip, so it is decoded once per asset at whatever
+ * size the row is currently drawn at, and kept until the row changes size.
+ *
+ * A video is a strip of its own frames, which is a decode per tile, so it
+ * comes in two layers like the peaks do. The asset layer decodes a handful of
+ * frames spread across the whole file, which is what any clip of it falls
+ * back on; the clip layer decodes the tiles the clip actually shows, for the
+ * stretch of it on screen, and re-decodes them when the zoom changes what a
+ * tile is worth. Nothing here blocks a frame: a draw pass asks for what it
+ * wants and draws the nearest picture that has arrived.
+ *
+ * Keyed by asset where two clips of the same footage would show the same
+ * picture, and by clip where they would not.
*/
-import { Library } from '@diffusionstudio/runtime';
+import { CanvasSink } from 'mediabunny';
+import { getAssetFile, getVideoTrack, secondsToFrames } from '@diffusionstudio/runtime';
+
+import { MAX_CLIP_HEIGHT } from './config';
+
+import type { InputVideoTrack } from 'mediabunny';
+import type { Asset, VideoAsset } from '@diffusionstudio/assets';
+
+/** One decoded picture, and where in the source it came from (source frames). */
+export type Frame = {
+ timestamp: number;
+ canvas: OffscreenCanvas | HTMLCanvasElement;
+ /** Superseded: kept on screen only until what replaces it has decoded. */
+ stale?: boolean;
+};
+
+/** A still, decoded at the size the row it is drawn in asked for. */
+export type Still = {
+ canvas: OffscreenCanvas;
+ width: number;
+ height: number;
+ hash: string;
+};
+
+/** What a video clip needs this frame: the tiles of the stretch on screen. */
+export type FrameRequest = {
+ /** The entity id of the clip asking. */
+ clip: number;
+ asset: VideoAsset;
+ /** How much source one tile covers, in source frames. May be fractional. */
+ interval: number;
+ /** Tile size in CSS pixels. */
+ width: number;
+ height: number;
+ /** The visible stretch of the source, in source frames. */
+ firstFrame: number;
+ lastFrame: number;
+ fps: number;
+};
-import type { Asset } from '@diffusionstudio/assets';
-import type { World } from 'koota';
+type AssetFrames = {
+ frames: Frame[];
+ decoding: boolean;
+ /** True once the spread has been decoded, whatever came of it. */
+ decoded: boolean;
+};
-/** How wide a poster is asked for: the widest tile any row height draws. */
-const POSTER_WIDTH = 300;
+type ClipFrames = {
+ assetId: string;
+ frames: Frame[];
+ /** The stretch `frames` covers, in source frames. */
+ start: number;
+ end: number;
+ request: FrameRequest | null;
+ pending: FrameRequest | null;
+ decoding: boolean;
+};
-const posters = new Map<string, ImageBitmap | null>();
-const loading = new Set<string>();
+/** How many frames of a video the fallback spread is made of. */
+const SPREAD_COUNT = 7;
+const LONG_SPREAD_COUNT = 10;
+
+/** Over this many seconds, a file gets the longer spread. */
+const LONG_DURATION = 60 * 10;
+
+/** How many tiles either side of the visible stretch are decoded ahead. */
+const TILE_MARGIN = 1;
+
+const stills = new Map<string, Still>();
+const stillsDecoding = new Set<string>();
+/** Assets whose file will not decode, so the draw pass stops asking. */
+const stillsFailed = new Set<string>();
+const assetFrames = new Map<string, AssetFrames>();
+const clipFrames = new Map<number, ClipFrames>();
+
+// ---------------------------------------------------------------------------
+// Stills
/**
- * The asset's poster, or null until it has loaded (the load is started here).
- * Null is also the answer for an asset that has no picture to show, and is
- * remembered as such so it is not asked for again.
+ * The picture an image or sequence clip is tiled with, at the size a row
+ * `width` wide draws it, or null until it has decoded (the decode is started
+ * here). Re-decoded when the row changes to a different tile size, which is
+ * one of three (see `../render/thumbnails`), so a drag of the row height
+ * costs at most two decodes.
*/
-export function resolvePoster(world: World, asset: Asset): ImageBitmap | null {
- const existing = posters.get(asset.id);
- if (existing !== undefined) return existing;
+export function resolveStill(asset: Asset, width: number): Still | null {
+ const hash = `${width}@${window.devicePixelRatio}`;
+ const existing = stills.get(asset.id);
+ if (existing?.hash === hash) return existing;
+
+ if (!stillsFailed.has(asset.id)) void decodeStill(asset, width, hash);
- if (loading.has(asset.id)) return null;
- loading.add(asset.id);
+ // Whatever is there at the wrong size still beats nothing at all: it is
+ // the same picture, and it is replaced the moment the right one lands.
+ return existing ?? null;
+}
- const cache = world.get(Library)?.cache;
- if (!cache) {
- loading.delete(asset.id);
- return null;
+async function decodeStill(asset: Asset, width: number, hash: string): Promise<void> {
+ if (stillsDecoding.has(asset.id)) return;
+ stillsDecoding.add(asset.id);
+
+ try {
+ const bitmap = await createImageBitmap(await getAssetFile(asset));
+
+ try {
+ // Big enough for the widest tile and the tallest row it could be
+ // drawn in, and never larger than the picture itself.
+ const dpr = window.devicePixelRatio;
+ const scale = Math.min(1, Math.max((width * dpr) / bitmap.width, (MAX_CLIP_HEIGHT * dpr) / bitmap.height));
+ const canvasWidth = Math.max(1, Math.round(bitmap.width * scale));
+ const canvasHeight = Math.max(1, Math.round(bitmap.height * scale));
+
+ const canvas = new OffscreenCanvas(canvasWidth, canvasHeight);
+ canvas.getContext('2d')!.drawImage(bitmap, 0, 0, canvasWidth, canvasHeight);
+
+ stills.set(asset.id, { canvas, width: canvasWidth, height: canvasHeight, hash });
+ } finally {
+ bitmap.close();
+ }
+ } catch (error) {
+ console.error('[timeline] could not decode still', error);
+ stillsFailed.add(asset.id);
+ } finally {
+ stillsDecoding.delete(asset.id);
}
+}
+
+// ---------------------------------------------------------------------------
+// Video frames
- void cache
- .thumbnail(asset, POSTER_WIDTH)
- .then((blob) => (blob ? createImageBitmap(blob) : null))
- .then((bitmap) => posters.set(asset.id, bitmap))
- .catch(() => posters.set(asset.id, null))
- .finally(() => loading.delete(asset.id));
+/**
+ * Asks for the tiles a video clip needs. Decodes the file's fallback spread
+ * the first time it is asked about, and re-decodes the clip's own tiles
+ * whenever the stretch on screen or the size of a tile has changed.
+ */
+export function requestFrames(request: FrameRequest): void {
+ const record = assetFrames.get(request.asset.id);
+
+ if (!record || (!record.decoding && !record.decoded)) {
+ void decodeSpread(request);
+ }
- return null;
+ if (record && record.frames.length > 0) {
+ void updateClip(request);
+ }
}
/**
- * Forgets an asset's poster, so the next frame asks for it again. Called when
- * an asset is re-pointed at another file, which the picture would outlive.
+ * The picture to draw for `timestamp` (source frames): the clip's own tile
+ * where its strip reaches that far, else the nearest of the file's fallback
+ * spread. Null while neither has decoded anything yet.
*/
-export function forgetPoster(assetId: string): void {
- posters.get(assetId)?.close();
- posters.delete(assetId);
+export function pickFrame(clip: number, assetId: string, timestamp: number, interval: number): Frame | null {
+ const cached = clipFrames.get(clip);
+ const covered = cached !== undefined
+ && timestamp >= cached.start - interval
+ && timestamp <= cached.end + interval;
+
+ const frames = (covered ? cached.frames : assetFrames.get(assetId)?.frames) ?? [];
+ const index = closestFrame(frames, timestamp);
+
+ return index < 0 ? null : frames[index]!;
+}
+
+/**
+ * Gives a clip's strip to the half that was cut off it, so the copy is drawn
+ * from the frames the original had rather than decoding them again. The
+ * canvases are shared; only the records saying what is on screen are copied.
+ */
+export function cloneFramesForSplit(clip: number, copy: number): void {
+ const source = clipFrames.get(clip);
+ if (!source) return;
+
+ clipFrames.set(copy, {
+ ...source,
+ frames: source.frames.map((frame) => ({ ...frame })),
+ pending: null,
+ decoding: false,
+ });
+}
+
+/** Forgets one clip's strip, or every clip's. */
+export function clearClipFrames(clip?: number): void {
+ if (clip === undefined) clipFrames.clear();
+ else clipFrames.delete(clip);
+}
+
+/** Forgets everything: the clips' strips, the spreads and the stills. */
+export function clearMedia(): void {
+ clipFrames.clear();
+ assetFrames.clear();
+ stills.clear();
+ stillsFailed.clear();
+}
+
+/**
+ * Forgets one asset's pictures, for a file that is no longer the one it was.
+ */
+export function forgetAssetMedia(assetId: string): void {
+ stills.delete(assetId);
+ stillsFailed.delete(assetId);
+ assetFrames.delete(assetId);
+ for (const [clip, frames] of clipFrames) {
+ if (frames.assetId === assetId) clipFrames.delete(clip);
+ }
+}
+
+/**
+ * A handful of frames from across the whole file. Coarse — they are a tenth
+ * of a long video apart — but they are what a clip shows the moment it lands
+ * on the timeline, and what it keeps showing wherever its own strip has not
+ * reached.
+ */
+async function decodeSpread(request: FrameRequest): Promise<void> {
+ const { asset } = request;
+
+ const record = assetFrames.get(asset.id) ?? { frames: [], decoding: false, decoded: false };
+ assetFrames.set(asset.id, record);
+ if (record.decoding) return;
+
+ record.decoding = true;
+
+ try {
+ const track = await getVideoTrack(asset);
+ if (!track) return;
+
+ const first = await track.getFirstTimestamp();
+ const last = await track.computeDuration();
+ const count = asset.duration < LONG_DURATION ? SPREAD_COUNT : LONG_SPREAD_COUNT;
+ const sink = createSink(track, request);
+
+ for (let i = 0; i < count; i++) {
+ try {
+ const wrapped = await sink.getCanvas(first + (i * (last - first)) / count);
+ if (!wrapped) continue;
+
+ record.frames.push({
+ timestamp: secondsToFrames(wrapped.timestamp, request.fps),
+ canvas: wrapped.canvas,
+ });
+ } catch {
+ // One frame that will not decode is not the file's whole spread.
+ }
+ }
+ } catch (error) {
+ console.error('[timeline] could not decode thumbnails', error);
+ } finally {
+ record.decoding = false;
+ record.decoded = true;
+ }
+}
+
+/**
+ * Decodes the tiles the clip is showing. Only what is missing: scrolling into
+ * new footage extends the strip at the end it moved towards, while a change
+ * of zoom or tile size invalidates it and decodes the visible stretch afresh.
+ *
+ * One decode runs at a time per clip, and a request that arrives while one is
+ * running replaces whatever was queued behind it — the work that finally runs
+ * is for where the timeline ended up, not everywhere it passed through.
+ */
+async function updateClip(request: FrameRequest): Promise<void> {
+ const cached = clipFrames.get(request.clip) ?? {
+ assetId: request.asset.id,
+ frames: [],
+ start: -1,
+ end: -1,
+ request: null,
+ pending: null,
+ decoding: false,
+ };
+ clipFrames.set(request.clip, cached);
+
+ if (cached.decoding) {
+ cached.pending = request;
+ return;
+ }
+
+ try {
+ cached.decoding = true;
+
+ if (rangeHash(cached.request) === rangeHash(request)) return;
+
+ const wanted = tileRange(request);
+ const resized = sizeHash(cached.request) !== sizeHash(request);
+ const rescaled = cached.request?.interval !== request.interval;
+ const overlaps = wanted.start < cached.end && cached.start < wanted.end;
+
+ let from = 0;
+ let to = 0;
+
+ if (rescaled || resized || !overlaps) {
+ // Nothing on screen is worth keeping: every tile is the wrong size
+ // or covers the wrong stretch, so the whole range is decoded again.
+ from = wanted.firstIndex;
+ to = wanted.lastIndex;
+ for (const frame of cached.frames) frame.stale = true;
+ } else if (wanted.start < cached.start) {
+ from = wanted.firstIndex;
+ to = Math.floor(cached.start / request.interval);
+ } else if (wanted.end > cached.end) {
+ from = Math.floor(cached.end / request.interval);
+ to = wanted.lastIndex;
+ }
+
+ for (const frame of cached.frames) {
+ if (frame.timestamp > wanted.end || frame.timestamp < wanted.start) frame.stale = true;
+ }
+
+ const count = to - from;
+
+ // The strip already reaches across everything on screen.
+ if (count <= 0) {
+ cached.frames = keep(cached.frames);
+ cached.start = wanted.start;
+ cached.end = wanted.end;
+ cached.request = request;
+ return;
+ }
+
+ const track = await getVideoTrack(request.asset);
+ if (!track) return;
+
+ const sink = createSink(track, request);
+
+ for (let i = 0; i < count; i++) {
+ try {
+ const wrapped = await sink.getCanvas(((from + i) * request.interval) / request.fps);
+ if (!wrapped) continue;
+
+ // The decoder may snap to the nearest keyframe, so the frame is
+ // filed under the timestamp it actually came back with.
+ const timestamp = secondsToFrames(wrapped.timestamp, request.fps);
+
+ // The frames it replaces are dropped only now, so the strip is
+ // never blank while the new ones are on their way.
+ cached.frames = keep(cached.frames, timestamp);
+ cached.frames.push({ timestamp, canvas: wrapped.canvas, stale: false });
+ } catch {
+ // One tile that will not decode is not the whole strip.
+ }
+ }
+
+ cached.frames = keep(cached.frames);
+ cached.start = wanted.start;
+ cached.end = wanted.end;
+ cached.request = request;
+ } catch (error) {
+ console.error('[timeline] could not update thumbnails', error);
+ } finally {
+ cached.decoding = false;
+ }
+
+ const pending = cached.pending;
+ if (pending) {
+ cached.pending = null;
+ void updateClip(pending);
+ }
+}
+
+/**
+ * The tiles to have decoded: the ones the visible stretch falls on, one
+ * either side, and never past the ends of the file.
+ */
+function tileRange(request: FrameRequest) {
+ const { firstFrame, lastFrame, interval, asset } = request;
+
+ const lastTile = Math.floor(secondsToFrames(asset.duration, request.fps) / interval);
+ const firstIndex = Math.max(Math.floor(firstFrame / interval) - TILE_MARGIN, 0);
+ const lastIndex = Math.min(Math.floor(lastFrame / interval) + TILE_MARGIN, lastTile);
+
+ return { firstIndex, lastIndex, start: firstIndex * interval, end: lastIndex * interval };
+}
+
+function createSink(track: InputVideoTrack, request: FrameRequest): CanvasSink {
+ const dpr = window.devicePixelRatio;
+ return new CanvasSink(track, { width: request.width * dpr, height: request.height * dpr, fit: 'cover' });
+}
+
+/** The strip without what has been superseded, in the order it is drawn. */
+function keep(frames: Frame[], upTo?: number): Frame[] {
+ return frames
+ .filter((frame) => !frame.stale || (upTo !== undefined && frame.timestamp > upTo))
+ .sort((a, b) => a.timestamp - b.timestamp);
+}
+
+function rangeHash(request: FrameRequest | null): string {
+ if (!request) return '';
+ return `${sizeHash(request)}-${request.interval}-${request.firstFrame}-${request.lastFrame}`;
+}
+
+function sizeHash(request: FrameRequest | null): string {
+ if (!request) return '';
+ return `${request.width}x${request.height}@${window.devicePixelRatio}`;
+}
+
+/** The frame nearest `timestamp`, or -1 when there are none. */
+function closestFrame(frames: Frame[], timestamp: number): number {
+ if (frames.length === 0) return -1;
+
+ let low = 0;
+ let high = frames.length - 1;
+ let closest = -1;
+ let distance = Infinity;
+
+ while (low <= high) {
+ const mid = (low + (high - low + 1) / 2) | 0;
+ const value = frames[mid]!.timestamp;
+ const delta = Math.abs(value - timestamp);
+
+ if (delta < distance) {
+ distance = delta;
+ closest = mid;
+ }
+
+ if (value < timestamp) low = mid + 1;
+ else if (value > timestamp) high = mid - 1;
+ else return mid;
+ }
+
+ return closest;
}
diff --git a/apps/web/src/engine/timeline/peaks.ts b/apps/web/src/engine/timeline/peaks.ts
new file mode 100644
index 00000000..25a7693e
--- /dev/null
+++ b/apps/web/src/engine/timeline/peaks.ts
@@ -0,0 +1,416 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The peaks the timeline's waveforms are drawn from, in two layers.
+ *
+ * The asset layer decodes a file once, at a fixed high resolution, and hands
+ * every couple of seconds of it to IndexedDB. Peaks that fine are the better
+ * part of a kilobyte a second, so a few long files would be tens of megabytes
+ * held for a whole session; the store is what keeps them off the JS heap. It
+ * is emptied when it is opened, since the index of which chunks exist lives
+ * in memory — nothing written in an earlier session could be found again.
+ *
+ * The clip layer takes whichever chunks the visible stretch of one clip
+ * covers and downsamples them to one peak per pixel, which is the only
+ * resolution a draw pass can use. Zoom changes what a pixel is worth, so this
+ * layer is rebuilt as the timeline is zoomed; the asset layer never is.
+ *
+ * Both are asked for by the draw pass and filled in behind it: a request says
+ * what a clip needs, and the next frame draws whatever has arrived. Nothing
+ * here blocks a frame.
+ */
+
+import { AudioSampleSink } from 'mediabunny';
+import { getAudioTrack } from '@diffusionstudio/runtime';
+
+import type { AudioSample } from 'mediabunny';
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
+
+/** Peaks a second the asset layer decodes at: finer than any zoom reaches. */
+const SOURCE_PEAKS_PER_SECOND = 800;
+
+/** How much audio one stored chunk covers, in seconds. */
+const CHUNK_DURATION = 2;
+
+/** How wide one drawn sample is, in pixels. */
+export const SAMPLE_WIDTH = 1;
+
+const IDB_NAME = 'peak-cache-db';
+const IDB_VERSION = 1;
+const IDB_STORE = 'data';
+
+/** Where a chunk of an asset's peaks sits in it, in seconds, and its key. */
+type PeakChunk = {
+ start: number;
+ end: number;
+ id: string;
+};
+
+/** A chunk of peaks, downsampled to what one clip draws it at. */
+export type ClipSamples = PeakChunk & {
+ data: Uint8ClampedArray;
+ peaksPerSecond: number;
+};
+
+/** What the decode produced for one stretch of a file, before it is stored. */
+type DecodedPeaks = {
+ start: number;
+ end: number;
+ data: Uint8ClampedArray;
+};
+
+/** The asset layer's record of one file: which chunks of it exist. */
+type AssetPeaks = {
+ decoding: boolean;
+ /** True once the file has been read through, whatever came of it. */
+ decoded: boolean;
+ chunks: PeakChunk[];
+};
+
+/** The clip layer's record of one clip: the chunks it can draw, at its zoom. */
+type ClipPeaks = {
+ assetId: string;
+ samples: ClipSamples[];
+ /** The range and zoom `samples` was built for; unchanged means no work. */
+ hash: string;
+ updating: boolean;
+ pending: PeakRequest | null;
+};
+
+/** What a clip needs this frame: a stretch of its asset, at a zoom. */
+export type PeakRequest = {
+ /** The entity id of the clip asking. */
+ clip: number;
+ asset: AudioAsset | VideoAsset;
+ peaksPerSecond: number;
+ /** The visible stretch of the source, in source seconds. */
+ start: number;
+ end: number;
+};
+
+const assetPeaks = new Map<string, AssetPeaks>();
+const clipPeaks = new Map<number, ClipPeaks>();
+
+/**
+ * The chunks `clip` can draw right now, ordered by where they start. Empty
+ * (or short) while the decode behind them is still running.
+ */
+export function getClipSamples(clip: number): ClipSamples[] | undefined {
+ return clipPeaks.get(clip)?.samples;
+}
+
+/**
+ * Asks for the peaks a clip needs. Starts the file's decode the first time it
+ * is asked about, and re-cuts the clip's own samples whenever the stretch it
+ * wants or the zoom it wants them at has changed.
+ */
+export function requestPeaks(request: PeakRequest): void {
+ const record = assetPeaks.get(request.asset.id);
+
+ if (!record || (!record.decoding && !record.decoded)) {
+ void decodeAsset(request);
+ }
+
+ if (record && record.chunks.length > 0) {
+ void updateClip(request, record);
+ }
+}
+
+/**
+ * Gives a clip's samples to the half that was cut off it. Both halves play
+ * the same source at the same zoom, so the copy starts with everything the
+ * original had — the peak buffers themselves are shared, only the records
+ * that say what is being drawn are its own.
+ */
+export function clonePeaksForSplit(clip: number, copy: number): void {
+ const source = clipPeaks.get(clip);
+ if (!source) return;
+
+ clipPeaks.set(copy, {
+ assetId: source.assetId,
+ samples: source.samples.map((sample) => ({ ...sample })),
+ hash: source.hash,
+ updating: false,
+ pending: null,
+ });
+}
+
+/** Forgets one clip's samples, or every clip's. */
+export function clearClipPeaks(clip?: number): void {
+ if (clip === undefined) clipPeaks.clear();
+ else clipPeaks.delete(clip);
+}
+
+/** Forgets everything: the clips' samples and the index of what was decoded. */
+export function clearPeaks(): void {
+ clipPeaks.clear();
+ assetPeaks.clear();
+}
+
+/** Forgets one asset's peaks, for a file that is no longer the one it was. */
+export function forgetAssetPeaks(assetId: string): void {
+ assetPeaks.delete(assetId);
+ for (const [clip, peaks] of clipPeaks) {
+ if (peaks.assetId === assetId) clipPeaks.delete(clip);
+ }
+}
+
+// ---------------------------------------------------------------------------
+// The asset layer
+
+/**
+ * Reads the file through once, writing its peaks to the store a couple of
+ * seconds at a time. Each chunk is usable the moment it lands, so a long file
+ * draws from the top down rather than all at once at the end.
+ */
+async function decodeAsset(request: PeakRequest): Promise<void> {
+ const { asset } = request;
+
+ const record = assetPeaks.get(asset.id) ?? { decoding: false, decoded: false, chunks: [] };
+ assetPeaks.set(asset.id, record);
+ if (record.decoding) return;
+
+ record.decoding = true;
+
+ try {
+ const track = await getAudioTrack(asset);
+ if (!track) return;
+
+ const sink = new AudioSampleSink(track);
+ const pending: DecodedPeaks[] = [];
+ let covered = 0;
+
+ const flush = async (): Promise<void> => {
+ const merged = mergePeaks(pending);
+ const id = `${asset.id}-${merged.start}`;
+ record.chunks.push({ start: merged.start, end: merged.end, id });
+ pending.length = 0;
+ covered = 0;
+ await storePeaks(id, merged.data);
+ };
+
+ for await (const sample of sink.samples()) {
+ covered += sample.duration;
+ try {
+ pending.push(peaksOfSample(sample));
+ } finally {
+ sample.close();
+ }
+
+ if (covered > CHUNK_DURATION) await flush();
+ }
+
+ if (pending.length > 0) await flush();
+
+ record.chunks.sort((a, b) => a.start - b.start);
+ } catch (error) {
+ console.error('[timeline] could not decode peaks', error);
+ } finally {
+ record.decoding = false;
+ record.decoded = true;
+ }
+}
+
+/**
+ * One decoded sample's peaks: the loudest frame of each slice, across every
+ * channel, raised to a power so quiet passages still show as something.
+ */
+function peaksOfSample(sample: AudioSample): DecodedPeaks {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+
+ const channels = sample.numberOfChannels;
+ const frames = floats.length / channels;
+ const target = Math.ceil(sample.duration * SOURCE_PEAKS_PER_SECOND);
+ const peaks = new Uint8ClampedArray(target);
+ const ratio = frames / target;
+
+ for (let i = 0; i < target; i++) {
+ const from = Math.floor(i * ratio);
+ const to = Math.floor((i + 1) * ratio);
+ let max = 0;
+
+ for (let frame = from; frame < to && frame < frames; frame++) {
+ for (let channel = 0; channel < channels; channel++) {
+ max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));
+ }
+ }
+
+ peaks[i] = Math.floor(255 * max);
+ }
+
+ return { start: sample.timestamp, end: sample.timestamp + sample.duration, data: peaks };
+}
+
+function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {
+ const length = peaks.reduce((total, peak) => total + peak.data.length, 0);
+ const merged = new Uint8ClampedArray(length);
+
+ let offset = 0;
+ for (const peak of peaks) {
+ merged.set(peak.data, offset);
+ offset += peak.data.length;
+ }
+
+ return { start: peaks[0]!.start, end: peaks[peaks.length - 1]!.end, data: merged };
+}
+
+// ---------------------------------------------------------------------------
+// The clip layer
+
+/**
+ * Re-cuts a clip's samples for the stretch and zoom it is asking about. Only
+ * one runs at a time per clip — a zoom or a scroll that arrives while one is
+ * reading the store replaces whatever was queued behind it, so the work that
+ * finally runs is for where the timeline ended up rather than where it went
+ * through.
+ */
+async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<void> {
+ const peaks = clipPeaks.get(request.clip)
+ ?? { assetId: request.asset.id, samples: [], hash: '', updating: false, pending: null };
+ clipPeaks.set(request.clip, peaks);
+
+ if (peaks.updating) {
+ peaks.pending = request;
+ return;
+ }
+
+ try {
+ const range = coveringRange(request, record);
+ if (range.hash === peaks.hash) return;
+
+ peaks.updating = true;
+ peaks.hash = range.hash;
+
+ for (const chunk of record.chunks) {
+ if (!inRange(chunk.start, range.start, range.end)) continue;
+
+ const data = await loadPeaks(chunk.id);
+ if (!data) continue;
+
+ peaks.samples = peaks.samples.filter((sample) => sample.id !== chunk.id);
+ peaks.samples.push({
+ ...chunk,
+ data: downsample(data, Math.ceil((chunk.end - chunk.start) * request.peaksPerSecond)),
+ peaksPerSecond: request.peaksPerSecond,
+ });
+ peaks.samples.sort((a, b) => a.start - b.start);
+ }
+
+ peaks.samples = peaks.samples.filter((sample) => inRange(sample.start, range.start, range.end));
+ } catch (error) {
+ console.error('[timeline] could not cut peaks', error);
+ } finally {
+ peaks.updating = false;
+ }
+
+ const pending = peaks.pending;
+ if (pending) {
+ peaks.pending = null;
+ void updateClip(pending, record);
+ }
+}
+
+/**
+ * The stretch of the asset to cut, snapped out to whole chunks: what is on
+ * screen, widened to the chunks its ends fall inside, so the waveform does
+ * not stop short of the edge of the viewport.
+ */
+function coveringRange(request: PeakRequest, record: AssetPeaks) {
+ const from = Math.max(0, request.start);
+ const to = Math.min(request.asset.duration, request.end);
+
+ const first = record.chunks.find((chunk) => inRange(from, chunk.start, chunk.end));
+ const last = record.chunks.find((chunk) => inRange(to, chunk.start, chunk.end));
+
+ const start = first?.start ?? record.chunks.at(0)?.start ?? 0;
+ const end = last?.end ?? record.chunks.at(-1)?.end ?? 0;
+
+ return { start, end, hash: `${request.peaksPerSecond}.${start}.${end}` };
+}
+
+/**
+ * The peaks of `data` at `target` positions: the loudest of each slice, and
+ * never fewer than one sample per position.
+ *
+ * Zoomed in past the resolution the peaks were stored at, a position's slice
+ * of the source is narrower than one stored sample, and every slice whose
+ * ends round to the same sample would read nothing at all — a fifth of them
+ * at the far end of the zoom, drawn as a regular pattern of gaps through the
+ * waveform. Holding the sample the slice starts in draws what there is to
+ * draw instead: past that zoom the file has no more detail to show.
+ */
+function downsample(data: Uint8ClampedArray, target: number): Uint8ClampedArray {
+ const peaks = new Uint8ClampedArray(target);
+ const ratio = data.length / target;
+
+ for (let i = 0; i < target; i++) {
+ const from = Math.floor(i * ratio);
+ const to = Math.max(from + 1, Math.floor((i + 1) * ratio));
+ let max = 0;
+
+ for (let index = from; index < to && index < data.length; index++) {
+ max = Math.max(max, data[index]!);
+ }
+
+ peaks[i] = max;
+ }
+
+ return peaks;
+}
+
+const inRange = (value: number, from: number, to: number): boolean => value >= from && value <= to;
+
+// ---------------------------------------------------------------------------
+// The store
+
+let database: Promise<IDBDatabase> | null = null;
+
+function getDatabase(): Promise<IDBDatabase> {
+ database ??= new Promise((resolve, reject) => {
+ const request = indexedDB.open(IDB_NAME, IDB_VERSION);
+
+ request.onupgradeneeded = () => request.result.createObjectStore(IDB_STORE);
+ request.onerror = () => reject(request.error);
+
+ request.onsuccess = () => {
+ // What an earlier session left behind is unreachable — the index of
+ // which chunks exist was never written — so the session starts on
+ // an empty store rather than one that only grows.
+ const transaction = request.result.transaction(IDB_STORE, 'readwrite');
+ transaction.objectStore(IDB_STORE).clear();
+ transaction.oncomplete = () => resolve(request.result);
+ transaction.onerror = () => reject(transaction.error);
+ };
+ });
+
+ return database;
+}
+
+async function storePeaks(key: string, peaks: Uint8ClampedArray): Promise<void> {
+ const db = await getDatabase();
+ const transaction = db.transaction(IDB_STORE, 'readwrite');
+
+ transaction.objectStore(IDB_STORE).put(new Blob([new Uint8Array(peaks)]), key);
+
+ return new Promise((resolve, reject) => {
+ transaction.oncomplete = () => resolve();
+ transaction.onerror = () => reject(transaction.error);
+ });
+}
+
+async function loadPeaks(key: string): Promise<Uint8ClampedArray | null> {
+ const db = await getDatabase();
+ const store = db.transaction(IDB_STORE, 'readonly').objectStore(IDB_STORE);
+
+ const blob = await new Promise<Blob | null>((resolve, reject) => {
+ const request = store.get(key);
+ request.onsuccess = () => resolve(request.result ?? null);
+ request.onerror = () => reject(request.error);
+ });
+
+ return blob ? new Uint8ClampedArray(await blob.arrayBuffer()) : null;
+}
diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.ts
index a1098769..30c5e9aa 100644
--- a/apps/web/src/engine/timeline/render/clip.ts
+++ b/apps/web/src/engine/timeline/render/clip.ts
@@ -20,7 +20,7 @@ import { truncateText } from '../text';
import { framesToPixels, getResolution, getViewport } from '../view';
import { renderCaption } from './caption';
import { renderGroup } from './group';
-import { renderThumbnails } from './thumbnails';
+import { renderStillThumbnails, renderVideoThumbnails } from './thumbnails';
import { renderWaveform } from './waveform';
import type { Entity, World } from 'koota';
@@ -165,34 +165,24 @@ function renderContent(
switch (asset?.type) {
case 'IMAGE':
case 'SEQUENCE':
- if (tall) renderThumbnails(world, scene, surface, entity, asset, row);
+ renderStillThumbnails(world, scene, surface, entity, asset, row);
return;
case 'AUDIO':
renderWaveform(world, scene, surface, entity, row, {
+ asset,
color: style.primary ?? style.foreground,
offsetY: tall ? CLIP_LABEL_HEIGHT : 2,
padding: 4,
});
return;
- case 'VIDEO': {
+ case 'VIDEO':
// A video shows both: its picture along the top of the row, its
- // sound along the bottom.
- const strip = tall ? Math.round((row.height - CLIP_LABEL_HEIGHT) / 2) : 0;
- if (strip > 0) {
- renderThumbnails(world, scene, surface, entity, asset, row, {
- top: CLIP_LABEL_HEIGHT,
- height: strip,
- });
- }
- renderWaveform(world, scene, surface, entity, row, {
- color: style.primary ?? style.foreground,
- offsetY: tall ? CLIP_LABEL_HEIGHT + strip : 2,
- padding: 2,
- });
+ // sound along the bottom. Which of the two the row has room for is
+ // the strip's own to work out.
+ renderVideoThumbnails(world, scene, surface, entity, asset, row, style.primary ?? style.foreground);
return;
- }
}
}
diff --git a/apps/web/src/engine/timeline/render/thumbnails.ts b/apps/web/src/engine/timeline/render/thumbnails.ts
index c4f2281f..6f3cf52a 100644
--- a/apps/web/src/engine/timeline/render/thumbnails.ts
+++ b/apps/web/src/engine/timeline/render/thumbnails.ts
@@ -2,13 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Computed, store } from '@diffusionstudio/runtime';
+import { Computed, getSourceWindow, store } from '@diffusionstudio/runtime';
-import { CLIP_LABEL_HEIGHT } from '../config';
-import { resolvePoster } from '../media';
-import { framesToPixels, getResolution, getViewport } from '../view';
+import { CLIP_BREAKPOINTS, CLIP_LABEL_HEIGHT } from '../config';
+import { pickFrame, requestFrames, resolveStill } from '../media';
+import { framesToPixels, getFrameRate, getResolution, getViewport, pixelsToFrames } from '../view';
+import { renderWaveform } from './waveform';
-import type { Asset } from '@diffusionstudio/assets';
+import type { Asset, VideoAsset } from '@diffusionstudio/assets';
import type { Entity, World } from 'koota';
import type { RowCursor } from '../layout';
import type { TimelineSurfaceState } from '../surface';
@@ -18,72 +19,203 @@ import type { TimelineSurfaceState } from '../surface';
* derived from the height, so dragging a row taller does not slide every
* tile along as it goes.
*/
-const TILE_WIDTHS = [80, 100, 120] as const;
+const STILL_WIDTHS = [80, 100, 120] as const;
-const TOP = 20;
-const INSET = 2;
+/** Where a still's tiles start, and where the strip they sit in does. */
+const STILL_TOP = 18;
+const STILL_MASK_TOP = 20;
/**
- * The clip's picture, tiled along it. An image shows the same frame however
- * long it runs, so tiling its poster is the whole truth about it; a video's
- * poster is only its first frame, so this says where the footage is rather
- * than what is in it.
+ * The least a video's picture strip is worth drawing at. Under it the row
+ * is all waveform, which says more about a clip that short than four rows of
+ * pixels would.
*/
-export function renderThumbnails(
+const MIN_STRIP_HEIGHT = 42;
+
+/** The tile a video's strip is drawn with, and the strip height it changes at. */
+const SMALL_TILE = { width: 56, height: 46 } as const;
+const LARGE_TILE = { width: 84, height: 52 } as const;
+const LARGE_TILE_STRIP_HEIGHT = 44;
+
+/**
+ * A still, tiled along the clip. An image shows the same frame however long
+ * it runs, so tiling one picture is the whole truth about it — a sequence is
+ * shown by its first frame for the same reason a folder of frames has a
+ * cover.
+ */
+export function renderStillThumbnails(
world: World,
scene: Entity,
surface: TimelineSurfaceState,
entity: Entity,
asset: Asset,
row: RowCursor,
- options: { top?: number; height?: number } = {},
): void {
const ctx = surface.ctx!;
- const poster = resolvePoster(world, asset);
- if (!poster) return;
+ // SVGs aren't supported by createImageBitmap on every browser, so skip
+ // the timeline thumbnail strip for them.
+ if (asset.mimeType === 'image/svg+xml') return;
- const top = options.top ?? TOP;
- const height = options.height ?? row.height - CLIP_LABEL_HEIGHT;
- if (height < 4) return;
+ // Too short a row to hold anything but its label.
+ if (row.height <= CLIP_BREAKPOINTS.sm) return;
+ const eid = entity.id();
const computed = store(world, Computed);
const resolution = getResolution(world, scene);
- const left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);
- const width = framesToPixels(computed.end[entity.id()] ?? 0, resolution) - left;
- if (width < 4) return;
+ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);
+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;
+
+ const tileWidth = stillWidth(row.height);
+ const still = resolveStill(asset, tileWidth);
+ if (!still) return;
- const tile = tileWidth(row.height);
- const crop = coverCrop(poster.width, poster.height, tile, height);
+ const maskHeight = row.height - CLIP_LABEL_HEIGHT;
// Only the tiles on screen: a clip an hour long is as cheap to draw as a
// clip a second long.
const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);
- const first = Math.max(0, Math.floor((viewportLeft - left) / tile));
- const last = Math.min(Math.ceil(width / tile) - 1, Math.floor((viewportRight - left) / tile));
+ const tiles = Math.ceil(clipWidth / tileWidth);
+ const first = Math.max(0, Math.floor((viewportLeft - clipLeft) / tileWidth));
+ const last = Math.min(tiles - 1, Math.floor((viewportRight - clipLeft) / tileWidth));
+
+ const crop = coverCrop(still.width, still.height, tileWidth, maskHeight);
ctx.save();
ctx.beginPath();
- ctx.roundRect(left + INSET, top, width - INSET * 2, height - INSET, 2);
+ ctx.roundRect(clipLeft + 2, STILL_MASK_TOP, clipWidth - 4, maskHeight - 2, 2);
ctx.clip();
for (let i = first; i <= last; i++) {
ctx.drawImage(
- poster,
+ still.canvas,
crop.x, crop.y, crop.width, crop.height,
- left + i * tile, top, tile, height,
+ clipLeft + i * tileWidth, STILL_TOP, tileWidth, maskHeight,
);
}
ctx.restore();
}
-function tileWidth(rowHeight: number): number {
- if (rowHeight >= 100) return TILE_WIDTHS[2];
- if (rowHeight >= 70) return TILE_WIDTHS[1];
- return TILE_WIDTHS[0];
+/**
+ * A video: its picture along the top of the row and its sound along the
+ * bottom, each with half of what the label leaves.
+ *
+ * The picture is a strip of the clip's own frames rather than one frame
+ * repeated, so what a tile shows is what the footage is doing at that point
+ * of the clip. Tiles are placed in the source's own time — the strip is
+ * drawn from the clip's origin, so trimming an end slides the footage under
+ * the clip instead of re-cutting the tiles.
+ */
+export function renderVideoThumbnails(
+ world: World,
+ scene: Entity,
+ surface: TimelineSurfaceState,
+ entity: Entity,
+ asset: VideoAsset,
+ row: RowCursor,
+ color: string,
+): void {
+ const ctx = surface.ctx!;
+
+ const eid = entity.id();
+ const computed = store(world, Computed);
+ const resolution = getResolution(world, scene);
+ const fps = getFrameRate(world);
+
+ const layout = videoLayout(row.height);
+
+ renderWaveform(world, scene, surface, entity, row, {
+ asset,
+ color,
+ offsetY: layout.waveformOffsetY,
+ padding: 0,
+ });
+
+ // Nothing left of the row for a picture once the label and the waveform
+ // have had theirs.
+ if (layout.stripHeight === 0) return;
+
+ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);
+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;
+
+ ctx.save();
+
+ ctx.beginPath();
+ ctx.roundRect(clipLeft + 2, CLIP_LABEL_HEIGHT, clipWidth - 4, layout.stripHeight, 2);
+ ctx.clip();
+
+ const playbackRate = computed.playbackRate[eid] || 1;
+ const origin = computed.origin[eid] ?? 0;
+
+ // How much source one tile covers, scaled by playbackRate so a tile's
+ // width in timeline pixels maps to the right chunk of source. Kept as a
+ // float: at high zoom this is sub-frame, and rounding it to 0 collapses
+ // the decode range and loses every tile.
+ const interval = (layout.tile.width / resolution) * playbackRate;
+
+ const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);
+
+ // The viewport is intersected with the source window directly in frames.
+ // Going through pixels first lets `framesToPixels` floor and
+ // `pixelsToFrames` round, which can shave a frame off the last tile.
+ const source = getSourceWindow(entity);
+ const firstFrame = Math.max(source.in, (pixelsToFrames(viewportLeft, resolution) - origin) * playbackRate);
+ const lastFrame = Math.min(source.out, (pixelsToFrames(viewportRight, resolution) - origin) * playbackRate);
+
+ requestFrames({
+ clip: eid,
+ asset,
+ fps,
+ firstFrame,
+ lastFrame,
+ width: layout.tile.width,
+ height: layout.tile.height,
+ interval,
+ });
+
+ // Tile indices are in source time, so they line up with the frames the
+ // decoder filed under source timestamps once the origin is translated to.
+ ctx.translate(framesToPixels(origin, resolution), 0);
+
+ // Only the tiles on screen — the rest are inside the clip but outside the
+ // viewport, and drawing them is work the clip path would throw away.
+ for (let i = Math.floor(firstFrame / interval); i <= Math.floor(lastFrame / interval); i++) {
+ const frame = pickFrame(eid, asset.id, i * interval, interval);
+ if (!frame) continue;
+
+ ctx.drawImage(frame.canvas, i * layout.tile.width, layout.tileTop, layout.tile.width, layout.tile.height);
+ }
+
+ ctx.restore();
+}
+
+/**
+ * How a video row is split. The picture takes half of what the label leaves,
+ * the waveform the other half; below the point where a picture would be too
+ * short to read, the waveform takes all of it.
+ */
+function videoLayout(rowHeight: number) {
+ const body = rowHeight - CLIP_LABEL_HEIGHT;
+ const stripHeight = body > MIN_STRIP_HEIGHT ? body * 0.5 : 0;
+ const tile = stripHeight > LARGE_TILE_STRIP_HEIGHT ? LARGE_TILE : SMALL_TILE;
+
+ return {
+ stripHeight,
+ tile,
+ // Bottom-aligned in the strip, so a tile taller than it is cropped at
+ // the top rather than dangling over the waveform.
+ tileTop: CLIP_LABEL_HEIGHT + (stripHeight - tile.height),
+ waveformOffsetY: rowHeight > CLIP_BREAKPOINTS.sm ? stripHeight + CLIP_LABEL_HEIGHT : 2,
+ };
+}
+
+function stillWidth(rowHeight: number): number {
+ if (rowHeight >= 100) return STILL_WIDTHS[2];
+ if (rowHeight >= 70) return STILL_WIDTHS[1];
+ return STILL_WIDTHS[0];
}
/**
diff --git a/apps/web/src/engine/timeline/render/waveform.ts b/apps/web/src/engine/timeline/render/waveform.ts
index 88514246..6b18c948 100644
--- a/apps/web/src/engine/timeline/render/waveform.ts
+++ b/apps/web/src/engine/timeline/render/waveform.ts
@@ -2,37 +2,41 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { AssetId, Cache, Computed, findAssetDuration, getSourceWindow, resolveWaveformPeaks, store } from '@diffusionstudio/runtime';
+import { Computed, getSourceWindow, store } from '@diffusionstudio/runtime';
-import { framesToPixels, getResolution } from '../view';
+import { CLIP_BREAKPOINTS, CLIP_CORNER_RADIUS } from '../config';
+import { SAMPLE_WIDTH, getClipSamples, requestPeaks } from '../peaks';
+import { framesToPixels, getFrameRate, getResolution, getViewport } from '../view';
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
import type { Entity, World } from 'koota';
+import type { ClipSamples } from '../peaks';
import type { RowCursor } from '../layout';
import type { TimelineSurfaceState } from '../surface';
-/** How wide a bar is, and how much of that is the gap after it. */
-const BAR_WIDTH = 2;
-const BAR_GAP = 1;
-const MIN_BAR_HEIGHT = 1;
+/** How far either side of the viewport peaks are asked for, in pixels. */
+const VIEWPORT_MARGIN = 5;
-type WaveformOptions = {
+export type WaveformOptions = {
+ asset: AudioAsset | VideoAsset;
color: string;
/** How far down the clip the waveform starts. */
offsetY: number;
- /** How much of the height to leave above and below it. */
+ /** How much of the height to leave for it to sit inside. */
padding: number;
};
/**
- * The clip's audio, as bars across the part of the waveform it plays.
+ * The clip's audio, one column of pixels at a time.
*
- * The peaks belong to the asset, not the clip: they summarise the whole file
- * in a fixed number of bars (`PEAK_BARS`), and the clip's own source window
- * says which stretch of them is its own. That is what makes a split cost
- * nothing — both halves read the same summary, from different ends of it.
+ * A column is a moment of the source, so what it is worth changes with the
+ * zoom: the peaks are asked for at one per pixel (see `../peaks`), and only
+ * for the stretch of the clip that is on screen. Drawing then reads them back
+ * by time rather than by index, so a column drawn before its peaks have
+ * arrived is simply left out and filled in on a later frame.
*
- * The summary is coarse, so a clip zoomed in far enough draws a smoothed
- * version of its audio rather than its samples.
+ * Tall rows show the waveform about its middle, short ones stand it on the
+ * bottom of the clip and fade it, so the label above it stays readable.
*/
export function renderWaveform(
world: World,
@@ -44,61 +48,100 @@ export function renderWaveform(
): void {
const ctx = surface.ctx!;
- const source = findPeakSource(entity);
- if (!source) return;
-
- const peaks = resolveWaveformPeaks(world, source);
- if (!peaks || peaks.length === 0) return;
-
- const height = row.height - options.offsetY - options.padding;
- if (height < 2) return;
+ // A video with no audio track has nothing to draw here.
+ if (!options.asset.sampleRate) return;
+ const eid = entity.id();
const computed = store(world, Computed);
const resolution = getResolution(world, scene);
+ const fps = getFrameRate(world);
- const left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);
- const width = framesToPixels(computed.end[entity.id()] ?? 0, resolution) - left;
- if (width < BAR_WIDTH) return;
+ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);
+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;
+ const maskHeight = row.height - options.offsetY;
+ if (maskHeight <= 0) return;
- // Which stretch of the asset this clip plays, as a fraction of it: the
- // source window is in source frames, and the peaks span all of them.
- const duration = findAssetDuration(world, entity) ?? 0;
- const window = getSourceWindow(entity);
- const from = duration > 0 ? window.in / duration : 0;
- const to = duration > 0 ? Math.min(1, window.out / duration) : 1;
+ const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);
- const step = BAR_WIDTH + BAR_GAP;
- const count = Math.floor(width / step);
- if (count <= 0) return;
+ ctx.save();
- const centerY = options.offsetY + height / 2;
+ // Everything below is drawn against the clip's own rounded body.
+ ctx.beginPath();
+ ctx.roundRect(clipLeft, options.offsetY, clipWidth, maskHeight, CLIP_CORNER_RADIUS);
+ ctx.clip();
+
+ // A little smaller than the mask, so the tallest sample does not touch it.
+ const waveformHeight = maskHeight - options.padding;
+
+ const playbackRate = computed.playbackRate[eid] || 1;
+ const pixelsToSeconds = (pixels: number): number => pixels / (fps * resolution);
+
+ // Each pixel covers playbackRate× more source audio when sped up, so we
+ // need proportionally fewer peaks per source second to keep one peak/pixel.
+ // Never none: zoomed far enough out a pixel is worth less than a second of
+ // source, and asking for no peaks a second is asking for no waveform.
+ const peaksPerSecond = Math.max(1, Math.floor(fps * resolution)) / SAMPLE_WIDTH / playbackRate;
+
+ // The source window is in source frames; the clip occupies
+ // window / playbackRate timeline frames, measured from its own origin.
+ const source = getSourceWindow(entity);
+ const inPx = framesToPixels(source.in / playbackRate, resolution);
+ const outPx = framesToPixels(source.out / playbackRate, resolution);
+ const originPx = framesToPixels(computed.origin[eid] ?? 0, resolution);
+
+ const firstSample = Math.max(inPx, viewportLeft - VIEWPORT_MARGIN - originPx);
+ const lastSample = Math.min(outPx, viewportRight + VIEWPORT_MARGIN - originPx);
+
+ requestPeaks({
+ clip: eid,
+ asset: options.asset,
+ start: pixelsToSeconds(firstSample) * playbackRate,
+ end: pixelsToSeconds(lastSample) * playbackRate,
+ peaksPerSecond,
+ });
- ctx.save();
ctx.fillStyle = options.color;
- for (let i = 0; i < count; i++) {
- // Where along the clip this bar is, mapped into the asset's summary.
- const position = from + ((i + 0.5) / count) * (to - from);
- const value = (peaks[Math.min(peaks.length - 1, Math.floor(position * peaks.length))] ?? 0) / 255;
- const barHeight = Math.max(value * height, MIN_BAR_HEIGHT);
+ // A tall row centres the waveform on the space it has; a short one stands
+ // it on the bottom, where it reads as a strip under the label.
+ const anchor = maskHeight > CLIP_BREAKPOINTS.sm ? 0.5 : 1;
+ const minSampleHeight = maskHeight > CLIP_BREAKPOINTS.sm ? 1 : 0;
+
+ // Tone the waveform down on short clip rows so the label stays readable.
+ if (row.height <= CLIP_BREAKPOINTS.sm) ctx.globalAlpha *= 0.6;
+
+ ctx.translate(originPx, options.offsetY + options.padding * anchor);
+
+ const samples = getClipSamples(eid);
- ctx.fillRect(left + i * step, centerY - barHeight / 2, BAR_WIDTH, barHeight);
+ // The chunks are ordered by where they start and the columns are drawn
+ // left to right, so the search for the chunk a column falls in only ever
+ // moves forwards.
+ let cursor = 0;
+
+ for (let x = firstSample; x < lastSample; x += SAMPLE_WIDTH) {
+ const time = pixelsToSeconds(x) * playbackRate;
+
+ cursor = findChunk(samples, time, cursor);
+ const chunk = samples?.[cursor];
+ if (!chunk || time < chunk.start || time > chunk.end) continue;
+
+ const peak = chunk.data[Math.floor((time - chunk.start) * chunk.peaksPerSecond)];
+ if (peak === undefined) continue;
+
+ const height = Math.max((peak / 255) * waveformHeight, minSampleHeight);
+ ctx.fillRect(x, (waveformHeight - height) * anchor, SAMPLE_WIDTH, height);
}
ctx.restore();
}
-/**
- * What holds the asset the peaks come from: the clip itself when it is its
- * own source (an audio clip), otherwise whichever fill names one. Whether
- * that asset has any audio is `resolveWaveformPeaks`'s to say.
- */
-function findPeakSource(entity: Entity): Entity | null {
- if (entity.has(AssetId)) return entity;
+/** The first chunk at or after `from` that has not already ended by `time`. */
+function findChunk(samples: ClipSamples[] | undefined, time: number, from: number): number {
+ if (!samples) return 0;
- for (const fill of entity.get(Cache)?.fills ?? []) {
- if (fill.has(AssetId)) return fill;
- }
+ let index = from;
+ while (index < samples.length && samples[index]!.end < time) index++;
- return null;
+ return index;
}
diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.ts
index 101f65f1..aad92952 100644
--- a/packages/runtime/src/media/audio-peaks.ts
+++ b/packages/runtime/src/media/audio-peaks.ts
@@ -29,8 +29,12 @@ export class Waveform {
}
private async initialize(): Promise<void> {
- const file = await getAssetFile(this.asset);
- const peaks = await derivePeaks(file);
+ // Through the cache where there is one, so a file's peaks are derived
+ // once per project rather than once per session.
+ const peaks = this.cache
+ ? await this.cache.peaks(this.asset)
+ : await derivePeaks(await getAssetFile(this.asset));
+
if (!this.disposed) {
this.peaks = peaks;
}