The VS Code of video editing, built for agents.
TypeScript
Refactor timeline media and peaks management
konstantin-paulus03f1078b3c52
diff --git a/apps/web/src/engine/asset-actions.ts b/apps/web/src/engine/asset-actions.tsindex 69b274c1..b79e238d 100644--- a/apps/web/src/engine/asset-actions.ts+++ b/apps/web/src/engine/asset-actions.ts@@ -8,7 +8,8 @@import { toast } from "somoto";import { importFiles as importFilesInto, pickFiles, saveAssetAs as saveAs } from "@diffusionstudio/assets";import { insertAsset } from "./insert-asset";-import { forgetPoster } from "./timeline/media";+import { forgetAssetMedia } from "./timeline/media";+import { forgetAssetPeaks } from "./timeline/peaks";import type { World } from "koota";@@ -55,8 +56,10 @@ export async function replaceAssetSource(library: AssetLibrary, asset: Asset): Pif (!path) return null;try {const relinked = await library.relink(asset, path);- // The picture the timeline is showing is of the file it used to be.- forgetPoster(asset.id);+ // The picture and the waveform the timeline is showing are of the file+ // it used to be.+ forgetAssetMedia(asset.id);+ forgetAssetPeaks(asset.id);return relinked;} catch (error) {toast.error("Failed to replace", { description: (error as Error).message });diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.tsindex e03d7cc6..30abafe6 100644--- a/apps/web/src/engine/create-engine.ts+++ b/apps/web/src/engine/create-engine.ts@@ -2,12 +2,12 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';+import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Geometry, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';import { hudSystem } from './hud';import { createSignal, type Accessor, type Setter } from 'solid-js';import { AssetSelection, Hud, Keys, Pointer, PointerEvents, ProjectConfig, SnapLines } from './traits';import { inputSystem } from './input/input-system';-import { timelineSystem, TimelineSurface } from './timeline';+import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';import { shortcutSystem } from './input/shortcuts';import { sourceErrorSystem } from './source-errors';@@ -54,6 +54,13 @@ class Engine {this.unsubscribe.push(this.world.onAdd(ChildOf('*'), () => (this.interactiveDirty = true)),this.world.onRemove(ChildOf('*'), () => (this.interactiveDirty = true)),+ // The timeline keeps a clip's waveform and thumbnails by entity id,+ // and koota hands ids back out: a clip that is gone has to take its+ // pictures with it, or the next clip to be given its id inherits them.+ this.world.onRemove(Geometry, (entity) => {+ clearClipPeaks(entity.id());+ clearClipFrames(entity.id());+ }),);[this.running, this.setRunning] = createSignal(false);@@ -268,6 +275,11 @@ class Engine {this.audioContext.close();}+ // The timeline's decoded pictures and peaks are of this project's+ // files, and are held outside the world: they go with the engine.+ clearPeaks();+ clearMedia();+this.unsubscribeEventListeners();this.unsubscribe.forEach(unsubscribe => unsubscribe());this.unsubscribe.length = 0;diff --git a/apps/web/src/engine/split.tsx b/apps/web/src/engine/split.tsxindex b2143667..dee88f1f 100644--- a/apps/web/src/engine/split.tsx+++ b/apps/web/src/engine/split.tsx@@ -29,6 +29,7 @@ import {import { Or } from 'koota';import { getDocumentEditor } from './editor';+import { cloneFramesForSplit, clonePeaksForSplit } from './timeline';import { trimIn, trimOut } from './timing';import type { Entity, World } from 'koota';@@ -105,6 +106,11 @@ export function splitAtPlayhead(world: World): Entity[] {trimOut(world, original, frame);trimIn(world, copy, frame);handOffDecoders(world, original, copy);+ // The two halves play the same source at the same zoom, so the copy+ // starts with the waveform and thumbnails the whole clip had rather+ // than decoding them all over again.+ clonePeaksForSplit(original.id(), copy.id());+ cloneFramesForSplit(original.id(), copy.id());}// Two clips side by side under a scene are two layers of the timeline,diff --git a/apps/web/src/engine/timeline/index.ts b/apps/web/src/engine/timeline/index.tsindex ad16eba3..f9d34035 100644--- a/apps/web/src/engine/timeline/index.ts+++ b/apps/web/src/engine/timeline/index.ts@@ -8,6 +8,8 @@ export * from './controller';export * from './drag';export * from './snapping';export * from './layout';+export * from './media';+export * from './peaks';export * from './style';export * from './surface';export * from './timeline';diff --git a/apps/web/src/engine/timeline/media.ts b/apps/web/src/engine/timeline/media.tsindex dba81144..24f4ec6e 100644--- a/apps/web/src/engine/timeline/media.ts+++ b/apps/web/src/engine/timeline/media.ts@@ -3,60 +3,444 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. *//**- * The pictures the timeline draws inside clips. The library derives and keeps- * an asset's poster (see `AssetCache.thumbnail`); what is needed here is the- * same picture decoded and ready to draw, which asking for it every frame- * would not give — so it is asked for once and kept until the asset changes.+ * The pictures the timeline draws inside clips, in the two kinds the two+ * kinds of clip need.*- * Keyed by asset rather than by clip: two clips of the same footage show the- * same picture, and a clip that is split into two does not need a second copy- * of anything.+ * A still — an image, or a frames directory shown by its first frame — is one+ * picture tiled along the clip, so it is decoded once per asset at whatever+ * size the row is currently drawn at, and kept until the row changes size.+ *+ * A video is a strip of its own frames, which is a decode per tile, so it+ * comes in two layers like the peaks do. The asset layer decodes a handful of+ * frames spread across the whole file, which is what any clip of it falls+ * back on; the clip layer decodes the tiles the clip actually shows, for the+ * stretch of it on screen, and re-decodes them when the zoom changes what a+ * tile is worth. Nothing here blocks a frame: a draw pass asks for what it+ * wants and draws the nearest picture that has arrived.+ *+ * Keyed by asset where two clips of the same footage would show the same+ * picture, and by clip where they would not.*/-import { Library } from '@diffusionstudio/runtime';+import { CanvasSink } from 'mediabunny';+import { getAssetFile, getVideoTrack, secondsToFrames } from '@diffusionstudio/runtime';++import { MAX_CLIP_HEIGHT } from './config';++import type { InputVideoTrack } from 'mediabunny';+import type { Asset, VideoAsset } from '@diffusionstudio/assets';++/** One decoded picture, and where in the source it came from (source frames). */+export type Frame = {+ timestamp: number;+ canvas: OffscreenCanvas | HTMLCanvasElement;+ /** Superseded: kept on screen only until what replaces it has decoded. */+ stale?: boolean;+};++/** A still, decoded at the size the row it is drawn in asked for. */+export type Still = {+ canvas: OffscreenCanvas;+ width: number;+ height: number;+ hash: string;+};++/** What a video clip needs this frame: the tiles of the stretch on screen. */+export type FrameRequest = {+ /** The entity id of the clip asking. */+ clip: number;+ asset: VideoAsset;+ /** How much source one tile covers, in source frames. May be fractional. */+ interval: number;+ /** Tile size in CSS pixels. */+ width: number;+ height: number;+ /** The visible stretch of the source, in source frames. */+ firstFrame: number;+ lastFrame: number;+ fps: number;+};-import type { Asset } from '@diffusionstudio/assets';-import type { World } from 'koota';+type AssetFrames = {+ frames: Frame[];+ decoding: boolean;+ /** True once the spread has been decoded, whatever came of it. */+ decoded: boolean;+};-/** How wide a poster is asked for: the widest tile any row height draws. */-const POSTER_WIDTH = 300;+type ClipFrames = {+ assetId: string;+ frames: Frame[];+ /** The stretch `frames` covers, in source frames. */+ start: number;+ end: number;+ request: FrameRequest | null;+ pending: FrameRequest | null;+ decoding: boolean;+};-const posters = new Map<string, ImageBitmap | null>();-const loading = new Set<string>();+/** How many frames of a video the fallback spread is made of. */+const SPREAD_COUNT = 7;+const LONG_SPREAD_COUNT = 10;++/** Over this many seconds, a file gets the longer spread. */+const LONG_DURATION = 60 * 10;++/** How many tiles either side of the visible stretch are decoded ahead. */+const TILE_MARGIN = 1;++const stills = new Map<string, Still>();+const stillsDecoding = new Set<string>();+/** Assets whose file will not decode, so the draw pass stops asking. */+const stillsFailed = new Set<string>();+const assetFrames = new Map<string, AssetFrames>();+const clipFrames = new Map<number, ClipFrames>();++// ---------------------------------------------------------------------------+// Stills/**- * The asset's poster, or null until it has loaded (the load is started here).- * Null is also the answer for an asset that has no picture to show, and is- * remembered as such so it is not asked for again.+ * The picture an image or sequence clip is tiled with, at the size a row+ * `width` wide draws it, or null until it has decoded (the decode is started+ * here). Re-decoded when the row changes to a different tile size, which is+ * one of three (see `../render/thumbnails`), so a drag of the row height+ * costs at most two decodes.*/-export function resolvePoster(world: World, asset: Asset): ImageBitmap | null {- const existing = posters.get(asset.id);- if (existing !== undefined) return existing;+export function resolveStill(asset: Asset, width: number): Still | null {+ const hash = `${width}@${window.devicePixelRatio}`;+ const existing = stills.get(asset.id);+ if (existing?.hash === hash) return existing;++ if (!stillsFailed.has(asset.id)) void decodeStill(asset, width, hash);- if (loading.has(asset.id)) return null;- loading.add(asset.id);+ // Whatever is there at the wrong size still beats nothing at all: it is+ // the same picture, and it is replaced the moment the right one lands.+ return existing ?? null;+}- const cache = world.get(Library)?.cache;- if (!cache) {- loading.delete(asset.id);- return null;+async function decodeStill(asset: Asset, width: number, hash: string): Promise<void> {+ if (stillsDecoding.has(asset.id)) return;+ stillsDecoding.add(asset.id);++ try {+ const bitmap = await createImageBitmap(await getAssetFile(asset));++ try {+ // Big enough for the widest tile and the tallest row it could be+ // drawn in, and never larger than the picture itself.+ const dpr = window.devicePixelRatio;+ const scale = Math.min(1, Math.max((width * dpr) / bitmap.width, (MAX_CLIP_HEIGHT * dpr) / bitmap.height));+ const canvasWidth = Math.max(1, Math.round(bitmap.width * scale));+ const canvasHeight = Math.max(1, Math.round(bitmap.height * scale));++ const canvas = new OffscreenCanvas(canvasWidth, canvasHeight);+ canvas.getContext('2d')!.drawImage(bitmap, 0, 0, canvasWidth, canvasHeight);++ stills.set(asset.id, { canvas, width: canvasWidth, height: canvasHeight, hash });+ } finally {+ bitmap.close();+ }+ } catch (error) {+ console.error('[timeline] could not decode still', error);+ stillsFailed.add(asset.id);+ } finally {+ stillsDecoding.delete(asset.id);}+}++// ---------------------------------------------------------------------------+// Video frames- void cache- .thumbnail(asset, POSTER_WIDTH)- .then((blob) => (blob ? createImageBitmap(blob) : null))- .then((bitmap) => posters.set(asset.id, bitmap))- .catch(() => posters.set(asset.id, null))- .finally(() => loading.delete(asset.id));+/**+ * Asks for the tiles a video clip needs. Decodes the file's fallback spread+ * the first time it is asked about, and re-decodes the clip's own tiles+ * whenever the stretch on screen or the size of a tile has changed.+ */+export function requestFrames(request: FrameRequest): void {+ const record = assetFrames.get(request.asset.id);++ if (!record || (!record.decoding && !record.decoded)) {+ void decodeSpread(request);+ }- return null;+ if (record && record.frames.length > 0) {+ void updateClip(request);+ }}/**- * Forgets an asset's poster, so the next frame asks for it again. Called when- * an asset is re-pointed at another file, which the picture would outlive.+ * The picture to draw for `timestamp` (source frames): the clip's own tile+ * where its strip reaches that far, else the nearest of the file's fallback+ * spread. Null while neither has decoded anything yet.*/-export function forgetPoster(assetId: string): void {- posters.get(assetId)?.close();- posters.delete(assetId);+export function pickFrame(clip: number, assetId: string, timestamp: number, interval: number): Frame | null {+ const cached = clipFrames.get(clip);+ const covered = cached !== undefined+ && timestamp >= cached.start - interval+ && timestamp <= cached.end + interval;++ const frames = (covered ? cached.frames : assetFrames.get(assetId)?.frames) ?? [];+ const index = closestFrame(frames, timestamp);++ return index < 0 ? null : frames[index]!;+}++/**+ * Gives a clip's strip to the half that was cut off it, so the copy is drawn+ * from the frames the original had rather than decoding them again. The+ * canvases are shared; only the records saying what is on screen are copied.+ */+export function cloneFramesForSplit(clip: number, copy: number): void {+ const source = clipFrames.get(clip);+ if (!source) return;++ clipFrames.set(copy, {+ ...source,+ frames: source.frames.map((frame) => ({ ...frame })),+ pending: null,+ decoding: false,+ });+}++/** Forgets one clip's strip, or every clip's. */+export function clearClipFrames(clip?: number): void {+ if (clip === undefined) clipFrames.clear();+ else clipFrames.delete(clip);+}++/** Forgets everything: the clips' strips, the spreads and the stills. */+export function clearMedia(): void {+ clipFrames.clear();+ assetFrames.clear();+ stills.clear();+ stillsFailed.clear();+}++/**+ * Forgets one asset's pictures, for a file that is no longer the one it was.+ */+export function forgetAssetMedia(assetId: string): void {+ stills.delete(assetId);+ stillsFailed.delete(assetId);+ assetFrames.delete(assetId);+ for (const [clip, frames] of clipFrames) {+ if (frames.assetId === assetId) clipFrames.delete(clip);+ }+}++/**+ * A handful of frames from across the whole file. Coarse — they are a tenth+ * of a long video apart — but they are what a clip shows the moment it lands+ * on the timeline, and what it keeps showing wherever its own strip has not+ * reached.+ */+async function decodeSpread(request: FrameRequest): Promise<void> {+ const { asset } = request;++ const record = assetFrames.get(asset.id) ?? { frames: [], decoding: false, decoded: false };+ assetFrames.set(asset.id, record);+ if (record.decoding) return;++ record.decoding = true;++ try {+ const track = await getVideoTrack(asset);+ if (!track) return;++ const first = await track.getFirstTimestamp();+ const last = await track.computeDuration();+ const count = asset.duration < LONG_DURATION ? SPREAD_COUNT : LONG_SPREAD_COUNT;+ const sink = createSink(track, request);++ for (let i = 0; i < count; i++) {+ try {+ const wrapped = await sink.getCanvas(first + (i * (last - first)) / count);+ if (!wrapped) continue;++ record.frames.push({+ timestamp: secondsToFrames(wrapped.timestamp, request.fps),+ canvas: wrapped.canvas,+ });+ } catch {+ // One frame that will not decode is not the file's whole spread.+ }+ }+ } catch (error) {+ console.error('[timeline] could not decode thumbnails', error);+ } finally {+ record.decoding = false;+ record.decoded = true;+ }+}++/**+ * Decodes the tiles the clip is showing. Only what is missing: scrolling into+ * new footage extends the strip at the end it moved towards, while a change+ * of zoom or tile size invalidates it and decodes the visible stretch afresh.+ *+ * One decode runs at a time per clip, and a request that arrives while one is+ * running replaces whatever was queued behind it — the work that finally runs+ * is for where the timeline ended up, not everywhere it passed through.+ */+async function updateClip(request: FrameRequest): Promise<void> {+ const cached = clipFrames.get(request.clip) ?? {+ assetId: request.asset.id,+ frames: [],+ start: -1,+ end: -1,+ request: null,+ pending: null,+ decoding: false,+ };+ clipFrames.set(request.clip, cached);++ if (cached.decoding) {+ cached.pending = request;+ return;+ }++ try {+ cached.decoding = true;++ if (rangeHash(cached.request) === rangeHash(request)) return;++ const wanted = tileRange(request);+ const resized = sizeHash(cached.request) !== sizeHash(request);+ const rescaled = cached.request?.interval !== request.interval;+ const overlaps = wanted.start < cached.end && cached.start < wanted.end;++ let from = 0;+ let to = 0;++ if (rescaled || resized || !overlaps) {+ // Nothing on screen is worth keeping: every tile is the wrong size+ // or covers the wrong stretch, so the whole range is decoded again.+ from = wanted.firstIndex;+ to = wanted.lastIndex;+ for (const frame of cached.frames) frame.stale = true;+ } else if (wanted.start < cached.start) {+ from = wanted.firstIndex;+ to = Math.floor(cached.start / request.interval);+ } else if (wanted.end > cached.end) {+ from = Math.floor(cached.end / request.interval);+ to = wanted.lastIndex;+ }++ for (const frame of cached.frames) {+ if (frame.timestamp > wanted.end || frame.timestamp < wanted.start) frame.stale = true;+ }++ const count = to - from;++ // The strip already reaches across everything on screen.+ if (count <= 0) {+ cached.frames = keep(cached.frames);+ cached.start = wanted.start;+ cached.end = wanted.end;+ cached.request = request;+ return;+ }++ const track = await getVideoTrack(request.asset);+ if (!track) return;++ const sink = createSink(track, request);++ for (let i = 0; i < count; i++) {+ try {+ const wrapped = await sink.getCanvas(((from + i) * request.interval) / request.fps);+ if (!wrapped) continue;++ // The decoder may snap to the nearest keyframe, so the frame is+ // filed under the timestamp it actually came back with.+ const timestamp = secondsToFrames(wrapped.timestamp, request.fps);++ // The frames it replaces are dropped only now, so the strip is+ // never blank while the new ones are on their way.+ cached.frames = keep(cached.frames, timestamp);+ cached.frames.push({ timestamp, canvas: wrapped.canvas, stale: false });+ } catch {+ // One tile that will not decode is not the whole strip.+ }+ }++ cached.frames = keep(cached.frames);+ cached.start = wanted.start;+ cached.end = wanted.end;+ cached.request = request;+ } catch (error) {+ console.error('[timeline] could not update thumbnails', error);+ } finally {+ cached.decoding = false;+ }++ const pending = cached.pending;+ if (pending) {+ cached.pending = null;+ void updateClip(pending);+ }+}++/**+ * The tiles to have decoded: the ones the visible stretch falls on, one+ * either side, and never past the ends of the file.+ */+function tileRange(request: FrameRequest) {+ const { firstFrame, lastFrame, interval, asset } = request;++ const lastTile = Math.floor(secondsToFrames(asset.duration, request.fps) / interval);+ const firstIndex = Math.max(Math.floor(firstFrame / interval) - TILE_MARGIN, 0);+ const lastIndex = Math.min(Math.floor(lastFrame / interval) + TILE_MARGIN, lastTile);++ return { firstIndex, lastIndex, start: firstIndex * interval, end: lastIndex * interval };+}++function createSink(track: InputVideoTrack, request: FrameRequest): CanvasSink {+ const dpr = window.devicePixelRatio;+ return new CanvasSink(track, { width: request.width * dpr, height: request.height * dpr, fit: 'cover' });+}++/** The strip without what has been superseded, in the order it is drawn. */+function keep(frames: Frame[], upTo?: number): Frame[] {+ return frames+ .filter((frame) => !frame.stale || (upTo !== undefined && frame.timestamp > upTo))+ .sort((a, b) => a.timestamp - b.timestamp);+}++function rangeHash(request: FrameRequest | null): string {+ if (!request) return '';+ return `${sizeHash(request)}-${request.interval}-${request.firstFrame}-${request.lastFrame}`;+}++function sizeHash(request: FrameRequest | null): string {+ if (!request) return '';+ return `${request.width}x${request.height}@${window.devicePixelRatio}`;+}++/** The frame nearest `timestamp`, or -1 when there are none. */+function closestFrame(frames: Frame[], timestamp: number): number {+ if (frames.length === 0) return -1;++ let low = 0;+ let high = frames.length - 1;+ let closest = -1;+ let distance = Infinity;++ while (low <= high) {+ const mid = (low + (high - low + 1) / 2) | 0;+ const value = frames[mid]!.timestamp;+ const delta = Math.abs(value - timestamp);++ if (delta < distance) {+ distance = delta;+ closest = mid;+ }++ if (value < timestamp) low = mid + 1;+ else if (value > timestamp) high = mid - 1;+ else return mid;+ }++ return closest;}diff --git a/apps/web/src/engine/timeline/peaks.ts b/apps/web/src/engine/timeline/peaks.tsnew file mode 100644index 00000000..25a7693e--- /dev/null+++ b/apps/web/src/engine/timeline/peaks.ts@@ -0,0 +1,416 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * The peaks the timeline's waveforms are drawn from, in two layers.+ *+ * The asset layer decodes a file once, at a fixed high resolution, and hands+ * every couple of seconds of it to IndexedDB. Peaks that fine are the better+ * part of a kilobyte a second, so a few long files would be tens of megabytes+ * held for a whole session; the store is what keeps them off the JS heap. It+ * is emptied when it is opened, since the index of which chunks exist lives+ * in memory — nothing written in an earlier session could be found again.+ *+ * The clip layer takes whichever chunks the visible stretch of one clip+ * covers and downsamples them to one peak per pixel, which is the only+ * resolution a draw pass can use. Zoom changes what a pixel is worth, so this+ * layer is rebuilt as the timeline is zoomed; the asset layer never is.+ *+ * Both are asked for by the draw pass and filled in behind it: a request says+ * what a clip needs, and the next frame draws whatever has arrived. Nothing+ * here blocks a frame.+ */++import { AudioSampleSink } from 'mediabunny';+import { getAudioTrack } from '@diffusionstudio/runtime';++import type { AudioSample } from 'mediabunny';+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';++/** Peaks a second the asset layer decodes at: finer than any zoom reaches. */+const SOURCE_PEAKS_PER_SECOND = 800;++/** How much audio one stored chunk covers, in seconds. */+const CHUNK_DURATION = 2;++/** How wide one drawn sample is, in pixels. */+export const SAMPLE_WIDTH = 1;++const IDB_NAME = 'peak-cache-db';+const IDB_VERSION = 1;+const IDB_STORE = 'data';++/** Where a chunk of an asset's peaks sits in it, in seconds, and its key. */+type PeakChunk = {+ start: number;+ end: number;+ id: string;+};++/** A chunk of peaks, downsampled to what one clip draws it at. */+export type ClipSamples = PeakChunk & {+ data: Uint8ClampedArray;+ peaksPerSecond: number;+};++/** What the decode produced for one stretch of a file, before it is stored. */+type DecodedPeaks = {+ start: number;+ end: number;+ data: Uint8ClampedArray;+};++/** The asset layer's record of one file: which chunks of it exist. */+type AssetPeaks = {+ decoding: boolean;+ /** True once the file has been read through, whatever came of it. */+ decoded: boolean;+ chunks: PeakChunk[];+};++/** The clip layer's record of one clip: the chunks it can draw, at its zoom. */+type ClipPeaks = {+ assetId: string;+ samples: ClipSamples[];+ /** The range and zoom `samples` was built for; unchanged means no work. */+ hash: string;+ updating: boolean;+ pending: PeakRequest | null;+};++/** What a clip needs this frame: a stretch of its asset, at a zoom. */+export type PeakRequest = {+ /** The entity id of the clip asking. */+ clip: number;+ asset: AudioAsset | VideoAsset;+ peaksPerSecond: number;+ /** The visible stretch of the source, in source seconds. */+ start: number;+ end: number;+};++const assetPeaks = new Map<string, AssetPeaks>();+const clipPeaks = new Map<number, ClipPeaks>();++/**+ * The chunks `clip` can draw right now, ordered by where they start. Empty+ * (or short) while the decode behind them is still running.+ */+export function getClipSamples(clip: number): ClipSamples[] | undefined {+ return clipPeaks.get(clip)?.samples;+}++/**+ * Asks for the peaks a clip needs. Starts the file's decode the first time it+ * is asked about, and re-cuts the clip's own samples whenever the stretch it+ * wants or the zoom it wants them at has changed.+ */+export function requestPeaks(request: PeakRequest): void {+ const record = assetPeaks.get(request.asset.id);++ if (!record || (!record.decoding && !record.decoded)) {+ void decodeAsset(request);+ }++ if (record && record.chunks.length > 0) {+ void updateClip(request, record);+ }+}++/**+ * Gives a clip's samples to the half that was cut off it. Both halves play+ * the same source at the same zoom, so the copy starts with everything the+ * original had — the peak buffers themselves are shared, only the records+ * that say what is being drawn are its own.+ */+export function clonePeaksForSplit(clip: number, copy: number): void {+ const source = clipPeaks.get(clip);+ if (!source) return;++ clipPeaks.set(copy, {+ assetId: source.assetId,+ samples: source.samples.map((sample) => ({ ...sample })),+ hash: source.hash,+ updating: false,+ pending: null,+ });+}++/** Forgets one clip's samples, or every clip's. */+export function clearClipPeaks(clip?: number): void {+ if (clip === undefined) clipPeaks.clear();+ else clipPeaks.delete(clip);+}++/** Forgets everything: the clips' samples and the index of what was decoded. */+export function clearPeaks(): void {+ clipPeaks.clear();+ assetPeaks.clear();+}++/** Forgets one asset's peaks, for a file that is no longer the one it was. */+export function forgetAssetPeaks(assetId: string): void {+ assetPeaks.delete(assetId);+ for (const [clip, peaks] of clipPeaks) {+ if (peaks.assetId === assetId) clipPeaks.delete(clip);+ }+}++// ---------------------------------------------------------------------------+// The asset layer++/**+ * Reads the file through once, writing its peaks to the store a couple of+ * seconds at a time. Each chunk is usable the moment it lands, so a long file+ * draws from the top down rather than all at once at the end.+ */+async function decodeAsset(request: PeakRequest): Promise<void> {+ const { asset } = request;++ const record = assetPeaks.get(asset.id) ?? { decoding: false, decoded: false, chunks: [] };+ assetPeaks.set(asset.id, record);+ if (record.decoding) return;++ record.decoding = true;++ try {+ const track = await getAudioTrack(asset);+ if (!track) return;++ const sink = new AudioSampleSink(track);+ const pending: DecodedPeaks[] = [];+ let covered = 0;++ const flush = async (): Promise<void> => {+ const merged = mergePeaks(pending);+ const id = `${asset.id}-${merged.start}`;+ record.chunks.push({ start: merged.start, end: merged.end, id });+ pending.length = 0;+ covered = 0;+ await storePeaks(id, merged.data);+ };++ for await (const sample of sink.samples()) {+ covered += sample.duration;+ try {+ pending.push(peaksOfSample(sample));+ } finally {+ sample.close();+ }++ if (covered > CHUNK_DURATION) await flush();+ }++ if (pending.length > 0) await flush();++ record.chunks.sort((a, b) => a.start - b.start);+ } catch (error) {+ console.error('[timeline] could not decode peaks', error);+ } finally {+ record.decoding = false;+ record.decoded = true;+ }+}++/**+ * One decoded sample's peaks: the loudest frame of each slice, across every+ * channel, raised to a power so quiet passages still show as something.+ */+function peaksOfSample(sample: AudioSample): DecodedPeaks {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });++ const channels = sample.numberOfChannels;+ const frames = floats.length / channels;+ const target = Math.ceil(sample.duration * SOURCE_PEAKS_PER_SECOND);+ const peaks = new Uint8ClampedArray(target);+ const ratio = frames / target;++ for (let i = 0; i < target; i++) {+ const from = Math.floor(i * ratio);+ const to = Math.floor((i + 1) * ratio);+ let max = 0;++ for (let frame = from; frame < to && frame < frames; frame++) {+ for (let channel = 0; channel < channels; channel++) {+ max = Math.max(max, Math.pow(Math.abs(floats[frame * channels + channel]!), 0.8));+ }+ }++ peaks[i] = Math.floor(255 * max);+ }++ return { start: sample.timestamp, end: sample.timestamp + sample.duration, data: peaks };+}++function mergePeaks(peaks: DecodedPeaks[]): DecodedPeaks {+ const length = peaks.reduce((total, peak) => total + peak.data.length, 0);+ const merged = new Uint8ClampedArray(length);++ let offset = 0;+ for (const peak of peaks) {+ merged.set(peak.data, offset);+ offset += peak.data.length;+ }++ return { start: peaks[0]!.start, end: peaks[peaks.length - 1]!.end, data: merged };+}++// ---------------------------------------------------------------------------+// The clip layer++/**+ * Re-cuts a clip's samples for the stretch and zoom it is asking about. Only+ * one runs at a time per clip — a zoom or a scroll that arrives while one is+ * reading the store replaces whatever was queued behind it, so the work that+ * finally runs is for where the timeline ended up rather than where it went+ * through.+ */+async function updateClip(request: PeakRequest, record: AssetPeaks): Promise<void> {+ const peaks = clipPeaks.get(request.clip)+ ?? { assetId: request.asset.id, samples: [], hash: '', updating: false, pending: null };+ clipPeaks.set(request.clip, peaks);++ if (peaks.updating) {+ peaks.pending = request;+ return;+ }++ try {+ const range = coveringRange(request, record);+ if (range.hash === peaks.hash) return;++ peaks.updating = true;+ peaks.hash = range.hash;++ for (const chunk of record.chunks) {+ if (!inRange(chunk.start, range.start, range.end)) continue;++ const data = await loadPeaks(chunk.id);+ if (!data) continue;++ peaks.samples = peaks.samples.filter((sample) => sample.id !== chunk.id);+ peaks.samples.push({+ ...chunk,+ data: downsample(data, Math.ceil((chunk.end - chunk.start) * request.peaksPerSecond)),+ peaksPerSecond: request.peaksPerSecond,+ });+ peaks.samples.sort((a, b) => a.start - b.start);+ }++ peaks.samples = peaks.samples.filter((sample) => inRange(sample.start, range.start, range.end));+ } catch (error) {+ console.error('[timeline] could not cut peaks', error);+ } finally {+ peaks.updating = false;+ }++ const pending = peaks.pending;+ if (pending) {+ peaks.pending = null;+ void updateClip(pending, record);+ }+}++/**+ * The stretch of the asset to cut, snapped out to whole chunks: what is on+ * screen, widened to the chunks its ends fall inside, so the waveform does+ * not stop short of the edge of the viewport.+ */+function coveringRange(request: PeakRequest, record: AssetPeaks) {+ const from = Math.max(0, request.start);+ const to = Math.min(request.asset.duration, request.end);++ const first = record.chunks.find((chunk) => inRange(from, chunk.start, chunk.end));+ const last = record.chunks.find((chunk) => inRange(to, chunk.start, chunk.end));++ const start = first?.start ?? record.chunks.at(0)?.start ?? 0;+ const end = last?.end ?? record.chunks.at(-1)?.end ?? 0;++ return { start, end, hash: `${request.peaksPerSecond}.${start}.${end}` };+}++/**+ * The peaks of `data` at `target` positions: the loudest of each slice, and+ * never fewer than one sample per position.+ *+ * Zoomed in past the resolution the peaks were stored at, a position's slice+ * of the source is narrower than one stored sample, and every slice whose+ * ends round to the same sample would read nothing at all — a fifth of them+ * at the far end of the zoom, drawn as a regular pattern of gaps through the+ * waveform. Holding the sample the slice starts in draws what there is to+ * draw instead: past that zoom the file has no more detail to show.+ */+function downsample(data: Uint8ClampedArray, target: number): Uint8ClampedArray {+ const peaks = new Uint8ClampedArray(target);+ const ratio = data.length / target;++ for (let i = 0; i < target; i++) {+ const from = Math.floor(i * ratio);+ const to = Math.max(from + 1, Math.floor((i + 1) * ratio));+ let max = 0;++ for (let index = from; index < to && index < data.length; index++) {+ max = Math.max(max, data[index]!);+ }++ peaks[i] = max;+ }++ return peaks;+}++const inRange = (value: number, from: number, to: number): boolean => value >= from && value <= to;++// ---------------------------------------------------------------------------+// The store++let database: Promise<IDBDatabase> | null = null;++function getDatabase(): Promise<IDBDatabase> {+ database ??= new Promise((resolve, reject) => {+ const request = indexedDB.open(IDB_NAME, IDB_VERSION);++ request.onupgradeneeded = () => request.result.createObjectStore(IDB_STORE);+ request.onerror = () => reject(request.error);++ request.onsuccess = () => {+ // What an earlier session left behind is unreachable — the index of+ // which chunks exist was never written — so the session starts on+ // an empty store rather than one that only grows.+ const transaction = request.result.transaction(IDB_STORE, 'readwrite');+ transaction.objectStore(IDB_STORE).clear();+ transaction.oncomplete = () => resolve(request.result);+ transaction.onerror = () => reject(transaction.error);+ };+ });++ return database;+}++async function storePeaks(key: string, peaks: Uint8ClampedArray): Promise<void> {+ const db = await getDatabase();+ const transaction = db.transaction(IDB_STORE, 'readwrite');++ transaction.objectStore(IDB_STORE).put(new Blob([new Uint8Array(peaks)]), key);++ return new Promise((resolve, reject) => {+ transaction.oncomplete = () => resolve();+ transaction.onerror = () => reject(transaction.error);+ });+}++async function loadPeaks(key: string): Promise<Uint8ClampedArray | null> {+ const db = await getDatabase();+ const store = db.transaction(IDB_STORE, 'readonly').objectStore(IDB_STORE);++ const blob = await new Promise<Blob | null>((resolve, reject) => {+ const request = store.get(key);+ request.onsuccess = () => resolve(request.result ?? null);+ request.onerror = () => reject(request.error);+ });++ return blob ? new Uint8ClampedArray(await blob.arrayBuffer()) : null;+}diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.tsindex a1098769..30c5e9aa 100644--- a/apps/web/src/engine/timeline/render/clip.ts+++ b/apps/web/src/engine/timeline/render/clip.ts@@ -20,7 +20,7 @@ import { truncateText } from '../text';import { framesToPixels, getResolution, getViewport } from '../view';import { renderCaption } from './caption';import { renderGroup } from './group';-import { renderThumbnails } from './thumbnails';+import { renderStillThumbnails, renderVideoThumbnails } from './thumbnails';import { renderWaveform } from './waveform';import type { Entity, World } from 'koota';@@ -165,34 +165,24 @@ function renderContent(switch (asset?.type) {case 'IMAGE':case 'SEQUENCE':- if (tall) renderThumbnails(world, scene, surface, entity, asset, row);+ renderStillThumbnails(world, scene, surface, entity, asset, row);return;case 'AUDIO':renderWaveform(world, scene, surface, entity, row, {+ asset,color: style.primary ?? style.foreground,offsetY: tall ? CLIP_LABEL_HEIGHT : 2,padding: 4,});return;- case 'VIDEO': {+ case 'VIDEO':// A video shows both: its picture along the top of the row, its- // sound along the bottom.- const strip = tall ? Math.round((row.height - CLIP_LABEL_HEIGHT) / 2) : 0;- if (strip > 0) {- renderThumbnails(world, scene, surface, entity, asset, row, {- top: CLIP_LABEL_HEIGHT,- height: strip,- });- }- renderWaveform(world, scene, surface, entity, row, {- color: style.primary ?? style.foreground,- offsetY: tall ? CLIP_LABEL_HEIGHT + strip : 2,- padding: 2,- });+ // sound along the bottom. Which of the two the row has room for is+ // the strip's own to work out.+ renderVideoThumbnails(world, scene, surface, entity, asset, row, style.primary ?? style.foreground);return;- }}}diff --git a/apps/web/src/engine/timeline/render/thumbnails.ts b/apps/web/src/engine/timeline/render/thumbnails.tsindex c4f2281f..6f3cf52a 100644--- a/apps/web/src/engine/timeline/render/thumbnails.ts+++ b/apps/web/src/engine/timeline/render/thumbnails.ts@@ -2,13 +2,14 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Computed, store } from '@diffusionstudio/runtime';+import { Computed, getSourceWindow, store } from '@diffusionstudio/runtime';-import { CLIP_LABEL_HEIGHT } from '../config';-import { resolvePoster } from '../media';-import { framesToPixels, getResolution, getViewport } from '../view';+import { CLIP_BREAKPOINTS, CLIP_LABEL_HEIGHT } from '../config';+import { pickFrame, requestFrames, resolveStill } from '../media';+import { framesToPixels, getFrameRate, getResolution, getViewport, pixelsToFrames } from '../view';+import { renderWaveform } from './waveform';-import type { Asset } from '@diffusionstudio/assets';+import type { Asset, VideoAsset } from '@diffusionstudio/assets';import type { Entity, World } from 'koota';import type { RowCursor } from '../layout';import type { TimelineSurfaceState } from '../surface';@@ -18,72 +19,203 @@ import type { TimelineSurfaceState } from '../surface';* derived from the height, so dragging a row taller does not slide every* tile along as it goes.*/-const TILE_WIDTHS = [80, 100, 120] as const;+const STILL_WIDTHS = [80, 100, 120] as const;-const TOP = 20;-const INSET = 2;+/** Where a still's tiles start, and where the strip they sit in does. */+const STILL_TOP = 18;+const STILL_MASK_TOP = 20;/**- * The clip's picture, tiled along it. An image shows the same frame however- * long it runs, so tiling its poster is the whole truth about it; a video's- * poster is only its first frame, so this says where the footage is rather- * than what is in it.+ * The least a video's picture strip is worth drawing at. Under it the row+ * is all waveform, which says more about a clip that short than four rows of+ * pixels would.*/-export function renderThumbnails(+const MIN_STRIP_HEIGHT = 42;++/** The tile a video's strip is drawn with, and the strip height it changes at. */+const SMALL_TILE = { width: 56, height: 46 } as const;+const LARGE_TILE = { width: 84, height: 52 } as const;+const LARGE_TILE_STRIP_HEIGHT = 44;++/**+ * A still, tiled along the clip. An image shows the same frame however long+ * it runs, so tiling one picture is the whole truth about it — a sequence is+ * shown by its first frame for the same reason a folder of frames has a+ * cover.+ */+export function renderStillThumbnails(world: World,scene: Entity,surface: TimelineSurfaceState,entity: Entity,asset: Asset,row: RowCursor,- options: { top?: number; height?: number } = {},): void {const ctx = surface.ctx!;- const poster = resolvePoster(world, asset);- if (!poster) return;+ // SVGs aren't supported by createImageBitmap on every browser, so skip+ // the timeline thumbnail strip for them.+ if (asset.mimeType === 'image/svg+xml') return;- const top = options.top ?? TOP;- const height = options.height ?? row.height - CLIP_LABEL_HEIGHT;- if (height < 4) return;+ // Too short a row to hold anything but its label.+ if (row.height <= CLIP_BREAKPOINTS.sm) return;+ const eid = entity.id();const computed = store(world, Computed);const resolution = getResolution(world, scene);- const left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);- const width = framesToPixels(computed.end[entity.id()] ?? 0, resolution) - left;- if (width < 4) return;+ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;++ const tileWidth = stillWidth(row.height);+ const still = resolveStill(asset, tileWidth);+ if (!still) return;- const tile = tileWidth(row.height);- const crop = coverCrop(poster.width, poster.height, tile, height);+ const maskHeight = row.height - CLIP_LABEL_HEIGHT;// Only the tiles on screen: a clip an hour long is as cheap to draw as a// clip a second long.const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);- const first = Math.max(0, Math.floor((viewportLeft - left) / tile));- const last = Math.min(Math.ceil(width / tile) - 1, Math.floor((viewportRight - left) / tile));+ const tiles = Math.ceil(clipWidth / tileWidth);+ const first = Math.max(0, Math.floor((viewportLeft - clipLeft) / tileWidth));+ const last = Math.min(tiles - 1, Math.floor((viewportRight - clipLeft) / tileWidth));++ const crop = coverCrop(still.width, still.height, tileWidth, maskHeight);ctx.save();ctx.beginPath();- ctx.roundRect(left + INSET, top, width - INSET * 2, height - INSET, 2);+ ctx.roundRect(clipLeft + 2, STILL_MASK_TOP, clipWidth - 4, maskHeight - 2, 2);ctx.clip();for (let i = first; i <= last; i++) {ctx.drawImage(- poster,+ still.canvas,crop.x, crop.y, crop.width, crop.height,- left + i * tile, top, tile, height,+ clipLeft + i * tileWidth, STILL_TOP, tileWidth, maskHeight,);}ctx.restore();}-function tileWidth(rowHeight: number): number {- if (rowHeight >= 100) return TILE_WIDTHS[2];- if (rowHeight >= 70) return TILE_WIDTHS[1];- return TILE_WIDTHS[0];+/**+ * A video: its picture along the top of the row and its sound along the+ * bottom, each with half of what the label leaves.+ *+ * The picture is a strip of the clip's own frames rather than one frame+ * repeated, so what a tile shows is what the footage is doing at that point+ * of the clip. Tiles are placed in the source's own time — the strip is+ * drawn from the clip's origin, so trimming an end slides the footage under+ * the clip instead of re-cutting the tiles.+ */+export function renderVideoThumbnails(+ world: World,+ scene: Entity,+ surface: TimelineSurfaceState,+ entity: Entity,+ asset: VideoAsset,+ row: RowCursor,+ color: string,+): void {+ const ctx = surface.ctx!;++ const eid = entity.id();+ const computed = store(world, Computed);+ const resolution = getResolution(world, scene);+ const fps = getFrameRate(world);++ const layout = videoLayout(row.height);++ renderWaveform(world, scene, surface, entity, row, {+ asset,+ color,+ offsetY: layout.waveformOffsetY,+ padding: 0,+ });++ // Nothing left of the row for a picture once the label and the waveform+ // have had theirs.+ if (layout.stripHeight === 0) return;++ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;++ ctx.save();++ ctx.beginPath();+ ctx.roundRect(clipLeft + 2, CLIP_LABEL_HEIGHT, clipWidth - 4, layout.stripHeight, 2);+ ctx.clip();++ const playbackRate = computed.playbackRate[eid] || 1;+ const origin = computed.origin[eid] ?? 0;++ // How much source one tile covers, scaled by playbackRate so a tile's+ // width in timeline pixels maps to the right chunk of source. Kept as a+ // float: at high zoom this is sub-frame, and rounding it to 0 collapses+ // the decode range and loses every tile.+ const interval = (layout.tile.width / resolution) * playbackRate;++ const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);++ // The viewport is intersected with the source window directly in frames.+ // Going through pixels first lets `framesToPixels` floor and+ // `pixelsToFrames` round, which can shave a frame off the last tile.+ const source = getSourceWindow(entity);+ const firstFrame = Math.max(source.in, (pixelsToFrames(viewportLeft, resolution) - origin) * playbackRate);+ const lastFrame = Math.min(source.out, (pixelsToFrames(viewportRight, resolution) - origin) * playbackRate);++ requestFrames({+ clip: eid,+ asset,+ fps,+ firstFrame,+ lastFrame,+ width: layout.tile.width,+ height: layout.tile.height,+ interval,+ });++ // Tile indices are in source time, so they line up with the frames the+ // decoder filed under source timestamps once the origin is translated to.+ ctx.translate(framesToPixels(origin, resolution), 0);++ // Only the tiles on screen — the rest are inside the clip but outside the+ // viewport, and drawing them is work the clip path would throw away.+ for (let i = Math.floor(firstFrame / interval); i <= Math.floor(lastFrame / interval); i++) {+ const frame = pickFrame(eid, asset.id, i * interval, interval);+ if (!frame) continue;++ ctx.drawImage(frame.canvas, i * layout.tile.width, layout.tileTop, layout.tile.width, layout.tile.height);+ }++ ctx.restore();+}++/**+ * How a video row is split. The picture takes half of what the label leaves,+ * the waveform the other half; below the point where a picture would be too+ * short to read, the waveform takes all of it.+ */+function videoLayout(rowHeight: number) {+ const body = rowHeight - CLIP_LABEL_HEIGHT;+ const stripHeight = body > MIN_STRIP_HEIGHT ? body * 0.5 : 0;+ const tile = stripHeight > LARGE_TILE_STRIP_HEIGHT ? LARGE_TILE : SMALL_TILE;++ return {+ stripHeight,+ tile,+ // Bottom-aligned in the strip, so a tile taller than it is cropped at+ // the top rather than dangling over the waveform.+ tileTop: CLIP_LABEL_HEIGHT + (stripHeight - tile.height),+ waveformOffsetY: rowHeight > CLIP_BREAKPOINTS.sm ? stripHeight + CLIP_LABEL_HEIGHT : 2,+ };+}++function stillWidth(rowHeight: number): number {+ if (rowHeight >= 100) return STILL_WIDTHS[2];+ if (rowHeight >= 70) return STILL_WIDTHS[1];+ return STILL_WIDTHS[0];}/**diff --git a/apps/web/src/engine/timeline/render/waveform.ts b/apps/web/src/engine/timeline/render/waveform.tsindex 88514246..6b18c948 100644--- a/apps/web/src/engine/timeline/render/waveform.ts+++ b/apps/web/src/engine/timeline/render/waveform.ts@@ -2,37 +2,41 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { AssetId, Cache, Computed, findAssetDuration, getSourceWindow, resolveWaveformPeaks, store } from '@diffusionstudio/runtime';+import { Computed, getSourceWindow, store } from '@diffusionstudio/runtime';-import { framesToPixels, getResolution } from '../view';+import { CLIP_BREAKPOINTS, CLIP_CORNER_RADIUS } from '../config';+import { SAMPLE_WIDTH, getClipSamples, requestPeaks } from '../peaks';+import { framesToPixels, getFrameRate, getResolution, getViewport } from '../view';+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';import type { Entity, World } from 'koota';+import type { ClipSamples } from '../peaks';import type { RowCursor } from '../layout';import type { TimelineSurfaceState } from '../surface';-/** How wide a bar is, and how much of that is the gap after it. */-const BAR_WIDTH = 2;-const BAR_GAP = 1;-const MIN_BAR_HEIGHT = 1;+/** How far either side of the viewport peaks are asked for, in pixels. */+const VIEWPORT_MARGIN = 5;-type WaveformOptions = {+export type WaveformOptions = {+ asset: AudioAsset | VideoAsset;color: string;/** How far down the clip the waveform starts. */offsetY: number;- /** How much of the height to leave above and below it. */+ /** How much of the height to leave for it to sit inside. */padding: number;};/**- * The clip's audio, as bars across the part of the waveform it plays.+ * The clip's audio, one column of pixels at a time.*- * The peaks belong to the asset, not the clip: they summarise the whole file- * in a fixed number of bars (`PEAK_BARS`), and the clip's own source window- * says which stretch of them is its own. That is what makes a split cost- * nothing — both halves read the same summary, from different ends of it.+ * A column is a moment of the source, so what it is worth changes with the+ * zoom: the peaks are asked for at one per pixel (see `../peaks`), and only+ * for the stretch of the clip that is on screen. Drawing then reads them back+ * by time rather than by index, so a column drawn before its peaks have+ * arrived is simply left out and filled in on a later frame.*- * The summary is coarse, so a clip zoomed in far enough draws a smoothed- * version of its audio rather than its samples.+ * Tall rows show the waveform about its middle, short ones stand it on the+ * bottom of the clip and fade it, so the label above it stays readable.*/export function renderWaveform(world: World,@@ -44,61 +48,100 @@ export function renderWaveform(): void {const ctx = surface.ctx!;- const source = findPeakSource(entity);- if (!source) return;-- const peaks = resolveWaveformPeaks(world, source);- if (!peaks || peaks.length === 0) return;-- const height = row.height - options.offsetY - options.padding;- if (height < 2) return;+ // A video with no audio track has nothing to draw here.+ if (!options.asset.sampleRate) return;+ const eid = entity.id();const computed = store(world, Computed);const resolution = getResolution(world, scene);+ const fps = getFrameRate(world);- const left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);- const width = framesToPixels(computed.end[entity.id()] ?? 0, resolution) - left;- if (width < BAR_WIDTH) return;+ const clipLeft = framesToPixels(computed.start[eid] ?? 0, resolution);+ const clipWidth = framesToPixels(computed.end[eid] ?? 0, resolution) - clipLeft;+ const maskHeight = row.height - options.offsetY;+ if (maskHeight <= 0) return;- // Which stretch of the asset this clip plays, as a fraction of it: the- // source window is in source frames, and the peaks span all of them.- const duration = findAssetDuration(world, entity) ?? 0;- const window = getSourceWindow(entity);- const from = duration > 0 ? window.in / duration : 0;- const to = duration > 0 ? Math.min(1, window.out / duration) : 1;+ const [viewportLeft, viewportRight] = getViewport(world, scene, surface.layout.width);- const step = BAR_WIDTH + BAR_GAP;- const count = Math.floor(width / step);- if (count <= 0) return;+ ctx.save();- const centerY = options.offsetY + height / 2;+ // Everything below is drawn against the clip's own rounded body.+ ctx.beginPath();+ ctx.roundRect(clipLeft, options.offsetY, clipWidth, maskHeight, CLIP_CORNER_RADIUS);+ ctx.clip();++ // A little smaller than the mask, so the tallest sample does not touch it.+ const waveformHeight = maskHeight - options.padding;++ const playbackRate = computed.playbackRate[eid] || 1;+ const pixelsToSeconds = (pixels: number): number => pixels / (fps * resolution);++ // Each pixel covers playbackRate× more source audio when sped up, so we+ // need proportionally fewer peaks per source second to keep one peak/pixel.+ // Never none: zoomed far enough out a pixel is worth less than a second of+ // source, and asking for no peaks a second is asking for no waveform.+ const peaksPerSecond = Math.max(1, Math.floor(fps * resolution)) / SAMPLE_WIDTH / playbackRate;++ // The source window is in source frames; the clip occupies+ // window / playbackRate timeline frames, measured from its own origin.+ const source = getSourceWindow(entity);+ const inPx = framesToPixels(source.in / playbackRate, resolution);+ const outPx = framesToPixels(source.out / playbackRate, resolution);+ const originPx = framesToPixels(computed.origin[eid] ?? 0, resolution);++ const firstSample = Math.max(inPx, viewportLeft - VIEWPORT_MARGIN - originPx);+ const lastSample = Math.min(outPx, viewportRight + VIEWPORT_MARGIN - originPx);++ requestPeaks({+ clip: eid,+ asset: options.asset,+ start: pixelsToSeconds(firstSample) * playbackRate,+ end: pixelsToSeconds(lastSample) * playbackRate,+ peaksPerSecond,+ });- ctx.save();ctx.fillStyle = options.color;- for (let i = 0; i < count; i++) {- // Where along the clip this bar is, mapped into the asset's summary.- const position = from + ((i + 0.5) / count) * (to - from);- const value = (peaks[Math.min(peaks.length - 1, Math.floor(position * peaks.length))] ?? 0) / 255;- const barHeight = Math.max(value * height, MIN_BAR_HEIGHT);+ // A tall row centres the waveform on the space it has; a short one stands+ // it on the bottom, where it reads as a strip under the label.+ const anchor = maskHeight > CLIP_BREAKPOINTS.sm ? 0.5 : 1;+ const minSampleHeight = maskHeight > CLIP_BREAKPOINTS.sm ? 1 : 0;++ // Tone the waveform down on short clip rows so the label stays readable.+ if (row.height <= CLIP_BREAKPOINTS.sm) ctx.globalAlpha *= 0.6;++ ctx.translate(originPx, options.offsetY + options.padding * anchor);++ const samples = getClipSamples(eid);- ctx.fillRect(left + i * step, centerY - barHeight / 2, BAR_WIDTH, barHeight);+ // The chunks are ordered by where they start and the columns are drawn+ // left to right, so the search for the chunk a column falls in only ever+ // moves forwards.+ let cursor = 0;++ for (let x = firstSample; x < lastSample; x += SAMPLE_WIDTH) {+ const time = pixelsToSeconds(x) * playbackRate;++ cursor = findChunk(samples, time, cursor);+ const chunk = samples?.[cursor];+ if (!chunk || time < chunk.start || time > chunk.end) continue;++ const peak = chunk.data[Math.floor((time - chunk.start) * chunk.peaksPerSecond)];+ if (peak === undefined) continue;++ const height = Math.max((peak / 255) * waveformHeight, minSampleHeight);+ ctx.fillRect(x, (waveformHeight - height) * anchor, SAMPLE_WIDTH, height);}ctx.restore();}-/**- * What holds the asset the peaks come from: the clip itself when it is its- * own source (an audio clip), otherwise whichever fill names one. Whether- * that asset has any audio is `resolveWaveformPeaks`'s to say.- */-function findPeakSource(entity: Entity): Entity | null {- if (entity.has(AssetId)) return entity;+/** The first chunk at or after `from` that has not already ended by `time`. */+function findChunk(samples: ClipSamples[] | undefined, time: number, from: number): number {+ if (!samples) return 0;- for (const fill of entity.get(Cache)?.fills ?? []) {- if (fill.has(AssetId)) return fill;- }+ let index = from;+ while (index < samples.length && samples[index]!.end < time) index++;- return null;+ return index;}diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.tsindex 101f65f1..aad92952 100644--- a/packages/runtime/src/media/audio-peaks.ts+++ b/packages/runtime/src/media/audio-peaks.ts@@ -29,8 +29,12 @@ export class Waveform {}private async initialize(): Promise<void> {- const file = await getAssetFile(this.asset);- const peaks = await derivePeaks(file);+ // Through the cache where there is one, so a file's peaks are derived+ // once per project rather than once per session.+ const peaks = this.cache+ ? await this.cache.peaks(this.asset)+ : await derivePeaks(await getAssetFile(this.asset));+if (!this.disposed) {this.peaks = peaks;}