The VS Code of video editing, built for agents.
TypeScript
Add caching for asset thumbnails and audio peaks
konstantin-paulusb7388afa7654
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex 7cd603c8..818bce17 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -188,6 +188,11 @@ const TSCONFIG = `{}`;+/** What a project folder produces but should not check in: installs, and derived data (thumbnails, waveforms). */+const GITIGNORE = `node_modules/+cache/+`;+const STARTER = `export default function Project() {return (<stage background="#161616">@@ -232,6 +237,7 @@ export async function scaffold(dir: string, displayName = basename(dir)): Promis}await ensurePackage(dir, name, displayName, entry);await writeIfMissing(dir, "tsconfig.json", TSCONFIG);+ await writeIfMissing(dir, ".gitignore", GITIGNORE);}/** Creates a fresh project folder under `root`. Fails if the folder exists. */diff --git a/apps/web/src/components/sidebar-left/asset-item.tsx b/apps/web/src/components/sidebar-left/asset-item.tsxindex 41f8a2bc..87c1fdf0 100644--- a/apps/web/src/components/sidebar-left/asset-item.tsx+++ b/apps/web/src/components/sidebar-left/asset-item.tsx@@ -130,7 +130,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {class="relative aspect-video w-full overflow-clip rounded bg-muted after:pointer-events-none after:absolute after:inset-0 after:rounded after:opacity-0 after:ring-2 after:ring-inset after:ring-ring after:z-10 data-[selected=true]:after:opacity-100"><Show when={isVisible()}>- <AssetThumbnail asset={props.asset} class="h-full w-full" />+ <AssetThumbnail asset={props.asset} class="h-full w-full" cache={library()?.cache} /></Show><Show when={assetDuration()}>{(duration) => (diff --git a/apps/web/src/components/ui/asset-thumbnail.tsx b/apps/web/src/components/ui/asset-thumbnail.tsxindex 4641f0f1..1efaad9a 100644--- a/apps/web/src/components/ui/asset-thumbnail.tsx+++ b/apps/web/src/components/ui/asset-thumbnail.tsx@@ -2,12 +2,12 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';-import { Show, For, createResource } from 'solid-js';+import { Show, For, createResource, onCleanup } from 'solid-js';import { cx } from '@/lib/cva';-import { getAssetFile, getAudioPeaksAsync } from '@diffusionstudio/runtime';+import { getAssetFile } from '@diffusionstudio/runtime';+import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH, derivePeaks } from '@diffusionstudio/assets';-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';+import type { Asset as LibraryAsset, AssetCache, AudioAsset, VideoAsset, ImageAsset } from '@diffusionstudio/assets';/*** What a thumbnail needs of an asset: the runtime's `Asset` and the legacy@@ -29,114 +29,56 @@ type Asset = ThumbnailAsset;/** Re-renders a thumbnail when the asset, or the file behind it, changes. */const keyOf = (asset: Asset): string => `${asset.id}:${asset.stat?.mtime ?? asset.lastModified ?? ''}`;-const DEFAULT_THUMBNAIL_SIZE = {- width: 300,- height: 168,-} as const;-type ThumbnailSize = {width: number;height: number;};-function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {- const [url] = createResource(- () => keyOf(props.asset),- async () => {- const objectUrl = URL.createObjectURL(await getAssetFile(props.asset));- try {- const image = await new Promise<HTMLImageElement>((resolve, reject) => {- const img = new Image();- img.onload = () => resolve(img);- img.onerror = () => reject(new Error('Failed to load image'));- img.src = objectUrl;- });-- const canvas = document.createElement('canvas');- canvas.width = props.size.width;- canvas.height = props.size.height;- const ctx = canvas.getContext('2d')!;-- const imgW = image.naturalWidth || props.asset.width || props.size.width;- const imgH = image.naturalHeight || props.asset.height || props.size.height;- const scale = Math.max(props.size.width / imgW, props.size.height / imgH);- const scaledW = imgW * scale;- const scaledH = imgH * scale;- const dx = (props.size.width - scaledW) / 2;- const dy = (props.size.height - scaledH) / 2;-- ctx.drawImage(image, dx, dy, scaledW, scaledH);- const url = canvas.toDataURL('image/webp', 0.7);-- canvas.width = 0;- canvas.height = 0;-- return url;- } catch {- return undefined;- } finally {- URL.revokeObjectURL(objectUrl);- }- }- );+/**+ * The thumbnail of an image or video: from the library's cache when there is+ * one (kept in the project's `cache/` across sessions), derived on the spot+ * otherwise. Scaled to `width` at the asset's own aspect ratio; the+ * container crops it.+ */+function MediaThumbnail(props: { asset: Asset; width: number; cache?: AssetCache }) {+ let objectUrl: string | undefined;- return (- <Show when={url()}>- <img- src={url()!}- alt=""+ const releaseUrl = () => {+ if (objectUrl) URL.revokeObjectURL(objectUrl);+ objectUrl = undefined;+ };- class="w-full h-full object-cover select-none"- />- </Show>- );-}+ const key = () => `${keyOf(props.asset)}:${props.width}:${props.cache ? 'cached' : ''}`;-function VideoThumbnail(props: { asset: Asset; size: ThumbnailSize }) {- const [src] = createResource(- () => keyOf(props.asset),- async () => {- try {- const file = await getAssetFile(props.asset);-- const input = new Input({- formats: ALL_FORMATS,- source: new BlobSource(file),- });-- const track = await input.getPrimaryVideoTrack();- if (!track) return;-- const sink = new CanvasSink(track, {- width: props.size.width,- height: props.size.height,- fit: 'cover',- });-- const ts = await track.getFirstTimestamp();- const wrappedCanvas = await sink.getCanvas(ts + 0.3);- if (!wrappedCanvas) return;-- const canvas = document.createElement('canvas');- canvas.width = wrappedCanvas.canvas.width;- canvas.height = wrappedCanvas.canvas.height;- canvas.getContext('2d')?.drawImage(wrappedCanvas.canvas, 0, 0);- const url = canvas.toDataURL('image/webp', 0.7);- canvas.width = 0;- canvas.height = 0;- return url;- } catch {- return undefined;+ const loadThumbnail = async () => {+ releaseUrl();++ try {+ let blob: Blob | null = null;++ if (props.cache) {+ blob = await props.cache.thumbnail(props.asset as LibraryAsset, props.width);+ } else {+ const file = await getAssetFile(props.asset as unknown as ImageAsset | VideoAsset);+ blob = await deriveThumbnail(file, props.asset.mimeType, props.width);}+ objectUrl = blob ? URL.createObjectURL(blob) : undefined;++ return objectUrl;+ } catch {+ return undefined;}- );+ }++ const [url] = createResource(key, loadThumbnail);++ onCleanup(releaseUrl);return (- <Show when={src()}>+ <Show when={url()}><img- src={src()!}+ src={url()!}alt=""-class="w-full h-full object-cover select-none"/></Show>@@ -159,11 +101,19 @@ function TranscriptThumbnail() {);}-function AudioThumbnail(props: { asset: Asset }) {- const [bins] = createResource(- () => keyOf(props.asset),- () => getAudioPeaksAsync(props.asset as unknown as AudioAsset | VideoAsset),- );+function AudioThumbnail(props: { asset: Asset; cache?: AssetCache }) {+ const loadPeaks = async () => {+ if (props.cache) {+ return props.cache.peaks(props.asset as unknown as AudioAsset | VideoAsset);+ }++ const file = await getAssetFile(props.asset as unknown as AudioAsset | VideoAsset);+ return derivePeaks(file);+ }++ const key = () => keyOf(props.asset);++ const [bins] = createResource(key, loadPeaks);return (<div class="absolute inset-0 pt-7 pb-1 bg-audio-background">@@ -186,21 +136,19 @@ type AssetThumbnailProps = {class?: string;draggable?: boolean;size?: ThumbnailSize;+ cache?: AssetCache;}export function AssetThumbnail(props: AssetThumbnailProps) {- const size = () => props.size ?? DEFAULT_THUMBNAIL_SIZE;+ const width = () => props.size?.width ?? DEFAULT_THUMBNAIL_WIDTH;return (<div class={cx(props.class, 'relative')} draggable={props.draggable}>- <Show when={props.asset.mimeType.startsWith('image')}>- <ImageThumbnail asset={props.asset} size={size()} />- </Show>- <Show when={props.asset.mimeType.startsWith('video')}>- <VideoThumbnail asset={props.asset} size={size()} />+ <Show when={props.asset.mimeType.startsWith('image') || props.asset.mimeType.startsWith('video')}>+ <MediaThumbnail asset={props.asset} width={width()} cache={props.cache} /></Show><Show when={props.asset.mimeType.startsWith('audio')}>- <AudioThumbnail asset={props.asset} />+ <AudioThumbnail asset={props.asset} cache={props.cache} /></Show><Show when={props.asset.type === 'TRANSCRIPT'}><TranscriptThumbnail />diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsxindex 37ab8223..1c23464e 100644--- a/apps/web/src/pages/editor.tsx+++ b/apps/web/src/pages/editor.tsx@@ -16,6 +16,7 @@ import { useWorld } from '@diffusionstudio/koota-solid';import { mount } from '@diffusionstudio/reconciler';import { getDocumentEditor } from '@/engine/editor';import { attachLibrary, isLibraryFile } from '@/engine/library';+import { isCacheFile } from '@diffusionstudio/assets';import { createEditWriter } from '@/projects/edits';import { compileProject, watchProject, projectDir } from '@/projects/host';import { useProjectId } from "@/hooks/use-project-id";@@ -84,7 +85,8 @@ export function EditorPage() {loadProject();const unwatch = watchProject(dir, (path) => {- if (isLibraryFile(path)) {+ if (isCacheFile(path)) return;+ if (isLibraryFile(path)) {library.load();} else {loadProject();diff --git a/packages/assets/src/cache.ts b/packages/assets/src/cache.tsnew file mode 100644index 00000000..df00a15b--- /dev/null+++ b/packages/assets/src/cache.ts@@ -0,0 +1,179 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */+++import { derivePeaks } from './derive/peaks';+import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH } from './derive/thumbnail';++import type { ProjectFS } from './fs';+import type { Asset } from './types';++/** The project directory derived data is kept in. */+export const CACHE_DIR = 'cache';++/** Whether a project-relative path is inside the cache directory. */+export const isCacheFile = (path: string): boolean => path === CACHE_DIR || path.startsWith(`${CACHE_DIR}/`);++/** A kind of cached value: where it lives under `cache/`, and how it is stored. */+export interface CacheKind<T> {+ /** Subdirectory of `cache/`. */+ dir: string;+ /** File extension (no dot). */+ ext: string;+ encode(value: T): Blob;+ decode(blob: Blob): Promise<T>;+}++/** The asset bar's thumbnail: a WebP, stored as is. */+export const THUMBNAIL: CacheKind<Blob> = {+ dir: 'thumbnails',+ ext: 'webp',+ encode: (blob) => blob,+ decode: async (blob) => blob,+};++/** The rough waveform: one byte per bar, stored raw. */+export const PEAKS: CacheKind<Uint8ClampedArray> = {+ dir: 'peaks',+ ext: 'u8',+ encode: (peaks) => new Blob([peaks as Uint8ClampedArray<ArrayBuffer>]),+ decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),+};++/** What an entry is keyed by: an asset, or just its id. */+type Keyed = string | Pick<Asset, 'id'>;++const idOf = (keyed: Keyed): string => (typeof keyed === 'string' ? keyed : keyed.id);++export class AssetCache {+ private readonly fs: ProjectFS;+ /** Reads and derivations under way, by `kind.dir/id[@variant]`. */+ private readonly inflight = new Map<string, Promise<unknown>>();+ /** Every kind used, so `remove` and `prune` know which directories to look in. */+ private readonly kinds = new Map<string, CacheKind<unknown>>();++ public constructor(fs: ProjectFS) {+ this.fs = fs;+ this.register(THUMBNAIL);+ this.register(PEAKS);+ }++ // -----------------------------------------------------------------------+ // Typed entries++ /**+ * The asset's thumbnail (IMAGE, VIDEO, or SEQUENCE by its first frame):+ * a WebP `width` wide at the asset's own aspect ratio, or null when none+ * can be made. Widths other than the default are kept as variants.+ */+ public thumbnail(asset: Asset, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {+ const variant = width === DEFAULT_THUMBNAIL_WIDTH ? undefined : `${width}`;+ return this.get(THUMBNAIL, asset, async () => deriveThumbnail(await asset.handle.getFile(), asset.mimeType, width), variant);+ }++ /** The asset's rough waveform, or null when it has no audio. */+ public peaks(asset: Asset): Promise<Uint8ClampedArray | null> {+ return this.get(PEAKS, asset, async () => derivePeaks(await asset.handle.getFile()));+ }++ // -----------------------------------------------------------------------+ // Generic entries++ /**+ * The cached value for (kind, asset[, variant]): read from disk, else+ * `produce()`d and written there. Concurrent calls share one result; a null+ * from `produce` (nothing to derive) is not stored.+ */+ public get<T>(kind: CacheKind<T>, keyed: Keyed, produce: () => Promise<T | null>, variant?: string): Promise<T | null> {+ this.register(kind);+ const key = this.key(kind, keyed, variant);+ const running = this.inflight.get(key);+ if (running) return running as Promise<T | null>;++ const promise = this.load(kind, key, produce).finally(() => this.inflight.delete(key));+ this.inflight.set(key, promise);+ return promise;+ }++ /** Drops every entry of an asset on disk. */+ public async remove(keyed: Keyed): Promise<void> {+ const id = idOf(keyed);+ for (const kind of this.kinds.values()) {+ const entries = await this.fs.list(`${CACHE_DIR}/${kind.dir}`).catch(() => []);+ for (const entry of entries) {+ if (entry.name === `${id}.${kind.ext}` || entry.name.startsWith(`${id}@`)) {+ await this.fs.remove(`${CACHE_DIR}/${kind.dir}/${entry.name}`).catch(() => {});+ }+ }+ }+ }++ /** Removes entries on disk whose asset is not among `keep`. */+ public async prune(keep: Iterable<string>): Promise<void> {+ const ids = new Set(keep);+ for (const kind of this.kinds.values()) {+ const entries = await this.fs.list(`${CACHE_DIR}/${kind.dir}`).catch(() => []);+ for (const entry of entries) {+ if (entry.kind !== 'file') continue;+ const stem = entry.name.split('.')[0]!.split('@')[0]!;+ if (ids.has(stem)) continue;+ await this.fs.remove(`${CACHE_DIR}/${kind.dir}/${entry.name}`).catch(() => {});+ }+ }+ }++ /** Deletes everything on disk; one kind, or all of it. */+ public async clear(kind?: CacheKind<unknown>): Promise<void> {+ await this.fs.remove(kind ? `${CACHE_DIR}/${kind.dir}` : CACHE_DIR).catch(() => {});+ }++ // -----------------------------------------------------------------------++ private register(kind: CacheKind<unknown>): void {+ if (!this.kinds.has(kind.dir)) this.kinds.set(kind.dir, kind);+ }++ private key(kind: CacheKind<unknown>, keyed: Keyed, variant?: string): string {+ return `${kind.dir}/${idOf(keyed)}${variant ? `@${variant}` : ''}`;+ }++ /** The file of an entry: `cache/<dir>/<id>[@variant].<ext>`. */+ private pathOf(kind: CacheKind<unknown>, key: string): string {+ return `${CACHE_DIR}/${key}.${kind.ext}`;+ }++ private async load<T>(kind: CacheKind<T>, key: string, produce: () => Promise<T | null>): Promise<T | null> {+ const stored = await this.read(kind, key);+ if (stored !== undefined) return stored;++ let value: T | null;+ try {+ value = await produce();+ } catch (error) {+ console.warn(`[assets] could not derive ${key}:`, error);+ value = null;+ }+ if (value !== null) void this.write(kind, key, value);+ return value;+ }++ /** The entry on disk, or undefined when missing or unreadable. */+ private async read<T>(kind: CacheKind<T>, key: string): Promise<T | undefined> {+ const path = this.pathOf(kind, key);+ try {+ if (!(await this.fs.stat(path))) return undefined;+ return await kind.decode(await this.fs.file(path));+ } catch {+ return undefined;+ }+ }++ private async write<T>(kind: CacheKind<T>, key: string, value: T): Promise<void> {+ try {+ await this.fs.write(this.pathOf(kind, key), kind.encode(value));+ } catch (error) {+ console.warn(`[assets] could not write cache entry ${key}:`, error);+ }+ }+}diff --git a/packages/assets/src/derive/peaks.ts b/packages/assets/src/derive/peaks.tsnew file mode 100644index 00000000..0667dff1--- /dev/null+++ b/packages/assets/src/derive/peaks.ts@@ -0,0 +1,73 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The rough waveform of a media file: a fixed number of bars, each the peak+// amplitude (across channels, gamma-lifted so quiet passages still show) of+// its slice of the file, as a byte. Decodes the whole audio track once; the+// AssetCache keeps the result.++import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';++/** How many bars a rough waveform has. */+export const PEAK_BARS = 128;++/** The peaks of `file`'s primary audio track, or null when it has none. */+export async function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });+ try {+ const track = await input.getPrimaryAudioTrack();+ if (!track) return null;++ const sink = new AudioSampleSink(track);+ const chunks: Uint8ClampedArray[] = [];++ for await (const sample of sink.samples()) {+ let floats: Float32Array;+ let channels: number;+ let frames: number;+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });+ channels = sample.numberOfChannels;+ frames = floats.length / channels;+ } finally {+ sample.close();+ }++ const framePeaks = new Uint8ClampedArray(frames);+ for (let f = 0; f < frames; f++) {+ let max = 0;+ for (let ch = 0; ch < channels; ch++) {+ max = Math.max(max, Math.pow(Math.abs(floats[f * channels + ch]!), 0.8));+ }+ framePeaks[f] = Math.floor(max * 255);+ }+ chunks.push(framePeaks);+ }++ if (chunks.length === 0) return null;++ const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);+ const merged = new Uint8ClampedArray(total);+ let offset = 0;+ for (const chunk of chunks) {+ merged.set(chunk, offset);+ offset += chunk.length;+ }++ const peaks = new Uint8ClampedArray(bars);+ const ratio = total / bars;+ for (let i = 0; i < bars; i++) {+ const start = Math.floor(i * ratio);+ const end = Math.min(total, Math.floor((i + 1) * ratio));+ let max = 0;+ for (let j = start; j < end; j++) max = Math.max(max, merged[j]!);+ peaks[i] = max;+ }+ return peaks;+ } finally {+ input.dispose();+ }+}diff --git a/packages/assets/src/derive/thumbnail.ts b/packages/assets/src/derive/thumbnail.tsnew file mode 100644index 00000000..c08f7a33--- /dev/null+++ b/packages/assets/src/derive/thumbnail.ts@@ -0,0 +1,58 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The thumbnail of an image or video file: scaled to a width, its aspect+// ratio kept, encoded as WebP. For a video it is the frame shortly after the+// start. The decoder does the scaling (mediabunny's CanvasSink for video,+// createImageBitmap for images), so the picture is encoded once, as it comes.++import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';++/** The width the asset bar shows; what the cache stores without a variant. */+export const DEFAULT_THUMBNAIL_WIDTH = 300;++/** How far into a video the thumbnail frame is taken from, in seconds. */+const VIDEO_THUMBNAIL_OFFSET = 0.3;++const WEBP_QUALITY = 0.7;++/** A WebP thumbnail of `file`, `width` wide, or null when it is neither an image nor a video with a picture. */+export async function deriveThumbnail(file: Blob, mimeType: string, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {+ if (mimeType.startsWith('image/')) return imageThumbnail(file, width);+ if (mimeType.startsWith('video/')) return videoThumbnail(file, width);+ return null;+}++async function imageThumbnail(file: Blob, width: number): Promise<Blob | null> {+ const bitmap = await createImageBitmap(file, { resizeWidth: width, resizeQuality: 'high' });+ try {+ const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);+ const ctx = canvas.getContext('bitmaprenderer');+ if (!ctx) throw new Error('Could not create a bitmaprenderer context');+ ctx.transferFromImageBitmap(bitmap);+ return await encode(canvas);+ } finally {+ bitmap.close();+ }+}++async function videoThumbnail(file: Blob, width: number): Promise<Blob | null> {+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });+ try {+ const track = await input.getPrimaryVideoTrack();+ if (!track) return null;+ const sink = new CanvasSink(track, { width });+ const first = await track.getFirstTimestamp();+ const wrapped = (await sink.getCanvas(first + VIDEO_THUMBNAIL_OFFSET)) ?? (await sink.getCanvas(first));+ if (!wrapped) return null;+ return await encode(wrapped.canvas);+ } finally {+ input.dispose();+ }+}++function encode(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob | null> {+ if (canvas instanceof OffscreenCanvas) return canvas.convertToBlob({ type: 'image/webp', quality: WEBP_QUALITY });+ return new Promise((resolve) => canvas.toBlob(resolve, 'image/webp', WEBP_QUALITY));+}diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.tsindex 641972bc..0eeeb641 100644--- a/packages/assets/src/index.ts+++ b/packages/assets/src/index.ts@@ -9,3 +9,6 @@ export * from './hash';export * from './probe';export * from './library';export * from './browser';+export * from './cache';+export * from './derive/peaks';+export * from './derive/thumbnail';diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.tsindex e3fc12ac..180408e1 100644--- a/packages/assets/src/library.ts+++ b/packages/assets/src/library.ts@@ -8,7 +8,8 @@// imports stay where they are (the manifest links to them by absolute path);// only bytes the app itself produces are written into the project's// `assets/` directory, and anything found there that the manifest does not-// know is taken into the library.+// know is taken into the library. What is derived from assets and worth+// keeping (thumbnails, waveforms) is the `cache`'s, under `cache/`.//// Knows nothing of the host: files come and go through a `ProjectFS`, and// what it cannot do itself (generate an asset, follow a rename into whatever@@ -18,6 +19,7 @@import { createSignal, type Accessor } from 'solid-js';+import { AssetCache } from './cache';import { hashBlob, hashSequence } from './hash';import {ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,@@ -71,6 +73,8 @@ export type AssetGenerator = (ref: object, library: AssetLibrary) => Promise<Assexport class AssetLibrary {public readonly fs: ProjectFS;+ /** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */+ public readonly cache: AssetCache;/** Library assets, newest first; transient ones excluded. Reactive. */public readonly assets: Accessor<Asset[]>;/** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */@@ -93,6 +97,7 @@ export class AssetLibrary {public constructor(fs: ProjectFS, options: LibraryOptions = {}) {this.fs = fs;+ this.cache = new AssetCache(fs);this.generator = options.generate;this.onRename = options.onRename;this.onRelink = options.onRelink;@@ -189,11 +194,14 @@ export class AssetLibrary {if (asset.transient && !next.has(id)) next.set(id, asset);}this.map.clear();- for (const [id, asset] of next) this.map.set(id, asset);+ for (const [id, asset] of next) {+ this.map.set(id, asset);+ }this.publish();await this.scanAssetsDir();this.publish();+ this.cache.prune(this.map.keys());}/** Attaches handles to a record; re-examines it when its source changed. */@@ -224,7 +232,7 @@ export class AssetLibrary {const walk = async (dir: string): Promise<void> => {const entries = await this.fs.list(dir);if (dir !== ASSETS_DIR && isSequenceListing(entries.map((entry) => entry.name))) {- if (!known.has(dir)) await this.link(dir, { folder: dirname(dir.slice(ASSETS_DIR.length + 1)) }).catch(() => {});+ if (!known.has(dir)) await this.link(dir, { folder: dirname(dir.slice(ASSETS_DIR.length + 1)) }).catch(() => { });return;}for (const entry of entries) {@@ -233,7 +241,7 @@ export class AssetLibrary {if (entry.kind === 'directory') {await walk(source);} else if (!known.has(source)) {- await this.link(source, { folder: dirname(source.slice(ASSETS_DIR.length + 1)) }).catch(() => {});+ await this.link(source, { folder: dirname(source.slice(ASSETS_DIR.length + 1)) }).catch(() => { });}}};@@ -417,6 +425,7 @@ export class AssetLibrary {this.map.delete(from);this.reorder(next);this.changed();+ this.cache.remove(from);this.onRelink?.(next, from);return next;}@@ -451,8 +460,9 @@ export class AssetLibrary {for (const asset of assets) {this.map.delete(asset.id);if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {- await this.fs.remove(asset.source).catch(() => {});+ await this.fs.remove(asset.source).catch(() => { });}+ this.cache.remove(asset);}this.changed();}diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.tsindex b4d8d4de..101f65f1 100644--- a/packages/runtime/src/media/audio-peaks.ts+++ b/packages/runtime/src/media/audio-peaks.ts@@ -2,128 +2,68 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { AudioSampleSink } from 'mediabunny';+import { derivePeaks } from '@diffusionstudio/assets';-import { getAudioTrack } from './audio';+import { Library, WaveformHandle, AssetId } from '../traits';+import { getAsset, getAssetFile } from '../actions/assets';-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';--const peakCache = new Map<string, Uint8ClampedArray>();-const inflight = new Map<string, Promise<Uint8ClampedArray | null>>();--const TARGET_BARS = 128;--export function clearAudioPeaksCache() {- peakCache.clear();- inflight.clear();-}+import type { Entity, World } from 'koota';+import type { AssetCache, AudioAsset, VideoAsset } from '@diffusionstudio/assets';/**- * Synchronous, allocation-free peak lookup for hot paths (render loop).- * Returns cached peaks or `null`. Kicks off a build on first call.+ * What a waveform paint's entity holds while it is on screen: the peaks of+ * its asset once they have loaded. Released on cull like the decoders.*/-export function getAudioPeaks(- asset: AudioAsset | VideoAsset,-): Uint8ClampedArray | null {- const key = asset.id;-- const cached = peakCache.get(key);- if (cached) return cached;+export class Waveform {+ public readonly asset: AudioAsset | VideoAsset;+ public readonly cache?: AssetCache;+ public peaks: Uint8ClampedArray | null = null;+ public initialized: Promise<void> | undefined;++ private disposed = false;++ public constructor(asset: AudioAsset | VideoAsset, cache?: AssetCache) {+ this.asset = asset;+ this.cache = cache;+ this.initialized = this.initialize();+ }- if (!inflight.has(key)) {- inflight.set(key, buildPeaks(key, asset));+ private async initialize(): Promise<void> {+ const file = await getAssetFile(this.asset);+ const peaks = await derivePeaks(file);+ if (!this.disposed) {+ this.peaks = peaks;+ }}- return null;+ public dispose(): void {+ this.disposed = true;+ this.peaks = null;+ }}/**- * Async peak lookup for UI components that can await.- * Returns peaks from cache or waits for the build to finish.+ * The peaks for a waveform paint, synchronously for the render loop: what the+ * paint entity holds, or null while they load (the load is started here).*/-export async function getAudioPeaksAsync(- asset: AudioAsset | VideoAsset,-): Promise<Uint8ClampedArray | null> {- const key = asset.id;+export function resolveWaveformPeaks(world: World, fill: Entity): Uint8ClampedArray | null {+ const assetId = fill.get(AssetId)?.value;+ if (!assetId) return null;- const cached = peakCache.get(key);- if (cached) return cached;+ const existing = fill.get(WaveformHandle);- let promise = inflight.get(key);- if (!promise) {- promise = buildPeaks(key, asset);- inflight.set(key, promise);+ if (existing && existing.asset.id === assetId) {+ return existing.peaks;}- return promise;-}--async function buildPeaks(- key: string,- asset: AudioAsset | VideoAsset,-): Promise<Uint8ClampedArray | null> {- try {- const track = await getAudioTrack(asset);- if (!track) return null;-- const sink = new AudioSampleSink(track);- const chunks: Uint8ClampedArray[] = [];+ existing?.dispose();- for await (const sample of sink.samples()) {- let floats: Float32Array;- let numChannels: number;- let numFrames: number;- try {- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });- floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });+ const asset = getAsset(world, assetId);+ // Both AUDIO and VIDEO assets carry peaks (video has audio).+ if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return null;- numChannels = sample.numberOfChannels;- numFrames = floats.length / numChannels;- } finally {- sample.close();- }-- const framePeaks = new Uint8ClampedArray(numFrames);- for (let f = 0; f < numFrames; f++) {- let max = 0;- for (let ch = 0; ch < numChannels; ch++) {- max = Math.max(max, Math.pow(Math.abs(floats[f * numChannels + ch]), 0.8));- }- framePeaks[f] = Math.floor(max * 255);- }- chunks.push(framePeaks);- }-- if (chunks.length === 0) return null;-- const totalFrames = chunks.reduce((s, c) => s + c.length, 0);- const merged = new Uint8ClampedArray(totalFrames);- let offset = 0;- for (const chunk of chunks) {- merged.set(chunk, offset);- offset += chunk.length;- }-- const peaks = new Uint8ClampedArray(TARGET_BARS);- const ratio = totalFrames / TARGET_BARS;-- for (let i = 0; i < TARGET_BARS; i++) {- const start = Math.floor(i * ratio);- const end = Math.floor((i + 1) * ratio);- let max = 0;- for (let j = start; j < end && j < totalFrames; j++) {- max = Math.max(max, merged[j]);- }- peaks[i] = max;- }-- peakCache.set(key, peaks);- return peaks;- } catch (e) {- console.error('Failed to build audio peaks', e);- return null;- } finally {- inflight.delete(key);- }+ const waveform = new Waveform(asset, world.get(Library)?.cache);+ fill.add(WaveformHandle);+ fill.set(WaveformHandle, waveform);+ return waveform.peaks;}diff --git a/packages/runtime/src/media/dispose.ts b/packages/runtime/src/media/dispose.tsindex ff53e61e..3b0aad51 100644--- a/packages/runtime/src/media/dispose.ts+++ b/packages/runtime/src/media/dispose.ts@@ -10,7 +10,7 @@import {ChildOf,ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,- AudioDecoderHandle, CaptionDecoderHandle, AudioBusHandle,+ AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle, AudioBusHandle,} from '../traits';import type { Entity, World } from 'koota';@@ -36,6 +36,10 @@ export function disposeDecoders(world: World, entity: Entity): void {entity.get(CaptionDecoderHandle)?.dispose();entity.set(CaptionDecoderHandle, null);}+ if (entity.has(WaveformHandle)) {+ entity.get(WaveformHandle)?.dispose();+ entity.set(WaveformHandle, null);+ }for (const child of world.query(ChildOf(entity))) {disposeDecoders(world, child);diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 85554fb0..798c0546 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -19,14 +19,13 @@ import {import {ChildOf, Hidden, Culled, Generating, Interactive, IsMask,ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,- BlendMode, Effect, AssetId, Transition, MixedCornerRadius,+ BlendMode, Effect, Transition, MixedCornerRadius,LocalTransform, WorldTransform, Computed, Cache,HtmlHostHandle, SurfaceHostHandle,Mode, Time, FrameRate, Camera, Background, RenderSurface,HitRegions,Root,} from '../traits';-import { getAsset } from '../actions/assets';import { getParentNode } from '../queries/hierarchy';import { getViewMatrix } from '../queries/camera';import { colorToHex } from '../utils/color';@@ -37,7 +36,7 @@ import { getIntrinsicPaint } from '../utils/time';import { createLinearGradient, createRadialGradient } from './gradients';import {resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,- resolveCaptionDecoder, resolveShaderHost, getAudioPeaks,+ resolveCaptionDecoder, resolveShaderHost, resolveWaveformPeaks,} from '../media';import type { Entity, World } from 'koota';@@ -620,14 +619,7 @@ function renderWaveform(world: World, entity: Entity, fill: Entity): void {const ctx = getCtx(world);const computed = store(world, Computed);- const assetId = store(world, AssetId).value[fill.id()];- if (!assetId) return;-- const asset = getAsset(world, assetId);- // Both AUDIO and VIDEO assets carry pre-computed peaks (video has audio).- if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return;-- const peaks = getAudioPeaks(asset);+ const peaks = resolveWaveformPeaks(world, fill);if (!peaks || peaks.length === 0) return;const w = computed.width[entity.id()]!;diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.tsindex 048e4fa9..01b5d9a4 100644--- a/packages/runtime/src/traits/media.ts+++ b/packages/runtime/src/traits/media.ts@@ -14,6 +14,7 @@ import type { VideoDecoderInstance } from '../media/video';import type { SequenceDecoder } from '../media/sequence';import type { AudioDecoder } from '../media/audio';import type { CaptionDecoder } from '../media/caption/types';+import type { Waveform } from '../media/audio-peaks';import type { AudioBus } from '../media/audio-bus';// Renderer-runtime state on mounted entities (never serialized).@@ -41,6 +42,8 @@ export const SequenceDecoderHandle = trait(() => null as SequenceDecoder | null)export const AudioDecoderHandle = trait(() => null as AudioDecoder | null);+export const WaveformHandle = trait(() => null as Waveform | null);+export const CaptionDecoderHandle = trait(() => null as CaptionDecoder | null);export const AudioBusHandle = trait(() => null as AudioBus | null);diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex 171ef65b..78e5e63d 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -24,7 +24,7 @@ import {Opacity, Color, Blur, Volume, Effect, CornerRadius, MixedCornerRadius,ColorStop, StrokeStyle, Size, Computed, Active, Stage, IsMask,ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,- AudioDecoderHandle, CaptionDecoderHandle,+ AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle,HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,} from '../traits';import { getParentEntity } from '../queries/hierarchy';@@ -77,6 +77,7 @@ export function observeWorld(world: World): () => void {subs.push(world.onRemove(SequenceDecoderHandle, (e) => e.get(SequenceDecoderHandle)?.dispose()));subs.push(world.onRemove(AudioDecoderHandle, (e) => e.get(AudioDecoderHandle)?.reset()));subs.push(world.onRemove(CaptionDecoderHandle, (e) => e.get(CaptionDecoderHandle)?.dispose()));+ subs.push(world.onRemove(WaveformHandle, (e) => e.get(WaveformHandle)?.dispose()));subs.push(world.onRemove(HtmlHostHandle, (e) => e.get(HtmlHostHandle)?.dispose()));subs.push(world.onRemove(SurfaceHostHandle, (e) => e.get(SurfaceHostHandle)?.dispose()));subs.push(world.onRemove(ShaderHostHandle, (e) => e.get(ShaderHostHandle)?.dispose()));