Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add caching for asset thumbnails and audio peaks

konstantin-paulusb7388afa7654
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts
index 7cd603c8..818bce17 100644
--- a/apps/desktop/src/projects.ts
+++ b/apps/desktop/src/projects.ts
@@ -188,6 +188,11 @@ const TSCONFIG = `{
}
`;
+/** What a project folder produces but should not check in: installs, and derived data (thumbnails, waveforms). */
+const GITIGNORE = `node_modules/
+cache/
+`;
+
const STARTER = `export default function Project() {
return (
<stage background="#161616">
@@ -232,6 +237,7 @@ export async function scaffold(dir: string, displayName = basename(dir)): Promis
}
await ensurePackage(dir, name, displayName, entry);
await writeIfMissing(dir, "tsconfig.json", TSCONFIG);
+ await writeIfMissing(dir, ".gitignore", GITIGNORE);
}
/** Creates a fresh project folder under `root`. Fails if the folder exists. */
diff --git a/apps/web/src/components/sidebar-left/asset-item.tsx b/apps/web/src/components/sidebar-left/asset-item.tsx
index 41f8a2bc..87c1fdf0 100644
--- a/apps/web/src/components/sidebar-left/asset-item.tsx
+++ b/apps/web/src/components/sidebar-left/asset-item.tsx
@@ -130,7 +130,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
class="relative aspect-video w-full overflow-clip rounded bg-muted after:pointer-events-none after:absolute after:inset-0 after:rounded after:opacity-0 after:ring-2 after:ring-inset after:ring-ring after:z-10 data-[selected=true]:after:opacity-100"
>
<Show when={isVisible()}>
- <AssetThumbnail asset={props.asset} class="h-full w-full" />
+ <AssetThumbnail asset={props.asset} class="h-full w-full" cache={library()?.cache} />
</Show>
<Show when={assetDuration()}>
{(duration) => (
diff --git a/apps/web/src/components/ui/asset-thumbnail.tsx b/apps/web/src/components/ui/asset-thumbnail.tsx
index 4641f0f1..1efaad9a 100644
--- a/apps/web/src/components/ui/asset-thumbnail.tsx
+++ b/apps/web/src/components/ui/asset-thumbnail.tsx
@@ -2,12 +2,12 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
-import { Show, For, createResource } from 'solid-js';
+import { Show, For, createResource, onCleanup } from 'solid-js';
import { cx } from '@/lib/cva';
-import { getAssetFile, getAudioPeaksAsync } from '@diffusionstudio/runtime';
+import { getAssetFile } from '@diffusionstudio/runtime';
+import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH, derivePeaks } from '@diffusionstudio/assets';
-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
+import type { Asset as LibraryAsset, AssetCache, AudioAsset, VideoAsset, ImageAsset } from '@diffusionstudio/assets';
/**
* What a thumbnail needs of an asset: the runtime's `Asset` and the legacy
@@ -29,114 +29,56 @@ type Asset = ThumbnailAsset;
/** Re-renders a thumbnail when the asset, or the file behind it, changes. */
const keyOf = (asset: Asset): string => `${asset.id}:${asset.stat?.mtime ?? asset.lastModified ?? ''}`;
-const DEFAULT_THUMBNAIL_SIZE = {
- width: 300,
- height: 168,
-} as const;
-
type ThumbnailSize = {
width: number;
height: number;
};
-function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
- const [url] = createResource(
- () => keyOf(props.asset),
- async () => {
- const objectUrl = URL.createObjectURL(await getAssetFile(props.asset));
- try {
- const image = await new Promise<HTMLImageElement>((resolve, reject) => {
- const img = new Image();
- img.onload = () => resolve(img);
- img.onerror = () => reject(new Error('Failed to load image'));
- img.src = objectUrl;
- });
-
- const canvas = document.createElement('canvas');
- canvas.width = props.size.width;
- canvas.height = props.size.height;
- const ctx = canvas.getContext('2d')!;
-
- const imgW = image.naturalWidth || props.asset.width || props.size.width;
- const imgH = image.naturalHeight || props.asset.height || props.size.height;
- const scale = Math.max(props.size.width / imgW, props.size.height / imgH);
- const scaledW = imgW * scale;
- const scaledH = imgH * scale;
- const dx = (props.size.width - scaledW) / 2;
- const dy = (props.size.height - scaledH) / 2;
-
- ctx.drawImage(image, dx, dy, scaledW, scaledH);
- const url = canvas.toDataURL('image/webp', 0.7);
-
- canvas.width = 0;
- canvas.height = 0;
-
- return url;
- } catch {
- return undefined;
- } finally {
- URL.revokeObjectURL(objectUrl);
- }
- }
- );
+/**
+ * The thumbnail of an image or video: from the library's cache when there is
+ * one (kept in the project's `cache/` across sessions), derived on the spot
+ * otherwise. Scaled to `width` at the asset's own aspect ratio; the
+ * container crops it.
+ */
+function MediaThumbnail(props: { asset: Asset; width: number; cache?: AssetCache }) {
+ let objectUrl: string | undefined;
- return (
- <Show when={url()}>
- <img
- src={url()!}
- alt=""
+ const releaseUrl = () => {
+ if (objectUrl) URL.revokeObjectURL(objectUrl);
+ objectUrl = undefined;
+ };
- class="w-full h-full object-cover select-none"
- />
- </Show>
- );
-}
+ const key = () => `${keyOf(props.asset)}:${props.width}:${props.cache ? 'cached' : ''}`;
-function VideoThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
- const [src] = createResource(
- () => keyOf(props.asset),
- async () => {
- try {
- const file = await getAssetFile(props.asset);
-
- const input = new Input({
- formats: ALL_FORMATS,
- source: new BlobSource(file),
- });
-
- const track = await input.getPrimaryVideoTrack();
- if (!track) return;
-
- const sink = new CanvasSink(track, {
- width: props.size.width,
- height: props.size.height,
- fit: 'cover',
- });
-
- const ts = await track.getFirstTimestamp();
- const wrappedCanvas = await sink.getCanvas(ts + 0.3);
- if (!wrappedCanvas) return;
-
- const canvas = document.createElement('canvas');
- canvas.width = wrappedCanvas.canvas.width;
- canvas.height = wrappedCanvas.canvas.height;
- canvas.getContext('2d')?.drawImage(wrappedCanvas.canvas, 0, 0);
- const url = canvas.toDataURL('image/webp', 0.7);
- canvas.width = 0;
- canvas.height = 0;
- return url;
- } catch {
- return undefined;
+ const loadThumbnail = async () => {
+ releaseUrl();
+
+ try {
+ let blob: Blob | null = null;
+
+ if (props.cache) {
+ blob = await props.cache.thumbnail(props.asset as LibraryAsset, props.width);
+ } else {
+ const file = await getAssetFile(props.asset as unknown as ImageAsset | VideoAsset);
+ blob = await deriveThumbnail(file, props.asset.mimeType, props.width);
}
+ objectUrl = blob ? URL.createObjectURL(blob) : undefined;
+
+ return objectUrl;
+ } catch {
+ return undefined;
}
- );
+ }
+
+ const [url] = createResource(key, loadThumbnail);
+
+ onCleanup(releaseUrl);
return (
- <Show when={src()}>
+ <Show when={url()}>
<img
- src={src()!}
+ src={url()!}
alt=""
-
class="w-full h-full object-cover select-none"
/>
</Show>
@@ -159,11 +101,19 @@ function TranscriptThumbnail() {
);
}
-function AudioThumbnail(props: { asset: Asset }) {
- const [bins] = createResource(
- () => keyOf(props.asset),
- () => getAudioPeaksAsync(props.asset as unknown as AudioAsset | VideoAsset),
- );
+function AudioThumbnail(props: { asset: Asset; cache?: AssetCache }) {
+ const loadPeaks = async () => {
+ if (props.cache) {
+ return props.cache.peaks(props.asset as unknown as AudioAsset | VideoAsset);
+ }
+
+ const file = await getAssetFile(props.asset as unknown as AudioAsset | VideoAsset);
+ return derivePeaks(file);
+ }
+
+ const key = () => keyOf(props.asset);
+
+ const [bins] = createResource(key, loadPeaks);
return (
<div class="absolute inset-0 pt-7 pb-1 bg-audio-background">
@@ -186,21 +136,19 @@ type AssetThumbnailProps = {
class?: string;
draggable?: boolean;
size?: ThumbnailSize;
+ cache?: AssetCache;
}
export function AssetThumbnail(props: AssetThumbnailProps) {
- const size = () => props.size ?? DEFAULT_THUMBNAIL_SIZE;
+ const width = () => props.size?.width ?? DEFAULT_THUMBNAIL_WIDTH;
return (
<div class={cx(props.class, 'relative')} draggable={props.draggable}>
- <Show when={props.asset.mimeType.startsWith('image')}>
- <ImageThumbnail asset={props.asset} size={size()} />
- </Show>
- <Show when={props.asset.mimeType.startsWith('video')}>
- <VideoThumbnail asset={props.asset} size={size()} />
+ <Show when={props.asset.mimeType.startsWith('image') || props.asset.mimeType.startsWith('video')}>
+ <MediaThumbnail asset={props.asset} width={width()} cache={props.cache} />
</Show>
<Show when={props.asset.mimeType.startsWith('audio')}>
- <AudioThumbnail asset={props.asset} />
+ <AudioThumbnail asset={props.asset} cache={props.cache} />
</Show>
<Show when={props.asset.type === 'TRANSCRIPT'}>
<TranscriptThumbnail />
diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsx
index 37ab8223..1c23464e 100644
--- a/apps/web/src/pages/editor.tsx
+++ b/apps/web/src/pages/editor.tsx
@@ -16,6 +16,7 @@ import { useWorld } from '@diffusionstudio/koota-solid';
import { mount } from '@diffusionstudio/reconciler';
import { getDocumentEditor } from '@/engine/editor';
import { attachLibrary, isLibraryFile } from '@/engine/library';
+import { isCacheFile } from '@diffusionstudio/assets';
import { createEditWriter } from '@/projects/edits';
import { compileProject, watchProject, projectDir } from '@/projects/host';
import { useProjectId } from "@/hooks/use-project-id";
@@ -84,7 +85,8 @@ export function EditorPage() {
loadProject();
const unwatch = watchProject(dir, (path) => {
- if (isLibraryFile(path)) {
+ if (isCacheFile(path)) return;
+ if (isLibraryFile(path)) {
library.load();
} else {
loadProject();
diff --git a/packages/assets/src/cache.ts b/packages/assets/src/cache.ts
new file mode 100644
index 00000000..df00a15b
--- /dev/null
+++ b/packages/assets/src/cache.ts
@@ -0,0 +1,179 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+import { derivePeaks } from './derive/peaks';
+import { deriveThumbnail, DEFAULT_THUMBNAIL_WIDTH } from './derive/thumbnail';
+
+import type { ProjectFS } from './fs';
+import type { Asset } from './types';
+
+/** The project directory derived data is kept in. */
+export const CACHE_DIR = 'cache';
+
+/** Whether a project-relative path is inside the cache directory. */
+export const isCacheFile = (path: string): boolean => path === CACHE_DIR || path.startsWith(`${CACHE_DIR}/`);
+
+/** A kind of cached value: where it lives under `cache/`, and how it is stored. */
+export interface CacheKind<T> {
+ /** Subdirectory of `cache/`. */
+ dir: string;
+ /** File extension (no dot). */
+ ext: string;
+ encode(value: T): Blob;
+ decode(blob: Blob): Promise<T>;
+}
+
+/** The asset bar's thumbnail: a WebP, stored as is. */
+export const THUMBNAIL: CacheKind<Blob> = {
+ dir: 'thumbnails',
+ ext: 'webp',
+ encode: (blob) => blob,
+ decode: async (blob) => blob,
+};
+
+/** The rough waveform: one byte per bar, stored raw. */
+export const PEAKS: CacheKind<Uint8ClampedArray> = {
+ dir: 'peaks',
+ ext: 'u8',
+ encode: (peaks) => new Blob([peaks as Uint8ClampedArray<ArrayBuffer>]),
+ decode: async (blob) => new Uint8ClampedArray(await blob.arrayBuffer()),
+};
+
+/** What an entry is keyed by: an asset, or just its id. */
+type Keyed = string | Pick<Asset, 'id'>;
+
+const idOf = (keyed: Keyed): string => (typeof keyed === 'string' ? keyed : keyed.id);
+
+export class AssetCache {
+ private readonly fs: ProjectFS;
+ /** Reads and derivations under way, by `kind.dir/id[@variant]`. */
+ private readonly inflight = new Map<string, Promise<unknown>>();
+ /** Every kind used, so `remove` and `prune` know which directories to look in. */
+ private readonly kinds = new Map<string, CacheKind<unknown>>();
+
+ public constructor(fs: ProjectFS) {
+ this.fs = fs;
+ this.register(THUMBNAIL);
+ this.register(PEAKS);
+ }
+
+ // -----------------------------------------------------------------------
+ // Typed entries
+
+ /**
+ * The asset's thumbnail (IMAGE, VIDEO, or SEQUENCE by its first frame):
+ * a WebP `width` wide at the asset's own aspect ratio, or null when none
+ * can be made. Widths other than the default are kept as variants.
+ */
+ public thumbnail(asset: Asset, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {
+ const variant = width === DEFAULT_THUMBNAIL_WIDTH ? undefined : `${width}`;
+ return this.get(THUMBNAIL, asset, async () => deriveThumbnail(await asset.handle.getFile(), asset.mimeType, width), variant);
+ }
+
+ /** The asset's rough waveform, or null when it has no audio. */
+ public peaks(asset: Asset): Promise<Uint8ClampedArray | null> {
+ return this.get(PEAKS, asset, async () => derivePeaks(await asset.handle.getFile()));
+ }
+
+ // -----------------------------------------------------------------------
+ // Generic entries
+
+ /**
+ * The cached value for (kind, asset[, variant]): read from disk, else
+ * `produce()`d and written there. Concurrent calls share one result; a null
+ * from `produce` (nothing to derive) is not stored.
+ */
+ public get<T>(kind: CacheKind<T>, keyed: Keyed, produce: () => Promise<T | null>, variant?: string): Promise<T | null> {
+ this.register(kind);
+ const key = this.key(kind, keyed, variant);
+ const running = this.inflight.get(key);
+ if (running) return running as Promise<T | null>;
+
+ const promise = this.load(kind, key, produce).finally(() => this.inflight.delete(key));
+ this.inflight.set(key, promise);
+ return promise;
+ }
+
+ /** Drops every entry of an asset on disk. */
+ public async remove(keyed: Keyed): Promise<void> {
+ const id = idOf(keyed);
+ for (const kind of this.kinds.values()) {
+ const entries = await this.fs.list(`${CACHE_DIR}/${kind.dir}`).catch(() => []);
+ for (const entry of entries) {
+ if (entry.name === `${id}.${kind.ext}` || entry.name.startsWith(`${id}@`)) {
+ await this.fs.remove(`${CACHE_DIR}/${kind.dir}/${entry.name}`).catch(() => {});
+ }
+ }
+ }
+ }
+
+ /** Removes entries on disk whose asset is not among `keep`. */
+ public async prune(keep: Iterable<string>): Promise<void> {
+ const ids = new Set(keep);
+ for (const kind of this.kinds.values()) {
+ const entries = await this.fs.list(`${CACHE_DIR}/${kind.dir}`).catch(() => []);
+ for (const entry of entries) {
+ if (entry.kind !== 'file') continue;
+ const stem = entry.name.split('.')[0]!.split('@')[0]!;
+ if (ids.has(stem)) continue;
+ await this.fs.remove(`${CACHE_DIR}/${kind.dir}/${entry.name}`).catch(() => {});
+ }
+ }
+ }
+
+ /** Deletes everything on disk; one kind, or all of it. */
+ public async clear(kind?: CacheKind<unknown>): Promise<void> {
+ await this.fs.remove(kind ? `${CACHE_DIR}/${kind.dir}` : CACHE_DIR).catch(() => {});
+ }
+
+ // -----------------------------------------------------------------------
+
+ private register(kind: CacheKind<unknown>): void {
+ if (!this.kinds.has(kind.dir)) this.kinds.set(kind.dir, kind);
+ }
+
+ private key(kind: CacheKind<unknown>, keyed: Keyed, variant?: string): string {
+ return `${kind.dir}/${idOf(keyed)}${variant ? `@${variant}` : ''}`;
+ }
+
+ /** The file of an entry: `cache/<dir>/<id>[@variant].<ext>`. */
+ private pathOf(kind: CacheKind<unknown>, key: string): string {
+ return `${CACHE_DIR}/${key}.${kind.ext}`;
+ }
+
+ private async load<T>(kind: CacheKind<T>, key: string, produce: () => Promise<T | null>): Promise<T | null> {
+ const stored = await this.read(kind, key);
+ if (stored !== undefined) return stored;
+
+ let value: T | null;
+ try {
+ value = await produce();
+ } catch (error) {
+ console.warn(`[assets] could not derive ${key}:`, error);
+ value = null;
+ }
+ if (value !== null) void this.write(kind, key, value);
+ return value;
+ }
+
+ /** The entry on disk, or undefined when missing or unreadable. */
+ private async read<T>(kind: CacheKind<T>, key: string): Promise<T | undefined> {
+ const path = this.pathOf(kind, key);
+ try {
+ if (!(await this.fs.stat(path))) return undefined;
+ return await kind.decode(await this.fs.file(path));
+ } catch {
+ return undefined;
+ }
+ }
+
+ private async write<T>(kind: CacheKind<T>, key: string, value: T): Promise<void> {
+ try {
+ await this.fs.write(this.pathOf(kind, key), kind.encode(value));
+ } catch (error) {
+ console.warn(`[assets] could not write cache entry ${key}:`, error);
+ }
+ }
+}
diff --git a/packages/assets/src/derive/peaks.ts b/packages/assets/src/derive/peaks.ts
new file mode 100644
index 00000000..0667dff1
--- /dev/null
+++ b/packages/assets/src/derive/peaks.ts
@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The rough waveform of a media file: a fixed number of bars, each the peak
+// amplitude (across channels, gamma-lifted so quiet passages still show) of
+// its slice of the file, as a byte. Decodes the whole audio track once; the
+// AssetCache keeps the result.
+
+import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
+
+/** How many bars a rough waveform has. */
+export const PEAK_BARS = 128;
+
+/** The peaks of `file`'s primary audio track, or null when it has none. */
+export async function derivePeaks(file: Blob, bars = PEAK_BARS): Promise<Uint8ClampedArray | null> {
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ try {
+ const track = await input.getPrimaryAudioTrack();
+ if (!track) return null;
+
+ const sink = new AudioSampleSink(track);
+ const chunks: Uint8ClampedArray[] = [];
+
+ for await (const sample of sink.samples()) {
+ let floats: Float32Array;
+ let channels: number;
+ let frames: number;
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+ channels = sample.numberOfChannels;
+ frames = floats.length / channels;
+ } finally {
+ sample.close();
+ }
+
+ const framePeaks = new Uint8ClampedArray(frames);
+ for (let f = 0; f < frames; f++) {
+ let max = 0;
+ for (let ch = 0; ch < channels; ch++) {
+ max = Math.max(max, Math.pow(Math.abs(floats[f * channels + ch]!), 0.8));
+ }
+ framePeaks[f] = Math.floor(max * 255);
+ }
+ chunks.push(framePeaks);
+ }
+
+ if (chunks.length === 0) return null;
+
+ const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+ const merged = new Uint8ClampedArray(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ merged.set(chunk, offset);
+ offset += chunk.length;
+ }
+
+ const peaks = new Uint8ClampedArray(bars);
+ const ratio = total / bars;
+ for (let i = 0; i < bars; i++) {
+ const start = Math.floor(i * ratio);
+ const end = Math.min(total, Math.floor((i + 1) * ratio));
+ let max = 0;
+ for (let j = start; j < end; j++) max = Math.max(max, merged[j]!);
+ peaks[i] = max;
+ }
+ return peaks;
+ } finally {
+ input.dispose();
+ }
+}
diff --git a/packages/assets/src/derive/thumbnail.ts b/packages/assets/src/derive/thumbnail.ts
new file mode 100644
index 00000000..c08f7a33
--- /dev/null
+++ b/packages/assets/src/derive/thumbnail.ts
@@ -0,0 +1,58 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The thumbnail of an image or video file: scaled to a width, its aspect
+// ratio kept, encoded as WebP. For a video it is the frame shortly after the
+// start. The decoder does the scaling (mediabunny's CanvasSink for video,
+// createImageBitmap for images), so the picture is encoded once, as it comes.
+
+import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
+
+/** The width the asset bar shows; what the cache stores without a variant. */
+export const DEFAULT_THUMBNAIL_WIDTH = 300;
+
+/** How far into a video the thumbnail frame is taken from, in seconds. */
+const VIDEO_THUMBNAIL_OFFSET = 0.3;
+
+const WEBP_QUALITY = 0.7;
+
+/** A WebP thumbnail of `file`, `width` wide, or null when it is neither an image nor a video with a picture. */
+export async function deriveThumbnail(file: Blob, mimeType: string, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {
+ if (mimeType.startsWith('image/')) return imageThumbnail(file, width);
+ if (mimeType.startsWith('video/')) return videoThumbnail(file, width);
+ return null;
+}
+
+async function imageThumbnail(file: Blob, width: number): Promise<Blob | null> {
+ const bitmap = await createImageBitmap(file, { resizeWidth: width, resizeQuality: 'high' });
+ try {
+ const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
+ const ctx = canvas.getContext('bitmaprenderer');
+ if (!ctx) throw new Error('Could not create a bitmaprenderer context');
+ ctx.transferFromImageBitmap(bitmap);
+ return await encode(canvas);
+ } finally {
+ bitmap.close();
+ }
+}
+
+async function videoThumbnail(file: Blob, width: number): Promise<Blob | null> {
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ try {
+ const track = await input.getPrimaryVideoTrack();
+ if (!track) return null;
+ const sink = new CanvasSink(track, { width });
+ const first = await track.getFirstTimestamp();
+ const wrapped = (await sink.getCanvas(first + VIDEO_THUMBNAIL_OFFSET)) ?? (await sink.getCanvas(first));
+ if (!wrapped) return null;
+ return await encode(wrapped.canvas);
+ } finally {
+ input.dispose();
+ }
+}
+
+function encode(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob | null> {
+ if (canvas instanceof OffscreenCanvas) return canvas.convertToBlob({ type: 'image/webp', quality: WEBP_QUALITY });
+ return new Promise((resolve) => canvas.toBlob(resolve, 'image/webp', WEBP_QUALITY));
+}
diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.ts
index 641972bc..0eeeb641 100644
--- a/packages/assets/src/index.ts
+++ b/packages/assets/src/index.ts
@@ -9,3 +9,6 @@ export * from './hash';
export * from './probe';
export * from './library';
export * from './browser';
+export * from './cache';
+export * from './derive/peaks';
+export * from './derive/thumbnail';
diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.ts
index e3fc12ac..180408e1 100644
--- a/packages/assets/src/library.ts
+++ b/packages/assets/src/library.ts
@@ -8,7 +8,8 @@
// imports stay where they are (the manifest links to them by absolute path);
// only bytes the app itself produces are written into the project's
// `assets/` directory, and anything found there that the manifest does not
-// know is taken into the library.
+// know is taken into the library. What is derived from assets and worth
+// keeping (thumbnails, waveforms) is the `cache`'s, under `cache/`.
//
// Knows nothing of the host: files come and go through a `ProjectFS`, and
// what it cannot do itself (generate an asset, follow a rename into whatever
@@ -18,6 +19,7 @@
import { createSignal, type Accessor } from 'solid-js';
+import { AssetCache } from './cache';
import { hashBlob, hashSequence } from './hash';
import {
ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,
@@ -71,6 +73,8 @@ export type AssetGenerator = (ref: object, library: AssetLibrary) => Promise<Ass
export class AssetLibrary {
public readonly fs: ProjectFS;
+ /** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */
+ public readonly cache: AssetCache;
/** Library assets, newest first; transient ones excluded. Reactive. */
public readonly assets: Accessor<Asset[]>;
/** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */
@@ -93,6 +97,7 @@ export class AssetLibrary {
public constructor(fs: ProjectFS, options: LibraryOptions = {}) {
this.fs = fs;
+ this.cache = new AssetCache(fs);
this.generator = options.generate;
this.onRename = options.onRename;
this.onRelink = options.onRelink;
@@ -189,11 +194,14 @@ export class AssetLibrary {
if (asset.transient && !next.has(id)) next.set(id, asset);
}
this.map.clear();
- for (const [id, asset] of next) this.map.set(id, asset);
+ for (const [id, asset] of next) {
+ this.map.set(id, asset);
+ }
this.publish();
await this.scanAssetsDir();
this.publish();
+ this.cache.prune(this.map.keys());
}
/** Attaches handles to a record; re-examines it when its source changed. */
@@ -224,7 +232,7 @@ export class AssetLibrary {
const walk = async (dir: string): Promise<void> => {
const entries = await this.fs.list(dir);
if (dir !== ASSETS_DIR && isSequenceListing(entries.map((entry) => entry.name))) {
- if (!known.has(dir)) await this.link(dir, { folder: dirname(dir.slice(ASSETS_DIR.length + 1)) }).catch(() => {});
+ if (!known.has(dir)) await this.link(dir, { folder: dirname(dir.slice(ASSETS_DIR.length + 1)) }).catch(() => { });
return;
}
for (const entry of entries) {
@@ -233,7 +241,7 @@ export class AssetLibrary {
if (entry.kind === 'directory') {
await walk(source);
} else if (!known.has(source)) {
- await this.link(source, { folder: dirname(source.slice(ASSETS_DIR.length + 1)) }).catch(() => {});
+ await this.link(source, { folder: dirname(source.slice(ASSETS_DIR.length + 1)) }).catch(() => { });
}
}
};
@@ -417,6 +425,7 @@ export class AssetLibrary {
this.map.delete(from);
this.reorder(next);
this.changed();
+ this.cache.remove(from);
this.onRelink?.(next, from);
return next;
}
@@ -451,8 +460,9 @@ export class AssetLibrary {
for (const asset of assets) {
this.map.delete(asset.id);
if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {
- await this.fs.remove(asset.source).catch(() => {});
+ await this.fs.remove(asset.source).catch(() => { });
}
+ this.cache.remove(asset);
}
this.changed();
}
diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.ts
index b4d8d4de..101f65f1 100644
--- a/packages/runtime/src/media/audio-peaks.ts
+++ b/packages/runtime/src/media/audio-peaks.ts
@@ -2,128 +2,68 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { AudioSampleSink } from 'mediabunny';
+import { derivePeaks } from '@diffusionstudio/assets';
-import { getAudioTrack } from './audio';
+import { Library, WaveformHandle, AssetId } from '../traits';
+import { getAsset, getAssetFile } from '../actions/assets';
-import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
-
-const peakCache = new Map<string, Uint8ClampedArray>();
-const inflight = new Map<string, Promise<Uint8ClampedArray | null>>();
-
-const TARGET_BARS = 128;
-
-export function clearAudioPeaksCache() {
- peakCache.clear();
- inflight.clear();
-}
+import type { Entity, World } from 'koota';
+import type { AssetCache, AudioAsset, VideoAsset } from '@diffusionstudio/assets';
/**
- * Synchronous, allocation-free peak lookup for hot paths (render loop).
- * Returns cached peaks or `null`. Kicks off a build on first call.
+ * What a waveform paint's entity holds while it is on screen: the peaks of
+ * its asset once they have loaded. Released on cull like the decoders.
*/
-export function getAudioPeaks(
- asset: AudioAsset | VideoAsset,
-): Uint8ClampedArray | null {
- const key = asset.id;
-
- const cached = peakCache.get(key);
- if (cached) return cached;
+export class Waveform {
+ public readonly asset: AudioAsset | VideoAsset;
+ public readonly cache?: AssetCache;
+ public peaks: Uint8ClampedArray | null = null;
+ public initialized: Promise<void> | undefined;
+
+ private disposed = false;
+
+ public constructor(asset: AudioAsset | VideoAsset, cache?: AssetCache) {
+ this.asset = asset;
+ this.cache = cache;
+ this.initialized = this.initialize();
+ }
- if (!inflight.has(key)) {
- inflight.set(key, buildPeaks(key, asset));
+ private async initialize(): Promise<void> {
+ const file = await getAssetFile(this.asset);
+ const peaks = await derivePeaks(file);
+ if (!this.disposed) {
+ this.peaks = peaks;
+ }
}
- return null;
+ public dispose(): void {
+ this.disposed = true;
+ this.peaks = null;
+ }
}
/**
- * Async peak lookup for UI components that can await.
- * Returns peaks from cache or waits for the build to finish.
+ * The peaks for a waveform paint, synchronously for the render loop: what the
+ * paint entity holds, or null while they load (the load is started here).
*/
-export async function getAudioPeaksAsync(
- asset: AudioAsset | VideoAsset,
-): Promise<Uint8ClampedArray | null> {
- const key = asset.id;
+export function resolveWaveformPeaks(world: World, fill: Entity): Uint8ClampedArray | null {
+ const assetId = fill.get(AssetId)?.value;
+ if (!assetId) return null;
- const cached = peakCache.get(key);
- if (cached) return cached;
+ const existing = fill.get(WaveformHandle);
- let promise = inflight.get(key);
- if (!promise) {
- promise = buildPeaks(key, asset);
- inflight.set(key, promise);
+ if (existing && existing.asset.id === assetId) {
+ return existing.peaks;
}
- return promise;
-}
-
-async function buildPeaks(
- key: string,
- asset: AudioAsset | VideoAsset,
-): Promise<Uint8ClampedArray | null> {
- try {
- const track = await getAudioTrack(asset);
- if (!track) return null;
-
- const sink = new AudioSampleSink(track);
- const chunks: Uint8ClampedArray[] = [];
+ existing?.dispose();
- for await (const sample of sink.samples()) {
- let floats: Float32Array;
- let numChannels: number;
- let numFrames: number;
- try {
- const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
- floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
- sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+ const asset = getAsset(world, assetId);
+ // Both AUDIO and VIDEO assets carry peaks (video has audio).
+ if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return null;
- numChannels = sample.numberOfChannels;
- numFrames = floats.length / numChannels;
- } finally {
- sample.close();
- }
-
- const framePeaks = new Uint8ClampedArray(numFrames);
- for (let f = 0; f < numFrames; f++) {
- let max = 0;
- for (let ch = 0; ch < numChannels; ch++) {
- max = Math.max(max, Math.pow(Math.abs(floats[f * numChannels + ch]), 0.8));
- }
- framePeaks[f] = Math.floor(max * 255);
- }
- chunks.push(framePeaks);
- }
-
- if (chunks.length === 0) return null;
-
- const totalFrames = chunks.reduce((s, c) => s + c.length, 0);
- const merged = new Uint8ClampedArray(totalFrames);
- let offset = 0;
- for (const chunk of chunks) {
- merged.set(chunk, offset);
- offset += chunk.length;
- }
-
- const peaks = new Uint8ClampedArray(TARGET_BARS);
- const ratio = totalFrames / TARGET_BARS;
-
- for (let i = 0; i < TARGET_BARS; i++) {
- const start = Math.floor(i * ratio);
- const end = Math.floor((i + 1) * ratio);
- let max = 0;
- for (let j = start; j < end && j < totalFrames; j++) {
- max = Math.max(max, merged[j]);
- }
- peaks[i] = max;
- }
-
- peakCache.set(key, peaks);
- return peaks;
- } catch (e) {
- console.error('Failed to build audio peaks', e);
- return null;
- } finally {
- inflight.delete(key);
- }
+ const waveform = new Waveform(asset, world.get(Library)?.cache);
+ fill.add(WaveformHandle);
+ fill.set(WaveformHandle, waveform);
+ return waveform.peaks;
}
diff --git a/packages/runtime/src/media/dispose.ts b/packages/runtime/src/media/dispose.ts
index ff53e61e..3b0aad51 100644
--- a/packages/runtime/src/media/dispose.ts
+++ b/packages/runtime/src/media/dispose.ts
@@ -10,7 +10,7 @@
import {
ChildOf,
ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,
- AudioDecoderHandle, CaptionDecoderHandle, AudioBusHandle,
+ AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle, AudioBusHandle,
} from '../traits';
import type { Entity, World } from 'koota';
@@ -36,6 +36,10 @@ export function disposeDecoders(world: World, entity: Entity): void {
entity.get(CaptionDecoderHandle)?.dispose();
entity.set(CaptionDecoderHandle, null);
}
+ if (entity.has(WaveformHandle)) {
+ entity.get(WaveformHandle)?.dispose();
+ entity.set(WaveformHandle, null);
+ }
for (const child of world.query(ChildOf(entity))) {
disposeDecoders(world, child);
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index 85554fb0..798c0546 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -19,14 +19,13 @@ import {
import {
ChildOf, Hidden, Culled, Generating, Interactive, IsMask,
ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,
- BlendMode, Effect, AssetId, Transition, MixedCornerRadius,
+ BlendMode, Effect, Transition, MixedCornerRadius,
LocalTransform, WorldTransform, Computed, Cache,
HtmlHostHandle, SurfaceHostHandle,
Mode, Time, FrameRate, Camera, Background, RenderSurface,
HitRegions,
Root,
} from '../traits';
-import { getAsset } from '../actions/assets';
import { getParentNode } from '../queries/hierarchy';
import { getViewMatrix } from '../queries/camera';
import { colorToHex } from '../utils/color';
@@ -37,7 +36,7 @@ import { getIntrinsicPaint } from '../utils/time';
import { createLinearGradient, createRadialGradient } from './gradients';
import {
resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,
- resolveCaptionDecoder, resolveShaderHost, getAudioPeaks,
+ resolveCaptionDecoder, resolveShaderHost, resolveWaveformPeaks,
} from '../media';
import type { Entity, World } from 'koota';
@@ -620,14 +619,7 @@ function renderWaveform(world: World, entity: Entity, fill: Entity): void {
const ctx = getCtx(world);
const computed = store(world, Computed);
- const assetId = store(world, AssetId).value[fill.id()];
- if (!assetId) return;
-
- const asset = getAsset(world, assetId);
- // Both AUDIO and VIDEO assets carry pre-computed peaks (video has audio).
- if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return;
-
- const peaks = getAudioPeaks(asset);
+ const peaks = resolveWaveformPeaks(world, fill);
if (!peaks || peaks.length === 0) return;
const w = computed.width[entity.id()]!;
diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.ts
index 048e4fa9..01b5d9a4 100644
--- a/packages/runtime/src/traits/media.ts
+++ b/packages/runtime/src/traits/media.ts
@@ -14,6 +14,7 @@ import type { VideoDecoderInstance } from '../media/video';
import type { SequenceDecoder } from '../media/sequence';
import type { AudioDecoder } from '../media/audio';
import type { CaptionDecoder } from '../media/caption/types';
+import type { Waveform } from '../media/audio-peaks';
import type { AudioBus } from '../media/audio-bus';
// Renderer-runtime state on mounted entities (never serialized).
@@ -41,6 +42,8 @@ export const SequenceDecoderHandle = trait(() => null as SequenceDecoder | null)
export const AudioDecoderHandle = trait(() => null as AudioDecoder | null);
+export const WaveformHandle = trait(() => null as Waveform | null);
+
export const CaptionDecoderHandle = trait(() => null as CaptionDecoder | null);
export const AudioBusHandle = trait(() => null as AudioBus | null);
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index 171ef65b..78e5e63d 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -24,7 +24,7 @@ import {
Opacity, Color, Blur, Volume, Effect, CornerRadius, MixedCornerRadius,
ColorStop, StrokeStyle, Size, Computed, Active, Stage, IsMask,
ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,
- AudioDecoderHandle, CaptionDecoderHandle,
+ AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle,
HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,
} from '../traits';
import { getParentEntity } from '../queries/hierarchy';
@@ -77,6 +77,7 @@ export function observeWorld(world: World): () => void {
subs.push(world.onRemove(SequenceDecoderHandle, (e) => e.get(SequenceDecoderHandle)?.dispose()));
subs.push(world.onRemove(AudioDecoderHandle, (e) => e.get(AudioDecoderHandle)?.reset()));
subs.push(world.onRemove(CaptionDecoderHandle, (e) => e.get(CaptionDecoderHandle)?.dispose()));
+ subs.push(world.onRemove(WaveformHandle, (e) => e.get(WaveformHandle)?.dispose()));
subs.push(world.onRemove(HtmlHostHandle, (e) => e.get(HtmlHostHandle)?.dispose()));
subs.push(world.onRemove(SurfaceHostHandle, (e) => e.get(SurfaceHostHandle)?.dispose()));
subs.push(world.onRemove(ShaderHostHandle, (e) => e.get(ShaderHostHandle)?.dispose()));