Skip to content

The VS Code of video editing, built for agents.

TypeScript

Update encoder and runtime packages to include new contact sheet and audio sync features

konstantin-paulusb1b2854bd9d3
diff --git a/packages/encoder/src/contact-sheet.ts b/packages/encoder/src/contact-sheet.ts
new file mode 100644
index 00000000..37259a8b
--- /dev/null
+++ b/packages/encoder/src/contact-sheet.ts
@@ -0,0 +1,149 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { stampTimestampLabel } from '@diffusionstudio/runtime';
+
+/**
+ * Contact sheets: a handful of frames merged into one labelled image, so a
+ * grab or a capture reads as a single picture instead of a directory of PNGs.
+ * The sheet never exceeds the largest image a vision model reads at full
+ * detail (92x52 patches of 28px), and frames are never upscaled past their
+ * source resolution.
+ */
+export const SHEET_MAX_WIDTH = 2576;
+export const SHEET_MAX_HEIGHT = 1456;
+
+// Owned here since sheets are composed from image-encoder captures; the CLI's
+// protocol constant mirrors this value for input validation.
+export const MAX_FRAMES_PER_SHEET = 12;
+
+const MARGIN = 4;
+const GUTTER = 8;
+const BACKGROUND = '#444';
+
+export type SheetPlan = {
+ columns: number;
+ rows: number;
+ /** Size each frame is drawn at, in px. */
+ cellWidth: number;
+ cellHeight: number;
+ width: number;
+ height: number;
+};
+
+/**
+ * Pick the grid for `count` frames of the given source size: the layout that
+ * draws each frame as large as possible within the sheet budget, preferring
+ * fewer blank cells and then wider rows when the sizes tie. The last row may be
+ * partially filled, and frames are never enlarged past `source`.
+ */
+export function planSheet(count: number, source: { width: number; height: number }): SheetPlan {
+ const cells = Math.max(1, Math.min(count, MAX_FRAMES_PER_SHEET));
+ let best: SheetPlan | undefined;
+
+ for (let columns = 1; columns <= cells; columns++) {
+ const rows = Math.ceil(cells / columns);
+ const availableWidth = SHEET_MAX_WIDTH - 2 * MARGIN - GUTTER * (columns - 1);
+ const availableHeight = SHEET_MAX_HEIGHT - 2 * MARGIN - GUTTER * (rows - 1);
+ if (availableWidth < columns || availableHeight < rows) continue;
+
+ // Fit the frame into its cell, never enlarging it past its own pixels.
+ const scale = Math.min(availableWidth / columns / source.width, availableHeight / rows / source.height, 1);
+ const cellWidth = Math.max(1, Math.floor(source.width * scale));
+ const cellHeight = Math.max(1, Math.floor(source.height * scale));
+
+ const candidate: SheetPlan = {
+ columns,
+ rows,
+ cellWidth,
+ cellHeight,
+ width: 2 * MARGIN + columns * cellWidth + GUTTER * (columns - 1),
+ height: 2 * MARGIN + rows * cellHeight + GUTTER * (rows - 1),
+ };
+ if (best === undefined || isBetterPlan(candidate, best, cells)) best = candidate;
+ }
+
+ // Every source fits at 1x1, so the loop always yields a plan.
+ return best!;
+}
+
+function isBetterPlan(candidate: SheetPlan, best: SheetPlan, cells: number): boolean {
+ const candidateArea = candidate.cellWidth * candidate.cellHeight;
+ const bestArea = best.cellWidth * best.cellHeight;
+ if (candidateArea !== bestArea) return candidateArea > bestArea;
+
+ const candidateBlanks = candidate.columns * candidate.rows - cells;
+ const bestBlanks = best.columns * best.rows - cells;
+ if (candidateBlanks !== bestBlanks) return candidateBlanks < bestBlanks;
+
+ // Same cells either way: fill rows before starting new ones.
+ return candidate.columns > best.columns;
+}
+
+/**
+ * A sheet is stamped with the span it covers, earliest to latest: `0f-08s10f`,
+ * or just the one timecode when every cell lands on the same frame. The cells
+ * themselves stay in the order they were requested in, hence `at`.
+ */
+export function sheetTimecode(cells: Array<{ at: number; timecode: string }>): string {
+ const sorted = [...cells].sort((a, b) => a.at - b.at);
+ const first = sorted[0]!.timecode;
+ const last = sorted[sorted.length - 1]!.timecode;
+ return first === last ? first : `${first}-${last}`;
+}
+
+/**
+ * Split `total` frames into balanced sheets of at most `perSheet`, so a run of
+ * 13 becomes 7 + 6 rather than 12 + 1. Returns the size of each sheet.
+ */
+export function planSheetSizes(total: number, perSheet?: number): number[] {
+ const max = Math.max(1, Math.min(Math.round(perSheet ?? MAX_FRAMES_PER_SHEET), MAX_FRAMES_PER_SHEET));
+ const sheets = Math.max(1, Math.ceil(total / max));
+ const base = Math.floor(total / sheets);
+ const extra = total % sheets;
+ return Array.from({ length: sheets }, (_, i) => base + (i < extra ? 1 : 0));
+}
+
+/**
+ * Draw frames into a sheet in reading order, each stamped with its timecode.
+ * The gutters are opaque, so transparent captures composite onto flat grey.
+ */
+export async function composeSheet(
+ frames: Array<{ image: CanvasImageSource; label: string }>,
+ plan: SheetPlan,
+): Promise<string> {
+ const canvas = new OffscreenCanvas(plan.width, plan.height);
+ const ctx = canvas.getContext('2d')!;
+
+ ctx.fillStyle = BACKGROUND;
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+ for (const [i, frame] of frames.entries()) {
+ const column = i % plan.columns;
+ const row = Math.floor(i / plan.columns);
+ const x = MARGIN + column * (plan.cellWidth + GUTTER);
+ const y = MARGIN + row * (plan.cellHeight + GUTTER);
+
+ ctx.drawImage(frame.image, x, y, plan.cellWidth, plan.cellHeight);
+
+ ctx.save();
+ ctx.translate(x, y);
+ stampTimestampLabel(ctx, plan.cellHeight, frame.label, { minBandHeight: 30 });
+ ctx.restore();
+ }
+
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
+ const dataUrl = await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+ return dataUrl.slice(dataUrl.indexOf(',') + 1);
+}
+
+export async function decodePngBase64(base64: string): Promise<ImageBitmap> {
+ const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));
+ return createImageBitmap(new Blob([bytes], { type: 'image/png' }));
+}
diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.ts
index 87d54003..ab8b77dd 100644
--- a/packages/encoder/src/index.ts
+++ b/packages/encoder/src/index.ts
@@ -3,6 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
export * from './buffer';
+export * from './contact-sheet';
export * from './encoder';
export * from './format';
export * from './image-encoder';
diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.md
index f38d7721..311bc030 100644
--- a/packages/runtime/MIGRATION.md
+++ b/packages/runtime/MIGRATION.md
@@ -42,6 +42,9 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon
| System schedule | `src/systems/schedule.ts` | `runSystems(world)`: playback → motion → transform → render, shared by the realtime tick and the capture loop |
| Encoders | `packages/encoder` (`@diffusionstudio/encoder`) | `engine/encode/*` extracted into its OWN package, not `src/capture/`: createEncoder + createImageEncoder as pure consumers of the runtime's public API. See semantic notes |
| Timecode helpers | `src/utils/time.ts` | `formatTimecode`/`formatTimestamp` (from `utils/transcode.ts`; the image encoder stamps captures with them) |
+| Audio sync | `src/media/audio-sync.ts` | Moved near-verbatim (FFT cross-correlation over RMS envelopes; pure math + mediabunny) |
+| Transcode + previews | `src/media/transcode.ts` | transcodeForTranscription/ForAnalysis, filmstripAsset, waveformAsset, stampTimestampLabel. Preview canvases became OffscreenCanvas + convertToBlob (worker-capable) instead of document.createElement + toDataURL |
+| Contact sheets | `packages/encoder/src/contact-sheet.ts` | Sheets compose image-encoder captures, so they live in the encoder package. It now OWNS `MAX_FRAMES_PER_SHEET` (was `@diffusionstudio/cli/channels`; the CLI imports it from here once rewired). composeSheet is async (OffscreenCanvas) |
| Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |
Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.
@@ -101,9 +104,8 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All
## Remaining work (suggested order)
-1. **Remaining utils**: captions (needs the encoder package, so it lands there or app-side), contact-sheet, audio-sync, transcode (timecode helpers already moved; audit the rest for DOM/app deps; DOM-touching ones stay app-side).
-2. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.
-3. **CLI**: point node capture at the runtime + encoder packages.
+1. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. `utils/captions.ts` (trpc/upload/encoder-driven) stays app-side, rewired to `@diffusionstudio/encoder`. Delete `engine/components`, `engine/api` progressively.
+2. **CLI**: point node capture at the runtime + encoder packages (incl. importing `MAX_FRAMES_PER_SHEET` from the encoder instead of defining it in `cli-channels.ts`).
## Layout reminders
diff --git a/packages/runtime/src/media/audio-sync.ts b/packages/runtime/src/media/audio-sync.ts
new file mode 100644
index 00000000..5fcca06e
--- /dev/null
+++ b/packages/runtime/src/media/audio-sync.ts
@@ -0,0 +1,233 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
+
+import { assert } from '../utils/assert';
+import { getAssetFile } from '../actions/assets';
+
+import type { Asset } from '../assets/types';
+
+// Envelope sample rate. 500 Hz → 2 ms resolution (well under one video frame),
+// and keeps the FFT small enough for multi-minute clips. Parabolic
+// interpolation around the peak refines below one bin.
+const SYNC_ENVELOPE_RATE_HZ = 500;
+// Require at least this much overlapping audio for a lag to be considered.
+const SYNC_MIN_OVERLAP_SEC = 1;
+// Below this correlation the match is treated as no match, and we fail loudly
+// rather than return a confident-looking but meaningless offset.
+const SYNC_NO_MATCH_FLOOR = 0.1;
+
+export type AudioSyncResult = {
+ // Place the audio at: audioStart = videoStart + offsetSeconds.
+ // Positive = the recording started after the camera rolled.
+ offsetSeconds: number;
+ confidence: number; // 0..1 (clamped Pearson correlation at the best lag)
+};
+
+// RMS energy envelope of an asset's primary audio track, at `rate` bins/sec.
+async function extractAudioEnvelope(asset: Asset, rate: number): Promise<Float32Array> {
+ const blob = await getAssetFile(asset);
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ const track = await input.getPrimaryAudioTrack();
+ assert(track, `Asset ${asset.id} has no audio track to sync.`);
+ const sink = new AudioSampleSink(track);
+ const bins: number[] = [];
+ for await (const sample of sink.samples()) {
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+
+ const numChannels = sample.numberOfChannels;
+ const numFrames = floats.length / numChannels;
+ const targetBins = Math.max(1, Math.round(sample.duration * rate));
+ const ratio = numFrames / targetBins;
+
+ for (let i = 0; i < targetBins; i++) {
+ const start = Math.floor(i * ratio);
+ const end = Math.max(start + 1, Math.floor((i + 1) * ratio));
+ let sumSq = 0;
+ let count = 0;
+ for (let f = start; f < end && f < numFrames; f++) {
+ for (let ch = 0; ch < numChannels; ch++) {
+ const v = floats[f * numChannels + ch];
+ sumSq += v * v;
+ count++;
+ }
+ }
+ bins.push(count ? Math.sqrt(sumSq / count) : 0);
+ }
+ } finally {
+ sample.close();
+ }
+ }
+ return Float32Array.from(bins);
+ } finally {
+ input.dispose();
+ }
+}
+
+function nextPow2(n: number): number {
+ let p = 1;
+ while (p < n) p <<= 1;
+ return p;
+}
+
+// In-place iterative radix-2 Cooley-Tukey FFT (inverse when `invert`).
+function fft(re: Float64Array, im: Float64Array, invert: boolean): void {
+ const n = re.length;
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ for (; j & bit; bit >>= 1) j ^= bit;
+ j ^= bit;
+ if (i < j) {
+ const tr = re[i]; re[i] = re[j]; re[j] = tr;
+ const ti = im[i]; im[i] = im[j]; im[j] = ti;
+ }
+ }
+ for (let len = 2; len <= n; len <<= 1) {
+ const ang = ((invert ? 2 : -2) * Math.PI) / len;
+ const wr = Math.cos(ang);
+ const wi = Math.sin(ang);
+ for (let i = 0; i < n; i += len) {
+ let cwr = 1;
+ let cwi = 0;
+ for (let k = 0; k < len >> 1; k++) {
+ const a = i + k;
+ const b = a + (len >> 1);
+ const vr = re[b] * cwr - im[b] * cwi;
+ const vi = re[b] * cwi + im[b] * cwr;
+ re[b] = re[a] - vr; im[b] = im[a] - vi;
+ re[a] += vr; im[a] += vi;
+ const ncwr = cwr * wr - cwi * wi;
+ cwi = cwr * wi + cwi * wr;
+ cwr = ncwr;
+ }
+ }
+ }
+ if (invert) {
+ for (let i = 0; i < n; i++) { re[i] /= n; im[i] /= n; }
+ }
+}
+
+// Linear cross-correlation via FFT. Returns cc where cc[m] = Σ a[i]·b[i+m]
+// (negative lags wrap to the high indices; safe because of zero padding).
+function crossCorrelate(a: Float32Array, b: Float32Array): Float64Array {
+ const n = nextPow2(a.length + b.length);
+ const ar = new Float64Array(n);
+ const ai = new Float64Array(n);
+ const br = new Float64Array(n);
+ const bi = new Float64Array(n);
+ ar.set(a);
+ br.set(b);
+ fft(ar, ai, false);
+ fft(br, bi, false);
+ // conj(FA) .* FB
+ for (let k = 0; k < n; k++) {
+ const cr = ar[k] * br[k] + ai[k] * bi[k];
+ const ci = ar[k] * bi[k] - ai[k] * br[k];
+ ar[k] = cr;
+ ai[k] = ci;
+ }
+ fft(ar, ai, true);
+ return ar; // real part holds the correlation
+}
+
+function prefixSums(x: Float32Array): { sum: Float64Array; sumSq: Float64Array } {
+ const sum = new Float64Array(x.length + 1);
+ const sumSq = new Float64Array(x.length + 1);
+ for (let i = 0; i < x.length; i++) {
+ sum[i + 1] = sum[i] + x[i];
+ sumSq[i + 1] = sumSq[i] + x[i] * x[i];
+ }
+ return { sum, sumSq };
+}
+
+const offsetCache = new Map<string, Promise<AudioSyncResult>>();
+
+export function computeAudioSyncOffsetCached(a: Asset, b: Asset): Promise<AudioSyncResult> {
+ const key = `${a.hash}:${b.hash}`;
+ const cached = offsetCache.get(key);
+ if (cached) return cached;
+ const promise = computeAudioSyncOffset(a, b);
+ offsetCache.set(key, promise);
+ promise.catch(() => offsetCache.delete(key));
+ return promise;
+}
+
+export async function computeAudioSyncOffset(
+ audioAsset: Asset,
+ videoAsset: Asset,
+): Promise<AudioSyncResult> {
+ const rate = SYNC_ENVELOPE_RATE_HZ;
+ const A = await extractAudioEnvelope(audioAsset, rate);
+ const B = await extractAudioEnvelope(videoAsset, rate);
+ const La = A.length;
+ const Lb = B.length;
+ assert(La > 0 && Lb > 0, "No reliable alignment found: an asset has no decodable audio.");
+
+ const cc = crossCorrelate(A, B);
+ const n = cc.length;
+ const pa = prefixSums(A);
+ const pb = prefixSums(B);
+ const minOverlap = Math.max(1, Math.floor(rate * SYNC_MIN_OVERLAP_SEC));
+
+ // Pearson correlation of the overlapping region at lag L (audio index i ↔
+ // video index i+L), computed in O(1) from prefix sums + the FFT dot product.
+ const nccAt = (L: number): number => {
+ const iLo = Math.max(0, -L);
+ const iHi = Math.min(La, Lb - L);
+ const cnt = iHi - iLo;
+ if (cnt < minOverlap) return -Infinity;
+ const jLo = iLo + L;
+ const jHi = iHi + L;
+ const sumA = pa.sum[iHi] - pa.sum[iLo];
+ const sumSqA = pa.sumSq[iHi] - pa.sumSq[iLo];
+ const sumB = pb.sum[jHi] - pb.sum[jLo];
+ const sumSqB = pb.sumSq[jHi] - pb.sumSq[jLo];
+ const meanA = sumA / cnt;
+ const meanB = sumB / cnt;
+ const varA = sumSqA - cnt * meanA * meanA;
+ const varB = sumSqB - cnt * meanB * meanB;
+ if (varA <= 1e-9 || varB <= 1e-9) return -Infinity;
+ const dot = cc[((L % n) + n) % n];
+ const cov = dot - cnt * meanA * meanB;
+ return cov / Math.sqrt(varA * varB);
+ };
+
+ let bestL = 0;
+ let bestNcc = -Infinity;
+ for (let L = -(La - 1); L <= Lb - 1; L++) {
+ const v = nccAt(L);
+ if (v > bestNcc) {
+ bestNcc = v;
+ bestL = L;
+ }
+ }
+ assert(
+ bestNcc > -Infinity,
+ "No reliable alignment found: the clips don't overlap enough to align.",
+ );
+ assert(
+ bestNcc >= SYNC_NO_MATCH_FLOOR,
+ "No reliable alignment found: the audio tracks share too little common sound.",
+ );
+
+ // Parabolic interpolation around the peak for sub-bin precision.
+ let delta = 0;
+ const ym = nccAt(bestL - 1);
+ const yp = nccAt(bestL + 1);
+ if (Number.isFinite(ym) && Number.isFinite(yp)) {
+ const denom = ym - 2 * bestNcc + yp;
+ if (denom !== 0) delta = (0.5 * (ym - yp)) / denom;
+ if (!Number.isFinite(delta) || Math.abs(delta) > 1) delta = 0;
+ }
+
+ return {
+ offsetSeconds: (bestL + delta) / rate,
+ confidence: Math.max(0, Math.min(1, bestNcc)),
+ };
+}
diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.ts
index 86367259..2b0025a5 100644
--- a/packages/runtime/src/media/index.ts
+++ b/packages/runtime/src/media/index.ts
@@ -5,6 +5,7 @@
export * from './audio';
export * from './audio-bus';
export * from './audio-peaks';
+export * from './audio-sync';
export * from './caption';
export * from './dispose';
export * from './frame-cache';
@@ -15,4 +16,5 @@ export * from './sequence';
export * from './shader';
export * from './surface';
export * from './time-stretcher';
+export * from './transcode';
export * from './video';
diff --git a/packages/runtime/src/media/transcode.ts b/packages/runtime/src/media/transcode.ts
new file mode 100644
index 00000000..36e3cc01
--- /dev/null
+++ b/packages/runtime/src/media/transcode.ts
@@ -0,0 +1,511 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Asset transcoding and preview rendering (was engine/utils/transcode.ts;
+// the timecode formatters moved to utils/time.ts). Filmstrips and waveforms
+// draw into an OffscreenCanvas, so everything here is worker-capable.
+
+import { ALL_FORMATS, AudioSampleSink, BlobSource, BufferTarget, CanvasSink, Conversion, Input, InputAudioTrack, Mp4OutputFormat, OggOutputFormat, Output, StreamTarget } from 'mediabunny';
+
+import { assert } from '../utils/assert';
+import { formatTimestamp } from '../utils/time';
+import { getAssetFile } from '../actions/assets';
+
+import type { StreamTargetChunk } from 'mediabunny';
+import type { Asset, VideoAsset, AudioAsset } from '../assets/types';
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+export async function transcodeForTranscription(asset: Asset): Promise<File> {
+ const blob = await getAssetFile(asset);
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ const output = new Output({ format: new OggOutputFormat(), target: new BufferTarget() });
+ const conversion = await Conversion.init({
+ input,
+ output,
+ video: { discard: true },
+ audio: { codec: "opus", numberOfChannels: 1, sampleRate: 16000 },
+ });
+ if (!conversion.isValid || conversion.utilizedTracks.length === 0) {
+ throw new Error("No audio found. The asset has no audio track to transcribe.");
+ }
+ await conversion.execute();
+
+ const buffer = output.target.buffer;
+ assert(buffer, "Transcoding produced no output.");
+ return new File([buffer], `${crypto.randomUUID()}.ogg`, { type: "audio/ogg" });
+ } finally {
+ input.dispose();
+ }
+}
+
+export async function transcodeForAnalysis(asset: VideoAsset | AudioAsset, window?: TimeWindow & { stripVideo?: boolean }) {
+ const blob = await getAssetFile(asset);
+ const hasWindow = window?.start !== undefined || window?.end !== undefined;
+
+ const isVideo = asset.type === "VIDEO" && !window?.stripVideo;
+ const trim = hasWindow ? resolveWindow(asset.duration, window) : undefined;
+
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+
+ const { writable, readable } = new TransformStream<StreamTargetChunk, Uint8Array<ArrayBuffer>>({
+ transform: (chunk, controller) => controller.enqueue(chunk.data),
+ });
+
+ const target = new StreamTarget(writable);
+ const format = isVideo
+ ? new Mp4OutputFormat({ fastStart: "fragmented" })
+ : new OggOutputFormat();
+ const output = new Output({ format, target });
+
+ const conversion = await Conversion.init({
+ input,
+ output,
+ trim: (trim && { start: trim.start, end: trim.end }),
+ video: { width: 640, frameRate: 24, bitrate: 500_000, discard: !isVideo },
+ audio: { codec: isVideo ? undefined : "opus", numberOfChannels: 1, sampleRate: 16_000 },
+ });
+
+ if (!conversion.isValid || conversion.utilizedTracks.length === 0) {
+ input.dispose();
+ throw new Error("Nothing to analyze. The asset has no usable video or audio track.");
+ }
+
+ const run = async () => {
+ try {
+ await conversion.execute();
+ } finally {
+ input.dispose();
+ }
+ };
+
+ return { readable, run };
+}
+
+const PATCH_SIZE = 28;
+const MAX_WIDTH_IN_TOKENS = 64; // 92
+const MAX_HEIGHT_IN_TOKENS = 52;
+const THUMBNAIL_HEIGHT_IN_TOKENS = 7;
+const RULER_HEIGHT_IN_TOKENS = 1;
+const AUDIO_WAVEFORM_HEIGHT_IN_TOKENS = 6;
+const AUDIO_COLUMN_WIDTH_IN_TOKENS = 6;
+const WAVEFORM_SAMPLE_WIDTH = 2; // pixels per peak; wider = less noisy
+const SILENCE_THRESHOLD = 12; // peak value (0-255) at/below which audio counts as silent
+const SILENCE_MIN_SECONDS = 0.4;
+const AUDIO_RULER_FRAME_RATE = 30;
+
+// Draws a timestamp label into the top-left corner of a canvas: white text with
+// a black outline, sized relative to the canvas so it reads at any resolution.
+export function stampTimestampLabel(
+ ctx: Ctx2D,
+ height: number,
+ label: string,
+ options?: { minBandHeight?: number },
+): void {
+ const bandHeight = Math.max(options?.minBandHeight ?? 20, Math.round(height * 0.06));
+ const fontSize = Math.round(bandHeight * 0.72);
+ const paddingLeft = Math.round(bandHeight * 0.4);
+ const y = paddingLeft + bandHeight / 2;
+
+ ctx.font = `normal ${fontSize}px sans-serif`;
+ ctx.textBaseline = 'middle';
+ ctx.textAlign = 'left';
+
+ ctx.lineJoin = 'round';
+ ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));
+ ctx.strokeStyle = '#000';
+ ctx.strokeText(label, paddingLeft, y);
+ ctx.fillStyle = '#FFF';
+ ctx.fillText(label, paddingLeft, y);
+}
+
+export type RenderedPreview = {
+ dataUrl: string;
+};
+
+export type RenderedWaveform = RenderedPreview & {
+ silences: Array<{ start: number; end: number }>;
+};
+
+export type TimeWindow = { start?: number; end?: number };
+
+export type PreviewOptions = TimeWindow & { scale?: number };
+
+const MIN_SCALE = 0.25;
+const MAX_SCALE = 4;
+
+function resolveScale(scale?: number): number {
+ if (scale === undefined) return 1;
+ assert(Number.isFinite(scale) && scale > 0, `scale must be a positive number (got ${scale}).`);
+ return Math.min(Math.max(scale, MIN_SCALE), MAX_SCALE);
+}
+
+// Clamp a requested [start, end] window (in seconds) to the asset's timeline.
+// `start` defaults to 0 and `end` to the full duration.
+function resolveWindow(duration: number, window?: TimeWindow): { start: number; end: number; duration: number } {
+ const start = Math.min(Math.max(window?.start ?? 0, 0), duration);
+ const end = Math.min(Math.max(window?.end ?? duration, start), duration);
+ assert(end > start, `The requested window is empty; start (${start.toFixed(2)}s) is at or past the asset's end (${duration.toFixed(2)}s).`);
+ return { start, end, duration: end - start };
+}
+
+/** PNG data URL of an OffscreenCanvas (toDataURL only exists on DOM canvases). */
+async function toPngDataUrl(canvas: OffscreenCanvas): Promise<string> {
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
+ return await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+}
+
+// Render a video as a grid of thumbnails sampled across the window, each row
+// stamped with a timestamp ruler. Video only; no audio.
+export async function filmstripAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedPreview> {
+ assert(asset.type === "VIDEO", "The filmstrip needs a video asset; use `waveform` for audio.");
+ const blob = await getAssetFile(asset);
+ const scale = resolveScale(options?.scale);
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ return await renderFilmstrip(asset, input, options, scale);
+ } finally {
+ input.dispose();
+ }
+}
+
+async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedPreview> {
+ const videoTrack = await input.getPrimaryVideoTrack();
+ assert(videoTrack, "Video track not found");
+ const { start: windowStart, duration } = resolveWindow(asset.duration, window);
+ const aspect = asset.width / asset.height;
+ const frameRate = asset.frameRate;
+
+ const thumbnailHeightInTokens = Math.min(
+ Math.max(Math.round(THUMBNAIL_HEIGHT_IN_TOKENS * scale), 1),
+ MAX_HEIGHT_IN_TOKENS - RULER_HEIGHT_IN_TOKENS,
+ );
+ const rowHeightInTokens = thumbnailHeightInTokens + RULER_HEIGHT_IN_TOKENS;
+
+ const columnWidthInTokens = Math.min(
+ Math.max(Math.floor(thumbnailHeightInTokens * aspect), 1),
+ MAX_WIDTH_IN_TOKENS,
+ );
+
+ const thumbnailWidthInPixels = columnWidthInTokens * PATCH_SIZE;
+ const thumbnailHeightInPixels = thumbnailHeightInTokens * PATCH_SIZE;
+ const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;
+
+ const rows = Math.floor(MAX_HEIGHT_IN_TOKENS / rowHeightInTokens);
+ const columns = Math.floor(MAX_WIDTH_IN_TOKENS / columnWidthInTokens);
+ const cells = rows * columns;
+
+ const secondsPerColumn = Math.max(duration / cells, 1 / frameRate);
+
+ const canvas = new OffscreenCanvas(
+ columnWidthInTokens * columns * PATCH_SIZE,
+ rowHeightInTokens * rows * PATCH_SIZE,
+ );
+ const ctx = canvas.getContext('2d')!;
+
+ ctx.fillStyle = '#000';
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+ const sink = new CanvasSink(videoTrack, {
+ width: thumbnailWidthInPixels,
+ height: thumbnailHeightInPixels,
+ fit: 'cover',
+ poolSize: 1,
+ });
+
+ const rulerHeight = RULER_HEIGHT_IN_TOKENS * PATCH_SIZE;
+
+ const timestamps = Array.from({ length: cells }, (_, i) => windowStart + i * secondsPerColumn);
+ const interator = sink.canvasesAtTimestamps(timestamps);
+ for (let i = 0; i < cells; i++) {
+ const wrapped = await interator.next();
+ if (!wrapped.value?.canvas) continue;
+
+ const column = i % columns;
+ const row = Math.floor(i / columns);
+
+ ctx.drawImage(
+ wrapped.value.canvas,
+ column * columnWidthInTokens * PATCH_SIZE,
+ rulerHeight + row * rowHeightInTokens * PATCH_SIZE,
+ thumbnailWidthInPixels,
+ thumbnailHeightInPixels
+ );
+ }
+
+ drawRuler(ctx, {
+ cells,
+ columns,
+ columnWidthInPixels: columnWidthInTokens * PATCH_SIZE,
+ rowHeightInPixels,
+ rulerHeight,
+ secondsPerColumn,
+ frameRate,
+ startOffset: windowStart,
+ });
+
+ return { dataUrl: await toPngDataUrl(canvas) };
+}
+
+// Render the audio track of a video or audio file as a loudness-over-time
+// waveform with a timestamp ruler and silent stretches highlighted.
+export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedWaveform> {
+ assert(asset.type === "VIDEO" || asset.type === "AUDIO", "The waveform needs a video or audio asset.");
+ const blob = await getAssetFile(asset);
+ const scale = resolveScale(options?.scale);
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ return await renderWaveform(asset, input, options, scale);
+ } finally {
+ input.dispose();
+ }
+}
+
+async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedWaveform> {
+ const audioTrack = await input.getPrimaryAudioTrack();
+ assert(audioTrack, "No audio track found, so no waveform can be rendered.");
+ assert(
+ await audioTrack.canDecode(),
+ `This browser's audio decoder can't handle the "${audioTrack.getCodec() ?? "unknown"}" codec, so no waveform can be rendered.`,
+ );
+ const { start: windowStart, duration } = resolveWindow(asset.duration, window);
+ const rulerFrameRate = asset.type === "VIDEO" ? asset.frameRate : AUDIO_RULER_FRAME_RATE;
+
+ const waveformHeightInTokens = Math.min(
+ Math.max(Math.round(AUDIO_WAVEFORM_HEIGHT_IN_TOKENS * scale), 1),
+ MAX_HEIGHT_IN_TOKENS - RULER_HEIGHT_IN_TOKENS,
+ );
+ const rowHeightInTokens = waveformHeightInTokens + RULER_HEIGHT_IN_TOKENS;
+ const columnWidthInTokens = Math.min(
+ Math.max(Math.round(AUDIO_COLUMN_WIDTH_IN_TOKENS * scale), 1),
+ MAX_WIDTH_IN_TOKENS,
+ );
+
+ const rows = Math.floor(MAX_HEIGHT_IN_TOKENS / rowHeightInTokens);
+ const columns = Math.floor(MAX_WIDTH_IN_TOKENS / columnWidthInTokens);
+ const cells = rows * columns;
+
+ const secondsPerColumn = duration / cells;
+
+ const columnWidthInPixels = columnWidthInTokens * PATCH_SIZE;
+ const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;
+ const rulerHeight = RULER_HEIGHT_IN_TOKENS * PATCH_SIZE;
+ const waveformHeight = waveformHeightInTokens * PATCH_SIZE;
+ const rowWidth = columns * columnWidthInPixels;
+
+ const canvas = new OffscreenCanvas(rowWidth, rowHeightInPixels * rows);
+ const ctx = canvas.getContext('2d')!;
+
+ ctx.fillStyle = '#000';
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+ const peaksPerSecond = Math.round(columnWidthInPixels / secondsPerColumn / WAVEFORM_SAMPLE_WIDTH);
+ const audioPeaks = await downsampleAudio(audioTrack, { peaksPerSecond, range: [windowStart, windowStart + duration] });
+
+ const silentSpans = findSilentSpans(audioPeaks, peaksPerSecond);
+ const silences = silentSpans.map((span) => ({
+ start: windowStart + span.start / peaksPerSecond,
+ end: windowStart + span.end / peaksPerSecond,
+ }));
+
+ drawWaveform(ctx, audioPeaks, silentSpans, {
+ rows,
+ rowWidth,
+ rowHeightInPixels,
+ waveformOffset: rulerHeight,
+ waveformHeight,
+ });
+
+ drawRuler(ctx, {
+ cells,
+ columns,
+ columnWidthInPixels,
+ rowHeightInPixels,
+ rulerHeight,
+ secondsPerColumn,
+ frameRate: rulerFrameRate,
+ startOffset: windowStart,
+ });
+
+ return { dataUrl: await toPngDataUrl(canvas), silences };
+}
+
+type WaveformLayout = {
+ rows: number;
+ rowWidth: number; // pixels spanning all columns of a row
+ rowHeightInPixels: number;
+ waveformOffset: number; // y offset within a row where the waveform begins
+ waveformHeight: number;
+};
+
+// Scan the peaks for runs at/below the silence threshold that last at least
+// SILENCE_MIN_SECONDS, returned as [start, end) peak-index spans.
+function findSilentSpans(audioPeaks: Uint8ClampedArray, peaksPerSecond: number): Array<{ start: number; end: number }> {
+ const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);
+ const silentSpans: Array<{ start: number; end: number }> = [];
+ let runStart = -1;
+ for (let i = 0; i <= audioPeaks.length; i++) {
+ const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;
+ if (silent && runStart === -1) {
+ runStart = i;
+ } else if (!silent && runStart !== -1) {
+ if (i - runStart >= minSilencePeaks) {
+ silentSpans.push({ start: runStart, end: i });
+ }
+ runStart = -1;
+ }
+ }
+ return silentSpans;
+}
+
+// Draws a center-aligned waveform, then overlays the given silent stretches in red.
+function drawWaveform(ctx: Ctx2D, audioPeaks: Uint8ClampedArray, silentSpans: Array<{ start: number; end: number }>, layout: WaveformLayout) {
+ const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight } = layout;
+ const peaksPerRow = Math.floor(rowWidth / WAVEFORM_SAMPLE_WIDTH);
+ const halfHeight = waveformHeight / 2;
+
+ const waveformTopOf = (row: number) => row * rowHeightInPixels + waveformOffset;
+
+ ctx.fillStyle = '#FFF';
+ for (let row = 0; row < rows; row++) {
+ const top = waveformTopOf(row);
+
+ for (let col = 0; col < peaksPerRow; col++) {
+ const peakIndex = row * peaksPerRow + col;
+ if (peakIndex >= audioPeaks.length) break;
+
+ const amplitude = (audioPeaks[peakIndex] / 255) * halfHeight;
+ const x = col * WAVEFORM_SAMPLE_WIDTH;
+ const centerY = top + halfHeight;
+ ctx.fillRect(x, centerY - amplitude, WAVEFORM_SAMPLE_WIDTH, Math.max(amplitude * 2, 1));
+ }
+ }
+
+ for (const span of silentSpans) {
+ let i = span.start;
+ while (i < span.end) {
+ const row = Math.floor(i / peaksPerRow);
+ const segEnd = Math.min(span.end, (row + 1) * peaksPerRow);
+ const x = (i % peaksPerRow) * WAVEFORM_SAMPLE_WIDTH;
+ const width = (segEnd - i) * WAVEFORM_SAMPLE_WIDTH;
+ const top = waveformTopOf(row);
+
+ ctx.fillStyle = 'rgba(255, 64, 64, 0.4)';
+ ctx.fillRect(x, top, width, waveformHeight);
+
+ i = segEnd;
+ }
+ }
+}
+
+type RulerLayout = {
+ cells: number;
+ columns: number;
+ columnWidthInPixels: number;
+ rowHeightInPixels: number;
+ rulerHeight: number;
+ secondsPerColumn: number;
+ frameRate: number;
+ startOffset?: number;
+};
+
+function drawRuler(ctx: Ctx2D, layout: RulerLayout) {
+ const { cells, columns, columnWidthInPixels, rowHeightInPixels, rulerHeight, secondsPerColumn, frameRate, startOffset = 0 } = layout;
+
+ const baseFontSize = Math.round(rulerHeight * 0.8);
+ const leadFontSize = Math.round(rulerHeight * 0.9);
+ const paddingLeft = 5;
+ const paddingTop = 2;
+ const tickHeight = Math.round(rulerHeight * 0.4);
+ const labelGap = 10;
+
+ ctx.fillStyle = '#FFF';
+ ctx.strokeStyle = '#FFF';
+ ctx.lineWidth = 2;
+ ctx.textBaseline = 'middle';
+ ctx.textAlign = 'left';
+
+ const rowWidth = columns * columnWidthInPixels;
+ let labelEndX = -Infinity;
+ for (let i = 0; i < cells; i++) {
+ const column = i % columns;
+ const row = Math.floor(i / columns);
+ const columnX = column * columnWidthInPixels;
+ const rowTop = row * rowHeightInPixels;
+
+ if (column === 0) labelEndX = -Infinity;
+
+ const tickX = columnX + 0.5;
+ if (tickX >= labelEndX) {
+ ctx.beginPath();
+ ctx.moveTo(tickX, rowTop + rulerHeight - tickHeight);
+ ctx.lineTo(tickX, rowTop + rulerHeight);
+ ctx.stroke();
+ }
+
+ ctx.font = column === 0
+ ? `bold ${leadFontSize}px sans-serif`
+ : `${baseFontSize}px sans-serif`;
+ const label = formatTimestamp(startOffset + i * secondsPerColumn, frameRate);
+ const labelX = columnX + paddingLeft;
+ const labelWidth = ctx.measureText(label).width;
+ if (labelX < labelEndX + labelGap || labelX + labelWidth > rowWidth) continue;
+
+ ctx.fillText(label, labelX, (rowTop + rulerHeight / 2) + paddingTop);
+ labelEndX = labelX + labelWidth;
+ }
+}
+
+type SamplerOptions = {
+ peaksPerSecond: number;
+ range?: [start: number, end: number]; // seconds; omitted = whole track
+}
+
+async function downsampleAudio(track: InputAudioTrack, options: SamplerOptions) {
+ const sink = new AudioSampleSink(track);
+ const iterator = options.range ? sink.samples(...options.range) : sink.samples();
+ const framesPerPeak = track.sampleRate / options.peaksPerSecond;
+
+ const peaks: number[] = [];
+ let frameCursor = 0; // frames consumed since the range start
+ let currentPeak = 0; // index of the peak currently being accumulated
+ let peakMax = 0;
+
+ for await (const sample of iterator) {
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+
+ const numChannels = sample.numberOfChannels;
+ const numFrames = floats.length / numChannels;
+
+ for (let frameIdx = 0; frameIdx < numFrames; frameIdx++) {
+ while (currentPeak < Math.floor(frameCursor / framesPerPeak)) {
+ peaks.push(Math.floor(255 * peakMax));
+ peakMax = 0;
+ currentPeak++;
+ }
+
+ for (let ch = 0; ch < numChannels; ch++) {
+ peakMax = Math.max(peakMax, Math.sqrt(Math.abs(floats[frameIdx * numChannels + ch])));
+ }
+ frameCursor++;
+ }
+ } finally {
+ sample.close();
+ }
+ }
+
+ // Flush the final in-progress peak.
+ if (frameCursor > 0) peaks.push(Math.floor(255 * peakMax));
+
+ return Uint8ClampedArray.from(peaks);
+}