The VS Code of video editing, built for agents.
TypeScript
Update encoder and runtime packages to include new contact sheet and audio sync features
konstantin-paulusb1b2854bd9d3
diff --git a/packages/encoder/src/contact-sheet.ts b/packages/encoder/src/contact-sheet.tsnew file mode 100644index 00000000..37259a8b--- /dev/null+++ b/packages/encoder/src/contact-sheet.ts@@ -0,0 +1,149 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { stampTimestampLabel } from '@diffusionstudio/runtime';++/**+ * Contact sheets: a handful of frames merged into one labelled image, so a+ * grab or a capture reads as a single picture instead of a directory of PNGs.+ * The sheet never exceeds the largest image a vision model reads at full+ * detail (92x52 patches of 28px), and frames are never upscaled past their+ * source resolution.+ */+export const SHEET_MAX_WIDTH = 2576;+export const SHEET_MAX_HEIGHT = 1456;++// Owned here since sheets are composed from image-encoder captures; the CLI's+// protocol constant mirrors this value for input validation.+export const MAX_FRAMES_PER_SHEET = 12;++const MARGIN = 4;+const GUTTER = 8;+const BACKGROUND = '#444';++export type SheetPlan = {+ columns: number;+ rows: number;+ /** Size each frame is drawn at, in px. */+ cellWidth: number;+ cellHeight: number;+ width: number;+ height: number;+};++/**+ * Pick the grid for `count` frames of the given source size: the layout that+ * draws each frame as large as possible within the sheet budget, preferring+ * fewer blank cells and then wider rows when the sizes tie. The last row may be+ * partially filled, and frames are never enlarged past `source`.+ */+export function planSheet(count: number, source: { width: number; height: number }): SheetPlan {+ const cells = Math.max(1, Math.min(count, MAX_FRAMES_PER_SHEET));+ let best: SheetPlan | undefined;++ for (let columns = 1; columns <= cells; columns++) {+ const rows = Math.ceil(cells / columns);+ const availableWidth = SHEET_MAX_WIDTH - 2 * MARGIN - GUTTER * (columns - 1);+ const availableHeight = SHEET_MAX_HEIGHT - 2 * MARGIN - GUTTER * (rows - 1);+ if (availableWidth < columns || availableHeight < rows) continue;++ // Fit the frame into its cell, never enlarging it past its own pixels.+ const scale = Math.min(availableWidth / columns / source.width, availableHeight / rows / source.height, 1);+ const cellWidth = Math.max(1, Math.floor(source.width * scale));+ const cellHeight = Math.max(1, Math.floor(source.height * scale));++ const candidate: SheetPlan = {+ columns,+ rows,+ cellWidth,+ cellHeight,+ width: 2 * MARGIN + columns * cellWidth + GUTTER * (columns - 1),+ height: 2 * MARGIN + rows * cellHeight + GUTTER * (rows - 1),+ };+ if (best === undefined || isBetterPlan(candidate, best, cells)) best = candidate;+ }++ // Every source fits at 1x1, so the loop always yields a plan.+ return best!;+}++function isBetterPlan(candidate: SheetPlan, best: SheetPlan, cells: number): boolean {+ const candidateArea = candidate.cellWidth * candidate.cellHeight;+ const bestArea = best.cellWidth * best.cellHeight;+ if (candidateArea !== bestArea) return candidateArea > bestArea;++ const candidateBlanks = candidate.columns * candidate.rows - cells;+ const bestBlanks = best.columns * best.rows - cells;+ if (candidateBlanks !== bestBlanks) return candidateBlanks < bestBlanks;++ // Same cells either way: fill rows before starting new ones.+ return candidate.columns > best.columns;+}++/**+ * A sheet is stamped with the span it covers, earliest to latest: `0f-08s10f`,+ * or just the one timecode when every cell lands on the same frame. The cells+ * themselves stay in the order they were requested in, hence `at`.+ */+export function sheetTimecode(cells: Array<{ at: number; timecode: string }>): string {+ const sorted = [...cells].sort((a, b) => a.at - b.at);+ const first = sorted[0]!.timecode;+ const last = sorted[sorted.length - 1]!.timecode;+ return first === last ? first : `${first}-${last}`;+}++/**+ * Split `total` frames into balanced sheets of at most `perSheet`, so a run of+ * 13 becomes 7 + 6 rather than 12 + 1. Returns the size of each sheet.+ */+export function planSheetSizes(total: number, perSheet?: number): number[] {+ const max = Math.max(1, Math.min(Math.round(perSheet ?? MAX_FRAMES_PER_SHEET), MAX_FRAMES_PER_SHEET));+ const sheets = Math.max(1, Math.ceil(total / max));+ const base = Math.floor(total / sheets);+ const extra = total % sheets;+ return Array.from({ length: sheets }, (_, i) => base + (i < extra ? 1 : 0));+}++/**+ * Draw frames into a sheet in reading order, each stamped with its timecode.+ * The gutters are opaque, so transparent captures composite onto flat grey.+ */+export async function composeSheet(+ frames: Array<{ image: CanvasImageSource; label: string }>,+ plan: SheetPlan,+): Promise<string> {+ const canvas = new OffscreenCanvas(plan.width, plan.height);+ const ctx = canvas.getContext('2d')!;++ ctx.fillStyle = BACKGROUND;+ ctx.fillRect(0, 0, canvas.width, canvas.height);++ for (const [i, frame] of frames.entries()) {+ const column = i % plan.columns;+ const row = Math.floor(i / plan.columns);+ const x = MARGIN + column * (plan.cellWidth + GUTTER);+ const y = MARGIN + row * (plan.cellHeight + GUTTER);++ ctx.drawImage(frame.image, x, y, plan.cellWidth, plan.cellHeight);++ ctx.save();+ ctx.translate(x, y);+ stampTimestampLabel(ctx, plan.cellHeight, frame.label, { minBandHeight: 30 });+ ctx.restore();+ }++ const blob = await canvas.convertToBlob({ type: 'image/png' });+ const dataUrl = await new Promise<string>((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(blob);+ });+ return dataUrl.slice(dataUrl.indexOf(',') + 1);+}++export async function decodePngBase64(base64: string): Promise<ImageBitmap> {+ const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));+ return createImageBitmap(new Blob([bytes], { type: 'image/png' }));+}diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.tsindex 87d54003..ab8b77dd 100644--- a/packages/encoder/src/index.ts+++ b/packages/encoder/src/index.ts@@ -3,6 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */export * from './buffer';+export * from './contact-sheet';export * from './encoder';export * from './format';export * from './image-encoder';diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex f38d7721..311bc030 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -42,6 +42,9 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon| System schedule | `src/systems/schedule.ts` | `runSystems(world)`: playback → motion → transform → render, shared by the realtime tick and the capture loop || Encoders | `packages/encoder` (`@diffusionstudio/encoder`) | `engine/encode/*` extracted into its OWN package, not `src/capture/`: createEncoder + createImageEncoder as pure consumers of the runtime's public API. See semantic notes || Timecode helpers | `src/utils/time.ts` | `formatTimecode`/`formatTimestamp` (from `utils/transcode.ts`; the image encoder stamps captures with them) |+| Audio sync | `src/media/audio-sync.ts` | Moved near-verbatim (FFT cross-correlation over RMS envelopes; pure math + mediabunny) |+| Transcode + previews | `src/media/transcode.ts` | transcodeForTranscription/ForAnalysis, filmstripAsset, waveformAsset, stampTimestampLabel. Preview canvases became OffscreenCanvas + convertToBlob (worker-capable) instead of document.createElement + toDataURL |+| Contact sheets | `packages/encoder/src/contact-sheet.ts` | Sheets compose image-encoder captures, so they live in the encoder package. It now OWNS `MAX_FRAMES_PER_SHEET` (was `@diffusionstudio/cli/channels`; the CLI imports it from here once rewired). composeSheet is async (OffscreenCanvas) || Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.@@ -101,9 +104,8 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All## Remaining work (suggested order)-1. **Remaining utils**: captions (needs the encoder package, so it lands there or app-side), contact-sheet, audio-sync, transcode (timecode helpers already moved; audit the rest for DOM/app deps; DOM-touching ones stay app-side).-2. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.-3. **CLI**: point node capture at the runtime + encoder packages.+1. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. `utils/captions.ts` (trpc/upload/encoder-driven) stays app-side, rewired to `@diffusionstudio/encoder`. Delete `engine/components`, `engine/api` progressively.+2. **CLI**: point node capture at the runtime + encoder packages (incl. importing `MAX_FRAMES_PER_SHEET` from the encoder instead of defining it in `cli-channels.ts`).## Layout remindersdiff --git a/packages/runtime/src/media/audio-sync.ts b/packages/runtime/src/media/audio-sync.tsnew file mode 100644index 00000000..5fcca06e--- /dev/null+++ b/packages/runtime/src/media/audio-sync.ts@@ -0,0 +1,233 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';++import { assert } from '../utils/assert';+import { getAssetFile } from '../actions/assets';++import type { Asset } from '../assets/types';++// Envelope sample rate. 500 Hz → 2 ms resolution (well under one video frame),+// and keeps the FFT small enough for multi-minute clips. Parabolic+// interpolation around the peak refines below one bin.+const SYNC_ENVELOPE_RATE_HZ = 500;+// Require at least this much overlapping audio for a lag to be considered.+const SYNC_MIN_OVERLAP_SEC = 1;+// Below this correlation the match is treated as no match, and we fail loudly+// rather than return a confident-looking but meaningless offset.+const SYNC_NO_MATCH_FLOOR = 0.1;++export type AudioSyncResult = {+ // Place the audio at: audioStart = videoStart + offsetSeconds.+ // Positive = the recording started after the camera rolled.+ offsetSeconds: number;+ confidence: number; // 0..1 (clamped Pearson correlation at the best lag)+};++// RMS energy envelope of an asset's primary audio track, at `rate` bins/sec.+async function extractAudioEnvelope(asset: Asset, rate: number): Promise<Float32Array> {+ const blob = await getAssetFile(asset);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const track = await input.getPrimaryAudioTrack();+ assert(track, `Asset ${asset.id} has no audio track to sync.`);+ const sink = new AudioSampleSink(track);+ const bins: number[] = [];+ for await (const sample of sink.samples()) {+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });++ const numChannels = sample.numberOfChannels;+ const numFrames = floats.length / numChannels;+ const targetBins = Math.max(1, Math.round(sample.duration * rate));+ const ratio = numFrames / targetBins;++ for (let i = 0; i < targetBins; i++) {+ const start = Math.floor(i * ratio);+ const end = Math.max(start + 1, Math.floor((i + 1) * ratio));+ let sumSq = 0;+ let count = 0;+ for (let f = start; f < end && f < numFrames; f++) {+ for (let ch = 0; ch < numChannels; ch++) {+ const v = floats[f * numChannels + ch];+ sumSq += v * v;+ count++;+ }+ }+ bins.push(count ? Math.sqrt(sumSq / count) : 0);+ }+ } finally {+ sample.close();+ }+ }+ return Float32Array.from(bins);+ } finally {+ input.dispose();+ }+}++function nextPow2(n: number): number {+ let p = 1;+ while (p < n) p <<= 1;+ return p;+}++// In-place iterative radix-2 Cooley-Tukey FFT (inverse when `invert`).+function fft(re: Float64Array, im: Float64Array, invert: boolean): void {+ const n = re.length;+ for (let i = 1, j = 0; i < n; i++) {+ let bit = n >> 1;+ for (; j & bit; bit >>= 1) j ^= bit;+ j ^= bit;+ if (i < j) {+ const tr = re[i]; re[i] = re[j]; re[j] = tr;+ const ti = im[i]; im[i] = im[j]; im[j] = ti;+ }+ }+ for (let len = 2; len <= n; len <<= 1) {+ const ang = ((invert ? 2 : -2) * Math.PI) / len;+ const wr = Math.cos(ang);+ const wi = Math.sin(ang);+ for (let i = 0; i < n; i += len) {+ let cwr = 1;+ let cwi = 0;+ for (let k = 0; k < len >> 1; k++) {+ const a = i + k;+ const b = a + (len >> 1);+ const vr = re[b] * cwr - im[b] * cwi;+ const vi = re[b] * cwi + im[b] * cwr;+ re[b] = re[a] - vr; im[b] = im[a] - vi;+ re[a] += vr; im[a] += vi;+ const ncwr = cwr * wr - cwi * wi;+ cwi = cwr * wi + cwi * wr;+ cwr = ncwr;+ }+ }+ }+ if (invert) {+ for (let i = 0; i < n; i++) { re[i] /= n; im[i] /= n; }+ }+}++// Linear cross-correlation via FFT. Returns cc where cc[m] = Σ a[i]·b[i+m]+// (negative lags wrap to the high indices; safe because of zero padding).+function crossCorrelate(a: Float32Array, b: Float32Array): Float64Array {+ const n = nextPow2(a.length + b.length);+ const ar = new Float64Array(n);+ const ai = new Float64Array(n);+ const br = new Float64Array(n);+ const bi = new Float64Array(n);+ ar.set(a);+ br.set(b);+ fft(ar, ai, false);+ fft(br, bi, false);+ // conj(FA) .* FB+ for (let k = 0; k < n; k++) {+ const cr = ar[k] * br[k] + ai[k] * bi[k];+ const ci = ar[k] * bi[k] - ai[k] * br[k];+ ar[k] = cr;+ ai[k] = ci;+ }+ fft(ar, ai, true);+ return ar; // real part holds the correlation+}++function prefixSums(x: Float32Array): { sum: Float64Array; sumSq: Float64Array } {+ const sum = new Float64Array(x.length + 1);+ const sumSq = new Float64Array(x.length + 1);+ for (let i = 0; i < x.length; i++) {+ sum[i + 1] = sum[i] + x[i];+ sumSq[i + 1] = sumSq[i] + x[i] * x[i];+ }+ return { sum, sumSq };+}++const offsetCache = new Map<string, Promise<AudioSyncResult>>();++export function computeAudioSyncOffsetCached(a: Asset, b: Asset): Promise<AudioSyncResult> {+ const key = `${a.hash}:${b.hash}`;+ const cached = offsetCache.get(key);+ if (cached) return cached;+ const promise = computeAudioSyncOffset(a, b);+ offsetCache.set(key, promise);+ promise.catch(() => offsetCache.delete(key));+ return promise;+}++export async function computeAudioSyncOffset(+ audioAsset: Asset,+ videoAsset: Asset,+): Promise<AudioSyncResult> {+ const rate = SYNC_ENVELOPE_RATE_HZ;+ const A = await extractAudioEnvelope(audioAsset, rate);+ const B = await extractAudioEnvelope(videoAsset, rate);+ const La = A.length;+ const Lb = B.length;+ assert(La > 0 && Lb > 0, "No reliable alignment found: an asset has no decodable audio.");++ const cc = crossCorrelate(A, B);+ const n = cc.length;+ const pa = prefixSums(A);+ const pb = prefixSums(B);+ const minOverlap = Math.max(1, Math.floor(rate * SYNC_MIN_OVERLAP_SEC));++ // Pearson correlation of the overlapping region at lag L (audio index i ↔+ // video index i+L), computed in O(1) from prefix sums + the FFT dot product.+ const nccAt = (L: number): number => {+ const iLo = Math.max(0, -L);+ const iHi = Math.min(La, Lb - L);+ const cnt = iHi - iLo;+ if (cnt < minOverlap) return -Infinity;+ const jLo = iLo + L;+ const jHi = iHi + L;+ const sumA = pa.sum[iHi] - pa.sum[iLo];+ const sumSqA = pa.sumSq[iHi] - pa.sumSq[iLo];+ const sumB = pb.sum[jHi] - pb.sum[jLo];+ const sumSqB = pb.sumSq[jHi] - pb.sumSq[jLo];+ const meanA = sumA / cnt;+ const meanB = sumB / cnt;+ const varA = sumSqA - cnt * meanA * meanA;+ const varB = sumSqB - cnt * meanB * meanB;+ if (varA <= 1e-9 || varB <= 1e-9) return -Infinity;+ const dot = cc[((L % n) + n) % n];+ const cov = dot - cnt * meanA * meanB;+ return cov / Math.sqrt(varA * varB);+ };++ let bestL = 0;+ let bestNcc = -Infinity;+ for (let L = -(La - 1); L <= Lb - 1; L++) {+ const v = nccAt(L);+ if (v > bestNcc) {+ bestNcc = v;+ bestL = L;+ }+ }+ assert(+ bestNcc > -Infinity,+ "No reliable alignment found: the clips don't overlap enough to align.",+ );+ assert(+ bestNcc >= SYNC_NO_MATCH_FLOOR,+ "No reliable alignment found: the audio tracks share too little common sound.",+ );++ // Parabolic interpolation around the peak for sub-bin precision.+ let delta = 0;+ const ym = nccAt(bestL - 1);+ const yp = nccAt(bestL + 1);+ if (Number.isFinite(ym) && Number.isFinite(yp)) {+ const denom = ym - 2 * bestNcc + yp;+ if (denom !== 0) delta = (0.5 * (ym - yp)) / denom;+ if (!Number.isFinite(delta) || Math.abs(delta) > 1) delta = 0;+ }++ return {+ offsetSeconds: (bestL + delta) / rate,+ confidence: Math.max(0, Math.min(1, bestNcc)),+ };+}diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.tsindex 86367259..2b0025a5 100644--- a/packages/runtime/src/media/index.ts+++ b/packages/runtime/src/media/index.ts@@ -5,6 +5,7 @@export * from './audio';export * from './audio-bus';export * from './audio-peaks';+export * from './audio-sync';export * from './caption';export * from './dispose';export * from './frame-cache';@@ -15,4 +16,5 @@ export * from './sequence';export * from './shader';export * from './surface';export * from './time-stretcher';+export * from './transcode';export * from './video';diff --git a/packages/runtime/src/media/transcode.ts b/packages/runtime/src/media/transcode.tsnew file mode 100644index 00000000..36e3cc01--- /dev/null+++ b/packages/runtime/src/media/transcode.ts@@ -0,0 +1,511 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Asset transcoding and preview rendering (was engine/utils/transcode.ts;+// the timecode formatters moved to utils/time.ts). Filmstrips and waveforms+// draw into an OffscreenCanvas, so everything here is worker-capable.++import { ALL_FORMATS, AudioSampleSink, BlobSource, BufferTarget, CanvasSink, Conversion, Input, InputAudioTrack, Mp4OutputFormat, OggOutputFormat, Output, StreamTarget } from 'mediabunny';++import { assert } from '../utils/assert';+import { formatTimestamp } from '../utils/time';+import { getAssetFile } from '../actions/assets';++import type { StreamTargetChunk } from 'mediabunny';+import type { Asset, VideoAsset, AudioAsset } from '../assets/types';++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++export async function transcodeForTranscription(asset: Asset): Promise<File> {+ const blob = await getAssetFile(asset);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const output = new Output({ format: new OggOutputFormat(), target: new BufferTarget() });+ const conversion = await Conversion.init({+ input,+ output,+ video: { discard: true },+ audio: { codec: "opus", numberOfChannels: 1, sampleRate: 16000 },+ });+ if (!conversion.isValid || conversion.utilizedTracks.length === 0) {+ throw new Error("No audio found. The asset has no audio track to transcribe.");+ }+ await conversion.execute();++ const buffer = output.target.buffer;+ assert(buffer, "Transcoding produced no output.");+ return new File([buffer], `${crypto.randomUUID()}.ogg`, { type: "audio/ogg" });+ } finally {+ input.dispose();+ }+}++export async function transcodeForAnalysis(asset: VideoAsset | AudioAsset, window?: TimeWindow & { stripVideo?: boolean }) {+ const blob = await getAssetFile(asset);+ const hasWindow = window?.start !== undefined || window?.end !== undefined;++ const isVideo = asset.type === "VIDEO" && !window?.stripVideo;+ const trim = hasWindow ? resolveWindow(asset.duration, window) : undefined;++ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });++ const { writable, readable } = new TransformStream<StreamTargetChunk, Uint8Array<ArrayBuffer>>({+ transform: (chunk, controller) => controller.enqueue(chunk.data),+ });++ const target = new StreamTarget(writable);+ const format = isVideo+ ? new Mp4OutputFormat({ fastStart: "fragmented" })+ : new OggOutputFormat();+ const output = new Output({ format, target });++ const conversion = await Conversion.init({+ input,+ output,+ trim: (trim && { start: trim.start, end: trim.end }),+ video: { width: 640, frameRate: 24, bitrate: 500_000, discard: !isVideo },+ audio: { codec: isVideo ? undefined : "opus", numberOfChannels: 1, sampleRate: 16_000 },+ });++ if (!conversion.isValid || conversion.utilizedTracks.length === 0) {+ input.dispose();+ throw new Error("Nothing to analyze. The asset has no usable video or audio track.");+ }++ const run = async () => {+ try {+ await conversion.execute();+ } finally {+ input.dispose();+ }+ };++ return { readable, run };+}++const PATCH_SIZE = 28;+const MAX_WIDTH_IN_TOKENS = 64; // 92+const MAX_HEIGHT_IN_TOKENS = 52;+const THUMBNAIL_HEIGHT_IN_TOKENS = 7;+const RULER_HEIGHT_IN_TOKENS = 1;+const AUDIO_WAVEFORM_HEIGHT_IN_TOKENS = 6;+const AUDIO_COLUMN_WIDTH_IN_TOKENS = 6;+const WAVEFORM_SAMPLE_WIDTH = 2; // pixels per peak; wider = less noisy+const SILENCE_THRESHOLD = 12; // peak value (0-255) at/below which audio counts as silent+const SILENCE_MIN_SECONDS = 0.4;+const AUDIO_RULER_FRAME_RATE = 30;++// Draws a timestamp label into the top-left corner of a canvas: white text with+// a black outline, sized relative to the canvas so it reads at any resolution.+export function stampTimestampLabel(+ ctx: Ctx2D,+ height: number,+ label: string,+ options?: { minBandHeight?: number },+): void {+ const bandHeight = Math.max(options?.minBandHeight ?? 20, Math.round(height * 0.06));+ const fontSize = Math.round(bandHeight * 0.72);+ const paddingLeft = Math.round(bandHeight * 0.4);+ const y = paddingLeft + bandHeight / 2;++ ctx.font = `normal ${fontSize}px sans-serif`;+ ctx.textBaseline = 'middle';+ ctx.textAlign = 'left';++ ctx.lineJoin = 'round';+ ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));+ ctx.strokeStyle = '#000';+ ctx.strokeText(label, paddingLeft, y);+ ctx.fillStyle = '#FFF';+ ctx.fillText(label, paddingLeft, y);+}++export type RenderedPreview = {+ dataUrl: string;+};++export type RenderedWaveform = RenderedPreview & {+ silences: Array<{ start: number; end: number }>;+};++export type TimeWindow = { start?: number; end?: number };++export type PreviewOptions = TimeWindow & { scale?: number };++const MIN_SCALE = 0.25;+const MAX_SCALE = 4;++function resolveScale(scale?: number): number {+ if (scale === undefined) return 1;+ assert(Number.isFinite(scale) && scale > 0, `scale must be a positive number (got ${scale}).`);+ return Math.min(Math.max(scale, MIN_SCALE), MAX_SCALE);+}++// Clamp a requested [start, end] window (in seconds) to the asset's timeline.+// `start` defaults to 0 and `end` to the full duration.+function resolveWindow(duration: number, window?: TimeWindow): { start: number; end: number; duration: number } {+ const start = Math.min(Math.max(window?.start ?? 0, 0), duration);+ const end = Math.min(Math.max(window?.end ?? duration, start), duration);+ assert(end > start, `The requested window is empty; start (${start.toFixed(2)}s) is at or past the asset's end (${duration.toFixed(2)}s).`);+ return { start, end, duration: end - start };+}++/** PNG data URL of an OffscreenCanvas (toDataURL only exists on DOM canvases). */+async function toPngDataUrl(canvas: OffscreenCanvas): Promise<string> {+ const blob = await canvas.convertToBlob({ type: 'image/png' });+ return await new Promise<string>((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(blob);+ });+}++// Render a video as a grid of thumbnails sampled across the window, each row+// stamped with a timestamp ruler. Video only; no audio.+export async function filmstripAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedPreview> {+ assert(asset.type === "VIDEO", "The filmstrip needs a video asset; use `waveform` for audio.");+ const blob = await getAssetFile(asset);+ const scale = resolveScale(options?.scale);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ return await renderFilmstrip(asset, input, options, scale);+ } finally {+ input.dispose();+ }+}++async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedPreview> {+ const videoTrack = await input.getPrimaryVideoTrack();+ assert(videoTrack, "Video track not found");+ const { start: windowStart, duration } = resolveWindow(asset.duration, window);+ const aspect = asset.width / asset.height;+ const frameRate = asset.frameRate;++ const thumbnailHeightInTokens = Math.min(+ Math.max(Math.round(THUMBNAIL_HEIGHT_IN_TOKENS * scale), 1),+ MAX_HEIGHT_IN_TOKENS - RULER_HEIGHT_IN_TOKENS,+ );+ const rowHeightInTokens = thumbnailHeightInTokens + RULER_HEIGHT_IN_TOKENS;++ const columnWidthInTokens = Math.min(+ Math.max(Math.floor(thumbnailHeightInTokens * aspect), 1),+ MAX_WIDTH_IN_TOKENS,+ );++ const thumbnailWidthInPixels = columnWidthInTokens * PATCH_SIZE;+ const thumbnailHeightInPixels = thumbnailHeightInTokens * PATCH_SIZE;+ const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;++ const rows = Math.floor(MAX_HEIGHT_IN_TOKENS / rowHeightInTokens);+ const columns = Math.floor(MAX_WIDTH_IN_TOKENS / columnWidthInTokens);+ const cells = rows * columns;++ const secondsPerColumn = Math.max(duration / cells, 1 / frameRate);++ const canvas = new OffscreenCanvas(+ columnWidthInTokens * columns * PATCH_SIZE,+ rowHeightInTokens * rows * PATCH_SIZE,+ );+ const ctx = canvas.getContext('2d')!;++ ctx.fillStyle = '#000';+ ctx.fillRect(0, 0, canvas.width, canvas.height);++ const sink = new CanvasSink(videoTrack, {+ width: thumbnailWidthInPixels,+ height: thumbnailHeightInPixels,+ fit: 'cover',+ poolSize: 1,+ });++ const rulerHeight = RULER_HEIGHT_IN_TOKENS * PATCH_SIZE;++ const timestamps = Array.from({ length: cells }, (_, i) => windowStart + i * secondsPerColumn);+ const interator = sink.canvasesAtTimestamps(timestamps);+ for (let i = 0; i < cells; i++) {+ const wrapped = await interator.next();+ if (!wrapped.value?.canvas) continue;++ const column = i % columns;+ const row = Math.floor(i / columns);++ ctx.drawImage(+ wrapped.value.canvas,+ column * columnWidthInTokens * PATCH_SIZE,+ rulerHeight + row * rowHeightInTokens * PATCH_SIZE,+ thumbnailWidthInPixels,+ thumbnailHeightInPixels+ );+ }++ drawRuler(ctx, {+ cells,+ columns,+ columnWidthInPixels: columnWidthInTokens * PATCH_SIZE,+ rowHeightInPixels,+ rulerHeight,+ secondsPerColumn,+ frameRate,+ startOffset: windowStart,+ });++ return { dataUrl: await toPngDataUrl(canvas) };+}++// Render the audio track of a video or audio file as a loudness-over-time+// waveform with a timestamp ruler and silent stretches highlighted.+export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedWaveform> {+ assert(asset.type === "VIDEO" || asset.type === "AUDIO", "The waveform needs a video or audio asset.");+ const blob = await getAssetFile(asset);+ const scale = resolveScale(options?.scale);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ return await renderWaveform(asset, input, options, scale);+ } finally {+ input.dispose();+ }+}++async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedWaveform> {+ const audioTrack = await input.getPrimaryAudioTrack();+ assert(audioTrack, "No audio track found, so no waveform can be rendered.");+ assert(+ await audioTrack.canDecode(),+ `This browser's audio decoder can't handle the "${audioTrack.getCodec() ?? "unknown"}" codec, so no waveform can be rendered.`,+ );+ const { start: windowStart, duration } = resolveWindow(asset.duration, window);+ const rulerFrameRate = asset.type === "VIDEO" ? asset.frameRate : AUDIO_RULER_FRAME_RATE;++ const waveformHeightInTokens = Math.min(+ Math.max(Math.round(AUDIO_WAVEFORM_HEIGHT_IN_TOKENS * scale), 1),+ MAX_HEIGHT_IN_TOKENS - RULER_HEIGHT_IN_TOKENS,+ );+ const rowHeightInTokens = waveformHeightInTokens + RULER_HEIGHT_IN_TOKENS;+ const columnWidthInTokens = Math.min(+ Math.max(Math.round(AUDIO_COLUMN_WIDTH_IN_TOKENS * scale), 1),+ MAX_WIDTH_IN_TOKENS,+ );++ const rows = Math.floor(MAX_HEIGHT_IN_TOKENS / rowHeightInTokens);+ const columns = Math.floor(MAX_WIDTH_IN_TOKENS / columnWidthInTokens);+ const cells = rows * columns;++ const secondsPerColumn = duration / cells;++ const columnWidthInPixels = columnWidthInTokens * PATCH_SIZE;+ const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;+ const rulerHeight = RULER_HEIGHT_IN_TOKENS * PATCH_SIZE;+ const waveformHeight = waveformHeightInTokens * PATCH_SIZE;+ const rowWidth = columns * columnWidthInPixels;++ const canvas = new OffscreenCanvas(rowWidth, rowHeightInPixels * rows);+ const ctx = canvas.getContext('2d')!;++ ctx.fillStyle = '#000';+ ctx.fillRect(0, 0, canvas.width, canvas.height);++ const peaksPerSecond = Math.round(columnWidthInPixels / secondsPerColumn / WAVEFORM_SAMPLE_WIDTH);+ const audioPeaks = await downsampleAudio(audioTrack, { peaksPerSecond, range: [windowStart, windowStart + duration] });++ const silentSpans = findSilentSpans(audioPeaks, peaksPerSecond);+ const silences = silentSpans.map((span) => ({+ start: windowStart + span.start / peaksPerSecond,+ end: windowStart + span.end / peaksPerSecond,+ }));++ drawWaveform(ctx, audioPeaks, silentSpans, {+ rows,+ rowWidth,+ rowHeightInPixels,+ waveformOffset: rulerHeight,+ waveformHeight,+ });++ drawRuler(ctx, {+ cells,+ columns,+ columnWidthInPixels,+ rowHeightInPixels,+ rulerHeight,+ secondsPerColumn,+ frameRate: rulerFrameRate,+ startOffset: windowStart,+ });++ return { dataUrl: await toPngDataUrl(canvas), silences };+}++type WaveformLayout = {+ rows: number;+ rowWidth: number; // pixels spanning all columns of a row+ rowHeightInPixels: number;+ waveformOffset: number; // y offset within a row where the waveform begins+ waveformHeight: number;+};++// Scan the peaks for runs at/below the silence threshold that last at least+// SILENCE_MIN_SECONDS, returned as [start, end) peak-index spans.+function findSilentSpans(audioPeaks: Uint8ClampedArray, peaksPerSecond: number): Array<{ start: number; end: number }> {+ const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);+ const silentSpans: Array<{ start: number; end: number }> = [];+ let runStart = -1;+ for (let i = 0; i <= audioPeaks.length; i++) {+ const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;+ if (silent && runStart === -1) {+ runStart = i;+ } else if (!silent && runStart !== -1) {+ if (i - runStart >= minSilencePeaks) {+ silentSpans.push({ start: runStart, end: i });+ }+ runStart = -1;+ }+ }+ return silentSpans;+}++// Draws a center-aligned waveform, then overlays the given silent stretches in red.+function drawWaveform(ctx: Ctx2D, audioPeaks: Uint8ClampedArray, silentSpans: Array<{ start: number; end: number }>, layout: WaveformLayout) {+ const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight } = layout;+ const peaksPerRow = Math.floor(rowWidth / WAVEFORM_SAMPLE_WIDTH);+ const halfHeight = waveformHeight / 2;++ const waveformTopOf = (row: number) => row * rowHeightInPixels + waveformOffset;++ ctx.fillStyle = '#FFF';+ for (let row = 0; row < rows; row++) {+ const top = waveformTopOf(row);++ for (let col = 0; col < peaksPerRow; col++) {+ const peakIndex = row * peaksPerRow + col;+ if (peakIndex >= audioPeaks.length) break;++ const amplitude = (audioPeaks[peakIndex] / 255) * halfHeight;+ const x = col * WAVEFORM_SAMPLE_WIDTH;+ const centerY = top + halfHeight;+ ctx.fillRect(x, centerY - amplitude, WAVEFORM_SAMPLE_WIDTH, Math.max(amplitude * 2, 1));+ }+ }++ for (const span of silentSpans) {+ let i = span.start;+ while (i < span.end) {+ const row = Math.floor(i / peaksPerRow);+ const segEnd = Math.min(span.end, (row + 1) * peaksPerRow);+ const x = (i % peaksPerRow) * WAVEFORM_SAMPLE_WIDTH;+ const width = (segEnd - i) * WAVEFORM_SAMPLE_WIDTH;+ const top = waveformTopOf(row);++ ctx.fillStyle = 'rgba(255, 64, 64, 0.4)';+ ctx.fillRect(x, top, width, waveformHeight);++ i = segEnd;+ }+ }+}++type RulerLayout = {+ cells: number;+ columns: number;+ columnWidthInPixels: number;+ rowHeightInPixels: number;+ rulerHeight: number;+ secondsPerColumn: number;+ frameRate: number;+ startOffset?: number;+};++function drawRuler(ctx: Ctx2D, layout: RulerLayout) {+ const { cells, columns, columnWidthInPixels, rowHeightInPixels, rulerHeight, secondsPerColumn, frameRate, startOffset = 0 } = layout;++ const baseFontSize = Math.round(rulerHeight * 0.8);+ const leadFontSize = Math.round(rulerHeight * 0.9);+ const paddingLeft = 5;+ const paddingTop = 2;+ const tickHeight = Math.round(rulerHeight * 0.4);+ const labelGap = 10;++ ctx.fillStyle = '#FFF';+ ctx.strokeStyle = '#FFF';+ ctx.lineWidth = 2;+ ctx.textBaseline = 'middle';+ ctx.textAlign = 'left';++ const rowWidth = columns * columnWidthInPixels;+ let labelEndX = -Infinity;+ for (let i = 0; i < cells; i++) {+ const column = i % columns;+ const row = Math.floor(i / columns);+ const columnX = column * columnWidthInPixels;+ const rowTop = row * rowHeightInPixels;++ if (column === 0) labelEndX = -Infinity;++ const tickX = columnX + 0.5;+ if (tickX >= labelEndX) {+ ctx.beginPath();+ ctx.moveTo(tickX, rowTop + rulerHeight - tickHeight);+ ctx.lineTo(tickX, rowTop + rulerHeight);+ ctx.stroke();+ }++ ctx.font = column === 0+ ? `bold ${leadFontSize}px sans-serif`+ : `${baseFontSize}px sans-serif`;+ const label = formatTimestamp(startOffset + i * secondsPerColumn, frameRate);+ const labelX = columnX + paddingLeft;+ const labelWidth = ctx.measureText(label).width;+ if (labelX < labelEndX + labelGap || labelX + labelWidth > rowWidth) continue;++ ctx.fillText(label, labelX, (rowTop + rulerHeight / 2) + paddingTop);+ labelEndX = labelX + labelWidth;+ }+}++type SamplerOptions = {+ peaksPerSecond: number;+ range?: [start: number, end: number]; // seconds; omitted = whole track+}++async function downsampleAudio(track: InputAudioTrack, options: SamplerOptions) {+ const sink = new AudioSampleSink(track);+ const iterator = options.range ? sink.samples(...options.range) : sink.samples();+ const framesPerPeak = track.sampleRate / options.peaksPerSecond;++ const peaks: number[] = [];+ let frameCursor = 0; // frames consumed since the range start+ let currentPeak = 0; // index of the peak currently being accumulated+ let peakMax = 0;++ for await (const sample of iterator) {+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ const floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });++ const numChannels = sample.numberOfChannels;+ const numFrames = floats.length / numChannels;++ for (let frameIdx = 0; frameIdx < numFrames; frameIdx++) {+ while (currentPeak < Math.floor(frameCursor / framesPerPeak)) {+ peaks.push(Math.floor(255 * peakMax));+ peakMax = 0;+ currentPeak++;+ }++ for (let ch = 0; ch < numChannels; ch++) {+ peakMax = Math.max(peakMax, Math.sqrt(Math.abs(floats[frameIdx * numChannels + ch])));+ }+ frameCursor++;+ }+ } finally {+ sample.close();+ }+ }++ // Flush the final in-progress peak.+ if (frameCursor > 0) peaks.push(Math.floor(255 * peakMax));++ return Uint8ClampedArray.from(peaks);+}