Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add WebGPU types and enhance media frame handling with auto selection feature

konstantin-paulus586cd781644e
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 5f2412ae..08cfb96d 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -129,6 +129,7 @@ export type MediaFrameRequest = AssetRef & {
end?: number;
quality?: FrameQuality;
timestamp?: boolean;
+ auto?: boolean;
};
export type MediaFrameResult = Array<{ time: number; base64: string }>;
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 0ff884ce..ad028c98 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -468,6 +468,7 @@ type MediaFrameOptions = {
timestamp?: boolean;
uncapped?: boolean;
output?: string;
+ auto?: boolean;
};
async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
@@ -475,6 +476,10 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
console.error("Pass either --time or --count, not both.");
process.exit(1);
}
+ if (opts.auto && opts.time !== undefined) {
+ console.error("--auto picks its own timestamps; it cannot be combined with --time.");
+ process.exit(1);
+ }
let times: number[] | undefined;
if (opts.time !== undefined) {
@@ -496,8 +501,8 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
console.error(`--start (${start}s) must be less than --end (${end}s).`);
process.exit(1);
}
- if ((start !== undefined || end !== undefined) && count === undefined) {
- console.error("--start and --end only apply together with --count.");
+ if ((start !== undefined || end !== undefined) && count === undefined && !opts.auto) {
+ console.error("--start and --end only apply together with --count or --auto.");
process.exit(1);
}
@@ -519,7 +524,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
const target = resolveAssetRef(ref);
const dir = opts.output ?? tmpdir();
try {
- const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp });
+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp, auto: opts.auto });
for (const { time, base64 } of frames) {
const path = join(dir, `${randomUUID()}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
@@ -1178,8 +1183,9 @@ media
.argument("<id|path>", "video asset id, or a local video file to grab frames from")
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)
.option("-c, --count <n>", "instead of --time, grab this many frames evenly spaced across the clip (or across the --start/--end window)")
- .option("-s, --start <time>", `with --count, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)
- .option("-e, --end <time>", `with --count, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)
+ .option("-a, --auto", "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most --count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU")
+ .option("-s, --start <time>", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)
+ .option("-e, --end <time>", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)
.option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")
.option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label")
.option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")
diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json
index 0dd3fce3..c3835eb7 100644
--- a/apps/cli/tsconfig.json
+++ b/apps/cli/tsconfig.json
@@ -17,7 +17,8 @@
"vite-plugin-solid-svg/types-component-solid",
"vite/client",
"@types/audioworklet",
- "@types/wicg-file-system-access"
+ "@types/wicg-file-system-access",
+ "@webgpu/types"
],
"resolveJsonModule": true,
"allowImportingTsExtensions": true,
diff --git a/apps/web/package.json b/apps/web/package.json
index 03828d3b..a912d390 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -52,6 +52,7 @@
"@types/wicg-file-system-access": "^2023.10.7",
"@typescript-eslint/eslint-plugin": "^8.51.0",
"@typescript-eslint/parser": "^8.51.0",
+ "@webgpu/types": "^0.1.71",
"eslint": "^9.39.2",
"eslint-plugin-solid": "^0.14.5",
"globals": "^17.0.0",
diff --git a/apps/web/src/context/editor-api/frame-triage.ts b/apps/web/src/context/editor-api/frame-triage.ts
new file mode 100644
index 00000000..18c20b55
--- /dev/null
+++ b/apps/web/src/context/editor-api/frame-triage.ts
@@ -0,0 +1,191 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { VideoSampleSink } from "mediabunny";
+import { assert } from "@/utils";
+
+import type { InputVideoTrack } from "mediabunny";
+
+// Triage scan rate; 2fps is enough to understand footage.
+const SCAN_FPS = 2;
+
+// Fingerprint: a GRID x GRID grid of block luma averages, computed on the GPU
+// so no full frame is ever read back (only GRID*GRID floats per frame).
+const GRID = 32;
+const DIM = GRID * GRID;
+const TAPS = 8; // bilinear taps per block axis; approximates a box filter
+
+// Distance between fingerprints is the fraction of meaningfully changed
+// blocks: per-block deltas below the deadzone count as encoder noise, deltas
+// past saturation as a fully changed block, ramping linearly in between. A
+// block-wise measure keeps small local changes (a new line of text in a
+// screen recording) from being averaged away by an otherwise static frame.
+const DEADZONE = 0.02;
+const SATURATION = 0.1;
+
+// A sample joins the result only when it differs from the last kept frame
+// (novelty) by at least about two blocks; static footage collapses to one frame.
+const THRESHOLD = 2 / DIM;
+
+// Adjacent samples farther apart than this mean the scene is mid-transition;
+// grabbing waits until it settles so frames are sharp and fully drawn instead
+// of caught inside a scroll, pan, or animation.
+const MOTION_EPS = 2 / DIM;
+
+// Footage that never settles (talking head, handheld) still yields a frame
+// per run of this many seconds: the calmest, least motion-blurred sample.
+const MAX_UNSETTLED = 4;
+
+const SHADER = /* wgsl */ `
+ @group(0) @binding(0) var samp: sampler;
+ @group(0) @binding(1) var frame: texture_external;
+ @group(0) @binding(2) var<storage, read_write> blocks: array<f32>;
+
+ @compute @workgroup_size(8, 8)
+ fn main(@builtin(global_invocation_id) id: vec3u) {
+ var sum = 0.0;
+ for (var y = 0u; y < ${TAPS}u; y++) {
+ for (var x = 0u; x < ${TAPS}u; x++) {
+ let uv = (vec2f(id.xy) + (vec2f(f32(x), f32(y)) + 0.5) / ${TAPS}.0) / ${GRID}.0;
+ sum += dot(textureSampleBaseClampToEdge(frame, samp, uv).rgb, vec3f(0.2126, 0.7152, 0.0722));
+ }
+ }
+ blocks[id.y * ${GRID}u + id.x] = sum / ${TAPS * TAPS}.0;
+ }
+`;
+
+/**
+ * Scans the track at 2fps within [from, to] (track time) and picks up to
+ * `max` timestamps covering the footage's distinct visual states. A frame is
+ * kept when the footage settles into a state that differs meaningfully from
+ * the last kept frame; grabbing waits out transitions, so picks land on
+ * sharp, at-rest frames rather than mid-motion. Static footage collapses to
+ * a single frame. Timestamps come back in presentation order.
+ */
+export async function pickInformativeTimes(
+ track: InputVideoTrack,
+ opts: { from: number; to: number; max: number },
+): Promise<number[]> {
+ const gpu = await createFingerprinter();
+ try {
+ const scan: number[] = [];
+ for (let t = opts.from; t < opts.to; t += 1 / SCAN_FPS) scan.push(t);
+
+ const kept: Array<{ time: number; novelty: number }> = [];
+ let lastKept: Float32Array | undefined;
+ const keep = (time: number, print: Float32Array, novelty: number) => {
+ kept.push({ time, novelty });
+ lastKept = print;
+ };
+
+ let prev: Float32Array | undefined;
+ let prevTime: number | undefined;
+ let runStart = 0;
+ let calmest: { time: number; print: Float32Array; motion: number } | undefined;
+
+ const sink = new VideoSampleSink(track);
+ for await (const sample of sink.samplesAtTimestamps(scan)) {
+ if (!sample) continue;
+ const time = sample.timestamp;
+ // Low-fps clips resolve adjacent scan times to the same frame; skip repeats.
+ if (time === prevTime) {
+ sample.close();
+ continue;
+ }
+ prevTime = time;
+ const frame = sample.toVideoFrame();
+ let print: Float32Array;
+ try {
+ print = await gpu.fingerprint(frame);
+ } finally {
+ frame.close();
+ sample.close();
+ }
+
+ const motion = prev ? distance(print, prev) : 0;
+ prev = print;
+
+ if (motion <= MOTION_EPS) {
+ // At rest: grab if the footage has drifted to a new state.
+ calmest = undefined;
+ const novelty = lastKept ? distance(print, lastKept) : Infinity;
+ if (novelty >= THRESHOLD) keep(time, print, novelty);
+ } else {
+ // Mid-transition: never grab here. Track the calmest sample so a run
+ // that never settles still yields its least blurred frame.
+ if (!calmest) runStart = time;
+ if (!calmest || motion < calmest.motion) calmest = { time, print, motion };
+ if (time - runStart >= MAX_UNSETTLED) {
+ const novelty = lastKept ? distance(calmest.print, lastKept) : Infinity;
+ if (novelty >= THRESHOLD) keep(calmest.time, calmest.print, novelty);
+ calmest = undefined; // start a new run window
+ }
+ }
+ }
+ assert(kept.length, "No frames could be decoded for auto frame selection.");
+
+ // Over budget: drop the most marginal state changes, then restore order.
+ return kept
+ .sort((a, b) => b.novelty - a.novelty)
+ .slice(0, opts.max)
+ .sort((a, b) => a.time - b.time)
+ .map((k) => k.time);
+ } finally {
+ gpu.dispose();
+ }
+}
+
+async function createFingerprinter() {
+ assert(navigator.gpu, "Auto frame selection requires WebGPU, which this environment does not support.");
+ const adapter = await navigator.gpu.requestAdapter();
+ assert(adapter, "Auto frame selection requires WebGPU, but no GPU adapter is available.");
+ const device = await adapter.requestDevice();
+ const pipeline = device.createComputePipeline({
+ layout: "auto",
+ compute: { module: device.createShaderModule({ code: SHADER }) },
+ });
+ const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });
+ const blocks = device.createBuffer({ size: DIM * 4, usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC });
+ const staging = device.createBuffer({ size: DIM * 4, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ });
+
+ return {
+ async fingerprint(frame: VideoFrame): Promise<Float32Array> {
+ // External textures are only valid within the current task, so import,
+ // encode, and submit synchronously for every frame.
+ const bindGroup = device.createBindGroup({
+ layout: pipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: sampler },
+ { binding: 1, resource: device.importExternalTexture({ source: frame }) },
+ { binding: 2, resource: { buffer: blocks } },
+ ],
+ });
+ const encoder = device.createCommandEncoder();
+ const pass = encoder.beginComputePass();
+ pass.setPipeline(pipeline);
+ pass.setBindGroup(0, bindGroup);
+ pass.dispatchWorkgroups(GRID / 8, GRID / 8);
+ pass.end();
+ encoder.copyBufferToBuffer(blocks, 0, staging, 0, DIM * 4);
+ device.queue.submit([encoder.finish()]);
+ await staging.mapAsync(GPUMapMode.READ);
+ const print = new Float32Array(staging.getMappedRange().slice(0));
+ staging.unmap();
+ return print;
+ },
+ dispose() {
+ device.destroy();
+ },
+ };
+}
+
+// Fraction of blocks whose luma moved meaningfully between two fingerprints.
+function distance(a: Float32Array, b: Float32Array): number {
+ let sum = 0;
+ for (let i = 0; i < DIM; i++) {
+ const d = Math.abs(a[i] - b[i]);
+ if (d > DEADZONE) sum += Math.min((d - DEADZONE) / (SATURATION - DEADZONE), 1);
+ }
+ return sum / DIM;
+}
diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.ts
index 97307558..550c7c76 100644
--- a/apps/web/src/context/editor-api/media.ts
+++ b/apps/web/src/context/editor-api/media.ts
@@ -4,6 +4,7 @@
import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
import { ElectronFileHandle } from '@/lib/electron-file-handle';
+import { pickInformativeTimes } from './frame-triage';
import { trpc } from '@/lib/trpc';
import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp, stampTimestampLabel } from '@/components/engine';
import { assert } from '@/utils';
@@ -107,23 +108,30 @@ const FRAME_QUALITY_BUDGETS = {
fullres: 0, // native
} as const;
+// Default cap on frames returned by auto selection when `count` is not given.
+const AUTO_MAX_FRAMES = 30;
+
export function handleMediaFrame(engine: Accessor<Engine>) {
return async (req: MediaFrameRequest) => {
- const { times, count, start, end, quality, timestamp } = req;
+ const { times, count, start, end, quality, timestamp, auto } = req;
const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);
- // `count` samples evenly across a window (default the whole clip); otherwise
- // grab the explicit `times` (falling back to a single frame at 0).
- let requested: number[];
- if (count !== undefined) {
- const from = Math.min(Math.max(start ?? 0, 0), asset.duration);
- const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);
+ // `count` samples evenly across a window (default the whole clip); `auto`
+ // scans the window and keeps frames where the footage settles into a new
+ // visual state, capped at `count` (resolved once the track is open).
+ // Otherwise grab the explicit `times` (falling back to a single frame at 0).
+ const from = Math.min(Math.max(start ?? 0, 0), asset.duration);
+ const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);
+ let requested: number[] = [];
+ if (auto || count !== undefined) {
assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`);
- const interval = (to - from) / count;
- requested = Array.from({ length: count }, (_, i) => from + i * interval);
+ if (!auto && count !== undefined) {
+ const interval = (to - from) / count;
+ requested = Array.from({ length: count }, (_, i) => from + i * interval);
+ }
} else {
const raw = times && times.length ? times : [0];
// A negative time is an offset back from the end of the clip: -1 is one
@@ -150,6 +158,15 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
// Track timestamps may not start at 0; offset content time by the first.
const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;
+ if (auto) {
+ const picked = await pickInformativeTimes(track, {
+ from: firstTimestamp + from,
+ to: firstTimestamp + to,
+ max: count ?? AUTO_MAX_FRAMES,
+ });
+ requested = picked.map((t) => Math.max(0, t - firstTimestamp));
+ }
+
// Downscale to fit the pixel budget while preserving aspect ratio; setting
// only the width lets the sink derive a matching height.
const displayWidth = await track.getDisplayWidth();
@@ -174,7 +191,8 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
const { time, index } = ordered[i++];
assert(wrapped, `No frame found at ${time}s.`);
- const ctx = wrapped.canvas.getContext("2d");
+ // The webgpu getContext overload muddies inference on the canvas union.
+ const ctx = wrapped.canvas.getContext("2d") as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null;
if (ctx && timestamp !== false) {
stampTimestampLabel(ctx, wrapped.canvas.height, formatTimestamp(time, asset.frameRate));
diff --git a/apps/web/tsconfig.app.json b/apps/web/tsconfig.app.json
index d6e87afb..6256a6ce 100644
--- a/apps/web/tsconfig.app.json
+++ b/apps/web/tsconfig.app.json
@@ -13,7 +13,8 @@
"vite-plugin-solid-svg/types-component-solid",
"vite/client",
"@types/audioworklet",
- "@types/wicg-file-system-access"
+ "@types/wicg-file-system-access",
+ "@webgpu/types"
],
"allowImportingTsExtensions": true,
"jsx": "preserve",
diff --git a/package-lock.json b/package-lock.json
index dae2bb4d..5c83e7b3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -100,6 +100,7 @@
"@types/wicg-file-system-access": "^2023.10.7",
"@typescript-eslint/eslint-plugin": "^8.51.0",
"@typescript-eslint/parser": "^8.51.0",
+ "@webgpu/types": "^0.1.71",
"eslint": "^9.39.2",
"eslint-plugin-solid": "^0.14.5",
"globals": "^17.0.0",
@@ -4514,6 +4515,13 @@
"@xtuc/long": "4.2.2"
}
},
+ "node_modules/@webgpu/types": {
+ "version": "0.1.71",
+ "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz",
+ "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==",
+ "dev": true,
+ "license": "BSD-3-Clause"
+ },
"node_modules/@xmldom/xmldom": {
"version": "0.9.10",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.9.10.tgz",
diff --git a/scripts/dev-desktop.mjs b/scripts/dev-desktop.mjs
index 553b0840..c9eccb52 100644
--- a/scripts/dev-desktop.mjs
+++ b/scripts/dev-desktop.mjs
@@ -11,15 +11,21 @@
import { spawn, execFileSync } from "node:child_process";
import { get } from "node:http";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+const ROOT = fileURLToPath(new URL("..", import.meta.url));
+const BIN = join(ROOT, "node_modules", ".bin");
const DEV_URL = "http://localhost:5173";
const children = [];
let shuttingDown = false;
-function run(name, args) {
- // Own process group (detached) so we can signal the npm process *and* its
- // grandchildren (vite, electron) in one shot on teardown.
- const child = spawn("npm", args, { stdio: "inherit", detached: true });
+function run(name, bin, args, cwd) {
+ // Spawn the tool binary directly rather than via `npm run`, so the teardown
+ // SIGTERM isn't dressed up as a "Lifecycle script failed" error by an npm
+ // wrapper. Own process group (detached) so we can signal the tool *and* its
+ // children (esbuild, electron) in one shot on teardown.
+ const child = spawn(join(BIN, bin), args, { cwd, stdio: "inherit", detached: true });
child.on("exit", (code) => {
if (shuttingDown) return;
// A child dying on its own (e.g. Vite crashed) should bring the rest down.
@@ -77,14 +83,17 @@ execFileSync("npm", ["run", "build", "--workspace=@diffusionstudio/cli"], { stdi
// 2. Start the web dev server.
console.log("[dev:desktop] starting web dev server…");
-run("web", ["run", "dev", "--workspace=@diffusionstudio/web"]);
+run("web", "vite", [], join(ROOT, "apps", "web"));
-// 3. Once it is up, launch Electron (its own build runs first, loads :5173).
+// 3. Once it is up, build the desktop app (blocking, mirrors its `dev`
+// script) and launch Electron, which loads :5173.
try {
await waitForServer(DEV_URL);
} catch (err) {
console.error(`[dev:desktop] ${err.message}`);
shutdown(1);
}
+console.log("[dev:desktop] building desktop app…");
+execFileSync("npm", ["run", "build", "--workspace=@diffusionstudio/desktop"], { stdio: "inherit" });
console.log("[dev:desktop] starting desktop app…");
-run("desktop", ["run", "dev", "--workspace=@diffusionstudio/desktop"]);
+run("desktop", "electron-forge", ["start"], join(ROOT, "apps", "desktop"));