The VS Code of video editing, built for agents.
TypeScript
Add WebGPU types and enhance media frame handling with auto selection feature
konstantin-paulus586cd781644e
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 5f2412ae..08cfb96d 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -129,6 +129,7 @@ export type MediaFrameRequest = AssetRef & {end?: number;quality?: FrameQuality;timestamp?: boolean;+ auto?: boolean;};export type MediaFrameResult = Array<{ time: number; base64: string }>;diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 0ff884ce..ad028c98 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -468,6 +468,7 @@ type MediaFrameOptions = {timestamp?: boolean;uncapped?: boolean;output?: string;+ auto?: boolean;};async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {@@ -475,6 +476,10 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {console.error("Pass either --time or --count, not both.");process.exit(1);}+ if (opts.auto && opts.time !== undefined) {+ console.error("--auto picks its own timestamps; it cannot be combined with --time.");+ process.exit(1);+ }let times: number[] | undefined;if (opts.time !== undefined) {@@ -496,8 +501,8 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {console.error(`--start (${start}s) must be less than --end (${end}s).`);process.exit(1);}- if ((start !== undefined || end !== undefined) && count === undefined) {- console.error("--start and --end only apply together with --count.");+ if ((start !== undefined || end !== undefined) && count === undefined && !opts.auto) {+ console.error("--start and --end only apply together with --count or --auto.");process.exit(1);}@@ -519,7 +524,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {const target = resolveAssetRef(ref);const dir = opts.output ?? tmpdir();try {- const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp });+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp, auto: opts.auto });for (const { time, base64 } of frames) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));@@ -1178,8 +1183,9 @@ media.argument("<id|path>", "video asset id, or a local video file to grab frames from").option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`).option("-c, --count <n>", "instead of --time, grab this many frames evenly spaced across the clip (or across the --start/--end window)")- .option("-s, --start <time>", `with --count, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)- .option("-e, --end <time>", `with --count, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)+ .option("-a, --auto", "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most --count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU")+ .option("-s, --start <time>", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)+ .option("-e, --end <time>", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`).option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)").option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label").option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.jsonindex 0dd3fce3..c3835eb7 100644--- a/apps/cli/tsconfig.json+++ b/apps/cli/tsconfig.json@@ -17,7 +17,8 @@"vite-plugin-solid-svg/types-component-solid","vite/client","@types/audioworklet",- "@types/wicg-file-system-access"+ "@types/wicg-file-system-access",+ "@webgpu/types"],"resolveJsonModule": true,"allowImportingTsExtensions": true,diff --git a/apps/web/package.json b/apps/web/package.jsonindex 03828d3b..a912d390 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -52,6 +52,7 @@"@types/wicg-file-system-access": "^2023.10.7","@typescript-eslint/eslint-plugin": "^8.51.0","@typescript-eslint/parser": "^8.51.0",+ "@webgpu/types": "^0.1.71","eslint": "^9.39.2","eslint-plugin-solid": "^0.14.5","globals": "^17.0.0",diff --git a/apps/web/src/context/editor-api/frame-triage.ts b/apps/web/src/context/editor-api/frame-triage.tsnew file mode 100644index 00000000..18c20b55--- /dev/null+++ b/apps/web/src/context/editor-api/frame-triage.ts@@ -0,0 +1,191 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { VideoSampleSink } from "mediabunny";+import { assert } from "@/utils";++import type { InputVideoTrack } from "mediabunny";++// Triage scan rate; 2fps is enough to understand footage.+const SCAN_FPS = 2;++// Fingerprint: a GRID x GRID grid of block luma averages, computed on the GPU+// so no full frame is ever read back (only GRID*GRID floats per frame).+const GRID = 32;+const DIM = GRID * GRID;+const TAPS = 8; // bilinear taps per block axis; approximates a box filter++// Distance between fingerprints is the fraction of meaningfully changed+// blocks: per-block deltas below the deadzone count as encoder noise, deltas+// past saturation as a fully changed block, ramping linearly in between. A+// block-wise measure keeps small local changes (a new line of text in a+// screen recording) from being averaged away by an otherwise static frame.+const DEADZONE = 0.02;+const SATURATION = 0.1;++// A sample joins the result only when it differs from the last kept frame+// (novelty) by at least about two blocks; static footage collapses to one frame.+const THRESHOLD = 2 / DIM;++// Adjacent samples farther apart than this mean the scene is mid-transition;+// grabbing waits until it settles so frames are sharp and fully drawn instead+// of caught inside a scroll, pan, or animation.+const MOTION_EPS = 2 / DIM;++// Footage that never settles (talking head, handheld) still yields a frame+// per run of this many seconds: the calmest, least motion-blurred sample.+const MAX_UNSETTLED = 4;++const SHADER = /* wgsl */ `+ @group(0) @binding(0) var samp: sampler;+ @group(0) @binding(1) var frame: texture_external;+ @group(0) @binding(2) var<storage, read_write> blocks: array<f32>;++ @compute @workgroup_size(8, 8)+ fn main(@builtin(global_invocation_id) id: vec3u) {+ var sum = 0.0;+ for (var y = 0u; y < ${TAPS}u; y++) {+ for (var x = 0u; x < ${TAPS}u; x++) {+ let uv = (vec2f(id.xy) + (vec2f(f32(x), f32(y)) + 0.5) / ${TAPS}.0) / ${GRID}.0;+ sum += dot(textureSampleBaseClampToEdge(frame, samp, uv).rgb, vec3f(0.2126, 0.7152, 0.0722));+ }+ }+ blocks[id.y * ${GRID}u + id.x] = sum / ${TAPS * TAPS}.0;+ }+`;++/**+ * Scans the track at 2fps within [from, to] (track time) and picks up to+ * `max` timestamps covering the footage's distinct visual states. A frame is+ * kept when the footage settles into a state that differs meaningfully from+ * the last kept frame; grabbing waits out transitions, so picks land on+ * sharp, at-rest frames rather than mid-motion. Static footage collapses to+ * a single frame. Timestamps come back in presentation order.+ */+export async function pickInformativeTimes(+ track: InputVideoTrack,+ opts: { from: number; to: number; max: number },+): Promise<number[]> {+ const gpu = await createFingerprinter();+ try {+ const scan: number[] = [];+ for (let t = opts.from; t < opts.to; t += 1 / SCAN_FPS) scan.push(t);++ const kept: Array<{ time: number; novelty: number }> = [];+ let lastKept: Float32Array | undefined;+ const keep = (time: number, print: Float32Array, novelty: number) => {+ kept.push({ time, novelty });+ lastKept = print;+ };++ let prev: Float32Array | undefined;+ let prevTime: number | undefined;+ let runStart = 0;+ let calmest: { time: number; print: Float32Array; motion: number } | undefined;++ const sink = new VideoSampleSink(track);+ for await (const sample of sink.samplesAtTimestamps(scan)) {+ if (!sample) continue;+ const time = sample.timestamp;+ // Low-fps clips resolve adjacent scan times to the same frame; skip repeats.+ if (time === prevTime) {+ sample.close();+ continue;+ }+ prevTime = time;+ const frame = sample.toVideoFrame();+ let print: Float32Array;+ try {+ print = await gpu.fingerprint(frame);+ } finally {+ frame.close();+ sample.close();+ }++ const motion = prev ? distance(print, prev) : 0;+ prev = print;++ if (motion <= MOTION_EPS) {+ // At rest: grab if the footage has drifted to a new state.+ calmest = undefined;+ const novelty = lastKept ? distance(print, lastKept) : Infinity;+ if (novelty >= THRESHOLD) keep(time, print, novelty);+ } else {+ // Mid-transition: never grab here. Track the calmest sample so a run+ // that never settles still yields its least blurred frame.+ if (!calmest) runStart = time;+ if (!calmest || motion < calmest.motion) calmest = { time, print, motion };+ if (time - runStart >= MAX_UNSETTLED) {+ const novelty = lastKept ? distance(calmest.print, lastKept) : Infinity;+ if (novelty >= THRESHOLD) keep(calmest.time, calmest.print, novelty);+ calmest = undefined; // start a new run window+ }+ }+ }+ assert(kept.length, "No frames could be decoded for auto frame selection.");++ // Over budget: drop the most marginal state changes, then restore order.+ return kept+ .sort((a, b) => b.novelty - a.novelty)+ .slice(0, opts.max)+ .sort((a, b) => a.time - b.time)+ .map((k) => k.time);+ } finally {+ gpu.dispose();+ }+}++async function createFingerprinter() {+ assert(navigator.gpu, "Auto frame selection requires WebGPU, which this environment does not support.");+ const adapter = await navigator.gpu.requestAdapter();+ assert(adapter, "Auto frame selection requires WebGPU, but no GPU adapter is available.");+ const device = await adapter.requestDevice();+ const pipeline = device.createComputePipeline({+ layout: "auto",+ compute: { module: device.createShaderModule({ code: SHADER }) },+ });+ const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });+ const blocks = device.createBuffer({ size: DIM * 4, usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC });+ const staging = device.createBuffer({ size: DIM * 4, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ });++ return {+ async fingerprint(frame: VideoFrame): Promise<Float32Array> {+ // External textures are only valid within the current task, so import,+ // encode, and submit synchronously for every frame.+ const bindGroup = device.createBindGroup({+ layout: pipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: sampler },+ { binding: 1, resource: device.importExternalTexture({ source: frame }) },+ { binding: 2, resource: { buffer: blocks } },+ ],+ });+ const encoder = device.createCommandEncoder();+ const pass = encoder.beginComputePass();+ pass.setPipeline(pipeline);+ pass.setBindGroup(0, bindGroup);+ pass.dispatchWorkgroups(GRID / 8, GRID / 8);+ pass.end();+ encoder.copyBufferToBuffer(blocks, 0, staging, 0, DIM * 4);+ device.queue.submit([encoder.finish()]);+ await staging.mapAsync(GPUMapMode.READ);+ const print = new Float32Array(staging.getMappedRange().slice(0));+ staging.unmap();+ return print;+ },+ dispose() {+ device.destroy();+ },+ };+}++// Fraction of blocks whose luma moved meaningfully between two fingerprints.+function distance(a: Float32Array, b: Float32Array): number {+ let sum = 0;+ for (let i = 0; i < DIM; i++) {+ const d = Math.abs(a[i] - b[i]);+ if (d > DEADZONE) sum += Math.min((d - DEADZONE) / (SATURATION - DEADZONE), 1);+ }+ return sum / DIM;+}diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.tsindex 97307558..550c7c76 100644--- a/apps/web/src/context/editor-api/media.ts+++ b/apps/web/src/context/editor-api/media.ts@@ -4,6 +4,7 @@import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';import { ElectronFileHandle } from '@/lib/electron-file-handle';+import { pickInformativeTimes } from './frame-triage';import { trpc } from '@/lib/trpc';import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp, stampTimestampLabel } from '@/components/engine';import { assert } from '@/utils';@@ -107,23 +108,30 @@ const FRAME_QUALITY_BUDGETS = {fullres: 0, // native} as const;+// Default cap on frames returned by auto selection when `count` is not given.+const AUTO_MAX_FRAMES = 30;+export function handleMediaFrame(engine: Accessor<Engine>) {return async (req: MediaFrameRequest) => {- const { times, count, start, end, quality, timestamp } = req;+ const { times, count, start, end, quality, timestamp, auto } = req;const { world } = engine();const asset = await resolveAssetRef(world, req);const id = asset.id;assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);- // `count` samples evenly across a window (default the whole clip); otherwise- // grab the explicit `times` (falling back to a single frame at 0).- let requested: number[];- if (count !== undefined) {- const from = Math.min(Math.max(start ?? 0, 0), asset.duration);- const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);+ // `count` samples evenly across a window (default the whole clip); `auto`+ // scans the window and keeps frames where the footage settles into a new+ // visual state, capped at `count` (resolved once the track is open).+ // Otherwise grab the explicit `times` (falling back to a single frame at 0).+ const from = Math.min(Math.max(start ?? 0, 0), asset.duration);+ const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);+ let requested: number[] = [];+ if (auto || count !== undefined) {assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`);- const interval = (to - from) / count;- requested = Array.from({ length: count }, (_, i) => from + i * interval);+ if (!auto && count !== undefined) {+ const interval = (to - from) / count;+ requested = Array.from({ length: count }, (_, i) => from + i * interval);+ }} else {const raw = times && times.length ? times : [0];// A negative time is an offset back from the end of the clip: -1 is one@@ -150,6 +158,15 @@ export function handleMediaFrame(engine: Accessor<Engine>) {// Track timestamps may not start at 0; offset content time by the first.const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;+ if (auto) {+ const picked = await pickInformativeTimes(track, {+ from: firstTimestamp + from,+ to: firstTimestamp + to,+ max: count ?? AUTO_MAX_FRAMES,+ });+ requested = picked.map((t) => Math.max(0, t - firstTimestamp));+ }+// Downscale to fit the pixel budget while preserving aspect ratio; setting// only the width lets the sink derive a matching height.const displayWidth = await track.getDisplayWidth();@@ -174,7 +191,8 @@ export function handleMediaFrame(engine: Accessor<Engine>) {const { time, index } = ordered[i++];assert(wrapped, `No frame found at ${time}s.`);- const ctx = wrapped.canvas.getContext("2d");+ // The webgpu getContext overload muddies inference on the canvas union.+ const ctx = wrapped.canvas.getContext("2d") as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null;if (ctx && timestamp !== false) {stampTimestampLabel(ctx, wrapped.canvas.height, formatTimestamp(time, asset.frameRate));diff --git a/apps/web/tsconfig.app.json b/apps/web/tsconfig.app.jsonindex d6e87afb..6256a6ce 100644--- a/apps/web/tsconfig.app.json+++ b/apps/web/tsconfig.app.json@@ -13,7 +13,8 @@"vite-plugin-solid-svg/types-component-solid","vite/client","@types/audioworklet",- "@types/wicg-file-system-access"+ "@types/wicg-file-system-access",+ "@webgpu/types"],"allowImportingTsExtensions": true,"jsx": "preserve",diff --git a/package-lock.json b/package-lock.jsonindex dae2bb4d..5c83e7b3 100644--- a/package-lock.json+++ b/package-lock.json@@ -100,6 +100,7 @@"@types/wicg-file-system-access": "^2023.10.7","@typescript-eslint/eslint-plugin": "^8.51.0","@typescript-eslint/parser": "^8.51.0",+ "@webgpu/types": "^0.1.71","eslint": "^9.39.2","eslint-plugin-solid": "^0.14.5","globals": "^17.0.0",@@ -4514,6 +4515,13 @@"@xtuc/long": "4.2.2"}},+ "node_modules/@webgpu/types": {+ "version": "0.1.71",+ "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz",+ "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==",+ "dev": true,+ "license": "BSD-3-Clause"+ },"node_modules/@xmldom/xmldom": {"version": "0.9.10","resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.9.10.tgz",diff --git a/scripts/dev-desktop.mjs b/scripts/dev-desktop.mjsindex 553b0840..c9eccb52 100644--- a/scripts/dev-desktop.mjs+++ b/scripts/dev-desktop.mjs@@ -11,15 +11,21 @@import { spawn, execFileSync } from "node:child_process";import { get } from "node:http";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+const ROOT = fileURLToPath(new URL("..", import.meta.url));+const BIN = join(ROOT, "node_modules", ".bin");const DEV_URL = "http://localhost:5173";const children = [];let shuttingDown = false;-function run(name, args) {- // Own process group (detached) so we can signal the npm process *and* its- // grandchildren (vite, electron) in one shot on teardown.- const child = spawn("npm", args, { stdio: "inherit", detached: true });+function run(name, bin, args, cwd) {+ // Spawn the tool binary directly rather than via `npm run`, so the teardown+ // SIGTERM isn't dressed up as a "Lifecycle script failed" error by an npm+ // wrapper. Own process group (detached) so we can signal the tool *and* its+ // children (esbuild, electron) in one shot on teardown.+ const child = spawn(join(BIN, bin), args, { cwd, stdio: "inherit", detached: true });child.on("exit", (code) => {if (shuttingDown) return;// A child dying on its own (e.g. Vite crashed) should bring the rest down.@@ -77,14 +83,17 @@ execFileSync("npm", ["run", "build", "--workspace=@diffusionstudio/cli"], { stdi// 2. Start the web dev server.console.log("[dev:desktop] starting web dev server…");-run("web", ["run", "dev", "--workspace=@diffusionstudio/web"]);+run("web", "vite", [], join(ROOT, "apps", "web"));-// 3. Once it is up, launch Electron (its own build runs first, loads :5173).+// 3. Once it is up, build the desktop app (blocking, mirrors its `dev`+// script) and launch Electron, which loads :5173.try {await waitForServer(DEV_URL);} catch (err) {console.error(`[dev:desktop] ${err.message}`);shutdown(1);}+console.log("[dev:desktop] building desktop app…");+execFileSync("npm", ["run", "build", "--workspace=@diffusionstudio/desktop"], { stdio: "inherit" });console.log("[dev:desktop] starting desktop app…");-run("desktop", ["run", "dev", "--workspace=@diffusionstudio/desktop"]);+run("desktop", "electron-forge", ["start"], join(ROOT, "apps", "desktop"));