The VS Code of video editing, built for agents.
TypeScript
Enhance asset frame handling by adding resolution parameter for improved frame processing
konstantin-paulus6f3b3e94279e
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 45cacf35..399344e0 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -146,7 +146,7 @@ export type NodeRenderResult = { path: string };export type AssetProbeRequest = { id: string };-export type AssetFrameRequest = { id: string; times?: number[] };+export type AssetFrameRequest = { id: string; times?: number[]; resolution?: number };export type AssetFrameResult = Array<{ time: number; base64: string }>;export type AssetTranscribeRequest = { id: string; start?: number; end?: number };diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex ae395ed3..cc8f871f 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -90,7 +90,8 @@ export const cliAPI = {moveFolders: (ids: string[], to?: string) => cliRequest(CLI_CHANNELS.FOLDERS_MOVE, { ids, to }),deleteFolders: (ids: string[]) => cliRequest(CLI_CHANNELS.FOLDERS_DELETE, { ids }),assetProbe: (id: string) => cliRequest(CLI_CHANNELS.ASSET_PROBE, { id }),- assetFrame: (id: string, times?: number[]) => cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times }),+ assetFrame: (id: string, times?: number[], resolution?: number) =>+ cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times, resolution }),assetTranscribe: (id: string, start?: number, end?: number) =>cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 22fcfe67..516f8cd6 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -447,7 +447,7 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):}}-type AssetFrameOptions = { time?: string[]; output?: string };+type AssetFrameOptions = { time?: string[]; resolution?: string; output?: string };async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {let times: number[] | undefined;@@ -455,10 +455,19 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {times = opts.time.map((t) => parseTimeArg(t, "--time"));}+ let resolution: number | undefined;+ if (opts.resolution !== undefined) {+ resolution = Number(opts.resolution);+ if (!Number.isInteger(resolution) || resolution < 0) {+ console.error(`--resolution must be a non-negative integer pixel count, or 0 for native (got "${opts.resolution}")`);+ process.exit(1);+ }+ }+const assetId = await resolveAssetRef(ref);const dir = opts.output ?? tmpdir();try {- const frames = await cliAPI.assetFrame(assetId, times);+ const frames = await cliAPI.assetFrame(assetId, times, resolution);for (const { time, base64 } of frames) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));@@ -1076,6 +1085,7 @@ asset.description(`Decode one or more frames of a video asset and write them as PNGs${docs("asset/frame")}`).argument("<id|path>", "video asset id, or a local file to add and grab frames from").option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`)+ .option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)").option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)").action((ref: string, opts: AssetFrameOptions) => assetFrame(ref, opts));diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.tsindex 108f8f36..a8e831c4 100644--- a/apps/web/src/context/electron/assets.ts+++ b/apps/web/src/context/electron/assets.ts@@ -20,7 +20,7 @@ import {import type { Engine } from "@/components/engine";import type { Asset } from "@/components/engine/db";-import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";+import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";import type { Accessor } from "solid-js";export function handleAssetsAdd(engine: Accessor<Engine>) {@@ -297,8 +297,12 @@ export function handleAssetProbe(engine: Accessor<Engine>) {};}+// Default cap on the pixel count of each decoded frame (384x384). Keeps frames+// small enough for vision models without the caller having to know the aspect ratio.+export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;+export function handleAssetFrame(engine: Accessor<Engine>) {- return async ({ id, times }: { id: string; times?: number[] }) => {+ return async ({ id, times, resolution }: AssetFrameRequest) => {const { world } = engine();const asset = world.assets.get(id);assert(asset, `Asset ${id} not found.`);@@ -309,6 +313,9 @@ export function handleAssetFrame(engine: Accessor<Engine>) {assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);}+ // A budget of 0 means "native resolution"; anything else caps the pixel count.+ const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;+const blob = await asset.handle.getFile();const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });try {@@ -318,13 +325,22 @@ export function handleAssetFrame(engine: Accessor<Engine>) {// Track timestamps may not start at 0; offset content time by the first.const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;+ // Downscale to fit the pixel budget while preserving aspect ratio; setting+ // only the width lets the sink derive a matching height.+ const displayWidth = await track.getDisplayWidth();+ const displayHeight = await track.getDisplayHeight();+ let width: number | undefined;+ if (budget > 0 && displayWidth * displayHeight > budget) {+ width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));+ }+// Decode in ascending order (the sink's fast path), remember each// entry's original slot so output mirrors the requested order.const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);// No pool: each yielded canvas is fresh, so converting to PNG can't race// the generator's read-ahead reusing a pooled canvas.- const sink = new CanvasSink(track);+ const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);const timestamps = ordered.map(({ time }) => firstTimestamp + time);const result: Array<{ time: number; base64: string }> = new Array(requested.length);