Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance asset frame handling by adding resolution parameter for improved frame processing

konstantin-paulus6f3b3e94279e
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 45cacf35..399344e0 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -146,7 +146,7 @@ export type NodeRenderResult = { path: string };
export type AssetProbeRequest = { id: string };
-export type AssetFrameRequest = { id: string; times?: number[] };
+export type AssetFrameRequest = { id: string; times?: number[]; resolution?: number };
export type AssetFrameResult = Array<{ time: number; base64: string }>;
export type AssetTranscribeRequest = { id: string; start?: number; end?: number };
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index ae395ed3..cc8f871f 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -90,7 +90,8 @@ export const cliAPI = {
moveFolders: (ids: string[], to?: string) => cliRequest(CLI_CHANNELS.FOLDERS_MOVE, { ids, to }),
deleteFolders: (ids: string[]) => cliRequest(CLI_CHANNELS.FOLDERS_DELETE, { ids }),
assetProbe: (id: string) => cliRequest(CLI_CHANNELS.ASSET_PROBE, { id }),
- assetFrame: (id: string, times?: number[]) => cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times }),
+ assetFrame: (id: string, times?: number[], resolution?: number) =>
+ cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times, resolution }),
assetTranscribe: (id: string, start?: number, end?: number) =>
cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),
assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 22fcfe67..516f8cd6 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -447,7 +447,7 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):
}
}
-type AssetFrameOptions = { time?: string[]; output?: string };
+type AssetFrameOptions = { time?: string[]; resolution?: string; output?: string };
async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {
let times: number[] | undefined;
@@ -455,10 +455,19 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {
times = opts.time.map((t) => parseTimeArg(t, "--time"));
}
+ let resolution: number | undefined;
+ if (opts.resolution !== undefined) {
+ resolution = Number(opts.resolution);
+ if (!Number.isInteger(resolution) || resolution < 0) {
+ console.error(`--resolution must be a non-negative integer pixel count, or 0 for native (got "${opts.resolution}")`);
+ process.exit(1);
+ }
+ }
+
const assetId = await resolveAssetRef(ref);
const dir = opts.output ?? tmpdir();
try {
- const frames = await cliAPI.assetFrame(assetId, times);
+ const frames = await cliAPI.assetFrame(assetId, times, resolution);
for (const { time, base64 } of frames) {
const path = join(dir, `${randomUUID()}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
@@ -1076,6 +1085,7 @@ asset
.description(`Decode one or more frames of a video asset and write them as PNGs${docs("asset/frame")}`)
.argument("<id|path>", "video asset id, or a local file to add and grab frames from")
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`)
+ .option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)")
.option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")
.action((ref: string, opts: AssetFrameOptions) => assetFrame(ref, opts));
diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.ts
index 108f8f36..a8e831c4 100644
--- a/apps/web/src/context/electron/assets.ts
+++ b/apps/web/src/context/electron/assets.ts
@@ -20,7 +20,7 @@ import {
import type { Engine } from "@/components/engine";
import type { Asset } from "@/components/engine/db";
-import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
+import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
import type { Accessor } from "solid-js";
export function handleAssetsAdd(engine: Accessor<Engine>) {
@@ -297,8 +297,12 @@ export function handleAssetProbe(engine: Accessor<Engine>) {
};
}
+// Default cap on the pixel count of each decoded frame (384x384). Keeps frames
+// small enough for vision models without the caller having to know the aspect ratio.
+export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;
+
export function handleAssetFrame(engine: Accessor<Engine>) {
- return async ({ id, times }: { id: string; times?: number[] }) => {
+ return async ({ id, times, resolution }: AssetFrameRequest) => {
const { world } = engine();
const asset = world.assets.get(id);
assert(asset, `Asset ${id} not found.`);
@@ -309,6 +313,9 @@ export function handleAssetFrame(engine: Accessor<Engine>) {
assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);
}
+ // A budget of 0 means "native resolution"; anything else caps the pixel count.
+ const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;
+
const blob = await asset.handle.getFile();
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
try {
@@ -318,13 +325,22 @@ export function handleAssetFrame(engine: Accessor<Engine>) {
// Track timestamps may not start at 0; offset content time by the first.
const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;
+ // Downscale to fit the pixel budget while preserving aspect ratio; setting
+ // only the width lets the sink derive a matching height.
+ const displayWidth = await track.getDisplayWidth();
+ const displayHeight = await track.getDisplayHeight();
+ let width: number | undefined;
+ if (budget > 0 && displayWidth * displayHeight > budget) {
+ width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));
+ }
+
// Decode in ascending order (the sink's fast path), remember each
// entry's original slot so output mirrors the requested order.
const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);
// No pool: each yielded canvas is fresh, so converting to PNG can't race
// the generator's read-ahead reusing a pooled canvas.
- const sink = new CanvasSink(track);
+ const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);
const timestamps = ordered.map(({ time }) => firstTimestamp + time);
const result: Array<{ time: number; base64: string }> = new Array(requested.length);