Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor media command structure and update asset handling for improved clarity

konstantin-paulus5c117fc4bc59
diff --git a/apps/cli/reference/README.md b/apps/cli/reference/README.md
index 1873c542..3676c2bf 100644
--- a/apps/cli/reference/README.md
+++ b/apps/cli/reference/README.md
@@ -21,7 +21,7 @@ How the surface is divided:
- Declarative composition happens through `mount`, which renders a Solid JSX project into the canvas (see [jsx/](./jsx/README.md)); `node insert` runs the same pipeline but inserts the rendered nodes into an existing parent entity instead of mounting document roots.
- Scenes are created declaratively via `mount` (`<scene key="...">`); there is no imperative scene command.
- AI asset generation (image / video / speech / audio) is declared in the project module (`generate.*`, see [jsx/generate.md](./jsx/generate.md)) and produced on mount. `models` and `voices` list what those declarations can reference.
-- Inspecting an existing asset (probe / transcribe / listen / visualize / frame) lives under `asset`; writing an asset's original file back to disk is `asset export`.
+- Inspecting an existing asset (probe / transcribe / listen / visualize / grab) lives under `media`; writing an asset's original file back to disk is `asset export`.
- Organizing the library lives under `folder`; moving assets between folders is `asset mv`.
## Commands
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 11a3e9e5..0c4eeb9c 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -20,7 +20,7 @@ export type CliHandshake = { port: number; token: string };
export type CliHandshakeReply = { ok: true } | { ok: false; error: string };
// One tRPC request/reply pair per WebSocket connection. `path` is the
-// dot-joined procedure path in the renderer's router (e.g. "asset.frame");
+// dot-joined procedure path in the renderer's router (e.g. "media.frame");
// procedure inputs and outputs are typed end-to-end via the AppRouter type,
// so the wire envelope stays untyped.
export type CliRequest = {
@@ -119,23 +119,23 @@ export type NodeRenderResult = { path: string };
export type AssetRef = { id: string } | { path: string };
-export type AssetProbeRequest = AssetRef;
+export type MediaProbeRequest = AssetRef;
-export type AssetFrameRequest = AssetRef & { times?: number[]; resolution?: number };
-export type AssetFrameResult = Array<{ time: number; base64: string }>;
+export type MediaFrameRequest = AssetRef & { times?: number[]; resolution?: number };
+export type MediaFrameResult = Array<{ time: number; base64: string }>;
-export type AssetTranscribeRequest = AssetRef & { start?: number; end?: number };
+export type MediaTranscribeRequest = AssetRef & { start?: number; end?: number };
export type TranscriptWord = { text: string; start: number; end: number };
export type TranscriptSegment = { text: string; words: TranscriptWord[] };
-export type AssetTranscribeResult = { segments: TranscriptSegment[] };
+export type MediaTranscribeResult = { segments: TranscriptSegment[] };
-export type AssetVisualizeRequest = AssetRef & { start?: number; end?: number; scale?: number };
-export type AssetVisualizeResult = {
+export type MediaVisualizeRequest = AssetRef & { start?: number; end?: number; scale?: number };
+export type MediaVisualizeResult = {
base64: string;
} & Record<string, unknown>;
-export type AssetListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };
-export type AssetListenResult = { analysis?: string; start?: number; end?: number };
+export type MediaListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };
+export type MediaListenResult = { analysis?: string; start?: number; end?: number };
export type GeneratedAsset = { id: string; name: string; type: string };
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index c54b5f52..3e02e5e7 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -450,9 +450,9 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):
}
}
-type AssetFrameOptions = { time?: string[]; resolution?: string; output?: string };
+type MediaFrameOptions = { time?: string[]; resolution?: string; output?: string };
-async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {
+async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
let times: number[] | undefined;
if (opts.time !== undefined) {
times = opts.time.map((t) => parseTimeArg(t, "--time"));
@@ -470,7 +470,7 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {
const target = resolveAssetRef(ref);
const dir = opts.output ?? tmpdir();
try {
- const frames = await editor.asset.frame.query({ ...target, times, resolution });
+ const frames = await editor.media.frame.query({ ...target, times, resolution });
for (const { time, base64 } of frames) {
const path = join(dir, `${randomUUID()}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
@@ -496,11 +496,11 @@ function resolveAssetRef(ref: string): AssetRef {
return { path: absPath };
}
-async function assetProbe(ref: string): Promise<void> {
+async function mediaProbe(ref: string): Promise<void> {
const target = resolveAssetRef(ref);
const stop = startSpinner("Probing asset");
try {
- const result = await editor.asset.probe.query(target);
+ const result = await editor.media.probe.query(target);
stop();
console.log(JSON.stringify(result));
} catch (e) {
@@ -509,9 +509,9 @@ async function assetProbe(ref: string): Promise<void> {
}
}
-type AssetTranscribeOptions = { start?: string; end?: string };
+type MediaTranscribeOptions = { start?: string; end?: string };
-async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promise<void> {
+async function mediaTranscribe(ref: string, opts: MediaTranscribeOptions): Promise<void> {
const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;
if (start !== undefined && end !== undefined && start >= end) {
@@ -522,7 +522,7 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promi
const target = resolveAssetRef(ref);
const stop = startSpinner("Transcribing asset");
try {
- const result = await editor.asset.transcribe.query({ ...target, start, end }, GENERATE);
+ const result = await editor.media.transcribe.query({ ...target, start, end }, GENERATE);
stop();
console.log(JSON.stringify(result));
} catch (e) {
@@ -531,9 +531,9 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promi
}
}
-type AssetListenOptions = { prompt?: string; start?: string; end?: string };
+type MediaListenOptions = { prompt?: string; start?: string; end?: string };
-async function assetListen(ref: string, opts: AssetListenOptions): Promise<void> {
+async function mediaListen(ref: string, opts: MediaListenOptions): Promise<void> {
const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;
if (start !== undefined && end !== undefined && start >= end) {
@@ -544,7 +544,7 @@ async function assetListen(ref: string, opts: AssetListenOptions): Promise<void>
const target = resolveAssetRef(ref);
const stop = startSpinner("Analyzing asset");
try {
- const result = await editor.asset.listen.query(
+ const result = await editor.media.listen.query(
{ ...target, prompt: opts.prompt, start, end, stripVideo: true },
GENERATE,
);
@@ -556,7 +556,7 @@ async function assetListen(ref: string, opts: AssetListenOptions): Promise<void>
}
}
-type AssetVisualizeOptions = { start?: string; end?: string; scale?: string; output?: string };
+type MediaVisualizeOptions = { start?: string; end?: string; scale?: string; output?: string };
function parseTimeArg(value: string, flag: string): number {
const seconds = parseTime(value);
@@ -569,7 +569,7 @@ function parseTimeArg(value: string, flag: string): number {
return seconds;
}
-async function assetVisualize(ref: string, opts: AssetVisualizeOptions): Promise<void> {
+async function mediaVisualize(ref: string, opts: MediaVisualizeOptions): Promise<void> {
const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;
if (start !== undefined && end !== undefined && start >= end) {
@@ -590,7 +590,7 @@ async function assetVisualize(ref: string, opts: AssetVisualizeOptions): Promise
const path = opts.output ?? join(tmpdir(), `${randomUUID()}.png`);
const stop = startSpinner("Rendering visualization");
try {
- const { base64, ...rest } = await editor.asset.visualize.query({ ...target, start, end, scale });
+ const { base64, ...rest } = await editor.media.visualize.query({ ...target, start, end, scale });
stop();
writeFileSync(path, Buffer.from(base64, "base64"));
console.log(JSON.stringify({ path, ...rest }));
@@ -1076,7 +1076,7 @@ media
.command("probe")
.description(`Read the container and per-track technical metadata of a media file. Commonly useful for a quick, free technical read (container, duration, per-track codec params), e.g. checking codec compatibility or duration before cutting${docs("media/probe")}`)
.argument("<id|path>", "asset id, or a local file")
- .action((ref: string) => assetProbe(ref));
+ .action((ref: string) => mediaProbe(ref));
media
.command("transcribe")
@@ -1084,7 +1084,7 @@ media
.argument("<id|path>", "video or audio asset id, or a local file")
.option("-s, --start <time>", `start of the range to print — seconds, "45f" frames, or "MM:SS" (default: 0)`)
.option("-e, --end <time>", `end of the range to print — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
- .action((ref: string, opts: AssetTranscribeOptions) => assetTranscribe(ref, opts));
+ .action((ref: string, opts: MediaTranscribeOptions) => mediaTranscribe(ref, opts));
media
.command("grab")
@@ -1093,7 +1093,7 @@ media
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`)
.option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)")
.option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")
- .action((ref: string, opts: AssetFrameOptions) => assetFrame(ref, opts));
+ .action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));
media
.command("visualize")
@@ -1104,7 +1104,7 @@ media
.option("-e, --end <time>", `end of the window to visualize — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
.option("-x, --scale <factor>", "scale factor for the thumbnails; smaller thumbnails fit more rows and columns, larger fit fewer (default: 1)")
.option("-o, --output <path>", "write the PNG here instead of a temp file")
- .action((ref: string, opts: AssetVisualizeOptions) => assetVisualize(ref, opts));
+ .action((ref: string, opts: MediaVisualizeOptions) => mediaVisualize(ref, opts));
media
.command("listen")
@@ -1113,7 +1113,7 @@ media
.option("-p, --prompt <str>", "question or instruction to guide the analysis")
.option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`)
.option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
- .action((ref: string, opts: AssetListenOptions) => assetListen(ref, opts));
+ .action((ref: string, opts: MediaListenOptions) => mediaListen(ref, opts));
const folder = program
.command("folder")
diff --git a/apps/web/src/context/editor-api/api.tsx b/apps/web/src/context/editor-api/api.tsx
index 97b7e287..12c950f1 100644
--- a/apps/web/src/context/editor-api/api.tsx
+++ b/apps/web/src/context/editor-api/api.tsx
@@ -8,7 +8,8 @@ import { useEngine } from '@/context/engine';
import { useAuth } from '@/context/auth';
import { t, q, m, q0, m0 } from "@/lib/cli-rpc";
import { handleContextGet } from "./context";
-import { handleAssetsAdd, handleAssetsList, handleAssetTree, handleAssetsDelete, handleAssetsMove, handleAssetsExport, handleAssetProbe, handleAssetFrame, handleAssetTranscribe, handleAssetVisualize, handleAssetListen } from "./assets";
+import { handleAssetsAdd, handleAssetsList, handleAssetTree, handleAssetsDelete, handleAssetsMove, handleAssetsExport } from "./assets";
+import { handleMediaProbe, handleMediaFrame, handleMediaTranscribe, handleMediaVisualize, handleMediaListen } from "./media";
import { handleFoldersList, handleFolderCreate, handleFolderRename, handleFoldersMove, handleFoldersDelete } from "./folders";
import { handleSelectionFocus, handleSelectionList, handleSelectionSet } from "./selection";
import { handleNodeList, handleNodeTree, handleNodeGrep, handleNodeScreenshot, handleNodeDelete, handleNodePatch, handleNodeDuplicate, handleNodeRender } from "./node";
@@ -104,11 +105,13 @@ function createAppRouter({ getEngine, getUser, requireAuth, setParams }: AppRout
delete: m(handleAssetsDelete(getEngine)),
move: m(handleAssetsMove(getEngine)),
export: m(handleAssetsExport(getEngine)),
- probe: q(handleAssetProbe(getEngine)),
- frame: q(handleAssetFrame(getEngine)),
- transcribe: q(handleAssetTranscribe(getEngine)),
- visualize: q(handleAssetVisualize(getEngine)),
- listen: q(requireAuth(handleAssetListen(getEngine))),
+ }),
+ media: t.router({
+ probe: q(handleMediaProbe(getEngine)),
+ frame: q(handleMediaFrame(getEngine)),
+ transcribe: q(handleMediaTranscribe(getEngine)),
+ visualize: q(handleMediaVisualize(getEngine)),
+ listen: q(requireAuth(handleMediaListen(getEngine))),
}),
folder: t.router({
list: q(handleFoldersList(getEngine)),
diff --git a/apps/web/src/context/editor-api/assets.ts b/apps/web/src/context/editor-api/assets.ts
index cd33150e..265e8869 100644
--- a/apps/web/src/context/editor-api/assets.ts
+++ b/apps/web/src/context/editor-api/assets.ts
@@ -2,17 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
import { ElectronFileHandle } from '@/lib/electron-file-handle';
import { ElectronWritableFileHandle } from '@/lib/electron-file-writable';
-import { trpc } from '@/lib/trpc';
-import { uploadBlob, visualizeAsset, loadAsset, removeAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';
+import { loadAsset, removeAsset, getAssetFile } from '@/components/engine';
import { assert, mimeTypeToExtension } from '@/utils';
import {
- transcodeForTranscription,
- transcodeForAnalysis,
- startResumableSession,
- uploadResumableStream,
assetFolderId,
moveAssetsToFolder,
useFolders,
@@ -20,7 +14,7 @@ import {
import type { Engine } from "@/components/engine";
import type { Asset } from "@/components/engine/db";
-import type { AssetListenRequest, AssetListenResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetRef, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
+import type { AssetExportResult, AssetListResult, AssetMoveResult, AssetRecord, AssetsExportRequest, AssetTreeEntry } from "@diffusionstudio/cli/channels";
import type { Accessor } from "solid-js";
export function handleAssetsAdd(engine: Accessor<Engine>) {
@@ -206,7 +200,7 @@ async function streamBlobToFile(blob: Blob, path: string): Promise<string> {
}
/**
- * Streams the blob chunk-by-chunk to the open writable
+ * Streams the blob chunk-by-chunk to the open writable
*/
async function pumpBlob(
blob: Blob,
@@ -229,286 +223,3 @@ async function pumpBlob(
throw e;
}
}
-
-/**
- * Resolves a command target: an id looks up the project's asset library, a
- * path describes the file in place as an ephemeral asset (never added to the
- * library). Ephemeral assets carry the path as their id.
- */
-async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {
- if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));
- const asset = world.assets.get(ref.id);
- assert(asset, `Asset ${ref.id} not found.`);
- return asset;
-}
-
-const PROBE_SAMPLE_PACKETS = 200;
-
-export function handleAssetProbe(engine: Accessor<Engine>) {
- return async (req: AssetProbeRequest): Promise<unknown> => {
- const { world } = engine();
- const asset = await resolveAssetRef(world, req);
-
- const blob = await getAssetFile(asset);
- const base = {
- id: asset.id,
- name: asset.name,
- type: asset.type,
- mimeType: asset.mimeType,
- size: blob.size,
- ...("width" in asset && { width: asset.width, height: asset.height }),
- };
-
- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
- try {
- const format = await input.getFormat();
- const mimeType = await input.getMimeType();
- const duration = await input.computeDuration();
- const { images, ...tags } = await input.getMetadataTags();
- delete tags.raw;
-
- const tracks: Array<Record<string, unknown>> = [];
- for (const track of await input.getTracks()) {
- const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);
- tracks.push({
- id: track.id,
- type: track.type,
- codec: track.codec,
- language: track.languageCode,
- firstTimestamp: await track.getFirstTimestamp(),
- duration: await track.computeDuration(),
- ...stats,
- ...(track.isVideoTrack() && {
- codedWidth: track.codedWidth,
- codedHeight: track.codedHeight,
- displayWidth: track.displayWidth,
- displayHeight: track.displayHeight,
- rotation: track.rotation,
- }),
- ...(track.isAudioTrack() && {
- sampleRate: track.sampleRate,
- channels: track.numberOfChannels,
- }),
- });
- }
-
- return {
- ...base,
- format: format.name,
- mimeType,
- duration,
- tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },
- tracks,
- };
- } catch {
- return { ...base, format: null, tracks: [] };
- } finally {
- input.dispose();
- }
- };
-}
-
-// Default cap on the pixel count of each decoded frame (384x384). Keeps frames
-// small enough for vision models without the caller having to know the aspect ratio.
-export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;
-
-export function handleAssetFrame(engine: Accessor<Engine>) {
- return async (req: AssetFrameRequest) => {
- const { times, resolution } = req;
- const { world } = engine();
- const asset = await resolveAssetRef(world, req);
- const id = asset.id;
- assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);
-
- const requested = times && times.length ? times : [0];
- for (const t of requested) {
- assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);
- }
-
- // A budget of 0 means "native resolution"; anything else caps the pixel count.
- const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;
-
- const blob = await getAssetFile(asset);
- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
- try {
- const track = await input.getPrimaryVideoTrack();
- assert(track, `Asset ${id} has no video track.`);
-
- // Track timestamps may not start at 0; offset content time by the first.
- const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;
-
- // Downscale to fit the pixel budget while preserving aspect ratio; setting
- // only the width lets the sink derive a matching height.
- const displayWidth = await track.getDisplayWidth();
- const displayHeight = await track.getDisplayHeight();
- let width: number | undefined;
- if (budget > 0 && displayWidth * displayHeight > budget) {
- width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));
- }
-
- // Decode in ascending order (the sink's fast path), remember each
- // entry's original slot so output mirrors the requested order.
- const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);
-
- // No pool: each yielded canvas is fresh, so converting to PNG can't race
- // the generator's read-ahead reusing a pooled canvas.
- const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);
- const timestamps = ordered.map(({ time }) => firstTimestamp + time);
-
- const result: Array<{ time: number; base64: string }> = new Array(requested.length);
- let i = 0;
- for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
- const { time, index } = ordered[i++];
- assert(wrapped, `No frame found at ${time}s.`);
-
- const ctx = wrapped.canvas.getContext("2d");
-
- if (ctx) {
- const label = formatTimestamp(time, asset.frameRate)
- const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));
- const fontSize = Math.round(bandHeight * 0.72);
- const paddingLeft = Math.round(bandHeight * 0.4);
- const y = paddingLeft + bandHeight / 2;
-
- ctx.font = `normal ${fontSize}px sans-serif`;
- ctx.textBaseline = "middle";
- ctx.textAlign = "left";
-
- ctx.lineJoin = "round";
- ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));
- ctx.strokeStyle = "#000";
- ctx.strokeText(label, paddingLeft, y);
- ctx.fillStyle = "#FFF";
- ctx.fillText(label, paddingLeft, y);
- }
-
- result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };
- }
- return result;
- } finally {
- input.dispose();
- }
- };
-}
-
-const transcripts = new Map<string, TranscriptSegment[]>();
-
-export function handleAssetTranscribe(engine: Accessor<Engine>) {
- return async (req: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {
- const { start, end } = req;
- const { world } = engine();
- const asset = await resolveAssetRef(world, req);
- const id = asset.id;
- assert(
- asset.type === "AUDIO" || asset.type === "VIDEO",
- `Asset ${id} is not a video or audio asset.`,
- );
-
- let transcript = transcripts.get(asset.hash);
- if (!transcript) {
- const uploadId = crypto.randomUUID();
- const audioFile = await transcodeForTranscription(asset);
- const fileRef = await uploadBlob(audioFile, uploadId);
- if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);
-
- ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));
- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {
- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");
- }
-
- transcripts.set(asset.hash, transcript);
- }
-
- return { segments: sliceTranscript(transcript, start, end) };
- };
-}
-
-function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {
- if (start === undefined && end === undefined) return segments;
- const from = start ?? 0;
- const to = end ?? Infinity;
- const sliced: TranscriptSegment[] = [];
- for (const segment of segments) {
- const words = segment.words.filter((w) => w.end > from && w.start < to);
- if (!words.length) continue;
- sliced.push(
- words.length === segment.words.length
- ? segment
- : { text: words.map((w) => w.text).join(" "), words },
- );
- }
- return sliced;
-}
-
-export function handleAssetVisualize(engine: Accessor<Engine>) {
- return async (req: AssetVisualizeRequest): Promise<AssetVisualizeResult> => {
- const { world } = engine();
- const asset = await resolveAssetRef(world, req);
- const { dataUrl, ...rest } = await visualizeAsset(asset, { start: req.start, end: req.end, scale: req.scale });
- const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
- return { base64, ...rest };
- };
-}
-
-export function handleAssetListen(engine: Accessor<Engine>) {
- return async (req: AssetListenRequest): Promise<AssetListenResult> => {
- const { prompt, start, end } = req;
- let { stripVideo } = req;
- const { world } = engine();
- const asset = await resolveAssetRef(world, req);
- const id = asset.id;
- assert(
- asset.type === "AUDIO" || asset.type === "VIDEO",
- `Asset ${id} is not a video or audio asset.`,
- );
-
- const hasWindow = start !== undefined || end !== undefined;
- stripVideo = stripVideo !== false && asset.type === "VIDEO";
-
- const contentType =
- asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";
-
- const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";
- const key = world.assets.has(id) ? id : asset.hash;
- const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`
- .replace(/[^A-Za-z0-9._-]/g, "_");
- const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({
- action: "resumable",
- id: uploadId,
- contentType,
- });
-
- // Transcoding pending
- if (uploadUrl) {
- const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });
- const sessionUrl = await startResumableSession(uploadUrl, contentType);
- const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);
- await transcoder.run?.();
- await uploadPromise;
- }
-
- const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });
-
- return { analysis, start, end };
- };
-}
-
-async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<string> {
- if (canvas instanceof OffscreenCanvas) {
- const blob = await canvas.convertToBlob({ type: "image/png" });
- return base64FromArrayBuffer(await blob.arrayBuffer());
- }
- return canvas.toDataURL("image/png").split(",")[1] ?? "";
-}
-
-function base64FromArrayBuffer(buf: ArrayBuffer): string {
- const bytes = new Uint8Array(buf);
- let binary = "";
- const chunk = 0x8000;
- for (let i = 0; i < bytes.length; i += chunk) {
- binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
- }
- return btoa(binary);
-}
-
-// hmr-probe-test
diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.ts
new file mode 100644
index 00000000..b5a78291
--- /dev/null
+++ b/apps/web/src/context/editor-api/media.ts
@@ -0,0 +1,301 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
+import { ElectronFileHandle } from '@/lib/electron-file-handle';
+import { trpc } from '@/lib/trpc';
+import { uploadBlob, visualizeAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';
+import { assert } from '@/utils';
+import {
+ transcodeForTranscription,
+ transcodeForAnalysis,
+ startResumableSession,
+ uploadResumableStream,
+} from '@/components/engine';
+
+import type { Engine } from "@/components/engine";
+import type { Asset } from "@/components/engine/db";
+import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaProbeRequest, AssetRef, MediaTranscribeRequest, MediaTranscribeResult, MediaVisualizeRequest, MediaVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
+import type { Accessor } from "solid-js";
+
+/**
+ * Resolves a command target: an id looks up the project's asset library, a
+ * path describes the file in place as an ephemeral asset (never added to the
+ * library). Ephemeral assets carry the path as their id.
+ */
+async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {
+ if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));
+ const asset = world.assets.get(ref.id);
+ assert(asset, `Asset ${ref.id} not found.`);
+ return asset;
+}
+
+const PROBE_SAMPLE_PACKETS = 200;
+
+export function handleMediaProbe(engine: Accessor<Engine>) {
+ return async (req: MediaProbeRequest): Promise<unknown> => {
+ const { world } = engine();
+ const asset = await resolveAssetRef(world, req);
+
+ const blob = await getAssetFile(asset);
+ const base = {
+ id: asset.id,
+ name: asset.name,
+ type: asset.type,
+ mimeType: asset.mimeType,
+ size: blob.size,
+ ...("width" in asset && { width: asset.width, height: asset.height }),
+ };
+
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ const format = await input.getFormat();
+ const mimeType = await input.getMimeType();
+ const duration = await input.computeDuration();
+ const { images, ...tags } = await input.getMetadataTags();
+ delete tags.raw;
+
+ const tracks: Array<Record<string, unknown>> = [];
+ for (const track of await input.getTracks()) {
+ const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);
+ tracks.push({
+ id: track.id,
+ type: track.type,
+ codec: track.codec,
+ language: track.languageCode,
+ firstTimestamp: await track.getFirstTimestamp(),
+ duration: await track.computeDuration(),
+ ...stats,
+ ...(track.isVideoTrack() && {
+ codedWidth: track.codedWidth,
+ codedHeight: track.codedHeight,
+ displayWidth: track.displayWidth,
+ displayHeight: track.displayHeight,
+ rotation: track.rotation,
+ }),
+ ...(track.isAudioTrack() && {
+ sampleRate: track.sampleRate,
+ channels: track.numberOfChannels,
+ }),
+ });
+ }
+
+ return {
+ ...base,
+ format: format.name,
+ mimeType,
+ duration,
+ tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },
+ tracks,
+ };
+ } catch {
+ return { ...base, format: null, tracks: [] };
+ } finally {
+ input.dispose();
+ }
+ };
+}
+
+// Default cap on the pixel count of each decoded frame (384x384). Keeps frames
+// small enough for vision models without the caller having to know the aspect ratio.
+export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;
+
+export function handleMediaFrame(engine: Accessor<Engine>) {
+ return async (req: MediaFrameRequest) => {
+ const { times, resolution } = req;
+ const { world } = engine();
+ const asset = await resolveAssetRef(world, req);
+ const id = asset.id;
+ assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);
+
+ const requested = times && times.length ? times : [0];
+ for (const t of requested) {
+ assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);
+ }
+
+ // A budget of 0 means "native resolution"; anything else caps the pixel count.
+ const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;
+
+ const blob = await getAssetFile(asset);
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ try {
+ const track = await input.getPrimaryVideoTrack();
+ assert(track, `Asset ${id} has no video track.`);
+
+ // Track timestamps may not start at 0; offset content time by the first.
+ const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;
+
+ // Downscale to fit the pixel budget while preserving aspect ratio; setting
+ // only the width lets the sink derive a matching height.
+ const displayWidth = await track.getDisplayWidth();
+ const displayHeight = await track.getDisplayHeight();
+ let width: number | undefined;
+ if (budget > 0 && displayWidth * displayHeight > budget) {
+ width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));
+ }
+
+ // Decode in ascending order (the sink's fast path), remember each
+ // entry's original slot so output mirrors the requested order.
+ const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);
+
+ // No pool: each yielded canvas is fresh, so converting to PNG can't race
+ // the generator's read-ahead reusing a pooled canvas.
+ const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);
+ const timestamps = ordered.map(({ time }) => firstTimestamp + time);
+
+ const result: Array<{ time: number; base64: string }> = new Array(requested.length);
+ let i = 0;
+ for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
+ const { time, index } = ordered[i++];
+ assert(wrapped, `No frame found at ${time}s.`);
+
+ const ctx = wrapped.canvas.getContext("2d");
+
+ if (ctx) {
+ const label = formatTimestamp(time, asset.frameRate)
+ const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));
+ const fontSize = Math.round(bandHeight * 0.72);
+ const paddingLeft = Math.round(bandHeight * 0.4);
+ const y = paddingLeft + bandHeight / 2;
+
+ ctx.font = `normal ${fontSize}px sans-serif`;
+ ctx.textBaseline = "middle";
+ ctx.textAlign = "left";
+
+ ctx.lineJoin = "round";
+ ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));
+ ctx.strokeStyle = "#000";
+ ctx.strokeText(label, paddingLeft, y);
+ ctx.fillStyle = "#FFF";
+ ctx.fillText(label, paddingLeft, y);
+ }
+
+ result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };
+ }
+ return result;
+ } finally {
+ input.dispose();
+ }
+ };
+}
+
+const transcripts = new Map<string, TranscriptSegment[]>();
+
+export function handleMediaTranscribe(engine: Accessor<Engine>) {
+ return async (req: MediaTranscribeRequest): Promise<MediaTranscribeResult> => {
+ const { start, end } = req;
+ const { world } = engine();
+ const asset = await resolveAssetRef(world, req);
+ const id = asset.id;
+ assert(
+ asset.type === "AUDIO" || asset.type === "VIDEO",
+ `Asset ${id} is not a video or audio asset.`,
+ );
+
+ let transcript = transcripts.get(asset.hash);
+ if (!transcript) {
+ const uploadId = crypto.randomUUID();
+ const audioFile = await transcodeForTranscription(asset);
+ const fileRef = await uploadBlob(audioFile, uploadId);
+ if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);
+
+ ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));
+ if (!transcript.length || transcript.every((s) => s.words.length === 0)) {
+ throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");
+ }
+
+ transcripts.set(asset.hash, transcript);
+ }
+
+ return { segments: sliceTranscript(transcript, start, end) };
+ };
+}
+
+function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {
+ if (start === undefined && end === undefined) return segments;
+ const from = start ?? 0;
+ const to = end ?? Infinity;
+ const sliced: TranscriptSegment[] = [];
+ for (const segment of segments) {
+ const words = segment.words.filter((w) => w.end > from && w.start < to);
+ if (!words.length) continue;
+ sliced.push(
+ words.length === segment.words.length
+ ? segment
+ : { text: words.map((w) => w.text).join(" "), words },
+ );
+ }
+ return sliced;
+}
+
+export function handleMediaVisualize(engine: Accessor<Engine>) {
+ return async (req: MediaVisualizeRequest): Promise<MediaVisualizeResult> => {
+ const { world } = engine();
+ const asset = await resolveAssetRef(world, req);
+ const { dataUrl, ...rest } = await visualizeAsset(asset, { start: req.start, end: req.end, scale: req.scale });
+ const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
+ return { base64, ...rest };
+ };
+}
+
+export function handleMediaListen(engine: Accessor<Engine>) {
+ return async (req: MediaListenRequest): Promise<MediaListenResult> => {
+ const { prompt, start, end } = req;
+ let { stripVideo } = req;
+ const { world } = engine();
+ const asset = await resolveAssetRef(world, req);
+ const id = asset.id;
+ assert(
+ asset.type === "AUDIO" || asset.type === "VIDEO",
+ `Asset ${id} is not a video or audio asset.`,
+ );
+
+ const hasWindow = start !== undefined || end !== undefined;
+ stripVideo = stripVideo !== false && asset.type === "VIDEO";
+
+ const contentType =
+ asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";
+
+ const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";
+ const key = world.assets.has(id) ? id : asset.hash;
+ const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`
+ .replace(/[^A-Za-z0-9._-]/g, "_");
+ const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({
+ action: "resumable",
+ id: uploadId,
+ contentType,
+ });
+
+ // Transcoding pending
+ if (uploadUrl) {
+ const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });
+ const sessionUrl = await startResumableSession(uploadUrl, contentType);
+ const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);
+ await transcoder.run?.();
+ await uploadPromise;
+ }
+
+ const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });
+
+ return { analysis, start, end };
+ };
+}
+
+async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<string> {
+ if (canvas instanceof OffscreenCanvas) {
+ const blob = await canvas.convertToBlob({ type: "image/png" });
+ return base64FromArrayBuffer(await blob.arrayBuffer());
+ }
+ return canvas.toDataURL("image/png").split(",")[1] ?? "";
+}
+
+function base64FromArrayBuffer(buf: ArrayBuffer): string {
+ const bytes = new Uint8Array(buf);
+ let binary = "";
+ const chunk = 0x8000;
+ for (let i = 0; i < bytes.length; i += chunk) {
+ binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+ }
+ return btoa(binary);
+}