The VS Code of video editing, built for agents.
TypeScript
Refactor media command structure and update asset handling for improved clarity
konstantin-paulus5c117fc4bc59
diff --git a/apps/cli/reference/README.md b/apps/cli/reference/README.mdindex 1873c542..3676c2bf 100644--- a/apps/cli/reference/README.md+++ b/apps/cli/reference/README.md@@ -21,7 +21,7 @@ How the surface is divided:- Declarative composition happens through `mount`, which renders a Solid JSX project into the canvas (see [jsx/](./jsx/README.md)); `node insert` runs the same pipeline but inserts the rendered nodes into an existing parent entity instead of mounting document roots.- Scenes are created declaratively via `mount` (`<scene key="...">`); there is no imperative scene command.- AI asset generation (image / video / speech / audio) is declared in the project module (`generate.*`, see [jsx/generate.md](./jsx/generate.md)) and produced on mount. `models` and `voices` list what those declarations can reference.-- Inspecting an existing asset (probe / transcribe / listen / visualize / frame) lives under `asset`; writing an asset's original file back to disk is `asset export`.+- Inspecting an existing asset (probe / transcribe / listen / visualize / grab) lives under `media`; writing an asset's original file back to disk is `asset export`.- Organizing the library lives under `folder`; moving assets between folders is `asset mv`.## Commandsdiff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 11a3e9e5..0c4eeb9c 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -20,7 +20,7 @@ export type CliHandshake = { port: number; token: string };export type CliHandshakeReply = { ok: true } | { ok: false; error: string };// One tRPC request/reply pair per WebSocket connection. `path` is the-// dot-joined procedure path in the renderer's router (e.g. "asset.frame");+// dot-joined procedure path in the renderer's router (e.g. "media.frame");// procedure inputs and outputs are typed end-to-end via the AppRouter type,// so the wire envelope stays untyped.export type CliRequest = {@@ -119,23 +119,23 @@ export type NodeRenderResult = { path: string };export type AssetRef = { id: string } | { path: string };-export type AssetProbeRequest = AssetRef;+export type MediaProbeRequest = AssetRef;-export type AssetFrameRequest = AssetRef & { times?: number[]; resolution?: number };-export type AssetFrameResult = Array<{ time: number; base64: string }>;+export type MediaFrameRequest = AssetRef & { times?: number[]; resolution?: number };+export type MediaFrameResult = Array<{ time: number; base64: string }>;-export type AssetTranscribeRequest = AssetRef & { start?: number; end?: number };+export type MediaTranscribeRequest = AssetRef & { start?: number; end?: number };export type TranscriptWord = { text: string; start: number; end: number };export type TranscriptSegment = { text: string; words: TranscriptWord[] };-export type AssetTranscribeResult = { segments: TranscriptSegment[] };+export type MediaTranscribeResult = { segments: TranscriptSegment[] };-export type AssetVisualizeRequest = AssetRef & { start?: number; end?: number; scale?: number };-export type AssetVisualizeResult = {+export type MediaVisualizeRequest = AssetRef & { start?: number; end?: number; scale?: number };+export type MediaVisualizeResult = {base64: string;} & Record<string, unknown>;-export type AssetListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };-export type AssetListenResult = { analysis?: string; start?: number; end?: number };+export type MediaListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };+export type MediaListenResult = { analysis?: string; start?: number; end?: number };export type GeneratedAsset = { id: string; name: string; type: string };diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex c54b5f52..3e02e5e7 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -450,9 +450,9 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):}}-type AssetFrameOptions = { time?: string[]; resolution?: string; output?: string };+type MediaFrameOptions = { time?: string[]; resolution?: string; output?: string };-async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {+async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {let times: number[] | undefined;if (opts.time !== undefined) {times = opts.time.map((t) => parseTimeArg(t, "--time"));@@ -470,7 +470,7 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {const target = resolveAssetRef(ref);const dir = opts.output ?? tmpdir();try {- const frames = await editor.asset.frame.query({ ...target, times, resolution });+ const frames = await editor.media.frame.query({ ...target, times, resolution });for (const { time, base64 } of frames) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));@@ -496,11 +496,11 @@ function resolveAssetRef(ref: string): AssetRef {return { path: absPath };}-async function assetProbe(ref: string): Promise<void> {+async function mediaProbe(ref: string): Promise<void> {const target = resolveAssetRef(ref);const stop = startSpinner("Probing asset");try {- const result = await editor.asset.probe.query(target);+ const result = await editor.media.probe.query(target);stop();console.log(JSON.stringify(result));} catch (e) {@@ -509,9 +509,9 @@ async function assetProbe(ref: string): Promise<void> {}}-type AssetTranscribeOptions = { start?: string; end?: string };+type MediaTranscribeOptions = { start?: string; end?: string };-async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promise<void> {+async function mediaTranscribe(ref: string, opts: MediaTranscribeOptions): Promise<void> {const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;if (start !== undefined && end !== undefined && start >= end) {@@ -522,7 +522,7 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promiconst target = resolveAssetRef(ref);const stop = startSpinner("Transcribing asset");try {- const result = await editor.asset.transcribe.query({ ...target, start, end }, GENERATE);+ const result = await editor.media.transcribe.query({ ...target, start, end }, GENERATE);stop();console.log(JSON.stringify(result));} catch (e) {@@ -531,9 +531,9 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promi}}-type AssetListenOptions = { prompt?: string; start?: string; end?: string };+type MediaListenOptions = { prompt?: string; start?: string; end?: string };-async function assetListen(ref: string, opts: AssetListenOptions): Promise<void> {+async function mediaListen(ref: string, opts: MediaListenOptions): Promise<void> {const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;if (start !== undefined && end !== undefined && start >= end) {@@ -544,7 +544,7 @@ async function assetListen(ref: string, opts: AssetListenOptions): Promise<void>const target = resolveAssetRef(ref);const stop = startSpinner("Analyzing asset");try {- const result = await editor.asset.listen.query(+ const result = await editor.media.listen.query({ ...target, prompt: opts.prompt, start, end, stripVideo: true },GENERATE,);@@ -556,7 +556,7 @@ async function assetListen(ref: string, opts: AssetListenOptions): Promise<void>}}-type AssetVisualizeOptions = { start?: string; end?: string; scale?: string; output?: string };+type MediaVisualizeOptions = { start?: string; end?: string; scale?: string; output?: string };function parseTimeArg(value: string, flag: string): number {const seconds = parseTime(value);@@ -569,7 +569,7 @@ function parseTimeArg(value: string, flag: string): number {return seconds;}-async function assetVisualize(ref: string, opts: AssetVisualizeOptions): Promise<void> {+async function mediaVisualize(ref: string, opts: MediaVisualizeOptions): Promise<void> {const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;if (start !== undefined && end !== undefined && start >= end) {@@ -590,7 +590,7 @@ async function assetVisualize(ref: string, opts: AssetVisualizeOptions): Promiseconst path = opts.output ?? join(tmpdir(), `${randomUUID()}.png`);const stop = startSpinner("Rendering visualization");try {- const { base64, ...rest } = await editor.asset.visualize.query({ ...target, start, end, scale });+ const { base64, ...rest } = await editor.media.visualize.query({ ...target, start, end, scale });stop();writeFileSync(path, Buffer.from(base64, "base64"));console.log(JSON.stringify({ path, ...rest }));@@ -1076,7 +1076,7 @@ media.command("probe").description(`Read the container and per-track technical metadata of a media file. Commonly useful for a quick, free technical read (container, duration, per-track codec params), e.g. checking codec compatibility or duration before cutting${docs("media/probe")}`).argument("<id|path>", "asset id, or a local file")- .action((ref: string) => assetProbe(ref));+ .action((ref: string) => mediaProbe(ref));media.command("transcribe")@@ -1084,7 +1084,7 @@ media.argument("<id|path>", "video or audio asset id, or a local file").option("-s, --start <time>", `start of the range to print — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the range to print — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)- .action((ref: string, opts: AssetTranscribeOptions) => assetTranscribe(ref, opts));+ .action((ref: string, opts: MediaTranscribeOptions) => mediaTranscribe(ref, opts));media.command("grab")@@ -1093,7 +1093,7 @@ media.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`).option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)").option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")- .action((ref: string, opts: AssetFrameOptions) => assetFrame(ref, opts));+ .action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));media.command("visualize")@@ -1104,7 +1104,7 @@ media.option("-e, --end <time>", `end of the window to visualize — seconds, "45f" frames, or "MM:SS" (default: asset duration)`).option("-x, --scale <factor>", "scale factor for the thumbnails; smaller thumbnails fit more rows and columns, larger fit fewer (default: 1)").option("-o, --output <path>", "write the PNG here instead of a temp file")- .action((ref: string, opts: AssetVisualizeOptions) => assetVisualize(ref, opts));+ .action((ref: string, opts: MediaVisualizeOptions) => mediaVisualize(ref, opts));media.command("listen")@@ -1113,7 +1113,7 @@ media.option("-p, --prompt <str>", "question or instruction to guide the analysis").option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`).option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)- .action((ref: string, opts: AssetListenOptions) => assetListen(ref, opts));+ .action((ref: string, opts: MediaListenOptions) => mediaListen(ref, opts));const folder = program.command("folder")diff --git a/apps/web/src/context/editor-api/api.tsx b/apps/web/src/context/editor-api/api.tsxindex 97b7e287..12c950f1 100644--- a/apps/web/src/context/editor-api/api.tsx+++ b/apps/web/src/context/editor-api/api.tsx@@ -8,7 +8,8 @@ import { useEngine } from '@/context/engine';import { useAuth } from '@/context/auth';import { t, q, m, q0, m0 } from "@/lib/cli-rpc";import { handleContextGet } from "./context";-import { handleAssetsAdd, handleAssetsList, handleAssetTree, handleAssetsDelete, handleAssetsMove, handleAssetsExport, handleAssetProbe, handleAssetFrame, handleAssetTranscribe, handleAssetVisualize, handleAssetListen } from "./assets";+import { handleAssetsAdd, handleAssetsList, handleAssetTree, handleAssetsDelete, handleAssetsMove, handleAssetsExport } from "./assets";+import { handleMediaProbe, handleMediaFrame, handleMediaTranscribe, handleMediaVisualize, handleMediaListen } from "./media";import { handleFoldersList, handleFolderCreate, handleFolderRename, handleFoldersMove, handleFoldersDelete } from "./folders";import { handleSelectionFocus, handleSelectionList, handleSelectionSet } from "./selection";import { handleNodeList, handleNodeTree, handleNodeGrep, handleNodeScreenshot, handleNodeDelete, handleNodePatch, handleNodeDuplicate, handleNodeRender } from "./node";@@ -104,11 +105,13 @@ function createAppRouter({ getEngine, getUser, requireAuth, setParams }: AppRoutdelete: m(handleAssetsDelete(getEngine)),move: m(handleAssetsMove(getEngine)),export: m(handleAssetsExport(getEngine)),- probe: q(handleAssetProbe(getEngine)),- frame: q(handleAssetFrame(getEngine)),- transcribe: q(handleAssetTranscribe(getEngine)),- visualize: q(handleAssetVisualize(getEngine)),- listen: q(requireAuth(handleAssetListen(getEngine))),+ }),+ media: t.router({+ probe: q(handleMediaProbe(getEngine)),+ frame: q(handleMediaFrame(getEngine)),+ transcribe: q(handleMediaTranscribe(getEngine)),+ visualize: q(handleMediaVisualize(getEngine)),+ listen: q(requireAuth(handleMediaListen(getEngine))),}),folder: t.router({list: q(handleFoldersList(getEngine)),diff --git a/apps/web/src/context/editor-api/assets.ts b/apps/web/src/context/editor-api/assets.tsindex cd33150e..265e8869 100644--- a/apps/web/src/context/editor-api/assets.ts+++ b/apps/web/src/context/editor-api/assets.ts@@ -2,17 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';import { ElectronFileHandle } from '@/lib/electron-file-handle';import { ElectronWritableFileHandle } from '@/lib/electron-file-writable';-import { trpc } from '@/lib/trpc';-import { uploadBlob, visualizeAsset, loadAsset, removeAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';+import { loadAsset, removeAsset, getAssetFile } from '@/components/engine';import { assert, mimeTypeToExtension } from '@/utils';import {- transcodeForTranscription,- transcodeForAnalysis,- startResumableSession,- uploadResumableStream,assetFolderId,moveAssetsToFolder,useFolders,@@ -20,7 +14,7 @@ import {import type { Engine } from "@/components/engine";import type { Asset } from "@/components/engine/db";-import type { AssetListenRequest, AssetListenResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetRef, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";+import type { AssetExportResult, AssetListResult, AssetMoveResult, AssetRecord, AssetsExportRequest, AssetTreeEntry } from "@diffusionstudio/cli/channels";import type { Accessor } from "solid-js";export function handleAssetsAdd(engine: Accessor<Engine>) {@@ -206,7 +200,7 @@ async function streamBlobToFile(blob: Blob, path: string): Promise<string> {}/**- * Streams the blob chunk-by-chunk to the open writable+ * Streams the blob chunk-by-chunk to the open writable*/async function pumpBlob(blob: Blob,@@ -229,286 +223,3 @@ async function pumpBlob(throw e;}}--/**- * Resolves a command target: an id looks up the project's asset library, a- * path describes the file in place as an ephemeral asset (never added to the- * library). Ephemeral assets carry the path as their id.- */-async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {- if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));- const asset = world.assets.get(ref.id);- assert(asset, `Asset ${ref.id} not found.`);- return asset;-}--const PROBE_SAMPLE_PACKETS = 200;--export function handleAssetProbe(engine: Accessor<Engine>) {- return async (req: AssetProbeRequest): Promise<unknown> => {- const { world } = engine();- const asset = await resolveAssetRef(world, req);-- const blob = await getAssetFile(asset);- const base = {- id: asset.id,- name: asset.name,- type: asset.type,- mimeType: asset.mimeType,- size: blob.size,- ...("width" in asset && { width: asset.width, height: asset.height }),- };-- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });- try {- const format = await input.getFormat();- const mimeType = await input.getMimeType();- const duration = await input.computeDuration();- const { images, ...tags } = await input.getMetadataTags();- delete tags.raw;-- const tracks: Array<Record<string, unknown>> = [];- for (const track of await input.getTracks()) {- const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);- tracks.push({- id: track.id,- type: track.type,- codec: track.codec,- language: track.languageCode,- firstTimestamp: await track.getFirstTimestamp(),- duration: await track.computeDuration(),- ...stats,- ...(track.isVideoTrack() && {- codedWidth: track.codedWidth,- codedHeight: track.codedHeight,- displayWidth: track.displayWidth,- displayHeight: track.displayHeight,- rotation: track.rotation,- }),- ...(track.isAudioTrack() && {- sampleRate: track.sampleRate,- channels: track.numberOfChannels,- }),- });- }-- return {- ...base,- format: format.name,- mimeType,- duration,- tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },- tracks,- };- } catch {- return { ...base, format: null, tracks: [] };- } finally {- input.dispose();- }- };-}--// Default cap on the pixel count of each decoded frame (384x384). Keeps frames-// small enough for vision models without the caller having to know the aspect ratio.-export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;--export function handleAssetFrame(engine: Accessor<Engine>) {- return async (req: AssetFrameRequest) => {- const { times, resolution } = req;- const { world } = engine();- const asset = await resolveAssetRef(world, req);- const id = asset.id;- assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);-- const requested = times && times.length ? times : [0];- for (const t of requested) {- assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);- }-- // A budget of 0 means "native resolution"; anything else caps the pixel count.- const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;-- const blob = await getAssetFile(asset);- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });- try {- const track = await input.getPrimaryVideoTrack();- assert(track, `Asset ${id} has no video track.`);-- // Track timestamps may not start at 0; offset content time by the first.- const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;-- // Downscale to fit the pixel budget while preserving aspect ratio; setting- // only the width lets the sink derive a matching height.- const displayWidth = await track.getDisplayWidth();- const displayHeight = await track.getDisplayHeight();- let width: number | undefined;- if (budget > 0 && displayWidth * displayHeight > budget) {- width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));- }-- // Decode in ascending order (the sink's fast path), remember each- // entry's original slot so output mirrors the requested order.- const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);-- // No pool: each yielded canvas is fresh, so converting to PNG can't race- // the generator's read-ahead reusing a pooled canvas.- const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);- const timestamps = ordered.map(({ time }) => firstTimestamp + time);-- const result: Array<{ time: number; base64: string }> = new Array(requested.length);- let i = 0;- for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {- const { time, index } = ordered[i++];- assert(wrapped, `No frame found at ${time}s.`);-- const ctx = wrapped.canvas.getContext("2d");-- if (ctx) {- const label = formatTimestamp(time, asset.frameRate)- const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));- const fontSize = Math.round(bandHeight * 0.72);- const paddingLeft = Math.round(bandHeight * 0.4);- const y = paddingLeft + bandHeight / 2;-- ctx.font = `normal ${fontSize}px sans-serif`;- ctx.textBaseline = "middle";- ctx.textAlign = "left";-- ctx.lineJoin = "round";- ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));- ctx.strokeStyle = "#000";- ctx.strokeText(label, paddingLeft, y);- ctx.fillStyle = "#FFF";- ctx.fillText(label, paddingLeft, y);- }-- result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };- }- return result;- } finally {- input.dispose();- }- };-}--const transcripts = new Map<string, TranscriptSegment[]>();--export function handleAssetTranscribe(engine: Accessor<Engine>) {- return async (req: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {- const { start, end } = req;- const { world } = engine();- const asset = await resolveAssetRef(world, req);- const id = asset.id;- assert(- asset.type === "AUDIO" || asset.type === "VIDEO",- `Asset ${id} is not a video or audio asset.`,- );-- let transcript = transcripts.get(asset.hash);- if (!transcript) {- const uploadId = crypto.randomUUID();- const audioFile = await transcodeForTranscription(asset);- const fileRef = await uploadBlob(audioFile, uploadId);- if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);-- ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");- }-- transcripts.set(asset.hash, transcript);- }-- return { segments: sliceTranscript(transcript, start, end) };- };-}--function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {- if (start === undefined && end === undefined) return segments;- const from = start ?? 0;- const to = end ?? Infinity;- const sliced: TranscriptSegment[] = [];- for (const segment of segments) {- const words = segment.words.filter((w) => w.end > from && w.start < to);- if (!words.length) continue;- sliced.push(- words.length === segment.words.length- ? segment- : { text: words.map((w) => w.text).join(" "), words },- );- }- return sliced;-}--export function handleAssetVisualize(engine: Accessor<Engine>) {- return async (req: AssetVisualizeRequest): Promise<AssetVisualizeResult> => {- const { world } = engine();- const asset = await resolveAssetRef(world, req);- const { dataUrl, ...rest } = await visualizeAsset(asset, { start: req.start, end: req.end, scale: req.scale });- const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);- return { base64, ...rest };- };-}--export function handleAssetListen(engine: Accessor<Engine>) {- return async (req: AssetListenRequest): Promise<AssetListenResult> => {- const { prompt, start, end } = req;- let { stripVideo } = req;- const { world } = engine();- const asset = await resolveAssetRef(world, req);- const id = asset.id;- assert(- asset.type === "AUDIO" || asset.type === "VIDEO",- `Asset ${id} is not a video or audio asset.`,- );-- const hasWindow = start !== undefined || end !== undefined;- stripVideo = stripVideo !== false && asset.type === "VIDEO";-- const contentType =- asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";-- const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";- const key = world.assets.has(id) ? id : asset.hash;- const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`- .replace(/[^A-Za-z0-9._-]/g, "_");- const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({- action: "resumable",- id: uploadId,- contentType,- });-- // Transcoding pending- if (uploadUrl) {- const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });- const sessionUrl = await startResumableSession(uploadUrl, contentType);- const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);- await transcoder.run?.();- await uploadPromise;- }-- const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });-- return { analysis, start, end };- };-}--async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<string> {- if (canvas instanceof OffscreenCanvas) {- const blob = await canvas.convertToBlob({ type: "image/png" });- return base64FromArrayBuffer(await blob.arrayBuffer());- }- return canvas.toDataURL("image/png").split(",")[1] ?? "";-}--function base64FromArrayBuffer(buf: ArrayBuffer): string {- const bytes = new Uint8Array(buf);- let binary = "";- const chunk = 0x8000;- for (let i = 0; i < bytes.length; i += chunk) {- binary += String.fromCharCode(...bytes.subarray(i, i + chunk));- }- return btoa(binary);-}--// hmr-probe-testdiff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.tsnew file mode 100644index 00000000..b5a78291--- /dev/null+++ b/apps/web/src/context/editor-api/media.ts@@ -0,0 +1,301 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';+import { ElectronFileHandle } from '@/lib/electron-file-handle';+import { trpc } from '@/lib/trpc';+import { uploadBlob, visualizeAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';+import { assert } from '@/utils';+import {+ transcodeForTranscription,+ transcodeForAnalysis,+ startResumableSession,+ uploadResumableStream,+} from '@/components/engine';++import type { Engine } from "@/components/engine";+import type { Asset } from "@/components/engine/db";+import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaProbeRequest, AssetRef, MediaTranscribeRequest, MediaTranscribeResult, MediaVisualizeRequest, MediaVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";+import type { Accessor } from "solid-js";++/**+ * Resolves a command target: an id looks up the project's asset library, a+ * path describes the file in place as an ephemeral asset (never added to the+ * library). Ephemeral assets carry the path as their id.+ */+async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {+ if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));+ const asset = world.assets.get(ref.id);+ assert(asset, `Asset ${ref.id} not found.`);+ return asset;+}++const PROBE_SAMPLE_PACKETS = 200;++export function handleMediaProbe(engine: Accessor<Engine>) {+ return async (req: MediaProbeRequest): Promise<unknown> => {+ const { world } = engine();+ const asset = await resolveAssetRef(world, req);++ const blob = await getAssetFile(asset);+ const base = {+ id: asset.id,+ name: asset.name,+ type: asset.type,+ mimeType: asset.mimeType,+ size: blob.size,+ ...("width" in asset && { width: asset.width, height: asset.height }),+ };++ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const format = await input.getFormat();+ const mimeType = await input.getMimeType();+ const duration = await input.computeDuration();+ const { images, ...tags } = await input.getMetadataTags();+ delete tags.raw;++ const tracks: Array<Record<string, unknown>> = [];+ for (const track of await input.getTracks()) {+ const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);+ tracks.push({+ id: track.id,+ type: track.type,+ codec: track.codec,+ language: track.languageCode,+ firstTimestamp: await track.getFirstTimestamp(),+ duration: await track.computeDuration(),+ ...stats,+ ...(track.isVideoTrack() && {+ codedWidth: track.codedWidth,+ codedHeight: track.codedHeight,+ displayWidth: track.displayWidth,+ displayHeight: track.displayHeight,+ rotation: track.rotation,+ }),+ ...(track.isAudioTrack() && {+ sampleRate: track.sampleRate,+ channels: track.numberOfChannels,+ }),+ });+ }++ return {+ ...base,+ format: format.name,+ mimeType,+ duration,+ tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },+ tracks,+ };+ } catch {+ return { ...base, format: null, tracks: [] };+ } finally {+ input.dispose();+ }+ };+}++// Default cap on the pixel count of each decoded frame (384x384). Keeps frames+// small enough for vision models without the caller having to know the aspect ratio.+export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;++export function handleMediaFrame(engine: Accessor<Engine>) {+ return async (req: MediaFrameRequest) => {+ const { times, resolution } = req;+ const { world } = engine();+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;+ assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);++ const requested = times && times.length ? times : [0];+ for (const t of requested) {+ assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);+ }++ // A budget of 0 means "native resolution"; anything else caps the pixel count.+ const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;++ const blob = await getAssetFile(asset);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const track = await input.getPrimaryVideoTrack();+ assert(track, `Asset ${id} has no video track.`);++ // Track timestamps may not start at 0; offset content time by the first.+ const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;++ // Downscale to fit the pixel budget while preserving aspect ratio; setting+ // only the width lets the sink derive a matching height.+ const displayWidth = await track.getDisplayWidth();+ const displayHeight = await track.getDisplayHeight();+ let width: number | undefined;+ if (budget > 0 && displayWidth * displayHeight > budget) {+ width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));+ }++ // Decode in ascending order (the sink's fast path), remember each+ // entry's original slot so output mirrors the requested order.+ const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);++ // No pool: each yielded canvas is fresh, so converting to PNG can't race+ // the generator's read-ahead reusing a pooled canvas.+ const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);+ const timestamps = ordered.map(({ time }) => firstTimestamp + time);++ const result: Array<{ time: number; base64: string }> = new Array(requested.length);+ let i = 0;+ for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {+ const { time, index } = ordered[i++];+ assert(wrapped, `No frame found at ${time}s.`);++ const ctx = wrapped.canvas.getContext("2d");++ if (ctx) {+ const label = formatTimestamp(time, asset.frameRate)+ const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));+ const fontSize = Math.round(bandHeight * 0.72);+ const paddingLeft = Math.round(bandHeight * 0.4);+ const y = paddingLeft + bandHeight / 2;++ ctx.font = `normal ${fontSize}px sans-serif`;+ ctx.textBaseline = "middle";+ ctx.textAlign = "left";++ ctx.lineJoin = "round";+ ctx.lineWidth = Math.max(2, Math.round(fontSize / 3));+ ctx.strokeStyle = "#000";+ ctx.strokeText(label, paddingLeft, y);+ ctx.fillStyle = "#FFF";+ ctx.fillText(label, paddingLeft, y);+ }++ result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };+ }+ return result;+ } finally {+ input.dispose();+ }+ };+}++const transcripts = new Map<string, TranscriptSegment[]>();++export function handleMediaTranscribe(engine: Accessor<Engine>) {+ return async (req: MediaTranscribeRequest): Promise<MediaTranscribeResult> => {+ const { start, end } = req;+ const { world } = engine();+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;+ assert(+ asset.type === "AUDIO" || asset.type === "VIDEO",+ `Asset ${id} is not a video or audio asset.`,+ );++ let transcript = transcripts.get(asset.hash);+ if (!transcript) {+ const uploadId = crypto.randomUUID();+ const audioFile = await transcodeForTranscription(asset);+ const fileRef = await uploadBlob(audioFile, uploadId);+ if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);++ ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));+ if (!transcript.length || transcript.every((s) => s.words.length === 0)) {+ throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");+ }++ transcripts.set(asset.hash, transcript);+ }++ return { segments: sliceTranscript(transcript, start, end) };+ };+}++function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {+ if (start === undefined && end === undefined) return segments;+ const from = start ?? 0;+ const to = end ?? Infinity;+ const sliced: TranscriptSegment[] = [];+ for (const segment of segments) {+ const words = segment.words.filter((w) => w.end > from && w.start < to);+ if (!words.length) continue;+ sliced.push(+ words.length === segment.words.length+ ? segment+ : { text: words.map((w) => w.text).join(" "), words },+ );+ }+ return sliced;+}++export function handleMediaVisualize(engine: Accessor<Engine>) {+ return async (req: MediaVisualizeRequest): Promise<MediaVisualizeResult> => {+ const { world } = engine();+ const asset = await resolveAssetRef(world, req);+ const { dataUrl, ...rest } = await visualizeAsset(asset, { start: req.start, end: req.end, scale: req.scale });+ const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);+ return { base64, ...rest };+ };+}++export function handleMediaListen(engine: Accessor<Engine>) {+ return async (req: MediaListenRequest): Promise<MediaListenResult> => {+ const { prompt, start, end } = req;+ let { stripVideo } = req;+ const { world } = engine();+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;+ assert(+ asset.type === "AUDIO" || asset.type === "VIDEO",+ `Asset ${id} is not a video or audio asset.`,+ );++ const hasWindow = start !== undefined || end !== undefined;+ stripVideo = stripVideo !== false && asset.type === "VIDEO";++ const contentType =+ asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";++ const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";+ const key = world.assets.has(id) ? id : asset.hash;+ const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`+ .replace(/[^A-Za-z0-9._-]/g, "_");+ const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({+ action: "resumable",+ id: uploadId,+ contentType,+ });++ // Transcoding pending+ if (uploadUrl) {+ const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });+ const sessionUrl = await startResumableSession(uploadUrl, contentType);+ const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);+ await transcoder.run?.();+ await uploadPromise;+ }++ const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });++ return { analysis, start, end };+ };+}++async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<string> {+ if (canvas instanceof OffscreenCanvas) {+ const blob = await canvas.convertToBlob({ type: "image/png" });+ return base64FromArrayBuffer(await blob.arrayBuffer());+ }+ return canvas.toDataURL("image/png").split(",")[1] ?? "";+}++function base64FromArrayBuffer(buf: ArrayBuffer): string {+ const bytes = new Uint8Array(buf);+ let binary = "";+ const chunk = 0x8000;+ for (let i = 0; i < bytes.length; i += chunk) {+ binary += String.fromCharCode(...bytes.subarray(i, i + chunk));+ }+ return btoa(binary);+}