The VS Code of video editing, built for agents.
TypeScript
Rename `dapi asset frame` command to `dapi asset grab` for clarity and update related documentation. Modify input parameters across various asset commands to indicate that local files can be analyzed or visualized without being added to the library. Enhance type safety in the CLI by introducing a unified `AssetRef` type for asset references, streamlining asset handling in the codebase.
konstantin-paulusf392e728190e
diff --git a/apps/cli/reference/README.md b/apps/cli/reference/README.mdindex e6d9e4b9..ca5e709f 100644--- a/apps/cli/reference/README.md+++ b/apps/cli/reference/README.md@@ -72,7 +72,7 @@ How the surface is divided:- [`dapi asset export`](./asset/export.md): write assets' original bytes to disk- [`dapi asset probe`](./asset/probe.md): container and track metadata- [`dapi asset transcribe`](./asset/transcribe.md): timed speech transcript-- [`dapi asset frame`](./asset/frame.md): decode video frames to PNGs+- [`dapi asset grab`](./asset/grab.md): decode video frames to PNGs- [`dapi asset visualize`](./asset/visualize.md): waveform / filmstrip / thumbnail PNG- [`dapi asset analyze`](./asset/analyze.md): AI description of an assetdiff --git a/apps/cli/reference/asset/analyze.md b/apps/cli/reference/asset/analyze.mdindex cfd1e842..113ed1cf 100644--- a/apps/cli/reference/asset/analyze.md+++ b/apps/cli/reference/asset/analyze.md@@ -4,7 +4,7 @@ Analyzes an image, video, or audio asset with a multimodal model. With no prompt## Input-- `<id|path>`: an image, audio, or video asset id, or a local file to add and analyze (required).+- `<id|path>`: an image, audio, or video asset id, or a local file to analyze in place without adding it to the library (required).- `-p, --prompt <str>`: question or instruction about the asset (optional; defaults to a general description).- `-s, --start <time>`: start of the segment to analyze, a `Time` value (optional; default `0`). Timestamps in the analysis are relative to this point. Ignored for images.- `-e, --end <time>`: end of the segment to analyze, a `Time` value (optional; default the asset's duration). Ignored for images.@@ -15,7 +15,7 @@ Analyzes an image, video, or audio asset with a multimodal model. With no promptOne JSON object, the model's answer:```ts-{ id: string; analysis: string } // id = the analyzed asset id+{ analysis: string }```## Errorsdiff --git a/apps/cli/reference/asset/frame.md b/apps/cli/reference/asset/grab.mdsimilarity index 86%rename from apps/cli/reference/asset/frame.mdrename to apps/cli/reference/asset/grab.mdindex 8035d3e9..078179a1 100644--- a/apps/cli/reference/asset/frame.md+++ b/apps/cli/reference/asset/grab.md@@ -1,10 +1,10 @@-# `dapi asset frame <id|path>`+# `dapi asset grab <id|path>`Decodes one or more frames of a video asset at the given times and writes each to a PNG file; like [`node screenshot`](../node/screenshot.md), but grabs the asset's own pixels at full resolution (unlike `node screenshot`, which captures the composited canvas). Renders locally; no credits.## Input-- `<id|path>`: a video asset id, or a local video file to add and grab frames from (required).+- `<id|path>`: a video asset id, or a local video file to grab frames from in place without adding it to the library (required).- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). Order is preserved in the output regardless of the order given.- `-o, --output <dir>`: directory to write the PNGs into (optional; default the system temp directory). Each frame is written as a uuid-named `.png`.diff --git a/apps/cli/reference/asset/probe.md b/apps/cli/reference/asset/probe.mdindex 54e0edc6..9650106e 100644--- a/apps/cli/reference/asset/probe.md+++ b/apps/cli/reference/asset/probe.md@@ -4,7 +4,7 @@ Reads the container and per-track technical metadata of an asset; like `ffprobe`## Input-- `<id|path>`: an asset id, or a local file to add and probe (required). Any asset type is accepted.+- `<id|path>`: an asset id, or a local file to probe in place without adding it to the library (required). Any asset type is accepted.## Outputdiff --git a/apps/cli/reference/asset/transcribe.md b/apps/cli/reference/asset/transcribe.mdindex 7a03b056..36d88bed 100644--- a/apps/cli/reference/asset/transcribe.md+++ b/apps/cli/reference/asset/transcribe.md@@ -4,14 +4,14 @@ Transcribes the speech in a video or audio asset and returns the timed transcrip## Input-- `<id|path>`: a video or audio asset id, or a local file to add and transcribe (required).+- `<id|path>`: a video or audio asset id, or a local file to transcribe in place without adding it to the library (required).## Options- `-s, --start <time>`: start of the range to print — seconds (`"1.5"`), frames (`"45f"`), or `"MM:SS"` (default: `0`).- `-e, --end <time>`: end of the range to print — same formats (default: the asset duration).-The whole asset is always transcribed once (and cached on the asset); the range only limits which words are printed. Words keep their absolute source-time timestamps, and segments that straddle a range boundary are trimmed to the words inside it. A range with no speech yields `segments: []`.+The whole asset is always transcribed once per app session (cached in memory, keyed by file content, for ids and paths alike; an app restart or an edited file re-transcribes); the range only limits which words are printed. Words keep their absolute source-time timestamps, and segments that straddle a range boundary are trimmed to the words inside it. A range with no speech yields `segments: []`.## Output@@ -19,7 +19,6 @@ One JSON object, the transcript:```ts{- id: string; // the asset idsegments: Array<{text: string; // spoken words only (no silence markers)words: Array<{ text: string; start: number; end: number }>; // secondsdiff --git a/apps/cli/reference/asset/visualize.md b/apps/cli/reference/asset/visualize.mdindex 8f70297a..961dbd8c 100644--- a/apps/cli/reference/asset/visualize.md+++ b/apps/cli/reference/asset/visualize.md@@ -12,7 +12,7 @@ Tick labels use `HH:MM:SS:FF` timecode (hours, minutes, seconds, frame within th## Input-- `<id|path>`: an image, audio, or video asset id, or a local file to add and visualize (required).+- `<id|path>`: an image, audio, or video asset id, or a local file to visualize in place without adding it to the library (required).- `-s, --start <time>`: start of the window to visualize, a `Time` value in source/content time (optional; default `0`). Ignored for images.- `-e, --end <time>`: end of the window to visualize, a `Time` value (optional; default the asset's duration). Ignored for images.- `-x, --scale <factor>`: scale factor for the thumbnails (optional; default `1`, clamped to `0.25`-`4`). The overall canvas size stays fixed, so smaller thumbnails fit **more rows and columns** (a denser grid sampling more moments) and larger thumbnails fit fewer but show more detail each. For images, scales the output resolution instead (never above the source size).diff --git a/apps/cli/reference/jsx/timing.md b/apps/cli/reference/jsx/timing.mdindex d15ac98b..2f092072 100644--- a/apps/cli/reference/jsx/timing.md+++ b/apps/cli/reference/jsx/timing.md@@ -31,4 +31,4 @@ The canonical internal unit is frames at **30 fps**; all formats are converted o| `"MM:SS"` | `"02:30"` | Minutes and seconds. || `"HH:MM:SS"` | `"01:02:30"` | Hours, minutes, seconds. |-CLI flags documented as taking a `Time` value (e.g. `--time` on [`node screenshot`](../node/screenshot.md) and [`asset frame`](../asset/frame.md)) accept the same formats.+CLI flags documented as taking a `Time` value (e.g. `--time` on [`node screenshot`](../node/screenshot.md) and [`asset grab`](../asset/grab.md)) accept the same formats.diff --git a/apps/cli/reference/node/screenshot.md b/apps/cli/reference/node/screenshot.mdindex 534b9677..e34631fc 100644--- a/apps/cli/reference/node/screenshot.md+++ b/apps/cli/reference/node/screenshot.md@@ -1,6 +1,6 @@# `dapi node screenshot [id] [-t, --time <time>]`-Focuses the node on the canvas and captures a screenshot, written to a PNG file in the system temp directory. If `[id]` is omitted, captures the canvas (the active scene). To grab a video asset's own pixels at full resolution instead of the composited canvas, use [`asset frame`](../asset/frame.md).+Focuses the node on the canvas and captures a screenshot, written to a PNG file in the system temp directory. If `[id]` is omitted, captures the canvas (the active scene). To grab a video asset's own pixels at full resolution instead of the composited canvas, use [`asset grab`](../asset/grab.md).## Inputdiff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 3f595774..bac80a4f 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -117,23 +117,25 @@ export type EncoderConfigInput = {export type NodeRenderRequest = { id?: number; output: string; config?: EncoderConfigInput };export type NodeRenderResult = { path: string };-export type AssetProbeRequest = { id: string };+export type AssetRef = { id: string } | { path: string };-export type AssetFrameRequest = { id: string; times?: number[]; resolution?: number };+export type AssetProbeRequest = AssetRef;++export type AssetFrameRequest = AssetRef & { times?: number[]; resolution?: number };export type AssetFrameResult = Array<{ time: number; base64: string }>;-export type AssetTranscribeRequest = { id: string; start?: number; end?: number };+export type AssetTranscribeRequest = AssetRef & { start?: number; end?: number };export type TranscriptWord = { text: string; start: number; end: number };export type TranscriptSegment = { text: string; words: TranscriptWord[] };-export type AssetTranscribeResult = { id: string; segments: TranscriptSegment[] };+export type AssetTranscribeResult = { segments: TranscriptSegment[] };-export type AssetVisualizeRequest = { id: string; start?: number; end?: number; scale?: number };+export type AssetVisualizeRequest = AssetRef & { start?: number; end?: number; scale?: number };export type AssetVisualizeResult = {base64: string;} & Record<string, unknown>;-export type AssetAnalyzeRequest = { id: string; prompt?: string; start?: number; end?: number; stripVideo?: boolean };-export type AssetAnalyzeResult = { id: string; analysis?: string; start?: number; end?: number };+export type AssetAnalyzeRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };+export type AssetAnalyzeResult = { analysis?: string; start?: number; end?: number };export type GeneratedAsset = { id: string; name: string; type: string };diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex b1879314..ad0ff3dc 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -16,7 +16,7 @@ import { compileProject } from "./compile-project";import { listLocalFonts } from "./fonts";import { openFolder } from "./open-folder";import { fetchVideo } from "./ytdlp";-import type { EncoderConfigInput, NodePatch } from "./protocol";+import type { AssetRef, EncoderConfigInput, NodePatch } from "./protocol";// Long-running commands (renders, AI generation) override the default 60s.const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } };@@ -467,10 +467,10 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {}}- const assetId = await resolveAssetRef(ref);+ const target = resolveAssetRef(ref);const dir = opts.output ?? tmpdir();try {- const frames = await editor.asset.frame.query({ id: assetId, times, resolution });+ const frames = await editor.asset.frame.query({ ...target, times, resolution });for (const { time, base64 } of frames) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));@@ -481,8 +481,8 @@ async function assetFrame(ref: string, opts: AssetFrameOptions): Promise<void> {}}-async function resolveAssetRef(ref: string): Promise<string> {- if (/^[A-Za-z0-9]+$/.test(ref)) return ref;+function resolveAssetRef(ref: string): AssetRef {+ if (/^[A-Za-z0-9]+$/.test(ref)) return { id: ref };const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref);if (!existsSync(absPath)) {@@ -493,32 +493,14 @@ async function resolveAssetRef(ref: string): Promise<string> {console.error(`Not a file: ${absPath}`);process.exit(1);}-- const stop = startSpinner("Adding asset");- try {- // The handler types per-item results as a loose union; narrow by hand.- const [result] = (await editor.asset.add.mutate({ paths: [absPath] })) as Array<{- status: string;- id?: unknown;- error?: string;- }>;- stop();- if (!result || result.status !== "fulfilled" || typeof result.id !== "string") {- console.error(result?.error ?? `Failed to add asset: ${absPath}`);- process.exit(1);- }- return result.id;- } catch (e) {- stop();- handleSocketError(e);- }+ return { path: absPath };}async function assetProbe(ref: string): Promise<void> {- const assetId = await resolveAssetRef(ref);+ const target = resolveAssetRef(ref);const stop = startSpinner("Probing asset");try {- const result = await editor.asset.probe.query({ id: assetId });+ const result = await editor.asset.probe.query(target);stop();console.log(JSON.stringify(result));} catch (e) {@@ -537,10 +519,10 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promiprocess.exit(1);}- const assetId = await resolveAssetRef(ref);+ const target = resolveAssetRef(ref);const stop = startSpinner("Transcribing asset");try {- const result = await editor.asset.transcribe.query({ id: assetId, start, end }, GENERATE);+ const result = await editor.asset.transcribe.query({ ...target, start, end }, GENERATE);stop();console.log(JSON.stringify(result));} catch (e) {@@ -559,11 +541,11 @@ async function assetAnalyze(ref: string, opts: AssetAnalyzeOptions): Promise<voiprocess.exit(1);}- const assetId = await resolveAssetRef(ref);+ const target = resolveAssetRef(ref);const stop = startSpinner("Analyzing asset");try {const result = await editor.asset.analyze.query(- { id: assetId, prompt: opts.prompt, start, end, stripVideo: opts.stripVideo },+ { ...target, prompt: opts.prompt, start, end, stripVideo: opts.stripVideo },GENERATE,);stop();@@ -604,11 +586,11 @@ async function assetVisualize(ref: string, opts: AssetVisualizeOptions): Promise}}- const assetId = await resolveAssetRef(ref);+ const target = resolveAssetRef(ref);const path = opts.output ?? join(tmpdir(), `${randomUUID()}.png`);const stop = startSpinner("Rendering visualization");try {- const { base64, ...rest } = await editor.asset.visualize.query({ id: assetId, start, end, scale });+ const { base64, ...rest } = await editor.asset.visualize.query({ ...target, start, end, scale });stop();writeFileSync(path, Buffer.from(base64, "base64"));console.log(JSON.stringify({ path, ...rest }));@@ -1080,21 +1062,21 @@ assetasset.command("probe").description(`Read the container and per-track technical metadata of an asset${docs("asset/probe")}`)- .argument("<id|path>", "asset id, or a local file to add and probe")+ .argument("<id|path>", "asset id, or a local file").action((ref: string) => assetProbe(ref));asset.command("transcribe").description(`Transcribe the speech in a video or audio asset and print the timed transcript${docs("asset/transcribe")}`)- .argument("<id|path>", "video or audio asset id, or a local file to add and transcribe")+ .argument("<id|path>", "video or audio asset id, or a local file").option("-s, --start <time>", `start of the range to print — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the range to print — seconds, "45f" frames, or "MM:SS" (default: asset duration)`).action((ref: string, opts: AssetTranscribeOptions) => assetTranscribe(ref, opts));asset- .command("frame")- .description(`Decode one or more frames of a video asset and write them as PNGs${docs("asset/frame")}`)- .argument("<id|path>", "video asset id, or a local file to add and grab frames from")+ .command("grab")+ .description(`Decode one or more frames of a video asset and write them as PNGs${docs("asset/grab")}`)+ .argument("<id|path>", "video asset id, or a local video file to grab frames from").option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`).option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)").option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")@@ -1104,7 +1086,7 @@ asset.command("visualize").alias("viz").description(`Render a visual preview of an asset (waveform, filmstrip, or thumbnail) to a PNG${docs("asset/visualize")}`)- .argument("<id|path>", "image, audio, or video asset id, or a local file to add and visualize")+ .argument("<id|path>", "image, audio, or video asset id, or a local file to visualize").option("-s, --start <time>", `start of the window to visualize — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the window to visualize — seconds, "45f" frames, or "MM:SS" (default: asset duration)`).option("-x, --scale <factor>", "scale factor for the thumbnails; smaller thumbnails fit more rows and columns, larger fit fewer (default: 1)")@@ -1114,7 +1096,7 @@ assetasset.command("analyze").description(`Analyze an image, audio, or video asset with AI and print a description of its contents${docs("asset/analyze")}`)- .argument("<id|path>", "image, audio, or video asset id, or a local file to add and analyze")+ .argument("<id|path>", "image, audio, or video asset id, or a local file to analyze").option("-p, --prompt <str>", "question or instruction to guide the analysis").option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`).option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)diff --git a/apps/web/src/components/engine/api/assets.ts b/apps/web/src/components/engine/api/assets.tsindex e6624663..56ade102 100644--- a/apps/web/src/components/engine/api/assets.ts+++ b/apps/web/src/components/engine/api/assets.ts@@ -293,22 +293,25 @@ export async function loadAsset(duration,directoryHandle: handle,});- } else if (mimeType.match(/^image\//)) {+ }++ const metadata = await probeMediaMetadata(file, mimeType);+ return saveAsset(world, { ...baseAsset, ...metadata });+}++/**+ * Derives the type-specific asset properties (dimensions, duration, tracks)+ * of a media file.+ */+async function probeMediaMetadata(file: Blob, mimeType: string) {+ if (mimeType.match(/^image\//)) {const { width, height } = mimeType === 'image/svg+xml'? await getSvgDimensions(file): await getRasterDimensions(file);- return saveAsset(world, {- ...baseAsset,- type: 'IMAGE',- width,- height,- });+ return { type: 'IMAGE', width, height } as const;} else if (mimeType === 'application/json') {- return saveAsset(world, {- ...baseAsset,- type: 'TRANSCRIPT',- });+ return { type: 'TRANSCRIPT' } as const;} else if (mimeType.match(/^(audio\/|video\/)/)) {const input = new Input({formats: ALL_FORMATS,@@ -325,13 +328,7 @@ export async function loadAsset(const duration = await trackContentDuration(audioTrack);- return saveAsset(world, {- ...baseAsset,- type: 'AUDIO',- duration,- sampleRate,- channels,- });+ return { type: 'AUDIO', duration, sampleRate, channels } as const;}const videoTrack = await input.getPrimaryVideoTrack();@@ -339,8 +336,7 @@ export async function loadAsset(const stats = await videoTrack.computePacketStats();const duration = await trackContentDuration(videoTrack);- return saveAsset(world, {- ...baseAsset,+ return {type: 'VIDEO',width: videoTrack.displayWidth,height: videoTrack.displayHeight,@@ -349,12 +345,38 @@ export async function loadAsset(sampleRate,duration,channels,- });+ } as const;}assert(false, 'Unsupported file type');}+/**+ * Describes a local file as an ephemeral Asset: full metadata, but no id+ * allocation and no persistence — the library is never touched. The path+ * doubles as the id so errors and results name the file.+ */+export async function describeFileAsset(handle: ElectronFileHandle): Promise<Asset> {+ const file = await handle.getFile();+ const mimeType = await detectMimeType(handle);+ assert(mimeType, 'Could not detect MIME type');++ return {+ id: handle.path,+ hash: `${file.name}-${file.size}-${file.lastModified}`,+ createdAt: new Date().toISOString(),+ lastModified: file.lastModified,+ name: handle.name,+ mimeType,+ size: file.size,+ handle,+ generationId: null,+ generationKey: null,+ folderId: null,+ ...(await probeMediaMetadata(file, mimeType)),+ };+}+/*** Reactive view over the asset set, for use *inside* Solid tracking scopes* (memos, effects, JSX). Each accessor reads the version signal so the scopediff --git a/apps/web/src/context/editor-api/assets.ts b/apps/web/src/context/editor-api/assets.tsindex a8e831c4..6ecc32af 100644--- a/apps/web/src/context/editor-api/assets.ts+++ b/apps/web/src/context/editor-api/assets.ts@@ -6,7 +6,7 @@ import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';import { ElectronFileHandle } from '@/lib/electron-file-handle';import { ElectronWritableFileHandle } from '@/lib/electron-file-writable';import { trpc } from '@/lib/trpc';-import { uploadBlob, visualizeAsset, loadAsset, removeAsset, saveAsset } from '@/components/engine';+import { uploadBlob, visualizeAsset, loadAsset, removeAsset, describeFileAsset } from '@/components/engine';import { assert, mimeTypeToExtension } from '@/utils';import {transcodeForTranscription,@@ -20,7 +20,7 @@ import {import type { Engine } from "@/components/engine";import type { Asset } from "@/components/engine/db";-import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";+import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetFrameRequest, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetRef, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";import type { Accessor } from "solid-js";export function handleAssetsAdd(engine: Accessor<Engine>) {@@ -217,7 +217,7 @@ async function pumpBlob(const reader = blob.stream().getReader();let position = 0;try {- for (;;) {+ for (; ;) {const { done, value } = await reader.read();if (done) break;await writer.write({ type: "write", data: value, position });@@ -225,22 +225,33 @@ async function pumpBlob(}await writer.close();} catch (e) {- await handle.dispose().catch(() => {});+ await handle.dispose().catch(() => { });throw e;}}+/**+ * Resolves a command target: an id looks up the project's asset library, a+ * path describes the file in place as an ephemeral asset (never added to the+ * library). Ephemeral assets carry the path as their id.+ */+async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {+ if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));+ const asset = world.assets.get(ref.id);+ assert(asset, `Asset ${ref.id} not found.`);+ return asset;+}+const PROBE_SAMPLE_PACKETS = 200;export function handleAssetProbe(engine: Accessor<Engine>) {- return async ({ id }: AssetProbeRequest): Promise<unknown> => {+ return async (req: AssetProbeRequest): Promise<unknown> => {const { world } = engine();- const asset = world.assets.get(id);- assert(asset, `Asset ${id} not found.`);+ const asset = await resolveAssetRef(world, req);const blob = await asset.handle.getFile();const base = {- id,+ id: asset.id,name: asset.name,type: asset.type,mimeType: asset.mimeType,@@ -302,10 +313,11 @@ export function handleAssetProbe(engine: Accessor<Engine>) {export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;export function handleAssetFrame(engine: Accessor<Engine>) {- return async ({ id, times, resolution }: AssetFrameRequest) => {+ return async (req: AssetFrameRequest) => {+ const { times, resolution } = req;const { world } = engine();- const asset = world.assets.get(id);- assert(asset, `Asset ${id} not found.`);+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);const requested = times && times.length ? times : [0];@@ -357,17 +369,20 @@ export function handleAssetFrame(engine: Accessor<Engine>) {};}+const transcripts = new Map<string, TranscriptSegment[]>();+export function handleAssetTranscribe(engine: Accessor<Engine>) {- return async ({ id, start, end }: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {+ return async (req: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {+ const { start, end } = req;const { world } = engine();- const asset = world.assets.get(id);- assert(asset, `Asset ${id} not found.`);+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;assert(asset.type === "AUDIO" || asset.type === "VIDEO",`Asset ${id} is not a video or audio asset.`,);- let transcript = asset.transcript;+ let transcript = transcripts.get(asset.hash);if (!transcript) {const uploadId = crypto.randomUUID();const audioFile = await transcodeForTranscription(asset);@@ -379,10 +394,10 @@ export function handleAssetTranscribe(engine: Accessor<Engine>) {throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");}- await saveAsset(world, { ...asset, transcript });+ transcripts.set(asset.hash, transcript);}- return { id, segments: sliceTranscript(transcript, start, end) };+ return { segments: sliceTranscript(transcript, start, end) };};}@@ -406,8 +421,7 @@ function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: nuexport function handleAssetVisualize(engine: Accessor<Engine>) {return async (req: AssetVisualizeRequest): Promise<AssetVisualizeResult> => {const { world } = engine();- const asset = world.assets.get(req.id);- assert(asset, `Asset ${req.id} not found.`);+ const asset = await resolveAssetRef(world, req);const { dataUrl, ...rest } = await visualizeAsset(asset, { start: req.start, end: req.end, scale: req.scale });const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);return { base64, ...rest };@@ -415,10 +429,12 @@ export function handleAssetVisualize(engine: Accessor<Engine>) {}export function handleAssetAnalyze(engine: Accessor<Engine>) {- return async ({ id, prompt, start, end, stripVideo }: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {+ return async (req: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {+ const { prompt, start, end } = req;+ let { stripVideo } = req;const { world } = engine();- const asset = world.assets.get(id);- assert(asset, `Asset ${id} not found.`);+ const asset = await resolveAssetRef(world, req);+ const id = asset.id;assert(asset.type === "IMAGE" || asset.type === "AUDIO" || asset.type === "VIDEO",`Asset ${id} is not an image, audio, or video asset.`,@@ -439,7 +455,9 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {}const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";- const uploadId = `${world.projectId}-${asset.id}-analyze${stripVideo ? "-audio" : ""}${window}`;+ const key = world.assets.has(id) ? id : asset.hash;+ const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`+ .replace(/[^A-Za-z0-9._-]/g, "_");const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({action: "resumable",id: uploadId,@@ -457,7 +475,7 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });- return { id, analysis, start, end };+ return { analysis, start, end };};}