The VS Code of video editing, built for agents.
TypeScript
Enhance `dapi asset analyze` command with `--strip-video` option for audio-only analysis
konstantin-paulus2e03b6349bb4
diff --git a/apps/cli/reference/asset/analyze.md b/apps/cli/reference/asset/analyze.mdindex abeca34e..cfd1e842 100644--- a/apps/cli/reference/asset/analyze.md+++ b/apps/cli/reference/asset/analyze.md@@ -8,6 +8,7 @@ Analyzes an image, video, or audio asset with a multimodal model. With no prompt- `-p, --prompt <str>`: question or instruction about the asset (optional; defaults to a general description).- `-s, --start <time>`: start of the segment to analyze, a `Time` value (optional; default `0`). Timestamps in the analysis are relative to this point. Ignored for images.- `-e, --end <time>`: end of the segment to analyze, a `Time` value (optional; default the asset's duration). Ignored for images.+- `--strip-video`: analyze only the audio track of a video asset (optional). Significantly faster, since the video stream is neither transcoded nor uploaded. No effect on audio assets; errors on images.## Outputdiff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex a8f41751..45cacf35 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -159,7 +159,7 @@ export type AssetVisualizeResult = {base64: string;} & Record<string, unknown>;-export type AssetAnalyzeRequest = { id: string; prompt?: string; start?: number; end?: number };+export type AssetAnalyzeRequest = { id: string; prompt?: string; start?: number; end?: number; stripVideo?: boolean };export type AssetAnalyzeResult = { id: string; analysis?: string; start?: number; end?: number };export type GeneratedAsset = { id: string; name: string; type: string };diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 30974eef..ae395ed3 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -95,8 +95,8 @@ export const cliAPI = {cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>cliRequest(CLI_CHANNELS.ASSET_VISUALIZE, { id, start, end, scale }),- assetAnalyze: (id: string, prompt?: string, start?: number, end?: number) =>- cliRequest(CLI_CHANNELS.ASSET_ANALYZE, { id, prompt, start, end }, GENERATE_TIMEOUT_MS),+ assetAnalyze: (id: string, prompt?: string, start?: number, end?: number, stripVideo?: boolean) =>+ cliRequest(CLI_CHANNELS.ASSET_ANALYZE, { id, prompt, start, end, stripVideo }, GENERATE_TIMEOUT_MS),listSelection: () => cliRequest(CLI_CHANNELS.SELECTION_LIST, undefined),setSelection: (ids: number[]) => cliRequest(CLI_CHANNELS.SELECTION_SET, { ids }),focusSelection: () => cliRequest(CLI_CHANNELS.SELECTION_FOCUS, undefined),diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 1b6e2f12..22fcfe67 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -532,7 +532,7 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promi}}-type AssetAnalyzeOptions = { prompt?: string; start?: string; end?: string };+type AssetAnalyzeOptions = { prompt?: string; start?: string; end?: string; stripVideo?: boolean };async function assetAnalyze(ref: string, opts: AssetAnalyzeOptions): Promise<void> {const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;@@ -545,7 +545,7 @@ async function assetAnalyze(ref: string, opts: AssetAnalyzeOptions): Promise<voiconst assetId = await resolveAssetRef(ref);const stop = startSpinner("Analyzing asset");try {- const result = await cliAPI.assetAnalyze(assetId, opts.prompt, start, end);+ const result = await cliAPI.assetAnalyze(assetId, opts.prompt, start, end, opts.stripVideo);stop();console.log(JSON.stringify(result));} catch (e) {@@ -1097,6 +1097,7 @@ asset.option("-p, --prompt <str>", "question or instruction to guide the analysis").option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`).option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)+ .option("--strip-video", "analyze only the audio track of a video; significantly faster since the video stream is not uploaded").action((ref: string, opts: AssetAnalyzeOptions) => assetAnalyze(ref, opts));const folder = programdiff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.tsindex 1474c7ed..c2595fc5 100644--- a/apps/web/src/components/engine/utils/transcode.ts+++ b/apps/web/src/components/engine/utils/transcode.ts@@ -31,7 +31,7 @@ export async function transcodeForTranscription(asset: Asset): Promise<File> {}}-export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {+export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow & { stripVideo?: boolean }) {const blob = await asset.handle.getFile();const hasWindow = window?.start !== undefined || window?.end !== undefined;@@ -39,7 +39,7 @@ export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {return { readable: blob.stream() as ReadableStream<Uint8Array<ArrayBuffer>> };}- const isVideo = asset.type === "VIDEO";+ const isVideo = asset.type === "VIDEO" && !window?.stripVideo;const trim = hasWindow ? resolveWindow(asset.duration, window) : undefined;const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });@@ -64,7 +64,11 @@ export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {if (!conversion.isValid || conversion.utilizedTracks.length === 0) {input.dispose();- throw new Error("Nothing to analyze. The asset has no usable video or audio track.");+ throw new Error(+ window?.stripVideo+ ? "Nothing to analyze. The asset has no audio track; retry without --strip-video."+ : "Nothing to analyze. The asset has no usable video or audio track.",+ );}const run = async () => {diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.tsindex 67f0f808..27289905 100644--- a/apps/web/src/context/electron/assets.ts+++ b/apps/web/src/context/electron/assets.ts@@ -399,7 +399,7 @@ export function handleAssetVisualize(engine: Accessor<Engine>) {}export function handleAssetAnalyze(engine: Accessor<Engine>) {- return async ({ id, prompt, start, end }: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {+ return async ({ id, prompt, start, end, stripVideo }: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {const { world } = engine();const asset = world.assets.get(id);assert(asset, `Asset ${id} not found.`);@@ -407,18 +407,23 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {asset.type === "IMAGE" || asset.type === "AUDIO" || asset.type === "VIDEO",`Asset ${id} is not an image, audio, or video asset.`,);+ assert(+ !stripVideo || asset.type === "AUDIO" || asset.type === "VIDEO",+ `Asset ${id} is an image; --strip-video requires an audio or video asset.`,+ );const hasWindow = start !== undefined || end !== undefined;+ stripVideo = stripVideo === true && asset.type === "VIDEO";let contentType = asset.mimeType;if (asset.type === "VIDEO") {- contentType = "video/mp4";+ contentType = stripVideo ? "audio/ogg" : "video/mp4";} else if (asset.type === "AUDIO") {contentType = "audio/ogg";}const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";- const uploadId = `${world.projectId}-${asset.id}-analyze${window}`;+ const uploadId = `${world.projectId}-${asset.id}-analyze${stripVideo ? "-audio" : ""}${window}`;const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({action: "resumable",id: uploadId,@@ -427,7 +432,7 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {// Transcoding pendingif (uploadUrl) {- const transcoder = await transcodeForAnalysis(asset, { start, end });+ const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });const sessionUrl = await startResumableSession(uploadUrl, contentType);const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);await transcoder.run?.();