Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance `dapi asset analyze` command with `--strip-video` option for audio-only analysis

konstantin-paulus2e03b6349bb4
diff --git a/apps/cli/reference/asset/analyze.md b/apps/cli/reference/asset/analyze.md
index abeca34e..cfd1e842 100644
--- a/apps/cli/reference/asset/analyze.md
+++ b/apps/cli/reference/asset/analyze.md
@@ -8,6 +8,7 @@ Analyzes an image, video, or audio asset with a multimodal model. With no prompt
- `-p, --prompt <str>`: question or instruction about the asset (optional; defaults to a general description).
- `-s, --start <time>`: start of the segment to analyze, a `Time` value (optional; default `0`). Timestamps in the analysis are relative to this point. Ignored for images.
- `-e, --end <time>`: end of the segment to analyze, a `Time` value (optional; default the asset's duration). Ignored for images.
+- `--strip-video`: analyze only the audio track of a video asset (optional). Significantly faster, since the video stream is neither transcoded nor uploaded. No effect on audio assets; errors on images.
## Output
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index a8f41751..45cacf35 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -159,7 +159,7 @@ export type AssetVisualizeResult = {
base64: string;
} & Record<string, unknown>;
-export type AssetAnalyzeRequest = { id: string; prompt?: string; start?: number; end?: number };
+export type AssetAnalyzeRequest = { id: string; prompt?: string; start?: number; end?: number; stripVideo?: boolean };
export type AssetAnalyzeResult = { id: string; analysis?: string; start?: number; end?: number };
export type GeneratedAsset = { id: string; name: string; type: string };
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index 30974eef..ae395ed3 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -95,8 +95,8 @@ export const cliAPI = {
cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),
assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>
cliRequest(CLI_CHANNELS.ASSET_VISUALIZE, { id, start, end, scale }),
- assetAnalyze: (id: string, prompt?: string, start?: number, end?: number) =>
- cliRequest(CLI_CHANNELS.ASSET_ANALYZE, { id, prompt, start, end }, GENERATE_TIMEOUT_MS),
+ assetAnalyze: (id: string, prompt?: string, start?: number, end?: number, stripVideo?: boolean) =>
+ cliRequest(CLI_CHANNELS.ASSET_ANALYZE, { id, prompt, start, end, stripVideo }, GENERATE_TIMEOUT_MS),
listSelection: () => cliRequest(CLI_CHANNELS.SELECTION_LIST, undefined),
setSelection: (ids: number[]) => cliRequest(CLI_CHANNELS.SELECTION_SET, { ids }),
focusSelection: () => cliRequest(CLI_CHANNELS.SELECTION_FOCUS, undefined),
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 1b6e2f12..22fcfe67 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -532,7 +532,7 @@ async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promi
}
}
-type AssetAnalyzeOptions = { prompt?: string; start?: string; end?: string };
+type AssetAnalyzeOptions = { prompt?: string; start?: string; end?: string; stripVideo?: boolean };
async function assetAnalyze(ref: string, opts: AssetAnalyzeOptions): Promise<void> {
const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
@@ -545,7 +545,7 @@ async function assetAnalyze(ref: string, opts: AssetAnalyzeOptions): Promise<voi
const assetId = await resolveAssetRef(ref);
const stop = startSpinner("Analyzing asset");
try {
- const result = await cliAPI.assetAnalyze(assetId, opts.prompt, start, end);
+ const result = await cliAPI.assetAnalyze(assetId, opts.prompt, start, end, opts.stripVideo);
stop();
console.log(JSON.stringify(result));
} catch (e) {
@@ -1097,6 +1097,7 @@ asset
.option("-p, --prompt <str>", "question or instruction to guide the analysis")
.option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`)
.option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
+ .option("--strip-video", "analyze only the audio track of a video; significantly faster since the video stream is not uploaded")
.action((ref: string, opts: AssetAnalyzeOptions) => assetAnalyze(ref, opts));
const folder = program
diff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.ts
index 1474c7ed..c2595fc5 100644
--- a/apps/web/src/components/engine/utils/transcode.ts
+++ b/apps/web/src/components/engine/utils/transcode.ts
@@ -31,7 +31,7 @@ export async function transcodeForTranscription(asset: Asset): Promise<File> {
}
}
-export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {
+export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow & { stripVideo?: boolean }) {
const blob = await asset.handle.getFile();
const hasWindow = window?.start !== undefined || window?.end !== undefined;
@@ -39,7 +39,7 @@ export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {
return { readable: blob.stream() as ReadableStream<Uint8Array<ArrayBuffer>> };
}
- const isVideo = asset.type === "VIDEO";
+ const isVideo = asset.type === "VIDEO" && !window?.stripVideo;
const trim = hasWindow ? resolveWindow(asset.duration, window) : undefined;
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
@@ -64,7 +64,11 @@ export async function transcodeForAnalysis(asset: Asset, window?: TimeWindow) {
if (!conversion.isValid || conversion.utilizedTracks.length === 0) {
input.dispose();
- throw new Error("Nothing to analyze. The asset has no usable video or audio track.");
+ throw new Error(
+ window?.stripVideo
+ ? "Nothing to analyze. The asset has no audio track; retry without --strip-video."
+ : "Nothing to analyze. The asset has no usable video or audio track.",
+ );
}
const run = async () => {
diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.ts
index 67f0f808..27289905 100644
--- a/apps/web/src/context/electron/assets.ts
+++ b/apps/web/src/context/electron/assets.ts
@@ -399,7 +399,7 @@ export function handleAssetVisualize(engine: Accessor<Engine>) {
}
export function handleAssetAnalyze(engine: Accessor<Engine>) {
- return async ({ id, prompt, start, end }: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {
+ return async ({ id, prompt, start, end, stripVideo }: AssetAnalyzeRequest): Promise<AssetAnalyzeResult> => {
const { world } = engine();
const asset = world.assets.get(id);
assert(asset, `Asset ${id} not found.`);
@@ -407,18 +407,23 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {
asset.type === "IMAGE" || asset.type === "AUDIO" || asset.type === "VIDEO",
`Asset ${id} is not an image, audio, or video asset.`,
);
+ assert(
+ !stripVideo || asset.type === "AUDIO" || asset.type === "VIDEO",
+ `Asset ${id} is an image; --strip-video requires an audio or video asset.`,
+ );
const hasWindow = start !== undefined || end !== undefined;
+ stripVideo = stripVideo === true && asset.type === "VIDEO";
let contentType = asset.mimeType;
if (asset.type === "VIDEO") {
- contentType = "video/mp4";
+ contentType = stripVideo ? "audio/ogg" : "video/mp4";
} else if (asset.type === "AUDIO") {
contentType = "audio/ogg";
}
const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";
- const uploadId = `${world.projectId}-${asset.id}-analyze${window}`;
+ const uploadId = `${world.projectId}-${asset.id}-analyze${stripVideo ? "-audio" : ""}${window}`;
const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({
action: "resumable",
id: uploadId,
@@ -427,7 +432,7 @@ export function handleAssetAnalyze(engine: Accessor<Engine>) {
// Transcoding pending
if (uploadUrl) {
- const transcoder = await transcodeForAnalysis(asset, { start, end });
+ const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });
const sessionUrl = await startResumableSession(uploadUrl, contentType);
const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);
await transcoder.run?.();