Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance `dapi asset transcribe` command with options for time range selection

konstantin-paulus7414220589a1
diff --git a/apps/cli/reference/asset/transcribe.md b/apps/cli/reference/asset/transcribe.md
index 8860b240..7a03b056 100644
--- a/apps/cli/reference/asset/transcribe.md
+++ b/apps/cli/reference/asset/transcribe.md
@@ -1,4 +1,4 @@
-# `dapi asset transcribe <id|path>`
+# `dapi asset transcribe [options] <id|path>`
Transcribes the speech in a video or audio asset and returns the timed transcript. Word-level start/end times are in **seconds** (source/content time).
@@ -6,6 +6,13 @@ Transcribes the speech in a video or audio asset and returns the timed transcrip
- `<id|path>`: a video or audio asset id, or a local file to add and transcribe (required).
+## Options
+
+- `-s, --start <time>`: start of the range to print — seconds (`"1.5"`), frames (`"45f"`), or `"MM:SS"` (default: `0`).
+- `-e, --end <time>`: end of the range to print — same formats (default: the asset duration).
+
+The whole asset is always transcribed once (and cached on the asset); the range only limits which words are printed. Words keep their absolute source-time timestamps, and segments that straddle a range boundary are trimmed to the words inside it. A range with no speech yields `segments: []`.
+
## Output
One JSON object, the transcript:
@@ -22,4 +29,4 @@ One JSON object, the transcript:
## Errors
-Exits non-zero if the id is unknown or the asset is not a video/audio asset, or if no speech is detected in the audio (`No speech detected`).
+Exits non-zero if the id is unknown or the asset is not a video/audio asset, if `--start`/`--end` are malformed or `--start >= --end`, or if no speech is detected in the audio at all (`No speech detected`).
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 26de46ab..a8f41751 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -149,7 +149,7 @@ export type AssetProbeRequest = { id: string };
export type AssetFrameRequest = { id: string; times?: number[] };
export type AssetFrameResult = Array<{ time: number; base64: string }>;
-export type AssetTranscribeRequest = { id: string };
+export type AssetTranscribeRequest = { id: string; start?: number; end?: number };
export type TranscriptWord = { text: string; start: number; end: number };
export type TranscriptSegment = { text: string; words: TranscriptWord[] };
export type AssetTranscribeResult = { id: string; segments: TranscriptSegment[] };
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index 1db8e01a..30974eef 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -91,8 +91,8 @@ export const cliAPI = {
deleteFolders: (ids: string[]) => cliRequest(CLI_CHANNELS.FOLDERS_DELETE, { ids }),
assetProbe: (id: string) => cliRequest(CLI_CHANNELS.ASSET_PROBE, { id }),
assetFrame: (id: string, times?: number[]) => cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times }),
- assetTranscribe: (id: string) =>
- cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id }, GENERATE_TIMEOUT_MS),
+ assetTranscribe: (id: string, start?: number, end?: number) =>
+ cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),
assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>
cliRequest(CLI_CHANNELS.ASSET_VISUALIZE, { id, start, end, scale }),
assetAnalyze: (id: string, prompt?: string, start?: number, end?: number) =>
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 10a3b618..1b6e2f12 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -510,11 +510,20 @@ async function assetProbe(ref: string): Promise<void> {
}
}
-async function assetTranscribe(ref: string): Promise<void> {
+type AssetTranscribeOptions = { start?: string; end?: string };
+
+async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promise<void> {
+ const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
+ const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;
+ if (start !== undefined && end !== undefined && start >= end) {
+ console.error(`--start (${start}s) must be less than --end (${end}s).`);
+ process.exit(1);
+ }
+
const assetId = await resolveAssetRef(ref);
const stop = startSpinner("Transcribing asset");
try {
- const result = await cliAPI.assetTranscribe(assetId);
+ const result = await cliAPI.assetTranscribe(assetId, start, end);
stop();
console.log(JSON.stringify(result));
} catch (e) {
@@ -1058,7 +1067,9 @@ asset
.command("transcribe")
.description(`Transcribe the speech in a video or audio asset and print the timed transcript${docs("asset/transcribe")}`)
.argument("<id|path>", "video or audio asset id, or a local file to add and transcribe")
- .action((ref: string) => assetTranscribe(ref));
+ .option("-s, --start <time>", `start of the range to print — seconds, "45f" frames, or "MM:SS" (default: 0)`)
+ .option("-e, --end <time>", `end of the range to print — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
+ .action((ref: string, opts: AssetTranscribeOptions) => assetTranscribe(ref, opts));
asset
.command("frame")
diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.ts
index 506f35ec..67f0f808 100644
--- a/apps/web/src/context/electron/assets.ts
+++ b/apps/web/src/context/electron/assets.ts
@@ -20,7 +20,7 @@ import {
import type { Engine } from "@/components/engine";
import type { Asset } from "@/components/engine/db";
-import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult } from "@diffusionstudio/cli/channels";
+import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
import type { Accessor } from "solid-js";
export function handleAssetsAdd(engine: Accessor<Engine>) {
@@ -342,7 +342,7 @@ export function handleAssetFrame(engine: Accessor<Engine>) {
}
export function handleAssetTranscribe(engine: Accessor<Engine>) {
- return async ({ id }: { id: string }): Promise<AssetTranscribeResult> => {
+ return async ({ id, start, end }: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {
const { world } = engine();
const asset = world.assets.get(id);
assert(asset, `Asset ${id} not found.`);
@@ -351,26 +351,42 @@ export function handleAssetTranscribe(engine: Accessor<Engine>) {
`Asset ${id} is not a video or audio asset.`,
);
- if (asset.transcript) {
- return { id, segments: asset.transcript };
- }
+ let transcript = asset.transcript;
+ if (!transcript) {
+ const uploadId = crypto.randomUUID();
+ const audioFile = await transcodeForTranscription(asset);
+ const fileRef = await uploadBlob(audioFile, uploadId);
+ if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);
- const uploadId = crypto.randomUUID();
- const audioFile = await transcodeForTranscription(asset);
- const fileRef = await uploadBlob(audioFile, uploadId);
- if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);
+ ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));
+ if (!transcript.length || transcript.every((s) => s.words.length === 0)) {
+ throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");
+ }
- const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });
- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {
- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");
+ await saveAsset(world, { ...asset, transcript });
}
- await saveAsset(world, { ...asset, transcript });
-
- return { id, segments: transcript };
+ return { id, segments: sliceTranscript(transcript, start, end) };
};
}
+function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {
+ if (start === undefined && end === undefined) return segments;
+ const from = start ?? 0;
+ const to = end ?? Infinity;
+ const sliced: TranscriptSegment[] = [];
+ for (const segment of segments) {
+ const words = segment.words.filter((w) => w.end > from && w.start < to);
+ if (!words.length) continue;
+ sliced.push(
+ words.length === segment.words.length
+ ? segment
+ : { text: words.map((w) => w.text).join(" "), words },
+ );
+ }
+ return sliced;
+}
+
export function handleAssetVisualize(engine: Accessor<Engine>) {
return async (req: AssetVisualizeRequest): Promise<AssetVisualizeResult> => {
const { world } = engine();