The VS Code of video editing, built for agents.
TypeScript
Enhance `dapi asset transcribe` command with options for time range selection
konstantin-paulus7414220589a1
diff --git a/apps/cli/reference/asset/transcribe.md b/apps/cli/reference/asset/transcribe.mdindex 8860b240..7a03b056 100644--- a/apps/cli/reference/asset/transcribe.md+++ b/apps/cli/reference/asset/transcribe.md@@ -1,4 +1,4 @@-# `dapi asset transcribe <id|path>`+# `dapi asset transcribe [options] <id|path>`Transcribes the speech in a video or audio asset and returns the timed transcript. Word-level start/end times are in **seconds** (source/content time).@@ -6,6 +6,13 @@ Transcribes the speech in a video or audio asset and returns the timed transcrip- `<id|path>`: a video or audio asset id, or a local file to add and transcribe (required).+## Options++- `-s, --start <time>`: start of the range to print — seconds (`"1.5"`), frames (`"45f"`), or `"MM:SS"` (default: `0`).+- `-e, --end <time>`: end of the range to print — same formats (default: the asset duration).++The whole asset is always transcribed once (and cached on the asset); the range only limits which words are printed. Words keep their absolute source-time timestamps, and segments that straddle a range boundary are trimmed to the words inside it. A range with no speech yields `segments: []`.+## OutputOne JSON object, the transcript:@@ -22,4 +29,4 @@ One JSON object, the transcript:## Errors-Exits non-zero if the id is unknown or the asset is not a video/audio asset, or if no speech is detected in the audio (`No speech detected`).+Exits non-zero if the id is unknown or the asset is not a video/audio asset, if `--start`/`--end` are malformed or `--start >= --end`, or if no speech is detected in the audio at all (`No speech detected`).diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 26de46ab..a8f41751 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -149,7 +149,7 @@ export type AssetProbeRequest = { id: string };export type AssetFrameRequest = { id: string; times?: number[] };export type AssetFrameResult = Array<{ time: number; base64: string }>;-export type AssetTranscribeRequest = { id: string };+export type AssetTranscribeRequest = { id: string; start?: number; end?: number };export type TranscriptWord = { text: string; start: number; end: number };export type TranscriptSegment = { text: string; words: TranscriptWord[] };export type AssetTranscribeResult = { id: string; segments: TranscriptSegment[] };diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 1db8e01a..30974eef 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -91,8 +91,8 @@ export const cliAPI = {deleteFolders: (ids: string[]) => cliRequest(CLI_CHANNELS.FOLDERS_DELETE, { ids }),assetProbe: (id: string) => cliRequest(CLI_CHANNELS.ASSET_PROBE, { id }),assetFrame: (id: string, times?: number[]) => cliRequest(CLI_CHANNELS.ASSET_FRAME, { id, times }),- assetTranscribe: (id: string) =>- cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id }, GENERATE_TIMEOUT_MS),+ assetTranscribe: (id: string, start?: number, end?: number) =>+ cliRequest(CLI_CHANNELS.ASSET_TRANSCRIBE, { id, start, end }, GENERATE_TIMEOUT_MS),assetVisualize: (id: string, start?: number, end?: number, scale?: number) =>cliRequest(CLI_CHANNELS.ASSET_VISUALIZE, { id, start, end, scale }),assetAnalyze: (id: string, prompt?: string, start?: number, end?: number) =>diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 10a3b618..1b6e2f12 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -510,11 +510,20 @@ async function assetProbe(ref: string): Promise<void> {}}-async function assetTranscribe(ref: string): Promise<void> {+type AssetTranscribeOptions = { start?: string; end?: string };++async function assetTranscribe(ref: string, opts: AssetTranscribeOptions): Promise<void> {+ const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;+ const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;+ if (start !== undefined && end !== undefined && start >= end) {+ console.error(`--start (${start}s) must be less than --end (${end}s).`);+ process.exit(1);+ }+const assetId = await resolveAssetRef(ref);const stop = startSpinner("Transcribing asset");try {- const result = await cliAPI.assetTranscribe(assetId);+ const result = await cliAPI.assetTranscribe(assetId, start, end);stop();console.log(JSON.stringify(result));} catch (e) {@@ -1058,7 +1067,9 @@ asset.command("transcribe").description(`Transcribe the speech in a video or audio asset and print the timed transcript${docs("asset/transcribe")}`).argument("<id|path>", "video or audio asset id, or a local file to add and transcribe")- .action((ref: string) => assetTranscribe(ref));+ .option("-s, --start <time>", `start of the range to print — seconds, "45f" frames, or "MM:SS" (default: 0)`)+ .option("-e, --end <time>", `end of the range to print — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)+ .action((ref: string, opts: AssetTranscribeOptions) => assetTranscribe(ref, opts));asset.command("frame")diff --git a/apps/web/src/context/electron/assets.ts b/apps/web/src/context/electron/assets.tsindex 506f35ec..67f0f808 100644--- a/apps/web/src/context/electron/assets.ts+++ b/apps/web/src/context/electron/assets.ts@@ -20,7 +20,7 @@ import {import type { Engine } from "@/components/engine";import type { Asset } from "@/components/engine/db";-import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult } from "@diffusionstudio/cli/channels";+import type { AssetAnalyzeRequest, AssetAnalyzeResult, AssetExportResult, AssetListResult, AssetMoveResult, AssetProbeRequest, AssetRecord, AssetsExportRequest, AssetTranscribeRequest, AssetTranscribeResult, AssetTreeEntry, AssetVisualizeRequest, AssetVisualizeResult, TranscriptSegment } from "@diffusionstudio/cli/channels";import type { Accessor } from "solid-js";export function handleAssetsAdd(engine: Accessor<Engine>) {@@ -342,7 +342,7 @@ export function handleAssetFrame(engine: Accessor<Engine>) {}export function handleAssetTranscribe(engine: Accessor<Engine>) {- return async ({ id }: { id: string }): Promise<AssetTranscribeResult> => {+ return async ({ id, start, end }: AssetTranscribeRequest): Promise<AssetTranscribeResult> => {const { world } = engine();const asset = world.assets.get(id);assert(asset, `Asset ${id} not found.`);@@ -351,26 +351,42 @@ export function handleAssetTranscribe(engine: Accessor<Engine>) {`Asset ${id} is not a video or audio asset.`,);- if (asset.transcript) {- return { id, segments: asset.transcript };- }+ let transcript = asset.transcript;+ if (!transcript) {+ const uploadId = crypto.randomUUID();+ const audioFile = await transcodeForTranscription(asset);+ const fileRef = await uploadBlob(audioFile, uploadId);+ if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);- const uploadId = crypto.randomUUID();- const audioFile = await transcodeForTranscription(asset);- const fileRef = await uploadBlob(audioFile, uploadId);- if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);+ ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));+ if (!transcript.length || transcript.every((s) => s.words.length === 0)) {+ throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");+ }- const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");+ await saveAsset(world, { ...asset, transcript });}- await saveAsset(world, { ...asset, transcript });-- return { id, segments: transcript };+ return { id, segments: sliceTranscript(transcript, start, end) };};}+function sliceTranscript(segments: TranscriptSegment[], start?: number, end?: number): TranscriptSegment[] {+ if (start === undefined && end === undefined) return segments;+ const from = start ?? 0;+ const to = end ?? Infinity;+ const sliced: TranscriptSegment[] = [];+ for (const segment of segments) {+ const words = segment.words.filter((w) => w.end > from && w.start < to);+ if (!words.length) continue;+ sliced.push(+ words.length === segment.words.length+ ? segment+ : { text: words.map((w) => w.text).join(" "), words },+ );+ }+ return sliced;+}+export function handleAssetVisualize(engine: Accessor<Engine>) {return async (req: AssetVisualizeRequest): Promise<AssetVisualizeResult> => {const { world } = engine();