Open source CapCut alternative.
TypeScript
feat(site): hosted transcription on Gemini 3.5 Transcribe [rebuild]
David Leb2015f0a8765
diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.tsindex 39570a34..e10abbe0 100644--- a/site/src/cut/lib/cloudTranscribe.ts+++ b/site/src/cut/lib/cloudTranscribe.ts@@ -7,9 +7,9 @@// result is the cut's audible mix in timeline time — whether it then goes to// this Mac (localStt.ts) or to the hosted route, which includes each// account's first chunks and meters the rest. The hosted path-// chunks it as 16 kHz mono WAV, POSTs each chunk, stitches the cues back into-// timeline time, and interpolates per-word timings. Mic dictation reuses the-// same chunk/post/stitch core on a MediaRecorder capture.+// chunks it as 16 kHz mono WAV, POSTs each chunk, and stitches the cues — each+// word carrying its own timing — back into timeline time. Mic dictation reuses+// the same chunk/post/stitch core on a MediaRecorder capture.import { renderMix as mixAudio } from "./audioMix";import { apiFetch } from "./backend";@@ -33,11 +33,10 @@ export interface CloudTranscribeSpec {}const RATE = 16000; // the wire format the hosted route expects-// The hosted model reads times off the audio by ear, and its clock drifts as a-// chunk runs on — cues open in sync and land seconds late by the end of a long-// one. Every chunk boundary re-anchors that clock at a known offset, so the-// chunk length is what bounds the drift: short enough to keep it inside the-// alignment pass's snap window, long enough that a sentence keeps its context.+// The hosted route's included allowance counts chunks and holds each to this+// length (its MAX_CHUNK_SECONDS leaves a little room), so the chunk is the+// unit of accounting: short enough that a few round trips overlap, long+// enough that a sentence keeps its context.const CHUNK_SECONDS = 20;const OVERLAP_SECONDS = 3; // audio shared across chunk seams for stitchingconst POSTS_IN_FLIGHT = 4; // short chunks mean more round-trips; overlap them@@ -105,6 +104,8 @@ interface WireCue {start: number;end: number;text: string;+ /** Per-word timings, seconds from the start of the chunk. */+ words?: { t0: number; t1: number; w: string }[];}// The route's error code for an account whose included chunks are used up.@@ -155,25 +156,6 @@ async function postChunk();}-/** Proportional word timings inside a cue: split the text into words and- * distribute [start, end] by each word's share of the characters. */-function interpolateWords(- text: string,- start: number,- end: number-): { t0: number; t1: number; w: string }[] {- const parts = text.split(/\s+/).filter(Boolean);- const total = parts.reduce((n, w) => n + w.length, 0) || 1;- const span = Math.max(0, end - start);- let at = start;- return parts.map((w) => {- const d = (w.length / total) * span;- const word = { t0: round(at), t1: round(at + d), w };- at += d;- return word;- });-}-/** Chunk mono 16 kHz samples, transcribe each chunk, and stitch: cue times* re-base by chunk offset, and in each 3s overlap the midpoint of the seam* decides ownership — cues whose center falls in the earlier chunk's half are@@ -226,12 +208,21 @@ export async function transcribeSamples(const text = c.text.trim();const mid = (start + end) / 2;if (!text || end <= start || mid < from || mid >= to) continue;+ // Word timings ride along when the route sends them; a cue that arrives+ // without them still shows, it just doesn't highlight word by word.+ const words = (Array.isArray(c.words) ? c.words : [])+ .filter((w) => typeof w?.t0 === "number" && typeof w?.t1 === "number" && typeof w?.w === "string")+ .map((w) => ({+ t0: round(Math.max(start, Math.min(w.t0 + offset, end))),+ t1: round(Math.max(start, Math.min(w.t1 + offset, end))),+ w: w.w,+ }));cues.push({id: uid(),start: round(start),end: round(end),text,- words: interpolateWords(text, start, end),+ ...(words.length > 0 ? { words } : {}),});}}diff --git a/site/src/cut/server/cloud/transcribe.test.ts b/site/src/cut/server/cloud/transcribe.test.tsnew file mode 100644index 00000000..0c5570b4--- /dev/null+++ b/site/src/cut/server/cloud/transcribe.test.ts@@ -0,0 +1,49 @@+import { describe, expect, test } from "bun:test";+import { groupWordsIntoCues, transcribeLanguageCode } from "./transcribe";++describe("transcribeLanguageCode", () => {+ test("keeps a locale the model lists", () => {+ expect(transcribeLanguageCode("en-GB")).toBe("en-GB");+ expect(transcribeLanguageCode("pt-BR")).toBe("pt-BR");+ expect(transcribeLanguageCode("es-419")).toBe("es-419");+ });++ test("maps a bare or unlisted-region locale to the language's default", () => {+ expect(transcribeLanguageCode("en")).toBe("en-US");+ expect(transcribeLanguageCode("ko-KR")).toBe("ko-KR");+ expect(transcribeLanguageCode("zh-CN")).toBe("cmn-Hans-CN");+ expect(transcribeLanguageCode("fr_BE")).toBe("fr-FR");+ });++ test("sends no hint for a language the model does not list", () => {+ expect(transcribeLanguageCode("")).toBeUndefined();+ expect(transcribeLanguageCode("tlh")).toBeUndefined();+ });+});++describe("groupWordsIntoCues", () => {+ const word = (w: string, t0: number, t1: number) => ({ w, t0, t1 });++ test("caps a cue at seven words", () => {+ const words = Array.from({ length: 9 }, (_, i) => word(`w${i}`, i * 0.5, i * 0.5 + 0.4));+ const cues = groupWordsIntoCues(words);+ expect(cues.map((c) => c.words.length)).toEqual([7, 2]);+ expect([cues[0].start, cues[0].end, cues[0].text]).toEqual([0, 3.4, "w0 w1 w2 w3 w4 w5 w6"]);+ });++ test("breaks on a pause and after a sentence", () => {+ const cues = groupWordsIntoCues([+ word("Hello", 0, 0.3),+ word("there.", 0.35, 0.7),+ word("Long", 0.8, 1.1),+ word("pause", 2.5, 2.9),+ word("here", 2.95, 3.2),+ ]);+ expect(cues.map((c) => c.text)).toEqual(["Hello there.", "Long", "pause here"]);+ expect([cues[2].start, cues[2].end]).toEqual([2.5, 3.2]);+ });++ test("returns nothing for no words", () => {+ expect(groupWordsIntoCues([])).toEqual([]);+ });+});diff --git a/site/src/cut/server/cloud/transcribe.ts b/site/src/cut/server/cloud/transcribe.tsindex 4da10f4a..13f82ffc 100644--- a/site/src/cut/server/cloud/transcribe.ts+++ b/site/src/cut/server/cloud/transcribe.ts@@ -1,11 +1,11 @@-// Hosted transcription: one short 16 kHz mono WAV chunk in, cue-level-// timestamps out. The client renders the timeline's audible mix, chunks it-// short enough to bound the model's timing drift, and stitches the results-// (lib/cloudTranscribe.ts); this route only turns one chunk of speech into-// cues with Gemini. Each account's allowance of chunks is included (Pro per-// billing period, free per month — see withinFreeAllowance); chunks past-// that meter against the user's inference credits like every /api/inference-// route.+// Hosted transcription: one short 16 kHz mono WAV chunk in, word-timed cues+// out. The client renders the timeline's audible mix, chunks it, and stitches+// the results (lib/cloudTranscribe.ts); this route only turns one chunk of+// speech into cues with the Gemini speech model, which returns every word+// with its own start and end. Each account's allowance of chunks is included+// (Pro per billing period, free per month — see withinFreeAllowance); chunks+// past that meter against the user's inference credits like every+// /api/inference route.import {creditErrorResponse,recordFailedInferenceUsage,@@ -23,7 +23,7 @@ import {stringValue,} from "@/lib/inference/adapters/gemini-client";import { geminiModelRoles } from "@/lib/inference/gemini-models";-import { toJsonValue } from "@/lib/inference/json";+import { isJsonObject, toJsonValue } from "@/lib/inference/json";import type { JsonValue } from "@/lib/inference/providers";import { err } from "./util";@@ -35,15 +35,14 @@ const MAX_CHUNK_SECONDS = 30;const MAX_AUDIO_BYTES = 44 + MAX_CHUNK_SECONDS * PCM_BYTES_PER_SECOND;const ROUTE = "/api/cut-cloud/transcribe/";const PROVIDER = "gemini";-// The registry's chat flash: multimodal, takes inline audio, priced in-// provider-pricing.ts. No dedicated STT role exists, so speech-to-text rides-// the chat model.-const MODEL = geminiModelRoles.chat;+// The registry's speech model: audio in, the words with their timings out,+// priced in provider-pricing.ts.+const MODEL = geminiModelRoles.transcription;// Each account's first chunks in its allowance window are on the house, so// transcripts fill in for every project — the background sweep included —// before any charge appears. A chunk is at most 20 seconds of speech-bearing-// audio (silent chunks never reach this route), at about $0.004 provider+// audio (silent chunks never reach this route), at about $0.001 provider// cost each. Pro includes 2 hours per billing period, aligned to the// subscription's own cycle; a free account gets 30 minutes per UTC calendar// month. Past the allowance, chunks bill the credit balance as usual; a@@ -89,59 +88,137 @@ async function withinFreeAllowance(userId: string, sweep: boolean): Promise<boolreturn sweepUsed < Math.floor(cap / 2);}+interface WireWord {+ t0: number;+ t1: number;+ w: string;+}+interface WireCue {start: number;end: number;text: string;+ words: WireWord[];}-const transcribePrompt = (locale: string) =>- [- "Transcribe the speech in this audio verbatim, in the language actually spoken" +- (locale ? ` (expected: ${locale})` : "") +- ".",- 'Return ONLY a JSON array of cues: [{"start": <seconds>, "end": <seconds>, "text": "<words>"}].',- "Times are seconds from the start of this audio.",- "Keep cues short — at most 7 words each — and in spoken order.",- "Return [] when there is no speech.",- ].join(" ");+// The speech model detects the language itself; a locale the caller expects+// rides along as a hint when it is one the model lists. Bare languages map to+// the model's default region for that language, and a language the model does+// not list sends no hint (auto-detect).+const TRANSCRIBE_LANGUAGE_CODES = new Set([+ "af-ZA", "am-ET", "ar-EG", "as-IN", "az-AZ", "be-BY", "bg-BG", "bn-BD", "bn-IN",+ "bs-BA", "ca-ES", "ceb", "cmn-Hans-CN", "cs-CZ", "da-DK", "de-DE", "el-GR",+ "en-AU", "en-GB", "en-IN", "en-US", "es-419", "es-ES", "es-US", "et-EE", "fa-IR",+ "fi-FI", "fil-PH", "fr-CA", "fr-FR", "gl-ES", "gu-IN", "ha-NG", "he-IL", "hi-IN",+ "hr-HR", "hu-HU", "hy-AM", "id-ID", "is-IS", "it-IT", "ja-JP", "jv-ID", "ka-GE",+ "kea-CV", "kk-KZ", "km-KH", "kn-IN", "ko-KR", "ky-KG", "ln-CD", "lt-LT", "lv-LV",+ "mk-MK", "ml-IN", "mn-MN", "mr-IN", "ms-MY", "mt-MT", "my-MM", "nb-NO", "ne-NP",+ "nl-NL", "or-IN", "pa-Guru-IN", "pa-IN", "pl-PL", "pt-BR", "pt-PT", "ro-RO",+ "ru-RU", "rup-BG", "sd-Arab-IN", "sk-SK", "sl-SI", "sr-RS", "sv-SE", "sw-KE",+ "te-IN", "tg-TJ", "th-TH", "tr-TR", "uk-UA", "uz-UZ", "vi-VN", "yue-Hant-HK",+]);+const TRANSCRIBE_LANGUAGE_DEFAULTS: Record<string, string> = {+ af: "af-ZA", am: "am-ET", ar: "ar-EG", as: "as-IN", az: "az-AZ", be: "be-BY",+ bg: "bg-BG", bn: "bn-IN", bs: "bs-BA", ca: "ca-ES", ceb: "ceb", cmn: "cmn-Hans-CN",+ cs: "cs-CZ", da: "da-DK", de: "de-DE", el: "el-GR", en: "en-US", es: "es-ES",+ et: "et-EE", fa: "fa-IR", fi: "fi-FI", fil: "fil-PH", fr: "fr-FR", gl: "gl-ES",+ gu: "gu-IN", ha: "ha-NG", he: "he-IL", hi: "hi-IN", hr: "hr-HR", hu: "hu-HU",+ hy: "hy-AM", id: "id-ID", is: "is-IS", it: "it-IT", ja: "ja-JP", jv: "jv-ID",+ ka: "ka-GE", kk: "kk-KZ", km: "km-KH", kn: "kn-IN", ko: "ko-KR", ky: "ky-KG",+ ln: "ln-CD", lt: "lt-LT", lv: "lv-LV", mk: "mk-MK", ml: "ml-IN", mn: "mn-MN",+ mr: "mr-IN", ms: "ms-MY", mt: "mt-MT", my: "my-MM", nb: "nb-NO", ne: "ne-NP",+ nl: "nl-NL", no: "nb-NO", or: "or-IN", pa: "pa-IN", pl: "pl-PL", pt: "pt-BR",+ ro: "ro-RO", ru: "ru-RU", sd: "sd-Arab-IN", sk: "sk-SK", sl: "sl-SI", sr: "sr-RS",+ sv: "sv-SE", sw: "sw-KE", te: "te-IN", tg: "tg-TJ", th: "th-TH", tr: "tr-TR",+ uk: "uk-UA", uz: "uz-UZ", vi: "vi-VN", yue: "yue-Hant-HK", zh: "cmn-Hans-CN",+};-/** The model's cue array, parsed defensively: code fences stripped, the- * outermost [...] recovered from surrounding prose, entries validated and- * clamped to the chunk. Null when no JSON array can be read at all. */-function parseCues(raw: string, maxEnd: number): WireCue[] | null {- const cleaned = raw- .trim()- .replace(/^```(?:json)?\s*/i, "")- .replace(/```\s*$/, "")- .trim();- let parsed: unknown;- try {- parsed = JSON.parse(cleaned);- } catch {- const s = cleaned.indexOf("[");- const e = cleaned.lastIndexOf("]");- if (s < 0 || e <= s) return null;- try {- parsed = JSON.parse(cleaned.slice(s, e + 1));- } catch {- return null;+/** The model's language code for a caller locale, or undefined to let the+ * model detect the language. */+export function transcribeLanguageCode(locale: string): string | undefined {+ const parts = locale.trim().split(/[-_]/).filter(Boolean);+ if (parts.length === 0) return undefined;+ const lang = parts[0].toLowerCase();+ if (parts.length > 1) {+ const region = parts[parts.length - 1];+ const exact = `${lang}-${region.length === 2 ? region.toUpperCase() : region}`;+ if (TRANSCRIBE_LANGUAGE_CODES.has(exact)) return exact;+ }+ return TRANSCRIBE_LANGUAGE_DEFAULTS[lang];+}++/** "1.250s" → 1.25; null for anything else. */+function offsetSeconds(value: JsonValue | undefined): number | null {+ const text = stringValue(value);+ if (!text) return null;+ const n = Number.parseFloat(text.replace(/s$/i, ""));+ return Number.isFinite(n) ? n : null;+}++/** Every timed word the response carries, in spoken order, clamped to the+ * chunk. Null when the response holds text but no timed words — the model+ * answered in a shape this route cannot time. */+function readWords(raw: JsonValue, maxEnd: number): WireWord[] | null {+ const parts = geminiCandidateParts(geminiCandidates(raw)[0]);+ const words: WireWord[] = [];+ let sawText = false;+ for (const part of parts) {+ const transcription = part.audioTranscription;+ if (!isJsonObject(transcription)) {+ if (stringValue(part.text)?.trim()) sawText = true;+ continue;+ }+ if (stringValue(transcription.text)?.trim()) sawText = true;+ const list = transcription.words;+ if (!Array.isArray(list)) continue;+ for (const item of list) {+ if (!isJsonObject(item)) continue;+ const w = stringValue(item.word)?.trim();+ const t0 = offsetSeconds(item.startOffset);+ const t1 = offsetSeconds(item.endOffset);+ if (!w || t0 === null || t1 === null) continue;+ const s = Math.max(0, Math.min(t0, maxEnd));+ const e = Math.max(s, Math.min(t1, maxEnd));+ words.push({ t0: s, t1: e, w });}}- if (!Array.isArray(parsed)) return null;+ if (words.length === 0 && sawText) return null;+ return words.sort((a, b) => a.t0 - b.t0);+}++// Words become short captions: at most this many per cue, a new cue on a+// pause longer than this, and a new cue after a sentence ends.+const CUE_MAX_WORDS = 7;+const CUE_BREAK_GAP_SECONDS = 0.6;++/** Timed words grouped into cues the caption track can show as they are. */+export function groupWordsIntoCues(words: WireWord[]): WireCue[] {const cues: WireCue[] = [];- for (const item of parsed) {- if (!item || typeof item !== "object") continue;- const { start, end, text } = item as Record<string, unknown>;- if (typeof start !== "number" || typeof end !== "number" || typeof text !== "string") continue;- if (!Number.isFinite(start) || !Number.isFinite(end)) continue;- const s = Math.max(0, Math.min(start, maxEnd));- const e = Math.max(0, Math.min(end, maxEnd));- const t = text.trim();- if (!t || e <= s) continue;- cues.push({ start: s, end: e, text: t });+ let current: WireWord[] = [];+ const flush = () => {+ if (current.length === 0) return;+ cues.push({+ start: current[0].t0,+ end: current[current.length - 1].t1,+ text: current.map((w) => w.w).join(" "),+ words: current,+ });+ current = [];+ };+ for (const word of words) {+ const prev = current[current.length - 1];+ if (+ prev &&+ (current.length >= CUE_MAX_WORDS ||+ word.t0 - prev.t1 > CUE_BREAK_GAP_SECONDS ||+ /[.!?]["'”’)]*$/.test(prev.w))+ ) {+ flush();+ }+ current.push(word);}- return cues.sort((a, b) => a.start - b.start);+ flush();+ return cues;}export const transcribeCloud = {@@ -197,19 +274,23 @@ export const transcribeCloud = {let raw: unknown;try {const client = defaultGeminiClientFactory(clientConfig.options);+ const languageCode = transcribeLanguageCode(locale);raw = await client.models.generateContent({model: MODEL,contents: [{role: "user",- parts: [- { inlineData: { mimeType: "audio/wav", data: wavBase64 } },- { text: transcribePrompt(locale) },- ],+ parts: [{ inlineData: { mimeType: "audio/wav", data: wavBase64 } }],},],- // JSON mode without a schema — constrained decoding degrades output.- config: { responseMimeType: "application/json", temperature: 0 },+ // The speech model takes no prompt, system instruction, or output+ // schema; the transcription config is the whole request.+ config: {+ audioTranscriptionConfig: {+ wordTimestamp: true,+ ...(languageCode ? { languageCodes: [languageCode] } : {}),+ },+ },});} catch (error) {await recordFailedInferenceUsage({@@ -249,13 +330,10 @@ export const transcribeCloud = {throw error;}- const text = geminiCandidateParts(geminiCandidates(raw as JsonValue)[0])- .map((p) => stringValue(p.text) ?? "")- .join("");- const cues = parseCues(text, audioSeconds);- if (cues === null) {- return err("The transcription model returned an unreadable response — try again.", 502);+ const words = readWords(raw as JsonValue, audioSeconds);+ if (words === null) {+ return err("The transcription model returned no word timings — try again.", 502);}- return Response.json({ cues });+ return Response.json({ cues: groupWordsIntoCues(words) });},};