Open source CapCut alternative.
TypeScript
"use client";
// On-device speech for a project the engine does not store. A cloud project's
// media lives in R2, so the browser renders its audible mix (cloudTranscribe.ts)
// and hands the audio to this Mac's engine instead of the metered hosted route.
// The engine answers with the same cues the local-project path produces —
// carrying real word timings, which the hosted route can only interpolate.
import { localBackend } from "./backend/local";
import { encodeWav } from "./cloudTranscribe";
import type { SubtitleCue } from "./types";
import { errorFromBody, UserError } from "./userError";
const POLL_MS = 700;
/** The engine transcribes one job at a time, so every caller in this client
* takes its turn here instead of racing for the slot and failing with
* "already running" — a user's caption run simply waits out the background
* sweep's current chunk, and the sweep waits out the user. */
let sttTurn: Promise<void> = Promise.resolve();
export function withEngineStt<T>(work: () => Promise<T>): Promise<T> {
const run = sttTurn.then(work, work);
sttTurn = run.then(
() => undefined,
() => undefined
);
return run;
}
/** Transcribe a rendered mix on this Mac. Returns null when `isStale` trips
* (the caller moved on); throws when the engine can't do it, which is the
* caller's cue to fall back to the hosted route. */
export function engineTranscribeSamples(
samples: Float32Array,
duration: number,
locale: string | undefined,
isStale?: () => boolean
): Promise<SubtitleCue[] | null> {
return withEngineStt(() => engineTranscribe(samples, duration, locale, isStale));
}
async function engineTranscribe(
samples: Float32Array,
duration: number,
locale: string | undefined,
isStale?: () => boolean
): Promise<SubtitleCue[] | null> {
const form = new FormData();
form.append("audio", new File([encodeWav(samples)], "mix.wav", { type: "audio/wav" }));
form.append("duration", String(duration));
if (locale) form.append("locale", locale);
const res = await localBackend.fetch("/api/cut/stt", { method: "POST", body: form });
const body = (await res.json().catch(() => null)) as { id?: string; error?: string } | null;
if (!res.ok || !body?.id) {
throw errorFromBody(body, new UserError("local_transcription_start_failed", "On-device transcription failed to start."));
}
for (;;) {
await new Promise((r) => setTimeout(r, POLL_MS));
if (isStale?.()) return null;
const st = await localBackend.fetch(`/api/cut/stt/${body.id}`);
if (!st.ok) throw new UserError("transcription_job_gone", "The transcription job was lost.");
const status = (await st.json()) as {
status: string;
error?: string;
cues?: SubtitleCue[];
};
if (status.status === "error") throw errorFromBody(status, new UserError("transcription_failed", "Transcription failed."));
if (status.status === "done") return status.cues ?? [];
}
}