Open source CapCut alternative.
TypeScript
fix(cut): transcribe and resume uploads in flight
David Lee47e7e87069c
diff --git a/site/src/cut/components/RequireSession.tsx b/site/src/cut/components/RequireSession.tsxindex 87fdb2b4..0834c04f 100644--- a/site/src/cut/components/RequireSession.tsx+++ b/site/src/cut/components/RequireSession.tsx@@ -1,5 +1,5 @@"use client";-import { bindChatRuntime, bindChatgptPolling, bindCutClip, bindCutJudge, bindCutNotes, bindCutTracking } from "@/cut/lib/chatRuntime";+import { bindChatRuntime, bindChatgptPolling, bindCutClip, bindCutJudge, bindCutNotes, bindCutTracking, bindCutUploads } from "@/cut/lib/chatRuntime";import { bindCutColor } from "@/cut/lib/colorSettings";import { bindCutExportPasses } from "@/cut/lib/exportPassSettings";import { bindCutSound } from "@/cut/lib/soundSettings";@@ -52,6 +52,7 @@ export function RequireSession({ children }: { children: ReactNode }) {bindCutClip(config.data.settings.cutClip);bindCutNotes(config.data.settings.cutNotes);bindCutTracking(config.data.settings.cutTracking);+ bindCutUploads(config.data.settings.cutUploads);bindCutColor(config.data.settings.cutColor);bindCutSound(config.data.settings.cutSound);bindCutMotion(config.data.settings.cutMotion);diff --git a/site/src/cut/components/TopBar.tsx b/site/src/cut/components/TopBar.tsxindex e96e136f..96826bf4 100644--- a/site/src/cut/components/TopBar.tsx+++ b/site/src/cut/components/TopBar.tsx@@ -34,7 +34,7 @@ import { cloudUsageQueryKey, useCutMode } from "@/cut/lib/backend/hooks";import { browserBackend } from "@/cut/lib/backend/browser";import { localBackend } from "@/cut/lib/backend/local";import { clearProjectThreads } from "@/cut/lib/chatThreads";-import { retryUpload } from "@/cut/lib/importQueue";+import { retryFailedUploads } from "@/cut/lib/importQueue";import { backTarget, projectHref, useCutBase } from "@/cut/lib/nav";import { copyProjectAcross } from "@/cut/lib/projectCopy";import { projectDuration, useEditor } from "@/cut/lib/store";@@ -296,9 +296,6 @@ export function TopBar({// dropped file, but a stock or library import lives on the timeline and has// no tile there — the save indicator is the one place every import reports.const failedImports = useEditor((s) => s.assets.filter((a) => a.upload?.error).length);- const retryFailedImports = () => {- for (const a of useEditor.getState().assets) if (a.upload?.error) retryUpload(a.id);- };// Uploads settle in bursts; refresh the storage pill's number when a burst// ends. Only the settling edge counts — on mount nothing has landed yet.const queryClient = useQueryClient();@@ -537,7 +534,7 @@ export function TopBar({<buttontype="button"className="font-medium underline underline-offset-2"- onClick={retryFailedImports}+ onClick={retryFailedUploads}>Retry</button>diff --git a/site/src/cut/lib/chatRuntime.ts b/site/src/cut/lib/chatRuntime.tsindex baf3e605..63c92b1a 100644--- a/site/src/cut/lib/chatRuntime.ts+++ b/site/src/cut/lib/chatRuntime.ts@@ -41,3 +41,10 @@ export function bindCutNotes(value: unknown): void {if (parsed.success) notes = parsed.data;}export function cutNotes(): Settings["cutNotes"] { return notes; }++let uploads: Settings["cutUploads"] = SETTINGS.cutUploads.default;+export function bindCutUploads(value: unknown): void {+ const parsed = SETTINGS.cutUploads.schema.safeParse(value);+ if (parsed.success) uploads = parsed.data;+}+export function cutUploads(): Settings["cutUploads"] { return uploads; }diff --git a/site/src/cut/lib/cloudTranscribe.test.ts b/site/src/cut/lib/cloudTranscribe.test.tsnew file mode 100644index 00000000..01bbb2ee--- /dev/null+++ b/site/src/cut/lib/cloudTranscribe.test.ts@@ -0,0 +1,28 @@+import { expect, test } from "bun:test";+import { stubModule } from "@/lib/testing/stubModule";+import type { MediaAsset } from "./types";++// The mix itself is the shared fold's to test; what is pinned here is where+// each file's sound is read from.+const resolved: string[] = [];+await stubModule<typeof import("./audioMix")>("./audioMix", import.meta.url, {+ renderMix: async (spec, opts) => {+ for (const c of spec.clips) resolved.push(opts.resolve(c.file));+ return null;+ },+});+const { renderMix } = await import("./cloudTranscribe");++test("a file still uploading is read from the bytes in hand", async () => {+ const uploading: MediaAsset = {+ id: "take",+ fileName: "take.mov",+ name: "take.mov",+ type: "video",+ duration: 12,+ url: "blob:take",+ upload: { progress: 0.4, stored: true },+ };+ await renderMix("p", { duration: 12, clips: [{ file: "take.mov", in: 0, out: 12, muted: false }], audio: [] }, [uploading]);+ expect(resolved).toEqual(["blob:take"]);+});diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.tsindex 5e38b57d..63098960 100644--- a/site/src/cut/lib/cloudTranscribe.ts+++ b/site/src/cut/lib/cloudTranscribe.ts@@ -14,7 +14,7 @@import { renderMix as mixAudio } from "./audioMix";import { apiFetch } from "./backend";import { cloudBackend } from "./backend/cloud";-import { mediaUrl, type SubtitleCue } from "./types";+import { mediaUrl, type MediaAsset, type SubtitleCue } from "./types";import { wavHeader } from "./wav";/** Mirror of the engine's TranscribeSpec (server/transcribe.ts) minus projectId. */@@ -56,8 +56,12 @@ const round = (n: number) => Math.round(n * 1000) / 1000;* different rate. Returns null when nothing audible exists. */export function renderMix(projectId: string,- spec: CloudTranscribeSpec+ spec: CloudTranscribeSpec,+ assets: readonly MediaAsset[]): Promise<AudioBuffer | null> {+ // Each file is read where its asset plays from: a dropped file still+ // uploading has no stored copy yet, and plays from the bytes in hand.+ const urls = new Map(assets.map((a) => [a.fileName, a.url]));return mixAudio({duration: spec.duration,@@ -66,7 +70,7 @@ export function renderMix(// shaping it does not change the timing of.items: spec.audio,},- { sampleRate: RATE, channels: 1, resolve: (file) => mediaUrl(projectId, file) }+ { sampleRate: RATE, channels: 1, resolve: (file) => urls.get(file) || mediaUrl(projectId, file) });}diff --git a/site/src/cut/lib/importQueue.test.ts b/site/src/cut/lib/importQueue.test.tsnew file mode 100644index 00000000..d78b98aa--- /dev/null+++ b/site/src/cut/lib/importQueue.test.ts@@ -0,0 +1,36 @@+import { expect, test } from "bun:test";+import { stubModule } from "@/lib/testing/stubModule";+import { ConnectionLostError } from "./network";+import type { PendingImport } from "./media";+import { useEditor } from "./store";+import type { MediaAsset } from "./types";++await stubModule<typeof import("./mediaSync")>("./mediaSync", import.meta.url, {+ storedMediaUrl: async (_projectId: string, fileName: string) => `stored:${fileName}`,+});+const { startUpload } = await import("./importQueue");++const settle = () => new Promise((r) => setTimeout(r, 20));++test("an upload the connection dropped picks back up when the connection returns", async () => {+ const asset: MediaAsset = { id: "take", fileName: "pending:take:take.mov", name: "take.mov", type: "video", duration: 12, url: "blob:take", upload: { progress: 0 } };+ useEditor.setState({ projectId: "p", assets: [asset] });+ let sends = 0;+ const pending: PendingImport = {+ asset,+ localUrl: "blob:take",+ send: async () => {+ sends++;+ if (sends === 1) throw new ConnectionLostError("Upload failed.");+ return "take.mov";+ },+ };+ startUpload("p", pending);+ await settle();+ expect(useEditor.getState().assets[0].upload?.error).toBe("Upload failed.");++ globalThis.dispatchEvent(new Event("online"));+ await settle();+ expect(sends).toBe(2);+ expect(useEditor.getState().assets[0]).toMatchObject({ fileName: "take.mov", url: "stored:take.mov", upload: undefined });+});diff --git a/site/src/cut/lib/importQueue.ts b/site/src/cut/lib/importQueue.tsindex 7101476e..c153fa56 100644--- a/site/src/cut/lib/importQueue.ts+++ b/site/src/cut/lib/importQueue.ts@@ -22,6 +22,8 @@ import { getBackend, type CutBackend } from "./backend";import { resolveRegisteredBlob } from "./backend/browser/registry";import type { PendingImport } from "./media";import { storedMediaUrl } from "./mediaSync";+import { cutUploads } from "./chatRuntime";+import { connectionLost, isOffline, onReconnect } from "./network";import { useEditor } from "./store";/** Blob URLs owned by the browser store's registry stay alive for the whole@@ -62,9 +64,78 @@ type Job = {const jobs = new Map<string, Job>();+// An upload the connection dropped, the way a phone hotspot does, picks back+// up by itself: after each pause the cutUploads setting lists, and the moment+// the browser reports the connection back, which also restarts the pauses. A+// refusal from the server waits for the user's retry.+const resumes = new Map<string, { attempt: number; timer: ReturnType<typeof setTimeout> | null }>();+let resumeArmed = false;++/** The connection is back: every failed upload retries with fresh pauses. */+function resumeAll() {+ for (const r of resumes.values()) {+ if (r.timer) {+ clearTimeout(r.timer);+ }+ }+ resumes.clear();+ retryFailedUploads();+}++/** Forget an asset's pending resume, when it lands or leaves. */+function dropResume(assetId: string) {+ const r = resumes.get(assetId);+ if (r?.timer) {+ clearTimeout(r.timer);+ }+ resumes.delete(assetId);+}++/** Retry every upload that failed, e.g. once the connection is back. */+export function retryFailedUploads() {+ for (const a of useEditor.getState().assets) {+ if (a.upload?.error) {+ retryUpload(a.id);+ }+ }+}++/** Book the next timed resume of a dropped upload, while any are left. One+ * waits per asset: a later failure replaces the booked one. */+function scheduleResume(job: Job) {+ const id = job.pending.asset.id;+ const delays = cutUploads().resumeDelaysMs;+ const prior = resumes.get(id);+ if (prior?.timer) {+ clearTimeout(prior.timer);+ }+ const attempt = prior?.attempt ?? 0;+ if (attempt >= delays.length) {+ resumes.set(id, { attempt, timer: null });+ return;+ }+ const timer = setTimeout(() => {+ const r = resumes.get(id);+ if (r) {+ r.timer = null;+ }+ // Offline, the reconnect does it; a retry already under way holds it.+ const failed = useEditor.getState().assets.find((a) => a.id === id)?.upload?.error;+ if (isOffline() || jobs.get(id) !== job || !failed) {+ return;+ }+ retryUpload(id);+ }, delays[attempt]);+ resumes.set(id, { attempt: attempt + 1, timer });+}+/** Start the background upload for a prepared import. The asset must already* be in the store. */export function startUpload(projectId: string, pending: PendingImport) {+ if (!resumeArmed) {+ resumeArmed = true;+ onReconnect(resumeAll);+ }const job: Job = {pending,projectId,@@ -129,6 +200,7 @@ export function cancelUpload(assetId: string) {const job = jobs.get(assetId);if (!job) return;jobs.delete(assetId);+ dropResume(assetId);job.controller.abort();revokeTabUrl(job.pending.localUrl);}@@ -192,6 +264,7 @@ async function run(job: Job) {return dropLanded(job, fileName);}jobs.delete(asset.id);+ dropResume(asset.id);// The stored file takes over from the source bytes, under the name the// copy resolved to (a dropped file reserved it up front; a remote import// learns it here), at the address every stored file plays from. Each move@@ -232,6 +305,9 @@ async function run(job: Job) {asset.id,uploadPatch(job, 0, err instanceof Error ? err.message : "Upload failed."));+ if (connectionLost(err)) {+ scheduleResume(job);+ }} finally {release();}diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 8549338d..31c4bf7c 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -11,6 +11,7 @@ import { quotaErrorMessage } from "./backend/cloud";import { encodeWav } from "./cloudTranscribe";import { allowance, canvasBytes, holdMemory } from "./memoryBudget";import { normalizeLink } from "./link";+import { ConnectionLostError } from "./network";import { downloadFromUrl } from "./download";import { FONT_RE, LUT_RE } from "./libraryFileType";import { markUploadStored, startUpload, uploadInFlight } from "./importQueue";@@ -280,7 +281,7 @@ export function putSigned(}xhr.onload = () =>xhr.status >= 200 && xhr.status < 300 ? resolve() : reject(new Error("Upload failed."));- xhr.onerror = () => reject(new Error("Upload failed."));+ xhr.onerror = () => reject(new ConnectionLostError("Upload failed."));xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });xhr.send(file);@@ -313,7 +314,7 @@ function postFormWatched(if (xhr.status >= 200 && xhr.status < 300 && body.fileName) resolve(body.fileName);else reject(new Error(body.error ?? "Upload failed."));};- xhr.onerror = () => reject(new Error("Upload failed."));+ xhr.onerror = () => reject(new ConnectionLostError("Upload failed."));xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });xhr.send(form);diff --git a/site/src/cut/lib/network.ts b/site/src/cut/lib/network.tsnew file mode 100644index 00000000..c74c11ae--- /dev/null+++ b/site/src/cut/lib/network.ts@@ -0,0 +1,28 @@+// The page's connection, as the browser reports it. A headless process has no+// browser connection state and reads as online: its network failures surface+// as the errors they are.++/** True while the browser says it has no connection. */+export function isOffline(): boolean {+ return globalThis.navigator?.onLine === false;+}++/** Run `work` each time the browser gets its connection back. */+export function onReconnect(work: () => void): void {+ globalThis.addEventListener?.("online", work);+}++/** A request the connection dropped: no answer came back at all. */+export class ConnectionLostError extends Error {}++/** Whether `err` is fetch giving up on the network, by the TypeError every+ * browser throws for it, e.g. Chrome's "Failed to fetch". */+export function fetchDropped(err: unknown): boolean {+ return err instanceof TypeError && /failed to fetch|network\s*error|load failed|connection/i.test(err.message);+}++/** Whether `err` is the connection failing rather than a server answering:+ * a dropped request, or any failure while offline. */+export function connectionLost(err: unknown): boolean {+ return err instanceof ConnectionLostError || fetchDropped(err) || isOffline();+}diff --git a/site/src/cut/lib/report.ts b/site/src/cut/lib/report.tsindex fd799457..87acb4d7 100644--- a/site/src/cut/lib/report.ts+++ b/site/src/cut/lib/report.ts@@ -1,5 +1,7 @@"use client";+import { fetchDropped } from "./network";+/*** Reporting a failure a feature carried on past.*@@ -54,10 +56,7 @@ function shape(err: unknown): boolean {err.name === "QuotaExceededError");}- return (- err instanceof TypeError &&- /failed to fetch|network\s*error|load failed|connection/i.test(err.message)- );+ return fetchDropped(err);}/**diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex d174a939..4a23253a 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -1446,7 +1446,7 @@ export async function runTranscription(projectId: string, spec: object): Promisethrow new Error("Add audio or video to the timeline first.");}const stale = () => useEditor.getState().projectId !== projectId;- const mix = await renderMix(projectId, s);+ const mix = await renderMix(projectId, s, useEditor.getState().assets);if (stale()) return null;if (!mix) return []; // nothing audible — no speech, like the engine's short-circuitconst samples = mix.getChannelData(0);@@ -4413,7 +4413,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (!s.projectId) return null;const spec = cutTranscribeSpec(s);if (!spec) return null;- const mix = await renderMix(s.projectId, spec);+ const mix = await renderMix(s.projectId, spec, s.assets);if (!mix) return null;return { samples: mix.getChannelData(0), sampleRate: mix.sampleRate };},diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.tsindex 1fb37e7f..6066fd14 100644--- a/site/src/cut/lib/watch/sweep.ts+++ b/site/src/cut/lib/watch/sweep.ts@@ -95,7 +95,7 @@ async function transcribeFree(): Promise<{ start: number; end: number; text: string }[] | null> {if (getBackend().kind === "local") return runTranscription(projectId, spec);const stale = () => useEditor.getState().projectId !== projectId;- const mix = await renderMix(projectId, spec);+ const mix = await renderMix(projectId, spec, useEditor.getState().assets);if (stale()) return null;if (!mix) return []; // nothing audible — no speech, like the engine's short-circuitconst samples = mix.getChannelData(0);diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.tsindex 9d1e9d95..c96b38d5 100644--- a/site/src/lib/config/registry.ts+++ b/site/src/lib/config/registry.ts@@ -292,6 +292,19 @@ export const SETTINGS = defineSettings({description:"How a talk or an interview is read for the moments worth cutting a short from: what a clip may run, how widely the first sweep looks, how many moments earn the close read, and what the ranking weighs.",},+ cutUploads: {+ schema: z+ .object({+ // The pause before each automatic retry of an upload the connection+ // dropped; the browser's reconnect retries it too.+ resumeDelaysMs: z.array(z.number().int().min(1000).max(600000)).max(10).readonly(),+ })+ .strict(),+ default: { resumeDelaysMs: [5000, 20000, 60000] },+ public: true,+ title: "Upload resume",+ description: "How an upload the connection dropped picks back up on its own: the pauses before each automatic retry.",+ },cutTracking: {schema: z.object({