Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): transcribe and resume uploads in flight

David Lee47e7e87069c
diff --git a/site/src/cut/components/RequireSession.tsx b/site/src/cut/components/RequireSession.tsx
index 87fdb2b4..0834c04f 100644
--- a/site/src/cut/components/RequireSession.tsx
+++ b/site/src/cut/components/RequireSession.tsx
@@ -1,5 +1,5 @@
"use client";
-import { bindChatRuntime, bindChatgptPolling, bindCutClip, bindCutJudge, bindCutNotes, bindCutTracking } from "@/cut/lib/chatRuntime";
+import { bindChatRuntime, bindChatgptPolling, bindCutClip, bindCutJudge, bindCutNotes, bindCutTracking, bindCutUploads } from "@/cut/lib/chatRuntime";
import { bindCutColor } from "@/cut/lib/colorSettings";
import { bindCutExportPasses } from "@/cut/lib/exportPassSettings";
import { bindCutSound } from "@/cut/lib/soundSettings";
@@ -52,6 +52,7 @@ export function RequireSession({ children }: { children: ReactNode }) {
bindCutClip(config.data.settings.cutClip);
bindCutNotes(config.data.settings.cutNotes);
bindCutTracking(config.data.settings.cutTracking);
+ bindCutUploads(config.data.settings.cutUploads);
bindCutColor(config.data.settings.cutColor);
bindCutSound(config.data.settings.cutSound);
bindCutMotion(config.data.settings.cutMotion);
diff --git a/site/src/cut/components/TopBar.tsx b/site/src/cut/components/TopBar.tsx
index e96e136f..96826bf4 100644
--- a/site/src/cut/components/TopBar.tsx
+++ b/site/src/cut/components/TopBar.tsx
@@ -34,7 +34,7 @@ import { cloudUsageQueryKey, useCutMode } from "@/cut/lib/backend/hooks";
import { browserBackend } from "@/cut/lib/backend/browser";
import { localBackend } from "@/cut/lib/backend/local";
import { clearProjectThreads } from "@/cut/lib/chatThreads";
-import { retryUpload } from "@/cut/lib/importQueue";
+import { retryFailedUploads } from "@/cut/lib/importQueue";
import { backTarget, projectHref, useCutBase } from "@/cut/lib/nav";
import { copyProjectAcross } from "@/cut/lib/projectCopy";
import { projectDuration, useEditor } from "@/cut/lib/store";
@@ -296,9 +296,6 @@ export function TopBar({
// dropped file, but a stock or library import lives on the timeline and has
// no tile there — the save indicator is the one place every import reports.
const failedImports = useEditor((s) => s.assets.filter((a) => a.upload?.error).length);
- const retryFailedImports = () => {
- for (const a of useEditor.getState().assets) if (a.upload?.error) retryUpload(a.id);
- };
// Uploads settle in bursts; refresh the storage pill's number when a burst
// ends. Only the settling edge counts — on mount nothing has landed yet.
const queryClient = useQueryClient();
@@ -537,7 +534,7 @@ export function TopBar({
<button
type="button"
className="font-medium underline underline-offset-2"
- onClick={retryFailedImports}
+ onClick={retryFailedUploads}
>
Retry
</button>
diff --git a/site/src/cut/lib/chatRuntime.ts b/site/src/cut/lib/chatRuntime.ts
index baf3e605..63c92b1a 100644
--- a/site/src/cut/lib/chatRuntime.ts
+++ b/site/src/cut/lib/chatRuntime.ts
@@ -41,3 +41,10 @@ export function bindCutNotes(value: unknown): void {
if (parsed.success) notes = parsed.data;
}
export function cutNotes(): Settings["cutNotes"] { return notes; }
+
+let uploads: Settings["cutUploads"] = SETTINGS.cutUploads.default;
+export function bindCutUploads(value: unknown): void {
+ const parsed = SETTINGS.cutUploads.schema.safeParse(value);
+ if (parsed.success) uploads = parsed.data;
+}
+export function cutUploads(): Settings["cutUploads"] { return uploads; }
diff --git a/site/src/cut/lib/cloudTranscribe.test.ts b/site/src/cut/lib/cloudTranscribe.test.ts
new file mode 100644
index 00000000..01bbb2ee
--- /dev/null
+++ b/site/src/cut/lib/cloudTranscribe.test.ts
@@ -0,0 +1,28 @@
+import { expect, test } from "bun:test";
+import { stubModule } from "@/lib/testing/stubModule";
+import type { MediaAsset } from "./types";
+
+// The mix itself is the shared fold's to test; what is pinned here is where
+// each file's sound is read from.
+const resolved: string[] = [];
+await stubModule<typeof import("./audioMix")>("./audioMix", import.meta.url, {
+ renderMix: async (spec, opts) => {
+ for (const c of spec.clips) resolved.push(opts.resolve(c.file));
+ return null;
+ },
+});
+const { renderMix } = await import("./cloudTranscribe");
+
+test("a file still uploading is read from the bytes in hand", async () => {
+ const uploading: MediaAsset = {
+ id: "take",
+ fileName: "take.mov",
+ name: "take.mov",
+ type: "video",
+ duration: 12,
+ url: "blob:take",
+ upload: { progress: 0.4, stored: true },
+ };
+ await renderMix("p", { duration: 12, clips: [{ file: "take.mov", in: 0, out: 12, muted: false }], audio: [] }, [uploading]);
+ expect(resolved).toEqual(["blob:take"]);
+});
diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.ts
index 5e38b57d..63098960 100644
--- a/site/src/cut/lib/cloudTranscribe.ts
+++ b/site/src/cut/lib/cloudTranscribe.ts
@@ -14,7 +14,7 @@
import { renderMix as mixAudio } from "./audioMix";
import { apiFetch } from "./backend";
import { cloudBackend } from "./backend/cloud";
-import { mediaUrl, type SubtitleCue } from "./types";
+import { mediaUrl, type MediaAsset, type SubtitleCue } from "./types";
import { wavHeader } from "./wav";
/** Mirror of the engine's TranscribeSpec (server/transcribe.ts) minus projectId. */
@@ -56,8 +56,12 @@ const round = (n: number) => Math.round(n * 1000) / 1000;
* different rate. Returns null when nothing audible exists. */
export function renderMix(
projectId: string,
- spec: CloudTranscribeSpec
+ spec: CloudTranscribeSpec,
+ assets: readonly MediaAsset[]
): Promise<AudioBuffer | null> {
+ // Each file is read where its asset plays from: a dropped file still
+ // uploading has no stored copy yet, and plays from the bytes in hand.
+ const urls = new Map(assets.map((a) => [a.fileName, a.url]));
return mixAudio(
{
duration: spec.duration,
@@ -66,7 +70,7 @@ export function renderMix(
// shaping it does not change the timing of.
items: spec.audio,
},
- { sampleRate: RATE, channels: 1, resolve: (file) => mediaUrl(projectId, file) }
+ { sampleRate: RATE, channels: 1, resolve: (file) => urls.get(file) || mediaUrl(projectId, file) }
);
}
diff --git a/site/src/cut/lib/importQueue.test.ts b/site/src/cut/lib/importQueue.test.ts
new file mode 100644
index 00000000..d78b98aa
--- /dev/null
+++ b/site/src/cut/lib/importQueue.test.ts
@@ -0,0 +1,36 @@
+import { expect, test } from "bun:test";
+import { stubModule } from "@/lib/testing/stubModule";
+import { ConnectionLostError } from "./network";
+import type { PendingImport } from "./media";
+import { useEditor } from "./store";
+import type { MediaAsset } from "./types";
+
+await stubModule<typeof import("./mediaSync")>("./mediaSync", import.meta.url, {
+ storedMediaUrl: async (_projectId: string, fileName: string) => `stored:${fileName}`,
+});
+const { startUpload } = await import("./importQueue");
+
+const settle = () => new Promise((r) => setTimeout(r, 20));
+
+test("an upload the connection dropped picks back up when the connection returns", async () => {
+ const asset: MediaAsset = { id: "take", fileName: "pending:take:take.mov", name: "take.mov", type: "video", duration: 12, url: "blob:take", upload: { progress: 0 } };
+ useEditor.setState({ projectId: "p", assets: [asset] });
+ let sends = 0;
+ const pending: PendingImport = {
+ asset,
+ localUrl: "blob:take",
+ send: async () => {
+ sends++;
+ if (sends === 1) throw new ConnectionLostError("Upload failed.");
+ return "take.mov";
+ },
+ };
+ startUpload("p", pending);
+ await settle();
+ expect(useEditor.getState().assets[0].upload?.error).toBe("Upload failed.");
+
+ globalThis.dispatchEvent(new Event("online"));
+ await settle();
+ expect(sends).toBe(2);
+ expect(useEditor.getState().assets[0]).toMatchObject({ fileName: "take.mov", url: "stored:take.mov", upload: undefined });
+});
diff --git a/site/src/cut/lib/importQueue.ts b/site/src/cut/lib/importQueue.ts
index 7101476e..c153fa56 100644
--- a/site/src/cut/lib/importQueue.ts
+++ b/site/src/cut/lib/importQueue.ts
@@ -22,6 +22,8 @@ import { getBackend, type CutBackend } from "./backend";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import type { PendingImport } from "./media";
import { storedMediaUrl } from "./mediaSync";
+import { cutUploads } from "./chatRuntime";
+import { connectionLost, isOffline, onReconnect } from "./network";
import { useEditor } from "./store";
/** Blob URLs owned by the browser store's registry stay alive for the whole
@@ -62,9 +64,78 @@ type Job = {
const jobs = new Map<string, Job>();
+// An upload the connection dropped, the way a phone hotspot does, picks back
+// up by itself: after each pause the cutUploads setting lists, and the moment
+// the browser reports the connection back, which also restarts the pauses. A
+// refusal from the server waits for the user's retry.
+const resumes = new Map<string, { attempt: number; timer: ReturnType<typeof setTimeout> | null }>();
+let resumeArmed = false;
+
+/** The connection is back: every failed upload retries with fresh pauses. */
+function resumeAll() {
+ for (const r of resumes.values()) {
+ if (r.timer) {
+ clearTimeout(r.timer);
+ }
+ }
+ resumes.clear();
+ retryFailedUploads();
+}
+
+/** Forget an asset's pending resume, when it lands or leaves. */
+function dropResume(assetId: string) {
+ const r = resumes.get(assetId);
+ if (r?.timer) {
+ clearTimeout(r.timer);
+ }
+ resumes.delete(assetId);
+}
+
+/** Retry every upload that failed, e.g. once the connection is back. */
+export function retryFailedUploads() {
+ for (const a of useEditor.getState().assets) {
+ if (a.upload?.error) {
+ retryUpload(a.id);
+ }
+ }
+}
+
+/** Book the next timed resume of a dropped upload, while any are left. One
+ * waits per asset: a later failure replaces the booked one. */
+function scheduleResume(job: Job) {
+ const id = job.pending.asset.id;
+ const delays = cutUploads().resumeDelaysMs;
+ const prior = resumes.get(id);
+ if (prior?.timer) {
+ clearTimeout(prior.timer);
+ }
+ const attempt = prior?.attempt ?? 0;
+ if (attempt >= delays.length) {
+ resumes.set(id, { attempt, timer: null });
+ return;
+ }
+ const timer = setTimeout(() => {
+ const r = resumes.get(id);
+ if (r) {
+ r.timer = null;
+ }
+ // Offline, the reconnect does it; a retry already under way holds it.
+ const failed = useEditor.getState().assets.find((a) => a.id === id)?.upload?.error;
+ if (isOffline() || jobs.get(id) !== job || !failed) {
+ return;
+ }
+ retryUpload(id);
+ }, delays[attempt]);
+ resumes.set(id, { attempt: attempt + 1, timer });
+}
+
/** Start the background upload for a prepared import. The asset must already
* be in the store. */
export function startUpload(projectId: string, pending: PendingImport) {
+ if (!resumeArmed) {
+ resumeArmed = true;
+ onReconnect(resumeAll);
+ }
const job: Job = {
pending,
projectId,
@@ -129,6 +200,7 @@ export function cancelUpload(assetId: string) {
const job = jobs.get(assetId);
if (!job) return;
jobs.delete(assetId);
+ dropResume(assetId);
job.controller.abort();
revokeTabUrl(job.pending.localUrl);
}
@@ -192,6 +264,7 @@ async function run(job: Job) {
return dropLanded(job, fileName);
}
jobs.delete(asset.id);
+ dropResume(asset.id);
// The stored file takes over from the source bytes, under the name the
// copy resolved to (a dropped file reserved it up front; a remote import
// learns it here), at the address every stored file plays from. Each move
@@ -232,6 +305,9 @@ async function run(job: Job) {
asset.id,
uploadPatch(job, 0, err instanceof Error ? err.message : "Upload failed.")
);
+ if (connectionLost(err)) {
+ scheduleResume(job);
+ }
} finally {
release();
}
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 8549338d..31c4bf7c 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -11,6 +11,7 @@ import { quotaErrorMessage } from "./backend/cloud";
import { encodeWav } from "./cloudTranscribe";
import { allowance, canvasBytes, holdMemory } from "./memoryBudget";
import { normalizeLink } from "./link";
+import { ConnectionLostError } from "./network";
import { downloadFromUrl } from "./download";
import { FONT_RE, LUT_RE } from "./libraryFileType";
import { markUploadStored, startUpload, uploadInFlight } from "./importQueue";
@@ -280,7 +281,7 @@ export function putSigned(
}
xhr.onload = () =>
xhr.status >= 200 && xhr.status < 300 ? resolve() : reject(new Error("Upload failed."));
- xhr.onerror = () => reject(new Error("Upload failed."));
+ xhr.onerror = () => reject(new ConnectionLostError("Upload failed."));
xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));
opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });
xhr.send(file);
@@ -313,7 +314,7 @@ function postFormWatched(
if (xhr.status >= 200 && xhr.status < 300 && body.fileName) resolve(body.fileName);
else reject(new Error(body.error ?? "Upload failed."));
};
- xhr.onerror = () => reject(new Error("Upload failed."));
+ xhr.onerror = () => reject(new ConnectionLostError("Upload failed."));
xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));
opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });
xhr.send(form);
diff --git a/site/src/cut/lib/network.ts b/site/src/cut/lib/network.ts
new file mode 100644
index 00000000..c74c11ae
--- /dev/null
+++ b/site/src/cut/lib/network.ts
@@ -0,0 +1,28 @@
+// The page's connection, as the browser reports it. A headless process has no
+// browser connection state and reads as online: its network failures surface
+// as the errors they are.
+
+/** True while the browser says it has no connection. */
+export function isOffline(): boolean {
+ return globalThis.navigator?.onLine === false;
+}
+
+/** Run `work` each time the browser gets its connection back. */
+export function onReconnect(work: () => void): void {
+ globalThis.addEventListener?.("online", work);
+}
+
+/** A request the connection dropped: no answer came back at all. */
+export class ConnectionLostError extends Error {}
+
+/** Whether `err` is fetch giving up on the network, by the TypeError every
+ * browser throws for it, e.g. Chrome's "Failed to fetch". */
+export function fetchDropped(err: unknown): boolean {
+ return err instanceof TypeError && /failed to fetch|network\s*error|load failed|connection/i.test(err.message);
+}
+
+/** Whether `err` is the connection failing rather than a server answering:
+ * a dropped request, or any failure while offline. */
+export function connectionLost(err: unknown): boolean {
+ return err instanceof ConnectionLostError || fetchDropped(err) || isOffline();
+}
diff --git a/site/src/cut/lib/report.ts b/site/src/cut/lib/report.ts
index fd799457..87acb4d7 100644
--- a/site/src/cut/lib/report.ts
+++ b/site/src/cut/lib/report.ts
@@ -1,5 +1,7 @@
"use client";
+import { fetchDropped } from "./network";
+
/**
* Reporting a failure a feature carried on past.
*
@@ -54,10 +56,7 @@ function shape(err: unknown): boolean {
err.name === "QuotaExceededError"
);
}
- return (
- err instanceof TypeError &&
- /failed to fetch|network\s*error|load failed|connection/i.test(err.message)
- );
+ return fetchDropped(err);
}
/**
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index d174a939..4a23253a 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -1446,7 +1446,7 @@ export async function runTranscription(projectId: string, spec: object): Promise
throw new Error("Add audio or video to the timeline first.");
}
const stale = () => useEditor.getState().projectId !== projectId;
- const mix = await renderMix(projectId, s);
+ const mix = await renderMix(projectId, s, useEditor.getState().assets);
if (stale()) return null;
if (!mix) return []; // nothing audible — no speech, like the engine's short-circuit
const samples = mix.getChannelData(0);
@@ -4413,7 +4413,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (!s.projectId) return null;
const spec = cutTranscribeSpec(s);
if (!spec) return null;
- const mix = await renderMix(s.projectId, spec);
+ const mix = await renderMix(s.projectId, spec, s.assets);
if (!mix) return null;
return { samples: mix.getChannelData(0), sampleRate: mix.sampleRate };
},
diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.ts
index 1fb37e7f..6066fd14 100644
--- a/site/src/cut/lib/watch/sweep.ts
+++ b/site/src/cut/lib/watch/sweep.ts
@@ -95,7 +95,7 @@ async function transcribeFree(
): Promise<{ start: number; end: number; text: string }[] | null> {
if (getBackend().kind === "local") return runTranscription(projectId, spec);
const stale = () => useEditor.getState().projectId !== projectId;
- const mix = await renderMix(projectId, spec);
+ const mix = await renderMix(projectId, spec, useEditor.getState().assets);
if (stale()) return null;
if (!mix) return []; // nothing audible — no speech, like the engine's short-circuit
const samples = mix.getChannelData(0);
diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.ts
index 9d1e9d95..c96b38d5 100644
--- a/site/src/lib/config/registry.ts
+++ b/site/src/lib/config/registry.ts
@@ -292,6 +292,19 @@ export const SETTINGS = defineSettings({
description:
"How a talk or an interview is read for the moments worth cutting a short from: what a clip may run, how widely the first sweep looks, how many moments earn the close read, and what the ranking weighs.",
},
+ cutUploads: {
+ schema: z
+ .object({
+ // The pause before each automatic retry of an upload the connection
+ // dropped; the browser's reconnect retries it too.
+ resumeDelaysMs: z.array(z.number().int().min(1000).max(600000)).max(10).readonly(),
+ })
+ .strict(),
+ default: { resumeDelaysMs: [5000, 20000, 60000] },
+ public: true,
+ title: "Upload resume",
+ description: "How an upload the connection dropped picks back up on its own: the pauses before each automatic retry.",
+ },
cutTracking: {
schema: z
.object({