Skip to content

Open source CapCut alternative.

TypeScript

fix(site): a link import lands once however often it is posted [rebuild]

David Lea822eb75278c
diff --git a/site/src/cut/components/LibraryView.tsx b/site/src/cut/components/LibraryView.tsx
index a3296dc8..091cf0f9 100644
--- a/site/src/cut/components/LibraryView.tsx
+++ b/site/src/cut/components/LibraryView.tsx
@@ -526,8 +526,13 @@ export function LibraryView() {
const id = crypto.randomUUID();
const run = async () => {
try {
- const imported = await importUrlToLibrary(value, residency, (stage) =>
- setStage(id, stage),
+ // The card's id goes with the link: a retry from this card gets the
+ // job it already queued.
+ const imported = await importUrlToLibrary(
+ value,
+ residency,
+ (stage) => setStage(id, stage),
+ id,
);
if (folderId) {
for (const asset of imported) {
diff --git a/site/src/cut/lib/library.ts b/site/src/cut/lib/library.ts
index 468e18ce..e0b25ef7 100644
--- a/site/src/cut/lib/library.ts
+++ b/site/src/cut/lib/library.ts
@@ -277,20 +277,24 @@ export function estimatedShape(url: string): { width: number; height: number } {
const SHORT_FORM_URL =
/^https?:\/\/[^/]*(?:youtube\.com\/shorts\/|tiktok\.com\/|instagram\.com\/(?:reels?|stories)\/)/i;
+/** Import a link into a shelf. `key` is the caller's own id for this import —
+ * the pending card's — and rides to the cloud with the link, so a retry of
+ * the same card gets the job it already queued and the link lands once. */
export async function importUrlToLibrary(
url: string,
residency: Residency = activeResidency(),
onStage?: (stage: ImportStage) => void,
+ key?: string,
): Promise<LibraryAsset[]> {
url = normalizeLink(url);
if (residency === "browser")
- return importUrlThroughCloud(url, "browser", onStage);
+ return importUrlThroughCloud(url, "browser", onStage, key);
onStage?.("downloading");
const backend = backendFor(residency);
const res = await backend.fetch("/api/cut/library/import-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ url }),
+ body: JSON.stringify({ url, ...(key ? { key } : {}) }),
});
// The engine downloads inside the request; the cloud answers {jobId} and a
// worker does the fetch, so that side waits on the job.
@@ -324,7 +328,7 @@ export async function importUrlToLibrary(
// either: it is the shelf the user was importing to.
const reason = body.error ?? "Could not import that URL.";
try {
- return await importUrlThroughCloud(url, residency, onStage);
+ return await importUrlThroughCloud(url, residency, onStage, key);
} catch {
throw new Error(reason);
}
@@ -345,12 +349,13 @@ async function importUrlThroughCloud(
url: string,
target: Residency,
onStage?: (stage: ImportStage) => void,
+ key?: string,
): Promise<LibraryAsset[]> {
onStage?.("downloading");
const res = await cloudBackend.fetch("/api/cut/library/import-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ url }),
+ body: JSON.stringify({ url, ...(key ? { key } : {}) }),
});
const started = await apiJson<{ jobId?: string }>(res);
if (!res.ok || !started.jobId)
diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.ts
index 44244bed..1b74376f 100644
--- a/site/src/cut/server/cloud/jobs.ts
+++ b/site/src/cut/server/cloud/jobs.ts
@@ -91,6 +91,29 @@ function needsWorker(row: JobRow): boolean {
);
}
+/** A client's own id for an import — the phone's inspiration item, the
+ * page's pending card. One path segment's worth of characters. */
+const validImportKey = (key: unknown): key is string | undefined =>
+ key === undefined || (typeof key === "string" && /^[A-Za-z0-9_-]{1,128}$/.test(key));
+
+/** The import the client already queued under this key. A phone that never
+ * heard the first answer, or a page retrying its card, posts the same link
+ * again with the key it made for the item and gets the job it already has —
+ * the link lands once. A job that failed or was canceled is not handed back:
+ * a repeat after one of those is the retry the client means. */
+async function importJobFor(userId: string, key: string | undefined): Promise<{ id: string } | null> {
+ if (!key) return null;
+ return prisma.cutRenderJob.findFirst({
+ where: {
+ userId,
+ kind: "import_url",
+ state: { in: ["queued", "running", "done"] },
+ spec: { path: ["key"], equals: key },
+ },
+ select: { id: true },
+ });
+}
+
/** The most bytes an import may bring in: what is left of the account's
* storage, under the hard ceiling. */
async function importByteCeiling(userId: string): Promise<number> {
@@ -622,9 +645,16 @@ export const jobsCloud = {
* engine's synchronous route. */
async importUrl(userId: string, projectId: string, req: Request) {
try {
- const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };
+ const { url, audio, key } = (await req.json()) as {
+ url?: string;
+ audio?: boolean;
+ key?: string;
+ };
if (!url) return err("No URL provided.", 400);
+ if (!validImportKey(key)) return err("Bad import key.", 400);
if (!(await getProject(userId, projectId))) return err("Project not found.", 404);
+ const open = await importJobFor(userId, key);
+ if (open) return Response.json({ jobId: open.id });
const capped = await renderJobCheck(userId);
if (capped) return capped;
// What lands counts against storage, so an account with none left gets
@@ -641,6 +671,7 @@ export const jobsCloud = {
url,
maxBytes,
...(audio === true ? { audio: true } : {}),
+ ...(key ? { key } : {}),
} as unknown as Prisma.InputJsonValue,
},
});
@@ -656,12 +687,16 @@ export const jobsCloud = {
* the assets, and the client polls this job for them. */
async importUrlToLibrary(userId: string, req: Request) {
try {
- const { url, origin, audio } = (await req.json()) as {
+ const { url, origin, audio, key } = (await req.json()) as {
url?: string;
origin?: string;
audio?: boolean;
+ key?: string;
};
if (!url) return err("No URL provided.", 400);
+ if (!validImportKey(key)) return err("Bad import key.", 400);
+ const open = await importJobFor(userId, key);
+ if (open) return Response.json({ jobId: open.id });
const capped = await renderJobCheck(userId);
if (capped) return capped;
const over = await quotaCheck(userId, 0);
@@ -679,6 +714,7 @@ export const jobsCloud = {
// An inspiration link's downloads land in the Inspiration folder,
// carrying the origin the Camera Roll and Library filters read.
...(origin === "inspiration" ? { origin } : {}),
+ ...(key ? { key } : {}),
} as unknown as Prisma.InputJsonValue,
},
});
diff --git a/site/src/cut/worker/importUrlJob.ts b/site/src/cut/worker/importUrlJob.ts
index 3e3a5e8b..71e5cb95 100644
--- a/site/src/cut/worker/importUrlJob.ts
+++ b/site/src/cut/worker/importUrlJob.ts
@@ -2,6 +2,7 @@ import { mkdtemp, rm, stat } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { Prisma } from "@/generated/prisma/client";
+import { deleteLibraryAssetCascade } from "../server/cloud/library";
import { dedupeName, inspirationFolderId, safeFileName, typeOf } from "../server/cloud/util";
import type { LibraryAsset } from "../server/library";
import { probeDuration } from "../server/frames";
@@ -65,6 +66,12 @@ export async function runImportUrlJob(
});
}
+ // A run of this job cut short — a worker drained mid-landing, a heartbeat
+ // lost — left rows behind for whichever worker claimed the retry. They go
+ // before this run lands anything, so the link lands once however many times
+ // the job runs.
+ if (toLibrary) await dropEarlierLanding(job);
+
const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-dl-"));
try {
const dl = await download(url.trim(), tmp, { audio: audio === true });
@@ -136,7 +143,7 @@ export async function runImportUrlJob(
});
}
const asset = await addLibraryRow(
- job.userId,
+ job,
objectId,
fileName,
f.file,
@@ -164,10 +171,22 @@ export async function runImportUrlJob(
}
}
+/** The library rows an earlier run of this job landed, taken down with their
+ * media. Tombstoned, so a phone that already listed one drops it too. */
+async function dropEarlierLanding(job: ClaimedJob): Promise<void> {
+ const rows = await prisma.cutLibraryAsset.findMany({
+ where: { userId: job.userId, deletedAt: null, meta: { path: ["importJobId"], equals: job.id } },
+ select: { id: true },
+ });
+ for (const row of rows)
+ await deleteLibraryAssetCascade(job.userId, row.id, { tombstone: true });
+}
+
/** Register one downloaded file as a library asset, probing the local copy for
- * the shape the library card reads (duration, pixel size). */
+ * the shape the library card reads (duration, pixel size). The row names the
+ * job that landed it, so a rerun of that job can find it. */
async function addLibraryRow(
- userId: string,
+ job: ClaimedJob,
mediaObjectId: string,
fileName: string,
localFile: string,
@@ -188,10 +207,11 @@ async function addLibraryRow(
...(source ? { source } : {}),
...(posterFile ? { posterFile } : {}),
...(filing ? { origin: filing.origin } : {}),
+ importJobId: job.id,
};
const row = await prisma.cutLibraryAsset.create({
data: {
- userId,
+ userId: job.userId,
mediaObjectId,
folderId: filing?.folderId ?? null,
meta: meta as unknown as Prisma.InputJsonValue,