Skip to content

Open source CapCut alternative.

TypeScript

fix(site): quota on every copy and a sweep for rowless objects

David Leb50bf86a0460
diff --git a/site/src/cut/server/cloud/gc.ts b/site/src/cut/server/cloud/gc.ts
index 69beddef..8e69cfa4 100644
--- a/site/src/cut/server/cloud/gc.ts
+++ b/site/src/cut/server/cloud/gc.ts
@@ -9,7 +9,15 @@ import { cutLimitsFor, FREE_STORAGE_BYTES } from "./limits";
import { collectable, listSweepableLadders, pruneLadder, readLadder } from "./ladderStore";
import { deleteLibraryAssetCascade } from "./library";
import { deleteProjectCascade } from "./projects";
-import { del, deletePrefix, INFERENCE_PREFIX, listOlderThan, projectHlsRoot } from "./r2";
+import {
+ del,
+ deletePrefix,
+ INFERENCE_PREFIX,
+ listCommonPrefixes,
+ listOlderThan,
+ projectHlsRoot,
+ USER_ROOT,
+} from "./r2";
import { addUsage, PENDING_CLAIM_MAX_AGE_MS } from "./usage";
// Scratch media a hosted inference call carried. No row tracks these — they are
@@ -21,7 +29,11 @@ const JOB_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
// clip off the phone. A phone that has not synced in a season has a bigger
// problem than a missed delete, so the row goes after that.
const LIBRARY_TOMBSTONE_MAX_AGE_MS = 90 * 24 * 60 * 60 * 1000;
-const TERMINAL_STATES = ["done", "error", "canceled"];
+const TERMINAL_STATES = ["done", "error", "canceled", "dismissed"];
+// Overlay PNGs are single-use render inputs the worker deletes when their job
+// settles. Ones presigned for a job that was never queued have no row and no
+// job, so age is what makes them garbage; two days is far past any render.
+const OVERLAY_MAX_AGE_MS = 2 * 24 * 60 * 60 * 1000;
// Reclamation converges over daily runs rather than risking one long one.
const RECLAIM_SCAN_LIMIT = 500;
const RECLAIM_MAX_USERS = 10;
@@ -228,6 +240,33 @@ async function reclaimOverQuota() {
return out;
}
+/** Overlay inputs older than any render could still be waiting on: presigned
+ * uploads whose export never started. Walks the bucket's user trees rather
+ * than the user table, so it touches only accounts that hold objects. */
+async function sweepStaleOverlays(): Promise<number> {
+ const before = new Date(Date.now() - OVERLAY_MAX_AGE_MS);
+ // A job still waiting to be claimed — through a wake outage, say — reads its
+ // overlays when it finally runs, however old they are by then.
+ const live = await prisma.cutRenderJob.findMany({
+ where: { state: { in: ["queued", "running"] } },
+ select: { spec: true },
+ });
+ const held = new Set(
+ live.flatMap((j) => {
+ const overlays = (j.spec as { overlays?: { key?: unknown }[] } | null)?.overlays ?? [];
+ return overlays.map((o) => o?.key).filter((k): k is string => typeof k === "string");
+ })
+ );
+ let removed = 0;
+ for (const user of await listCommonPrefixes(USER_ROOT)) {
+ if (user === INFERENCE_PREFIX) continue;
+ const keys = (await listOlderThan(`${user}overlays/`, before)).filter((k) => !held.has(k));
+ await del(keys);
+ removed += keys.length;
+ }
+ return removed;
+}
+
export async function runGc(): Promise<Response> {
const pending = await prisma.cutMediaObject.findMany({
where: {
@@ -327,6 +366,12 @@ export async function runGc(): Promise<Response> {
).catch(() => [] as string[]);
await del(scratch);
+ // Stale overlays, one listing per user tree that exists in the bucket.
+ const overlays = await sweepStaleOverlays().catch((e: unknown) => {
+ console.error("[cut-gc] overlay sweep failed:", e instanceof Error ? e.message : e);
+ return 0;
+ });
+
// Guarded like the prefix sweeps above it: the ladder store is optional
// infrastructure whose loss costs a re-render, so a missing token or an
// unreachable KV must not take down the run that reclaims storage.
@@ -347,6 +392,7 @@ export async function runGc(): Promise<Response> {
pendingObjects: pending.length,
orphanedMedia: orphans.length,
inferenceScratch: scratch.length,
+ staleOverlays: overlays,
renderJobs: jobs.count,
libraryTombstones: tombstones.count,
staleLadderProjects: ladders.projects,
diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.ts
index 65cda454..ec99a61a 100644
--- a/site/src/cut/server/cloud/library.ts
+++ b/site/src/cut/server/cloud/library.ts
@@ -19,16 +19,7 @@ import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl, mediaUrlLifetime } from "./medi
import { getProject, takenMediaNames } from "./projects";
import { copy, del, head, libraryKey, presignPut, projectMediaKey } from "./r2";
import { addUsage, quotaCheck } from "./usage";
-import {
- caught,
- decodeFileParam,
- dedupeName,
- err,
- inspirationFolderId,
- redirect,
- safeFileName,
- typeOf,
-} from "./util";
+import { caught, decodeFileParam, dedupeName, err, HttpResponseError, inspirationFolderId, redirect, safeFileName, typeOf } from "./util";
/** Cap on one signed-URL batch, matching the project media batch. */
const PRESIGN_GET_BATCH_MAX = 500;
@@ -168,6 +159,9 @@ async function copyIntoLibrary(
src: MediaObjectRow,
taken: Set<string>,
): Promise<string> {
+ // A copy is new bytes on the account, held to the quota like an upload.
+ const over = await quotaCheck(userId, Number(src.bytes));
+ if (over) throw new HttpResponseError(over);
const dest = dedupeName(safeFileName(src.fileName), taken);
taken.add(dest);
const key = libraryKey(userId, dest);
@@ -249,6 +243,8 @@ async function copyIntoProject(
): Promise<string> {
const src = await libraryMediaObject(userId, libFileName);
if (!src) throw new Error("Library asset not found.");
+ const over = await quotaCheck(userId, Number(src.bytes));
+ if (over) throw new HttpResponseError(over);
const dest = dedupeName(safeFileName(libFileName), taken);
taken.add(dest);
const key = projectMediaKey(userId, projectId, dest);
@@ -791,20 +787,39 @@ export const libraryCloud = {
if (!(await getProject(userId, projectId)))
throw new Error("Project not found.");
if (templateEmpty(input)) throw new Error("Nothing to save.");
+ // Every source is resolved and the whole batch checked against the quota
+ // before the first copy, so the wall is met with nothing written; a
+ // copy that still fails takes the ones before it back with it, since a
+ // library object no template names is one nothing could ever find.
+ const sources = await Promise.all(
+ input.media.map(async (m) => {
+ const src = await projectMediaObject(userId, projectId, m.fileName);
+ if (!src) throw new Error("Media file not found in project.");
+ return { m, src };
+ }),
+ );
+ const over = await quotaCheck(
+ userId,
+ sources.reduce((sum, { src }) => sum + Number(src.bytes), 0),
+ );
+ if (over) throw new HttpResponseError(over);
const taken = await takenLibraryNames(userId);
const media: TemplateMedia[] = [];
- for (const m of input.media) {
- const src = await projectMediaObject(userId, projectId, m.fileName);
- if (!src) throw new Error("Media file not found in project.");
- const dest = await copyIntoLibrary(userId, src, taken);
- media.push({
- fileName: dest,
- name: m.name,
- type: m.type,
- duration: m.duration,
- width: m.width,
- height: m.height,
- });
+ try {
+ for (const { m, src } of sources) {
+ const dest = await copyIntoLibrary(userId, src, taken);
+ media.push({
+ fileName: dest,
+ name: m.name,
+ type: m.type,
+ duration: m.duration,
+ width: m.width,
+ height: m.height,
+ });
+ }
+ } catch (e) {
+ for (const done of media) await deleteLibraryObject(userId, done.fileName).catch(() => {});
+ throw e;
}
const doc: TemplateDoc = {
folderId: null,
@@ -897,6 +912,17 @@ export const libraryCloud = {
const row = await findTemplate(userId, id);
if (!row) throw new Error("Template not found.");
const template = templateView(row);
+ // The whole batch against the quota before the first copy; a project
+ // copy that still fails is unreferenced project media, which the sweep
+ // reclaims.
+ let incoming = 0;
+ for (const m of template.media) {
+ const src = await libraryMediaObject(userId, m.fileName);
+ if (!src) throw new Error("Library asset not found.");
+ incoming += Number(src.bytes);
+ }
+ const over = await quotaCheck(userId, incoming);
+ if (over) throw new HttpResponseError(over);
const taken = await takenMediaNames(userId, projectId);
const media: TemplateMedia[] = [];
for (const m of template.media) {
diff --git a/site/src/cut/server/cloud/projects.ts b/site/src/cut/server/cloud/projects.ts
index 422c0a2a..6d6c93b4 100644
--- a/site/src/cut/server/cloud/projects.ts
+++ b/site/src/cut/server/cloud/projects.ts
@@ -6,7 +6,7 @@ import type { Prisma } from "@/generated/prisma/client";
import { prisma } from "@/lib/prisma";
import { deleteLadder } from "./ladderStore";
import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl } from "./mediaCdn";
-import { del, deletePrefix, projectExportKey, projectHlsRoot, projectMediaKey } from "./r2";
+import { del, deletePrefix, projectExportKey, projectPrefix, projectMediaKey } from "./r2";
import { addUsage } from "./usage";
import { caught, decodeFileParam, err, redirect } from "./util";
@@ -126,15 +126,16 @@ export async function deleteProjectCascade(userId: string, id: string): Promise<
await addUsage(tx, userId, -freed);
});
await del(objects.map((o) => o.r2Key));
- // The HLS ladders have no media rows at all — they are found by prefix and
- // recorded in KV — so both have to be cleared by hand here. Neither affects
- // `freed`: a ladder never counted toward the quota.
+ // Then the whole tree, rows or no rows: the HLS ladders (found by prefix and
+ // recorded in KV), an export a tab uploaded and never registered, a proxy
+ // whose registration failed after its upload. None of it affects `freed`;
+ // nothing without a row ever counted toward the quota.
//
- // The bytes go before the record does. A long 4K cut runs to tens of
+ // The bytes go before the ladder record does. A long 4K cut runs to tens of
// thousands of objects, so this pages rather than listing once; and pruning
// the record first would leave anything the delete missed with no index that
// could ever find it again.
- await deletePrefix(projectHlsRoot(userId, id)).catch(() => 0);
+ await deletePrefix(projectPrefix(userId, id)).catch(() => 0);
await deleteLadder(id);
return freed;
}
diff --git a/site/src/cut/server/cloud/r2.ts b/site/src/cut/server/cloud/r2.ts
index 3a1fff8c..7f05c260 100644
--- a/site/src/cut/server/cloud/r2.ts
+++ b/site/src/cut/server/cloud/r2.ts
@@ -40,8 +40,13 @@ function r2(): S3Client {
// --- Key scheme: everything a user owns lives under cut/<userId>/. ---
+/** Everything a project owns in the bucket — media, exports, the hover proxy,
+ * the card, every ladder — lives under this one prefix, so a project delete
+ * can sweep it whole. */
+export const projectPrefix = (userId: string, projectId: string) =>
+ `cut/${userId}/projects/${projectId}/`;
export const projectMediaKey = (userId: string, projectId: string, fileName: string) =>
- `cut/${userId}/projects/${projectId}/media/${fileName}`;
+ `${projectPrefix(userId, projectId)}media/${fileName}`;
export const projectExportKey = (userId: string, projectId: string, fileName: string) =>
`cut/${userId}/projects/${projectId}/exports/${fileName}`;
export const projectPreviewKey = (userId: string, projectId: string) =>
@@ -62,8 +67,13 @@ export const projectCardKey = (userId: string, projectId: string, ext: "jpg" | "
`cut/${userId}/projects/${projectId}/card.${ext}`;
export const libraryKey = (userId: string, fileName: string) =>
`cut/${userId}/library/${fileName}`;
+/** Per-user root of the bucket, and the overlay tree under it. `overlayPrefix`
+ * is what the sweep walks: one listing per user that has any objects, found
+ * through `listCommonPrefixes(USER_ROOT)`. */
+export const USER_ROOT = "cut/";
+export const overlayPrefix = (userId: string) => `cut/${userId}/overlays/`;
export const overlayKey = (userId: string, batchId: string, name: string) =>
- `cut/${userId}/overlays/${batchId}/${name}`;
+ `${overlayPrefix(userId)}${batchId}/${name}`;
/** Scratch media a hosted inference call carries — reference sheets, keyframes,
* chat attachments. Keyed by the SHA-256 of the bytes, so a sheet that rides
@@ -208,6 +218,28 @@ export async function listObjectsWithDates(
return out;
}
+/** The "directories" directly under `prefix` — one entry per distinct next
+ * path segment, without listing the objects inside. A sweep that walks every
+ * user's tree starts here instead of from a table scan. */
+export async function listCommonPrefixes(prefix: string): Promise<string[]> {
+ const out: string[] = [];
+ let token: string | undefined;
+ do {
+ const res = await r2().send(
+ new ListObjectsV2Command({
+ Bucket: R2_BUCKET,
+ Prefix: prefix,
+ Delimiter: "/",
+ ContinuationToken: token,
+ MaxKeys: 1000,
+ })
+ );
+ for (const p of res.CommonPrefixes ?? []) if (p.Prefix) out.push(p.Prefix);
+ token = res.IsTruncated ? res.NextContinuationToken : undefined;
+ } while (token);
+ return out;
+}
+
/** Keys under `prefix` last written before `before`, paged to `limit` objects.
* For prefix sweeps of objects no database row tracks. */
export async function listOlderThan(
diff --git a/site/src/cut/server/cloud/util.ts b/site/src/cut/server/cloud/util.ts
index 79e36b0d..a25b39b9 100644
--- a/site/src/cut/server/cloud/util.ts
+++ b/site/src/cut/server/cloud/util.ts
@@ -6,7 +6,17 @@ import { R2NotConfiguredError } from "./r2";
export const err = (message: string, status: number) =>
Response.json({ error: message }, { status });
+/** A refusal raised deep inside a helper — a quota wall met halfway through a
+ * copy — carrying the exact response the route should answer with, so the
+ * wall reads the same wherever it is hit. `caught` unwraps it. */
+export class HttpResponseError extends Error {
+ constructor(public readonly response: Response) {
+ super(`HTTP ${response.status}`);
+ }
+}
+
export const caught = (e: unknown, fallback: string, status = 500) => {
+ if (e instanceof HttpResponseError) return e.response;
if (e instanceof R2NotConfiguredError) return err(e.message, 500);
// The real reason belongs in the server's logs, not in a response an
// anonymous share viewer reads: it names the environment variable to set.