Open source CapCut alternative.
TypeScript
fix(site): quota on every copy and a sweep for rowless objects
David Leb50bf86a0460
diff --git a/site/src/cut/server/cloud/gc.ts b/site/src/cut/server/cloud/gc.tsindex 69beddef..8e69cfa4 100644--- a/site/src/cut/server/cloud/gc.ts+++ b/site/src/cut/server/cloud/gc.ts@@ -9,7 +9,15 @@ import { cutLimitsFor, FREE_STORAGE_BYTES } from "./limits";import { collectable, listSweepableLadders, pruneLadder, readLadder } from "./ladderStore";import { deleteLibraryAssetCascade } from "./library";import { deleteProjectCascade } from "./projects";-import { del, deletePrefix, INFERENCE_PREFIX, listOlderThan, projectHlsRoot } from "./r2";+import {+ del,+ deletePrefix,+ INFERENCE_PREFIX,+ listCommonPrefixes,+ listOlderThan,+ projectHlsRoot,+ USER_ROOT,+} from "./r2";import { addUsage, PENDING_CLAIM_MAX_AGE_MS } from "./usage";// Scratch media a hosted inference call carried. No row tracks these — they are@@ -21,7 +29,11 @@ const JOB_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;// clip off the phone. A phone that has not synced in a season has a bigger// problem than a missed delete, so the row goes after that.const LIBRARY_TOMBSTONE_MAX_AGE_MS = 90 * 24 * 60 * 60 * 1000;-const TERMINAL_STATES = ["done", "error", "canceled"];+const TERMINAL_STATES = ["done", "error", "canceled", "dismissed"];+// Overlay PNGs are single-use render inputs the worker deletes when their job+// settles. Ones presigned for a job that was never queued have no row and no+// job, so age is what makes them garbage; two days is far past any render.+const OVERLAY_MAX_AGE_MS = 2 * 24 * 60 * 60 * 1000;// Reclamation converges over daily runs rather than risking one long one.const RECLAIM_SCAN_LIMIT = 500;const RECLAIM_MAX_USERS = 10;@@ -228,6 +240,33 @@ async function reclaimOverQuota() {return out;}+/** Overlay inputs older than any render could still be waiting on: presigned+ * uploads whose export never started. Walks the bucket's user trees rather+ * than the user table, so it touches only accounts that hold objects. */+async function sweepStaleOverlays(): Promise<number> {+ const before = new Date(Date.now() - OVERLAY_MAX_AGE_MS);+ // A job still waiting to be claimed — through a wake outage, say — reads its+ // overlays when it finally runs, however old they are by then.+ const live = await prisma.cutRenderJob.findMany({+ where: { state: { in: ["queued", "running"] } },+ select: { spec: true },+ });+ const held = new Set(+ live.flatMap((j) => {+ const overlays = (j.spec as { overlays?: { key?: unknown }[] } | null)?.overlays ?? [];+ return overlays.map((o) => o?.key).filter((k): k is string => typeof k === "string");+ })+ );+ let removed = 0;+ for (const user of await listCommonPrefixes(USER_ROOT)) {+ if (user === INFERENCE_PREFIX) continue;+ const keys = (await listOlderThan(`${user}overlays/`, before)).filter((k) => !held.has(k));+ await del(keys);+ removed += keys.length;+ }+ return removed;+}+export async function runGc(): Promise<Response> {const pending = await prisma.cutMediaObject.findMany({where: {@@ -327,6 +366,12 @@ export async function runGc(): Promise<Response> {).catch(() => [] as string[]);await del(scratch);+ // Stale overlays, one listing per user tree that exists in the bucket.+ const overlays = await sweepStaleOverlays().catch((e: unknown) => {+ console.error("[cut-gc] overlay sweep failed:", e instanceof Error ? e.message : e);+ return 0;+ });+// Guarded like the prefix sweeps above it: the ladder store is optional// infrastructure whose loss costs a re-render, so a missing token or an// unreachable KV must not take down the run that reclaims storage.@@ -347,6 +392,7 @@ export async function runGc(): Promise<Response> {pendingObjects: pending.length,orphanedMedia: orphans.length,inferenceScratch: scratch.length,+ staleOverlays: overlays,renderJobs: jobs.count,libraryTombstones: tombstones.count,staleLadderProjects: ladders.projects,diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.tsindex 65cda454..ec99a61a 100644--- a/site/src/cut/server/cloud/library.ts+++ b/site/src/cut/server/cloud/library.ts@@ -19,16 +19,7 @@ import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl, mediaUrlLifetime } from "./mediimport { getProject, takenMediaNames } from "./projects";import { copy, del, head, libraryKey, presignPut, projectMediaKey } from "./r2";import { addUsage, quotaCheck } from "./usage";-import {- caught,- decodeFileParam,- dedupeName,- err,- inspirationFolderId,- redirect,- safeFileName,- typeOf,-} from "./util";+import { caught, decodeFileParam, dedupeName, err, HttpResponseError, inspirationFolderId, redirect, safeFileName, typeOf } from "./util";/** Cap on one signed-URL batch, matching the project media batch. */const PRESIGN_GET_BATCH_MAX = 500;@@ -168,6 +159,9 @@ async function copyIntoLibrary(src: MediaObjectRow,taken: Set<string>,): Promise<string> {+ // A copy is new bytes on the account, held to the quota like an upload.+ const over = await quotaCheck(userId, Number(src.bytes));+ if (over) throw new HttpResponseError(over);const dest = dedupeName(safeFileName(src.fileName), taken);taken.add(dest);const key = libraryKey(userId, dest);@@ -249,6 +243,8 @@ async function copyIntoProject(): Promise<string> {const src = await libraryMediaObject(userId, libFileName);if (!src) throw new Error("Library asset not found.");+ const over = await quotaCheck(userId, Number(src.bytes));+ if (over) throw new HttpResponseError(over);const dest = dedupeName(safeFileName(libFileName), taken);taken.add(dest);const key = projectMediaKey(userId, projectId, dest);@@ -791,20 +787,39 @@ export const libraryCloud = {if (!(await getProject(userId, projectId)))throw new Error("Project not found.");if (templateEmpty(input)) throw new Error("Nothing to save.");+ // Every source is resolved and the whole batch checked against the quota+ // before the first copy, so the wall is met with nothing written; a+ // copy that still fails takes the ones before it back with it, since a+ // library object no template names is one nothing could ever find.+ const sources = await Promise.all(+ input.media.map(async (m) => {+ const src = await projectMediaObject(userId, projectId, m.fileName);+ if (!src) throw new Error("Media file not found in project.");+ return { m, src };+ }),+ );+ const over = await quotaCheck(+ userId,+ sources.reduce((sum, { src }) => sum + Number(src.bytes), 0),+ );+ if (over) throw new HttpResponseError(over);const taken = await takenLibraryNames(userId);const media: TemplateMedia[] = [];- for (const m of input.media) {- const src = await projectMediaObject(userId, projectId, m.fileName);- if (!src) throw new Error("Media file not found in project.");- const dest = await copyIntoLibrary(userId, src, taken);- media.push({- fileName: dest,- name: m.name,- type: m.type,- duration: m.duration,- width: m.width,- height: m.height,- });+ try {+ for (const { m, src } of sources) {+ const dest = await copyIntoLibrary(userId, src, taken);+ media.push({+ fileName: dest,+ name: m.name,+ type: m.type,+ duration: m.duration,+ width: m.width,+ height: m.height,+ });+ }+ } catch (e) {+ for (const done of media) await deleteLibraryObject(userId, done.fileName).catch(() => {});+ throw e;}const doc: TemplateDoc = {folderId: null,@@ -897,6 +912,17 @@ export const libraryCloud = {const row = await findTemplate(userId, id);if (!row) throw new Error("Template not found.");const template = templateView(row);+ // The whole batch against the quota before the first copy; a project+ // copy that still fails is unreferenced project media, which the sweep+ // reclaims.+ let incoming = 0;+ for (const m of template.media) {+ const src = await libraryMediaObject(userId, m.fileName);+ if (!src) throw new Error("Library asset not found.");+ incoming += Number(src.bytes);+ }+ const over = await quotaCheck(userId, incoming);+ if (over) throw new HttpResponseError(over);const taken = await takenMediaNames(userId, projectId);const media: TemplateMedia[] = [];for (const m of template.media) {diff --git a/site/src/cut/server/cloud/projects.ts b/site/src/cut/server/cloud/projects.tsindex 422c0a2a..6d6c93b4 100644--- a/site/src/cut/server/cloud/projects.ts+++ b/site/src/cut/server/cloud/projects.ts@@ -6,7 +6,7 @@ import type { Prisma } from "@/generated/prisma/client";import { prisma } from "@/lib/prisma";import { deleteLadder } from "./ladderStore";import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl } from "./mediaCdn";-import { del, deletePrefix, projectExportKey, projectHlsRoot, projectMediaKey } from "./r2";+import { del, deletePrefix, projectExportKey, projectPrefix, projectMediaKey } from "./r2";import { addUsage } from "./usage";import { caught, decodeFileParam, err, redirect } from "./util";@@ -126,15 +126,16 @@ export async function deleteProjectCascade(userId: string, id: string): Promise<await addUsage(tx, userId, -freed);});await del(objects.map((o) => o.r2Key));- // The HLS ladders have no media rows at all — they are found by prefix and- // recorded in KV — so both have to be cleared by hand here. Neither affects- // `freed`: a ladder never counted toward the quota.+ // Then the whole tree, rows or no rows: the HLS ladders (found by prefix and+ // recorded in KV), an export a tab uploaded and never registered, a proxy+ // whose registration failed after its upload. None of it affects `freed`;+ // nothing without a row ever counted toward the quota.//- // The bytes go before the record does. A long 4K cut runs to tens of+ // The bytes go before the ladder record does. A long 4K cut runs to tens of// thousands of objects, so this pages rather than listing once; and pruning// the record first would leave anything the delete missed with no index that// could ever find it again.- await deletePrefix(projectHlsRoot(userId, id)).catch(() => 0);+ await deletePrefix(projectPrefix(userId, id)).catch(() => 0);await deleteLadder(id);return freed;}diff --git a/site/src/cut/server/cloud/r2.ts b/site/src/cut/server/cloud/r2.tsindex 3a1fff8c..7f05c260 100644--- a/site/src/cut/server/cloud/r2.ts+++ b/site/src/cut/server/cloud/r2.ts@@ -40,8 +40,13 @@ function r2(): S3Client {// --- Key scheme: everything a user owns lives under cut/<userId>/. ---+/** Everything a project owns in the bucket — media, exports, the hover proxy,+ * the card, every ladder — lives under this one prefix, so a project delete+ * can sweep it whole. */+export const projectPrefix = (userId: string, projectId: string) =>+ `cut/${userId}/projects/${projectId}/`;export const projectMediaKey = (userId: string, projectId: string, fileName: string) =>- `cut/${userId}/projects/${projectId}/media/${fileName}`;+ `${projectPrefix(userId, projectId)}media/${fileName}`;export const projectExportKey = (userId: string, projectId: string, fileName: string) =>`cut/${userId}/projects/${projectId}/exports/${fileName}`;export const projectPreviewKey = (userId: string, projectId: string) =>@@ -62,8 +67,13 @@ export const projectCardKey = (userId: string, projectId: string, ext: "jpg" | "`cut/${userId}/projects/${projectId}/card.${ext}`;export const libraryKey = (userId: string, fileName: string) =>`cut/${userId}/library/${fileName}`;+/** Per-user root of the bucket, and the overlay tree under it. `overlayPrefix`+ * is what the sweep walks: one listing per user that has any objects, found+ * through `listCommonPrefixes(USER_ROOT)`. */+export const USER_ROOT = "cut/";+export const overlayPrefix = (userId: string) => `cut/${userId}/overlays/`;export const overlayKey = (userId: string, batchId: string, name: string) =>- `cut/${userId}/overlays/${batchId}/${name}`;+ `${overlayPrefix(userId)}${batchId}/${name}`;/** Scratch media a hosted inference call carries — reference sheets, keyframes,* chat attachments. Keyed by the SHA-256 of the bytes, so a sheet that rides@@ -208,6 +218,28 @@ export async function listObjectsWithDates(return out;}+/** The "directories" directly under `prefix` — one entry per distinct next+ * path segment, without listing the objects inside. A sweep that walks every+ * user's tree starts here instead of from a table scan. */+export async function listCommonPrefixes(prefix: string): Promise<string[]> {+ const out: string[] = [];+ let token: string | undefined;+ do {+ const res = await r2().send(+ new ListObjectsV2Command({+ Bucket: R2_BUCKET,+ Prefix: prefix,+ Delimiter: "/",+ ContinuationToken: token,+ MaxKeys: 1000,+ })+ );+ for (const p of res.CommonPrefixes ?? []) if (p.Prefix) out.push(p.Prefix);+ token = res.IsTruncated ? res.NextContinuationToken : undefined;+ } while (token);+ return out;+}+/** Keys under `prefix` last written before `before`, paged to `limit` objects.* For prefix sweeps of objects no database row tracks. */export async function listOlderThan(diff --git a/site/src/cut/server/cloud/util.ts b/site/src/cut/server/cloud/util.tsindex 79e36b0d..a25b39b9 100644--- a/site/src/cut/server/cloud/util.ts+++ b/site/src/cut/server/cloud/util.ts@@ -6,7 +6,17 @@ import { R2NotConfiguredError } from "./r2";export const err = (message: string, status: number) =>Response.json({ error: message }, { status });+/** A refusal raised deep inside a helper — a quota wall met halfway through a+ * copy — carrying the exact response the route should answer with, so the+ * wall reads the same wherever it is hit. `caught` unwraps it. */+export class HttpResponseError extends Error {+ constructor(public readonly response: Response) {+ super(`HTTP ${response.status}`);+ }+}+export const caught = (e: unknown, fallback: string, status = 500) => {+ if (e instanceof HttpResponseError) return e.response;if (e instanceof R2NotConfiguredError) return err(e.message, 500);// The real reason belongs in the server's logs, not in a response an// anonymous share viewer reads: it names the environment variable to set.