Open source CapCut alternative.
TypeScript
// Storage garbage collection: presigned-but-never-completed media objects,
// project media no doc references, long-terminal render jobs, and storage
// reclamation for lapsed subscriptions past their grace window. Runs from
// the Vercel daily cron (vercel.json) or a superuser hitting
// GET /api/cut-cloud/gc.
import { prisma } from "@/lib/prisma";
import { graceDeadline } from "./grace";
import { cutLimitsFor, FREE_STORAGE_BYTES } from "./limits";
import { collectable, listSweepableLadders, pruneLadder, readLadder } from "./ladderStore";
import { deleteLibraryAssetCascade } from "./library";
import { deleteProjectCascade } from "./projects";
import {
del,
deletePrefix,
INFERENCE_PREFIX,
listCommonPrefixes,
listOlderThan,
projectHlsRoot,
USER_ROOT,
} from "./r2";
import { addUsage, PENDING_CLAIM_MAX_AGE_MS } from "./usage";
// Scratch media a hosted inference call carried. No row tracks these — they are
// found by prefix and live only as long as the calls that might reuse them.
const SCRATCH_MAX_AGE_MS = 24 * 60 * 60 * 1000;
const ORPHAN_MAX_AGE_MS = 24 * 60 * 60 * 1000;
const JOB_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
// A deleted library asset leaves a row behind so the iOS app can take the same
// clip off the phone. A phone that has not synced in a season has a bigger
// problem than a missed delete, so the row goes after that.
const LIBRARY_TOMBSTONE_MAX_AGE_MS = 90 * 24 * 60 * 60 * 1000;
const TERMINAL_STATES = ["done", "error", "canceled", "dismissed"];
// Overlay PNGs are single-use render inputs the worker deletes when their job
// settles. Ones presigned for a job that was never queued have no row and no
// job, so age is what makes them garbage; two days is far past any render.
const OVERLAY_MAX_AGE_MS = 2 * 24 * 60 * 60 * 1000;
// Reclamation converges over daily runs rather than risking one long one.
const RECLAIM_SCAN_LIMIT = 500;
const RECLAIM_MAX_USERS = 10;
const RECLAIM_MAX_DELETES_PER_USER = 25;
// A re-rendered HLS ladder is written to a NEW version prefix so viewers on
// the old one keep playing off the edge cache. That makes the previous ladder
// garbage the moment the new one publishes — but not immediately collectable,
// because someone may still be watching it. A day of grace clears any single
// sitting, timed from when the version was RETIRED (the record carries that
// stamp) rather than from when its bytes were written, which for a ladder built
// weeks ago is already long past and would mean no grace at all.
const HLS_RETIRED_GRACE_MS = 24 * 60 * 60 * 1000;
// A version claimed but never published belongs to a render that is either
// still going or dead. The longest a ladder can legitimately take is bounded by
// the worker's own job lifetime, so past this it is a dead render's leftovers.
const HLS_PENDING_TIMEOUT_MS = 6 * 60 * 60 * 1000;
type ReclaimPlan = { projects: string[]; assets: string[]; bytes: number };
/** The proxy file names a stored document's assets carry — what keeps a
* kind "proxy" object alive. */
export function proxyNamesIn(doc: unknown): Set<string> {
const assets = (doc as { assets?: { proxy?: { fileName?: string } }[] } | null)?.assets ?? [];
return new Set(assets.flatMap((a) => (a.proxy?.fileName ? [a.proxy.fileName] : [])));
}
/** Delete stored objects and their rows, one transaction per account.
* Quota-exempt rows never added usage, so only the counted ones subtract. */
async function dropMediaRows(
rows: { id: string; userId: string; r2Key: string; bytes: bigint; quotaExempt: boolean }[]
): Promise<void> {
const byUser = new Map<string, typeof rows>();
for (const o of rows) {
const list = byUser.get(o.userId) ?? [];
list.push(o);
byUser.set(o.userId, list);
}
for (const [userId, list] of byUser) {
const counted = list.filter((r) => !r.quotaExempt);
await prisma.$transaction(async (tx) => {
await tx.cutMediaObject.deleteMany({ where: { id: { in: list.map((r) => r.id) } } });
await addUsage(tx, userId, -counted.reduce((sum, r) => sum + Number(r.bytes), 0));
});
await del(list.map((r) => r.r2Key));
}
}
/**
* Drop ladder versions a project has moved on from, and the leftovers of
* renders that died partway.
*
* Ladders carry no media rows and no storage quota, so the record in KV is the
* only index of them and this is a pure prefix delete. Two clocks decide what
* goes: a retired version leaves a day after it stopped being current, and a
* pending version — one claimed before upload whose render never published —
* leaves once it is too old to still be running.
*
* The work scales with the garbage, not the project count: only records that
* carry retired or pending versions are listed at all, and a swept record drops
* out of that set. So there is no per-run project cap to strand the tail of the
* namespace behind the same first hundred keys every night.
*/
async function sweepStaleLadders() {
const out = { projects: 0, versions: 0, objects: 0, failed: 0 };
const now = Date.now();
for (const { projectId } of await listSweepableLadders()) {
try {
const record = await readLadder(projectId);
if (!record) continue;
const drop = collectable(record, now, HLS_RETIRED_GRACE_MS, HLS_PENDING_TIMEOUT_MS);
if (drop.length === 0) continue;
const root = projectHlsRoot(record.userId, projectId);
const dropped = new Set<string>();
for (const version of drop) {
out.objects += await deletePrefix(`${root}${version}/`);
dropped.add(version);
}
// Only now does the record forget them: pruning first would unindex the
// objects if the delete then failed.
await pruneLadder(projectId, record, dropped);
out.projects += 1;
out.versions += dropped.size;
} catch {
// One project's sweep failing must not fail the cron, and the record
// still names its garbage, so the next run tries again.
out.failed += 1;
}
}
return out;
}
/** What the sweep can actually free for one account, oldest first: whole
* projects, then library assets, stopping as soon as the plan covers `target`.
* Sizes come from the media rows themselves, so a drifted usage counter can
* never widen a deletion, and anything that would free nothing is left out —
* it costs the user their work and buys no space. */
async function reclaimPlan(userId: string, target: number): Promise<ReclaimPlan> {
const plan: ReclaimPlan = { projects: [], assets: [], bytes: 0 };
const projects = await prisma.cutProject.findMany({
where: { userId },
orderBy: { updatedAt: "asc" },
select: { id: true },
});
const sums = await prisma.cutMediaObject.groupBy({
by: ["projectId"],
where: {
userId,
projectId: { in: projects.map((p) => p.id) },
uploadState: "complete",
// Exempt bytes never counted toward the quota, so deleting them frees
// nothing the quota can see — they must not pad a project's claim.
quotaExempt: false,
},
_sum: { bytes: true },
});
const projectBytes = new Map(sums.map((s) => [s.projectId as string, Number(s._sum.bytes ?? 0)]));
for (const p of projects) {
if (plan.bytes >= target) return plan;
const size = projectBytes.get(p.id) ?? 0;
if (size <= 0) continue;
plan.projects.push(p.id);
plan.bytes += size;
}
const assets = await prisma.cutLibraryAsset.findMany({
where: { userId, deletedAt: null },
orderBy: { createdAt: "asc" },
select: { id: true, mediaObjectId: true },
});
const objects = await prisma.cutMediaObject.findMany({
where: {
userId,
uploadState: "complete",
id: { in: assets.map((a) => a.mediaObjectId) },
},
select: { id: true, bytes: true },
});
const assetBytes = new Map(objects.map((o) => [o.id, Number(o.bytes)]));
for (const a of assets) {
if (plan.bytes >= target) break;
const size = assetBytes.get(a.mediaObjectId) ?? 0;
if (size <= 0) continue;
plan.assets.push(a.id);
plan.bytes += size;
}
return plan;
}
/** Bring accounts whose Pro ended and whose grace window has passed back under
* the free storage cap: least-recently-updated projects first, then the oldest
* library assets. The plan is built and checked before anything is deleted, so
* an account whose overage the sweep cannot reach is reported rather than
* emptied, and content under the cap is never touched. */
async function reclaimOverQuota() {
const out = {
users: 0,
projects: 0,
libraryAssets: 0,
bytes: 0,
// Left for tomorrow by the per-run caps: a truncated sweep has to look
// truncated in the summary, not finished.
deferred: 0,
// Over the cap with nothing worth deleting — needs a human, not a sweep.
unreclaimable: 0,
failed: 0,
};
const overCap = await prisma.cutStorageUsage.findMany({
where: { bytes: { gt: BigInt(FREE_STORAGE_BYTES) } },
orderBy: { bytes: "desc" },
take: RECLAIM_SCAN_LIMIT,
select: { userId: true },
});
if (overCap.length === 0) return out;
// One read for the whole batch: most accounts over the free cap are paying
// ones the sweep will never touch, and they are not worth a query each.
const subs = await prisma.proSubscription.findMany({
where: { userId: { in: overCap.map((r) => r.userId) } },
select: { userId: true, status: true, endedAt: true, cancelAt: true, currentPeriodEnd: true },
});
const now = Date.now();
const lapsed = subs.filter((s) => {
const deadline = graceDeadline(s);
return deadline !== null && now >= deadline.getTime();
});
for (const { userId } of lapsed) {
if (out.users >= RECLAIM_MAX_USERS) {
out.deferred += 1;
continue;
}
try {
// Re-read the tier: a superuser flag or a resubscribe since the scan
// takes the account back out of reach.
const limits = await cutLimitsFor(userId);
if (limits.storageBytes === null || limits.storageBytes > FREE_STORAGE_BYTES) continue;
const stored = await prisma.cutMediaObject.aggregate({
where: { userId, uploadState: "complete", quotaExempt: false },
_sum: { bytes: true },
});
const target = Number(stored._sum.bytes ?? 0) - limits.storageBytes;
if (target <= 0) continue;
const plan = await reclaimPlan(userId, target);
if (plan.bytes < target) {
out.unreclaimable += 1;
continue;
}
out.users += 1;
let budget = RECLAIM_MAX_DELETES_PER_USER;
for (const projectId of plan.projects) {
if (budget <= 0) break;
out.bytes += await deleteProjectCascade(userId, projectId);
out.projects += 1;
budget -= 1;
}
for (const assetId of plan.assets) {
if (budget <= 0) break;
out.bytes += (await deleteLibraryAssetCascade(userId, assetId)) ?? 0;
out.libraryAssets += 1;
budget -= 1;
}
if (plan.projects.length + plan.assets.length > RECLAIM_MAX_DELETES_PER_USER) {
out.deferred += 1;
}
} catch {
// One account's failure — a project deleted underneath us, an R2 blip —
// must not stop the sweep or fail the cron.
out.failed += 1;
}
}
return out;
}
/** Overlay inputs older than any render could still be waiting on: presigned
* uploads whose export never started. Walks the bucket's user trees rather
* than the user table, so it touches only accounts that hold objects. */
async function sweepStaleOverlays(): Promise<number> {
const before = new Date(Date.now() - OVERLAY_MAX_AGE_MS);
// A job still waiting to be claimed — through a wake outage, say — reads its
// overlays when it finally runs, however old they are by then.
const live = await prisma.cutRenderJob.findMany({
where: { state: { in: ["queued", "running"] } },
select: { spec: true },
});
const held = new Set(
live.flatMap((j) => {
const overlays = (j.spec as { overlays?: { key?: unknown }[] } | null)?.overlays ?? [];
return overlays.map((o) => o?.key).filter((k): k is string => typeof k === "string");
})
);
let removed = 0;
for (const user of await listCommonPrefixes(USER_ROOT)) {
if (user === INFERENCE_PREFIX) continue;
const keys = (await listOlderThan(`${user}overlays/`, before)).filter((k) => !held.has(k));
await del(keys);
removed += keys.length;
}
return removed;
}
export async function runGc(): Promise<Response> {
// Existing derived files adopt the same accounting as newly rendered ones.
const chargedDerived = await prisma.cutMediaObject.findMany({
where: { kind: { in: ["preview", "card"] }, quotaExempt: false, uploadState: "complete" },
take: RECLAIM_SCAN_LIMIT,
select: { id: true, userId: true, bytes: true },
});
for (const object of chargedDerived) {
await prisma.$transaction(async (tx) => {
const changed = await tx.cutMediaObject.updateMany({
where: { id: object.id, quotaExempt: false, bytes: object.bytes },
data: { quotaExempt: true },
});
if (changed.count) await addUsage(tx, object.userId, -Number(object.bytes));
});
}
const preparing = await prisma.cutRenderJob.findMany({
where: { kind: "convert", state: { in: ["queued", "running"] }, spec: { path: ["target"], equals: "library" } },
select: { spec: true },
});
const heldUploads = preparing.flatMap((job) => {
const key = (job.spec as { key?: unknown } | null)?.key;
return typeof key === "string" ? [key] : [];
});
const pending = await prisma.cutMediaObject.findMany({
where: {
uploadState: "pending",
r2Key: { notIn: heldUploads },
createdAt: { lt: new Date(Date.now() - PENDING_CLAIM_MAX_AGE_MS) },
},
select: { id: true, r2Key: true },
});
// Pending rows never counted toward usage, so no usage adjustment here.
await del(pending.map((o) => o.r2Key));
if (pending.length > 0) {
await prisma.cutMediaObject.deleteMany({
where: { id: { in: pending.map((o) => o.id) } },
});
}
// Unreferenced project media: bytes a worker or client landed in a project
// whose doc never adopted them — an abandoned import, a tab closed before
// the save. The doc is the sole owner of project media, adoption happens
// within seconds of landing, and the one delayed delivery path (the cloud
// import poll) caps at ten minutes — so after a day of grace, unreferenced
// means garbage. Docs of deleted projects are gone too, which orphans (and
// sweeps) any rows their deletion missed.
const mediaCandidates = await prisma.cutMediaObject.findMany({
where: {
kind: "media",
uploadState: "complete",
projectId: { not: null },
updatedAt: { lt: new Date(Date.now() - ORPHAN_MAX_AGE_MS) },
},
select: {
id: true,
userId: true,
projectId: true,
fileName: true,
r2Key: true,
bytes: true,
quotaExempt: true,
},
});
const docProjects = await prisma.cutProject.findMany({
where: { id: { in: [...new Set(mediaCandidates.map((c) => c.projectId!))] } },
select: { id: true, doc: true },
});
const referenced = new Map(
docProjects.map((p) => {
const assets = (p.doc as { assets?: { fileName?: string }[] } | null)?.assets ?? [];
return [p.id, new Set(assets.map((a) => a.fileName))];
})
);
const activeRenders = await prisma.cutRenderJob.findMany({
where: { state: { in: ["queued", "running"] }, projectId: { in: docProjects.map((p) => p.id) } },
select: { projectId: true, spec: true },
});
for (const job of activeRenders) {
if (!job.projectId) continue;
const spec = job.spec as {
fromDoc?: { snapshot?: { doc?: { assets?: { fileName: string }[] } } };
spec?: { clips?: { file?: string }[]; audio?: { file?: string }[]; overlayVideos?: { file?: string }[] };
} | null;
const files = referenced.get(job.projectId) ?? new Set<string | undefined>();
for (const asset of spec?.fromDoc?.snapshot?.doc?.assets ?? []) files.add(asset.fileName);
for (const item of [...(spec?.spec?.clips ?? []), ...(spec?.spec?.audio ?? []), ...(spec?.spec?.overlayVideos ?? [])])
if (item.file) files.add(item.file);
referenced.set(job.projectId, files);
}
const orphans = mediaCandidates.filter((c) => !referenced.get(c.projectId!)?.has(c.fileName));
await dropMediaRows(orphans);
// Preview proxies no asset names any more — the master's asset was deleted
// by a client that never got to drop the proxy, or the doc is gone. An
// asset carries its proxy's file name, so that is the reference.
const proxyCandidates = await prisma.cutMediaObject.findMany({
where: {
kind: "proxy",
uploadState: "complete",
projectId: { not: null },
updatedAt: { lt: new Date(Date.now() - ORPHAN_MAX_AGE_MS) },
},
select: { id: true, userId: true, projectId: true, fileName: true, r2Key: true, bytes: true, quotaExempt: true },
});
const proxyProjects = await prisma.cutProject.findMany({
where: { id: { in: [...new Set(proxyCandidates.map((c) => c.projectId!))] } },
select: { id: true, doc: true },
});
const referencedProxies = new Map(proxyProjects.map((p) => [p.id, proxyNamesIn(p.doc)]));
const orphanProxies = proxyCandidates.filter((c) => !referencedProxies.get(c.projectId!)?.has(c.fileName));
await dropMediaRows(orphanProxies);
// Retired preview revisions get the same grace as other unreferenced media.
// The current preview remains available for an idle project indefinitely.
const previews = await prisma.cutMediaObject.findMany({
where: { kind: "preview", updatedAt: { lt: new Date(Date.now() - ORPHAN_MAX_AGE_MS) } },
select: { id: true, userId: true, projectId: true, r2Key: true, bytes: true, quotaExempt: true },
});
const previewProjects = await prisma.cutProject.findMany({
where: { id: { in: previews.flatMap((p) => p.projectId ? [p.projectId] : []) } },
select: { previewKey: true },
});
const livePreviewKeys = new Set(previewProjects.map((p) => p.previewKey));
for (const preview of previews) {
if (livePreviewKeys.has(preview.r2Key)) continue;
await prisma.$transaction(async (tx) => {
const removed = await tx.cutMediaObject.deleteMany({ where: { id: preview.id } });
if (removed.count && !preview.quotaExempt) await addUsage(tx, preview.userId, -Number(preview.bytes));
});
await del([preview.r2Key]);
}
// Old terminal jobs go together with their overlay PNGs — the worker deletes
// overlays when it settles a job, but a job canceled while still queued was
// never claimed, so its overlays only die here.
const oldJobs = await prisma.cutRenderJob.findMany({
where: {
state: { in: TERMINAL_STATES },
createdAt: { lt: new Date(Date.now() - JOB_MAX_AGE_MS) },
},
select: { id: true, userId: true, spec: true },
});
const overlayKeys = oldJobs.flatMap((j) => {
const overlays = (j.spec as { overlays?: { key?: unknown }[] } | null)?.overlays ?? [];
const prefix = `cut/${j.userId}/overlays/`;
return overlays
.map((o) => o?.key)
.filter((k): k is string => typeof k === "string" && k.startsWith(prefix));
});
await del(overlayKeys);
const jobs =
oldJobs.length > 0
? await prisma.cutRenderJob.deleteMany({ where: { id: { in: oldJobs.map((j) => j.id) } } })
: { count: 0 };
// Scratch inference media, swept by prefix: it never counted toward usage and
// no row points at it, so age is the only thing that makes it garbage. The
// nightly analytics pipeline also keeps rowless JSON under analytics/ — any
// future sweep of unknown prefixes must leave that tree alone.
const scratch = await listOlderThan(
INFERENCE_PREFIX,
new Date(Date.now() - SCRATCH_MAX_AGE_MS)
).catch(() => [] as string[]);
await del(scratch);
// Stale overlays, one listing per user tree that exists in the bucket.
const overlays = await sweepStaleOverlays().catch((e: unknown) => {
console.error("[cut-gc] overlay sweep failed:", e instanceof Error ? e.message : e);
return 0;
});
// Guarded like the prefix sweeps above it: the ladder store is optional
// infrastructure whose loss costs a re-render, so a missing token or an
// unreachable KV must not take down the run that reclaims storage.
const ladders = await sweepStaleLadders().catch((e: unknown) => {
console.error("[cut-gc] ladder sweep failed:", e instanceof Error ? e.message : e);
return { projects: 0, versions: 0, objects: 0, failed: 1 };
});
// Library tombstones past the window every client has had to see them in.
// Their bytes went with the delete; this is the empty row.
const tombstones = await prisma.cutLibraryAsset.deleteMany({
where: { deletedAt: { lt: new Date(Date.now() - LIBRARY_TOMBSTONE_MAX_AGE_MS) } },
});
const reclaimed = await reclaimOverQuota();
return Response.json({
pendingObjects: pending.length,
orphanedMedia: orphans.length,
orphanedProxies: orphanProxies.length,
inferenceScratch: scratch.length,
staleOverlays: overlays,
renderJobs: jobs.count,
libraryTombstones: tombstones.count,
staleLadderProjects: ladders.projects,
staleLadderVersions: ladders.versions,
staleLadderObjects: ladders.objects,
staleLadderFailures: ladders.failed,
reclaimedUsers: reclaimed.users,
reclaimedProjects: reclaimed.projects,
reclaimedLibraryAssets: reclaimed.libraryAssets,
reclaimedBytes: reclaimed.bytes,
reclaimDeferredUsers: reclaimed.deferred,
reclaimUnreclaimableUsers: reclaimed.unreclaimable,
reclaimFailedUsers: reclaimed.failed,
});
}