Skip to content

Open source CapCut alternative.

TypeScript

fix(site): a presign reserves its bytes against the storage quota

David Leb344d7214426
diff --git a/site/src/cut/server/cloud/gc.ts b/site/src/cut/server/cloud/gc.ts
index 55c7bb4f..69beddef 100644
--- a/site/src/cut/server/cloud/gc.ts
+++ b/site/src/cut/server/cloud/gc.ts
@@ -10,9 +10,8 @@ import { collectable, listSweepableLadders, pruneLadder, readLadder } from "./la
import { deleteLibraryAssetCascade } from "./library";
import { deleteProjectCascade } from "./projects";
import { del, deletePrefix, INFERENCE_PREFIX, listOlderThan, projectHlsRoot } from "./r2";
-import { addUsage } from "./usage";
+import { addUsage, PENDING_CLAIM_MAX_AGE_MS } from "./usage";
-const PENDING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// Scratch media a hosted inference call carried. No row tracks these — they are
// found by prefix and live only as long as the calls that might reuse them.
const SCRATCH_MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -233,7 +232,7 @@ export async function runGc(): Promise<Response> {
const pending = await prisma.cutMediaObject.findMany({
where: {
uploadState: "pending",
- createdAt: { lt: new Date(Date.now() - PENDING_MAX_AGE_MS) },
+ createdAt: { lt: new Date(Date.now() - PENDING_CLAIM_MAX_AGE_MS) },
},
select: { id: true, r2Key: true },
});
diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.ts
index 53b20e7a..6243a772 100644
--- a/site/src/cut/server/cloud/library.ts
+++ b/site/src/cut/server/cloud/library.ts
@@ -364,7 +364,7 @@ export const libraryCloud = {
if (row.uploadState === "complete") {
return Response.json({ fileName: row.fileName, key: row.r2Key, done: true });
}
- const url = await presignPut(row.r2Key, body.mime ?? "application/octet-stream");
+ const url = await presignPut(row.r2Key, body.mime ?? "application/octet-stream", Number(row.bytes));
return Response.json({ fileName: row.fileName, key: row.r2Key, url });
}
}
@@ -375,17 +375,15 @@ export const libraryCloud = {
await takenLibraryNames(userId),
);
const key = libraryKey(userId, fileName);
- const url = await presignPut(
- key,
- body.mime ?? "application/octet-stream",
- );
+ const bytes = Math.round(body.bytes);
+ const url = await presignPut(key, body.mime ?? "application/octet-stream", bytes);
await prisma.cutMediaObject.create({
data: {
userId,
r2Key: key,
fileName,
mime: body.mime ?? "",
- bytes: BigInt(Math.round(body.bytes)),
+ bytes: BigInt(bytes),
kind: "library",
uploadState: "pending",
},
diff --git a/site/src/cut/server/cloud/media.ts b/site/src/cut/server/cloud/media.ts
index ffd94b96..7ffd3b6d 100644
--- a/site/src/cut/server/cloud/media.ts
+++ b/site/src/cut/server/cloud/media.ts
@@ -84,7 +84,7 @@ export const mediaCloud = {
if (row.uploadState === "complete") {
return Response.json({ fileName: row.fileName, key: row.r2Key, done: true });
}
- const url = await presignPut(row.r2Key, body.mime ?? "application/octet-stream");
+ const url = await presignPut(row.r2Key, body.mime ?? "application/octet-stream", Number(row.bytes));
return Response.json({ fileName: row.fileName, key: row.r2Key, url });
}
}
@@ -101,7 +101,8 @@ export const mediaCloud = {
new Set([...(await takenMediaNames(userId, projectId)), ...avoid])
);
const key = projectMediaKey(userId, projectId, fileName);
- const url = await presignPut(key, body.mime ?? "application/octet-stream");
+ const bytes = Math.round(body.bytes);
+ const url = await presignPut(key, body.mime ?? "application/octet-stream", bytes);
await prisma.cutMediaObject.create({
data: {
userId,
@@ -109,7 +110,7 @@ export const mediaCloud = {
r2Key: key,
fileName,
mime: body.mime ?? "",
- bytes: BigInt(Math.round(body.bytes)),
+ bytes: BigInt(bytes),
kind: "media",
uploadState: "pending",
},
diff --git a/site/src/cut/server/cloud/r2.ts b/site/src/cut/server/cloud/r2.ts
index f2fb3d18..3a1fff8c 100644
--- a/site/src/cut/server/cloud/r2.ts
+++ b/site/src/cut/server/cloud/r2.ts
@@ -75,10 +75,19 @@ export const INFERENCE_PREFIX = "cut/inference/";
export const inferenceBlobKey = (userId: string, sha256: string, ext: string) =>
`${INFERENCE_PREFIX}${userId}/${sha256}.${ext}`;
-export function presignPut(key: string, mime: string): Promise<string> {
+/** A time-limited PUT URL. `bytes` pins the object size into the signature:
+ * the claim the quota was checked against is the most R2 will accept, and a
+ * larger body is a 403 at the bucket. Left out only for transient inputs the
+ * quota never sees. */
+export function presignPut(key: string, mime: string, bytes?: number): Promise<string> {
return getSignedUrl(
r2(),
- new PutObjectCommand({ Bucket: R2_BUCKET, Key: key, ContentType: mime }),
+ new PutObjectCommand({
+ Bucket: R2_BUCKET,
+ Key: key,
+ ContentType: mime,
+ ...(bytes !== undefined ? { ContentLength: bytes } : {}),
+ }),
{ expiresIn: PUT_EXPIRY_SECONDS }
);
}
diff --git a/site/src/cut/server/cloud/usage.ts b/site/src/cut/server/cloud/usage.ts
index 825635e5..a5acc8ba 100644
--- a/site/src/cut/server/cloud/usage.ts
+++ b/site/src/cut/server/cloud/usage.ts
@@ -8,6 +8,27 @@ export async function usageBytes(userId: string): Promise<number> {
return row ? Number(row.bytes) : 0;
}
+/** How long a presigned claim holds its bytes. A claim older than this never
+ * completed; the daily sweep drops it, and the quota stops counting it at the
+ * same age so the two agree. */
+export const PENDING_CLAIM_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+/** Bytes claimed by presigns still in flight. A presign is a reservation: the
+ * quota counts these alongside stored bytes, so a batch of uploads is held to
+ * the cap as a whole and the check is not a race against the completes. */
+export async function reservedBytes(userId: string): Promise<number> {
+ const sum = await prisma.cutMediaObject.aggregate({
+ _sum: { bytes: true },
+ where: {
+ userId,
+ uploadState: "pending",
+ quotaExempt: false,
+ createdAt: { gte: new Date(Date.now() - PENDING_CLAIM_MAX_AGE_MS) },
+ },
+ });
+ return Number(sum._sum.bytes ?? 0);
+}
+
/** Adjust the user's stored-bytes counter inside the caller's transaction, so
* it moves with the CutMediaObject rows it mirrors. Clamped at zero. */
export async function addUsage(tx: Prisma.TransactionClient, userId: string, delta: number) {
@@ -21,7 +42,8 @@ export async function addUsage(tx: Prisma.TransactionClient, userId: string, del
}
/** 413 when `incoming` more bytes would break the account's storage quota,
- * else null. Superusers are unquotaed. `margin` widens the ceiling for a write
+ * else null. Stored bytes and in-flight claims both count. Superusers are
+ * unquotaed. `margin` widens the ceiling for a write
* that gets something out rather than in (see EXPORT_QUOTA_MARGIN); the quota
* the caller is told about stays the real one, so the wall reads the same
* wherever it is raised. */
@@ -32,7 +54,8 @@ export async function quotaCheck(
): Promise<Response | null> {
const limits = await cutLimitsFor(userId);
if (limits.storageBytes === null) return null;
- const bytes = await usageBytes(userId);
+ const [stored, reserved] = await Promise.all([usageBytes(userId), reservedBytes(userId)]);
+ const bytes = stored + reserved;
if (bytes + incoming > limits.storageBytes * margin) {
return Response.json(
{ error: "storage_quota_exceeded", bytes, quotaBytes: limits.storageBytes },