Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): bound concurrent GCS deletes in ZDR request cleanup (#5742)

Gergő Móricz2b15ea232f5c
diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.ts
index bc6632e90..391d9db57 100644
--- a/apps/api/src/lib/zdrcleaner.test.ts
+++ b/apps/api/src/lib/zdrcleaner.test.ts
@@ -124,6 +124,46 @@ describe("ZDR cleaner", () => {
);
});
+ it("bounds concurrent blob deletes and still removes every blob", async () => {
+ const ids = Array.from({ length: 500 }, (_, i) => `blob-${i}`);
+ mockChildren(ids);
+ let inFlight = 0;
+ let maxInFlight = 0;
+ removeJobFromGCS.mockImplementation(async () => {
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise(resolve => setTimeout(resolve, 1));
+ inFlight--;
+ });
+
+ await cleanZdrRequest("request-large");
+
+ expect(maxInFlight).toBeLessThanOrEqual(50);
+ expect(maxInFlight).toBeGreaterThan(1);
+ expect(new Set(removeJobFromGCS.mock.calls.map(([id]) => id))).toEqual(
+ new Set(ids),
+ );
+ expect(removeJobFromGCS).toHaveBeenCalledTimes(ids.length);
+ });
+
+ it("keeps deleting after a failure and reports every failed blob", async () => {
+ const ids = Array.from({ length: 200 }, (_, i) => `blob-${i}`);
+ mockChildren(ids);
+ removeJobFromGCS.mockImplementation(async id => {
+ if (id === "blob-0" || id === "blob-199") {
+ throw new Error("GCS unavailable");
+ }
+ });
+
+ await expect(cleanZdrRequest("request-partial")).rejects.toThrow(
+ "Failed to remove 2 blobs for ZDR request request-partial",
+ );
+ expect(new Set(removeJobFromGCS.mock.calls.map(([id]) => id))).toEqual(
+ new Set(ids),
+ );
+ expect(removeJobFromGCS).toHaveBeenCalledTimes(ids.length);
+ });
+
it("fails a queued request when ClickHouse is not configured", async () => {
client.current = null;
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
index d48fe5f87..20b391341 100644
--- a/apps/api/src/lib/zdrcleaner.ts
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -12,18 +12,35 @@ export async function sendZdrHeartbeat() {
}
}
+// A request can resolve to a very large number of blobs; issuing every delete
+// at once exhausts the worker's memory.
+const BLOB_DELETE_CONCURRENCY = 50;
+
async function removeBlobs(blobIds: string[]): Promise<unknown[]> {
const logger = _logger.child({
module: "zdrcleaner",
method: "removeBlobs",
zeroDataRetention: true,
});
- const results = await Promise.allSettled(
- blobIds.map(blobId => removeJobFromGCS(blobId, logger)),
- );
- return results.flatMap(result =>
- result.status === "rejected" ? [result.reason] : [],
+ const errors: unknown[] = [];
+ let next = 0;
+ const worker = async () => {
+ while (next < blobIds.length) {
+ const blobId = blobIds[next++];
+ try {
+ await removeJobFromGCS(blobId, logger);
+ } catch (error) {
+ errors.push(error);
+ }
+ }
+ };
+ await Promise.all(
+ Array.from(
+ { length: Math.min(BLOB_DELETE_CONCURRENCY, blobIds.length) },
+ worker,
+ ),
);
+ return errors;
}
/**