Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): bound concurrent GCS deletes in ZDR request cleanup (#5742)
Gergő Móricz2b15ea232f5c
diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.tsindex bc6632e90..391d9db57 100644--- a/apps/api/src/lib/zdrcleaner.test.ts+++ b/apps/api/src/lib/zdrcleaner.test.ts@@ -124,6 +124,46 @@ describe("ZDR cleaner", () => {);});+ it("bounds concurrent blob deletes and still removes every blob", async () => {+ const ids = Array.from({ length: 500 }, (_, i) => `blob-${i}`);+ mockChildren(ids);+ let inFlight = 0;+ let maxInFlight = 0;+ removeJobFromGCS.mockImplementation(async () => {+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await new Promise(resolve => setTimeout(resolve, 1));+ inFlight--;+ });++ await cleanZdrRequest("request-large");++ expect(maxInFlight).toBeLessThanOrEqual(50);+ expect(maxInFlight).toBeGreaterThan(1);+ expect(new Set(removeJobFromGCS.mock.calls.map(([id]) => id))).toEqual(+ new Set(ids),+ );+ expect(removeJobFromGCS).toHaveBeenCalledTimes(ids.length);+ });++ it("keeps deleting after a failure and reports every failed blob", async () => {+ const ids = Array.from({ length: 200 }, (_, i) => `blob-${i}`);+ mockChildren(ids);+ removeJobFromGCS.mockImplementation(async id => {+ if (id === "blob-0" || id === "blob-199") {+ throw new Error("GCS unavailable");+ }+ });++ await expect(cleanZdrRequest("request-partial")).rejects.toThrow(+ "Failed to remove 2 blobs for ZDR request request-partial",+ );+ expect(new Set(removeJobFromGCS.mock.calls.map(([id]) => id))).toEqual(+ new Set(ids),+ );+ expect(removeJobFromGCS).toHaveBeenCalledTimes(ids.length);+ });+it("fails a queued request when ClickHouse is not configured", async () => {client.current = null;diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsindex d48fe5f87..20b391341 100644--- a/apps/api/src/lib/zdrcleaner.ts+++ b/apps/api/src/lib/zdrcleaner.ts@@ -12,18 +12,35 @@ export async function sendZdrHeartbeat() {}}+// A request can resolve to a very large number of blobs; issuing every delete+// at once exhausts the worker's memory.+const BLOB_DELETE_CONCURRENCY = 50;+async function removeBlobs(blobIds: string[]): Promise<unknown[]> {const logger = _logger.child({module: "zdrcleaner",method: "removeBlobs",zeroDataRetention: true,});- const results = await Promise.allSettled(- blobIds.map(blobId => removeJobFromGCS(blobId, logger)),- );- return results.flatMap(result =>- result.status === "rejected" ? [result.reason] : [],+ const errors: unknown[] = [];+ let next = 0;+ const worker = async () => {+ while (next < blobIds.length) {+ const blobId = blobIds[next++];+ try {+ await removeJobFromGCS(blobId, logger);+ } catch (error) {+ errors.push(error);+ }+ }+ };+ await Promise.all(+ Array.from(+ { length: Math.min(BLOB_DELETE_CONCURRENCY, blobIds.length) },+ worker,+ ),);+ return errors;}/**