Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: new zdr cleanup method (#3611)

Gergő Móricz16e04daa0fe2
diff --git a/apps/api/package.json b/apps/api/package.json
index 1e54d284b..41f4628a9 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -33,6 +33,8 @@
"extract-worker:production": "node dist/src/services/extract-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
"index-worker:production": "node dist/src/services/indexing/index-worker.js",
+ "zdr-worker": "tsc-watch --onSuccess \"node dist/src/services/zdr-worker.js\"",
+ "zdr-worker:production": "node dist/src/services/zdr-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
"sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",
diff --git a/apps/api/src/controllers/v0/admin/zdrcleaner.ts b/apps/api/src/controllers/v0/admin/zdrcleaner.ts
deleted file mode 100644
index 687bf7b8d..000000000
--- a/apps/api/src/controllers/v0/admin/zdrcleaner.ts
+++ /dev/null
@@ -1,134 +0,0 @@
-import "dotenv/config";
-import { supabase_service } from "../../../services/supabase";
-import { removeJobFromGCS } from "../../../lib/gcs-jobs";
-import { Request, Response } from "express";
-import { logger as _logger } from "../../../lib/logger";
-import type { Logger } from "winston";
-
-async function cleanUpJob(jobId: string) {
- await removeJobFromGCS(jobId);
-}
-
-async function cleanUpRequests(specificTeamId: string | null, _logger: Logger) {
- const logger = _logger.child({
- ...(specificTeamId ? { teamId: specificTeamId } : {}),
- method: "cleanUpRequests",
- });
-
- // Track all blobs that need cleanup for each request
- const requestBlobs = new Map<string, Set<string>>();
- // Track which blobs were successfully cleaned up
- const successfulBlobs = new Set<string>();
-
- try {
- for (let i = 0; ; i++) {
- // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)
- // associated with requests that need cleanup.
- // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)
- const { data: rows, error } = await supabase_service.rpc(
- "get_zdr_cleanup_batch",
- {
- p_team_id: specificTeamId,
- p_limit: 1000,
- p_offset: i * 1000,
- },
- );
-
- if (error) {
- logger.error("Error calling get_zdr_cleanup_batch RPC", { error, i });
- throw error;
- }
-
- if (!rows || rows.length === 0) {
- break;
- }
-
- // Track all blobs for each request before processing
- for (const row of rows as { request_id: string; blob_id: string }[]) {
- if (!requestBlobs.has(row.request_id)) {
- requestBlobs.set(row.request_id, new Set());
- }
- requestBlobs.get(row.request_id)!.add(row.blob_id);
- }
-
- // Process in batches of 50 for GCS cleanup
- for (let j = 0; j < Math.ceil(rows.length / 50); j++) {
- const batch = rows.slice(j * 50, (j + 1) * 50);
- await Promise.allSettled(
- batch.map(async (row: { request_id: string; blob_id: string }) => {
- try {
- await cleanUpJob(row.blob_id);
- successfulBlobs.add(row.blob_id);
- } catch (error) {
- logger.error(`Error cleaning up blob`, {
- method: "cleanUpJob",
- blobId: row.blob_id,
- requestId: row.request_id,
- error,
- });
- // Don't throw - continue with other blobs
- }
- }),
- );
- }
-
- if (rows.length < 1000) {
- break;
- }
- }
- } catch (error) {
- logger.error(`Error looping through cleanup batch`, {
- error,
- });
- }
-
- // Only clear dr_clean_by on requests where ALL blobs were successfully cleaned up
- const fullyCleanedRequests: string[] = [];
- for (const [requestId, blobIds] of requestBlobs) {
- const allBlobsCleaned = [...blobIds].every(blobId =>
- successfulBlobs.has(blobId),
- );
- if (allBlobsCleaned) {
- fullyCleanedRequests.push(requestId);
- }
- }
-
- if (fullyCleanedRequests.length > 0) {
- // Process in batches of 100 to deal with URL limits on PostgREST
- for (let i = 0; i < fullyCleanedRequests.length; i += 100) {
- const batch = fullyCleanedRequests.slice(i, i + 100);
- try {
- await supabase_service
- .from("requests")
- .update({
- dr_clean_by: null,
- })
- .in("id", batch)
- .throwOnError();
-
- logger.info(`Cleaned up ${fullyCleanedRequests.length} requests`);
- } catch (error) {
- logger.error(`Error clearing dr_clean_by on requests`, {
- error,
- requestCount: fullyCleanedRequests.length,
- });
- }
- }
- }
-}
-
-export async function zdrcleanerController(req: Request, res: Response) {
- const logger = _logger.child({
- module: "zdrcleaner",
- method: "zdrcleanerController",
- });
-
- const teamId = (req.query.teamId as string | undefined) ?? null;
-
- // Clean up requests/scrapes tables
- await cleanUpRequests(teamId, logger);
-
- logger.info("ZDR Cleaner finished!");
-
- res.json({ ok: true });
-}
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
new file mode 100644
index 000000000..2f02e2dc6
--- /dev/null
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -0,0 +1,123 @@
+import "dotenv/config";
+import { supabase_service } from "../services/supabase";
+import { removeJobFromGCS } from "./gcs-jobs";
+import { logger as _logger } from "./logger";
+
+export async function zdrcleaner() {
+ const logger = _logger.child({
+ module: "zdrcleaner",
+ method: "zdrcleaner",
+ canonicalLog: "zdrcleaner",
+ });
+
+ const start = Date.now();
+ try {
+ // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)
+ // associated with requests that need cleanup.
+ // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)
+ const { data: _rows, error } = await supabase_service.rpc(
+ "get_zdr_cleanup_batch_2",
+ {
+ p_limit: 1000,
+ },
+ );
+
+ if (error) {
+ throw new Error("Error calling get_zdr_cleanup_batch_2 RPC", {
+ cause: error,
+ });
+ }
+
+ const rows: { request_id: string; ids: string[] }[] = _rows;
+
+ if (!rows || rows.length === 0) {
+ logger.debug("zdrcleaner batch completed with no rows to process", {
+ canonicalLog: "zdrcleaner",
+ success: true,
+ timeMs: Date.now() - start,
+ });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ const requestBlobs = new Map<string, Set<string>>();
+ const successfulBlobs = new Set<string>();
+
+ // Track all blobs for each request before processing
+ for (const row of rows) {
+ requestBlobs.set(row.request_id, new Set(row.ids));
+ }
+
+ // Process in batches of 50 for GCS cleanup
+ const deleteErrors: any[] = [];
+ for (let j = 0; j < Math.ceil(rows.length / 50); j++) {
+ const batch = rows.slice(j * 50, (j + 1) * 50);
+ await Promise.allSettled(
+ batch
+ .flatMap(x => x.ids)
+ .map(async blob_id => {
+ try {
+ await removeJobFromGCS(blob_id);
+ successfulBlobs.add(blob_id);
+ } catch (error) {
+ deleteErrors.push(error);
+ }
+ }),
+ );
+ }
+
+ const fullyCleanedRequests: string[] = [];
+ for (const [requestId, blobIds] of requestBlobs) {
+ const allBlobsCleaned = [...blobIds].every(blobId =>
+ successfulBlobs.has(blobId),
+ );
+ if (allBlobsCleaned) {
+ fullyCleanedRequests.push(requestId);
+ }
+ }
+
+ const updateErrors: any[] = [];
+
+ if (fullyCleanedRequests.length > 0) {
+ // Process in batches of 100 to deal with URL limits on PostgREST
+ for (let i = 0; i < fullyCleanedRequests.length; i += 100) {
+ const batch = fullyCleanedRequests.slice(i, i + 100);
+ try {
+ await supabase_service
+ .from("requests")
+ .update({
+ dr_clean_by: null,
+ })
+ .in("id", batch)
+ .throwOnError();
+ } catch (error) {
+ updateErrors.push(error);
+ }
+ }
+ }
+
+ if (deleteErrors.length > 0 || updateErrors.length > 0) {
+ logger.warn("zdrcleaner batch completed with errors", {
+ canonicalLog: "zdrcleaner",
+ success: true,
+ deleteErrors,
+ updateErrors,
+ timeMs: Date.now() - start,
+ });
+ } else {
+ logger.debug("zdrcleaner batch completed", {
+ canonicalLog: "zdrcleaner",
+ success: true,
+ deleteErrors,
+ timeMs: Date.now() - start,
+ });
+ }
+ } catch (error) {
+ logger.error(`Error looping through cleanup batch`, {
+ canonicalLog: "zdrcleaner",
+ success: false,
+ timeMs: Date.now() - start,
+ error,
+ });
+ }
+}
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 4bccf2336..aebdf1a0a 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -7,7 +7,6 @@ import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-cle
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
import { cclogController } from "../controllers/v0/admin/cclog";
import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
-import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";
import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
import {
metricsController,
@@ -47,11 +46,6 @@ adminRouter.get(
adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/cclog`, wrap(cclogController));
-adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/zdrcleaner`,
- wrap(zdrcleanerController),
-);
-
adminRouter.get(
`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,
wrap(indexQueuePrometheus),
diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.ts
new file mode 100644
index 000000000..5dde4ddec
--- /dev/null
+++ b/apps/api/src/services/zdr-worker.ts
@@ -0,0 +1,28 @@
+import "dotenv/config";
+import "./sentry";
+import { setSentryServiceTag } from "./sentry";
+import { logger } from "../lib/logger";
+import { zdrcleaner } from "../lib/zdrcleaner";
+
+let isShuttingDown = false;
+
+process.on("SIGINT", () => {
+ logger.info("Received SIGINT. Shutting down gracefully...");
+ isShuttingDown = true;
+});
+
+process.on("SIGTERM", () => {
+ logger.info("Received SIGTERM. Shutting down gracefully...");
+ isShuttingDown = true;
+});
+
+(async () => {
+ setSentryServiceTag("zdr-worker");
+
+ while (!isShuttingDown) {
+ await zdrcleaner();
+ }
+
+ logger.info("zdr-worker exiting");
+ process.exit(0);
+})();