Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: new zdr cleanup method (#3611)
Gergő Móricz16e04daa0fe2
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 1e54d284b..41f4628a9 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -33,6 +33,8 @@"extract-worker:production": "node dist/src/services/extract-worker.js","index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"","index-worker:production": "node dist/src/services/indexing/index-worker.js",+ "zdr-worker": "tsc-watch --onSuccess \"node dist/src/services/zdr-worker.js\"",+ "zdr-worker:production": "node dist/src/services/zdr-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest","mongo-docker-console": "docker exec -it mongodb mongosh","sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",diff --git a/apps/api/src/controllers/v0/admin/zdrcleaner.ts b/apps/api/src/controllers/v0/admin/zdrcleaner.tsdeleted file mode 100644index 687bf7b8d..000000000--- a/apps/api/src/controllers/v0/admin/zdrcleaner.ts+++ /dev/null@@ -1,134 +0,0 @@-import "dotenv/config";-import { supabase_service } from "../../../services/supabase";-import { removeJobFromGCS } from "../../../lib/gcs-jobs";-import { Request, Response } from "express";-import { logger as _logger } from "../../../lib/logger";-import type { Logger } from "winston";--async function cleanUpJob(jobId: string) {- await removeJobFromGCS(jobId);-}--async function cleanUpRequests(specificTeamId: string | null, _logger: Logger) {- const logger = _logger.child({- ...(specificTeamId ? { teamId: specificTeamId } : {}),- method: "cleanUpRequests",- });-- // Track all blobs that need cleanup for each request- const requestBlobs = new Map<string, Set<string>>();- // Track which blobs were successfully cleaned up- const successfulBlobs = new Set<string>();-- try {- for (let i = 0; ; i++) {- // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)- // associated with requests that need cleanup.- // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)- const { data: rows, error } = await supabase_service.rpc(- "get_zdr_cleanup_batch",- {- p_team_id: specificTeamId,- p_limit: 1000,- p_offset: i * 1000,- },- );-- if (error) {- logger.error("Error calling get_zdr_cleanup_batch RPC", { error, i });- throw error;- }-- if (!rows || rows.length === 0) {- break;- }-- // Track all blobs for each request before processing- for (const row of rows as { request_id: string; blob_id: string }[]) {- if (!requestBlobs.has(row.request_id)) {- requestBlobs.set(row.request_id, new Set());- }- requestBlobs.get(row.request_id)!.add(row.blob_id);- }-- // Process in batches of 50 for GCS cleanup- for (let j = 0; j < Math.ceil(rows.length / 50); j++) {- const batch = rows.slice(j * 50, (j + 1) * 50);- await Promise.allSettled(- batch.map(async (row: { request_id: string; blob_id: string }) => {- try {- await cleanUpJob(row.blob_id);- successfulBlobs.add(row.blob_id);- } catch (error) {- logger.error(`Error cleaning up blob`, {- method: "cleanUpJob",- blobId: row.blob_id,- requestId: row.request_id,- error,- });- // Don't throw - continue with other blobs- }- }),- );- }-- if (rows.length < 1000) {- break;- }- }- } catch (error) {- logger.error(`Error looping through cleanup batch`, {- error,- });- }-- // Only clear dr_clean_by on requests where ALL blobs were successfully cleaned up- const fullyCleanedRequests: string[] = [];- for (const [requestId, blobIds] of requestBlobs) {- const allBlobsCleaned = [...blobIds].every(blobId =>- successfulBlobs.has(blobId),- );- if (allBlobsCleaned) {- fullyCleanedRequests.push(requestId);- }- }-- if (fullyCleanedRequests.length > 0) {- // Process in batches of 100 to deal with URL limits on PostgREST- for (let i = 0; i < fullyCleanedRequests.length; i += 100) {- const batch = fullyCleanedRequests.slice(i, i + 100);- try {- await supabase_service- .from("requests")- .update({- dr_clean_by: null,- })- .in("id", batch)- .throwOnError();-- logger.info(`Cleaned up ${fullyCleanedRequests.length} requests`);- } catch (error) {- logger.error(`Error clearing dr_clean_by on requests`, {- error,- requestCount: fullyCleanedRequests.length,- });- }- }- }-}--export async function zdrcleanerController(req: Request, res: Response) {- const logger = _logger.child({- module: "zdrcleaner",- method: "zdrcleanerController",- });-- const teamId = (req.query.teamId as string | undefined) ?? null;-- // Clean up requests/scrapes tables- await cleanUpRequests(teamId, logger);-- logger.info("ZDR Cleaner finished!");-- res.json({ ok: true });-}diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsnew file mode 100644index 000000000..2f02e2dc6--- /dev/null+++ b/apps/api/src/lib/zdrcleaner.ts@@ -0,0 +1,123 @@+import "dotenv/config";+import { supabase_service } from "../services/supabase";+import { removeJobFromGCS } from "./gcs-jobs";+import { logger as _logger } from "./logger";++export async function zdrcleaner() {+ const logger = _logger.child({+ module: "zdrcleaner",+ method: "zdrcleaner",+ canonicalLog: "zdrcleaner",+ });++ const start = Date.now();+ try {+ // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)+ // associated with requests that need cleanup.+ // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)+ const { data: _rows, error } = await supabase_service.rpc(+ "get_zdr_cleanup_batch_2",+ {+ p_limit: 1000,+ },+ );++ if (error) {+ throw new Error("Error calling get_zdr_cleanup_batch_2 RPC", {+ cause: error,+ });+ }++ const rows: { request_id: string; ids: string[] }[] = _rows;++ if (!rows || rows.length === 0) {+ logger.debug("zdrcleaner batch completed with no rows to process", {+ canonicalLog: "zdrcleaner",+ success: true,+ timeMs: Date.now() - start,+ });+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ const requestBlobs = new Map<string, Set<string>>();+ const successfulBlobs = new Set<string>();++ // Track all blobs for each request before processing+ for (const row of rows) {+ requestBlobs.set(row.request_id, new Set(row.ids));+ }++ // Process in batches of 50 for GCS cleanup+ const deleteErrors: any[] = [];+ for (let j = 0; j < Math.ceil(rows.length / 50); j++) {+ const batch = rows.slice(j * 50, (j + 1) * 50);+ await Promise.allSettled(+ batch+ .flatMap(x => x.ids)+ .map(async blob_id => {+ try {+ await removeJobFromGCS(blob_id);+ successfulBlobs.add(blob_id);+ } catch (error) {+ deleteErrors.push(error);+ }+ }),+ );+ }++ const fullyCleanedRequests: string[] = [];+ for (const [requestId, blobIds] of requestBlobs) {+ const allBlobsCleaned = [...blobIds].every(blobId =>+ successfulBlobs.has(blobId),+ );+ if (allBlobsCleaned) {+ fullyCleanedRequests.push(requestId);+ }+ }++ const updateErrors: any[] = [];++ if (fullyCleanedRequests.length > 0) {+ // Process in batches of 100 to deal with URL limits on PostgREST+ for (let i = 0; i < fullyCleanedRequests.length; i += 100) {+ const batch = fullyCleanedRequests.slice(i, i + 100);+ try {+ await supabase_service+ .from("requests")+ .update({+ dr_clean_by: null,+ })+ .in("id", batch)+ .throwOnError();+ } catch (error) {+ updateErrors.push(error);+ }+ }+ }++ if (deleteErrors.length > 0 || updateErrors.length > 0) {+ logger.warn("zdrcleaner batch completed with errors", {+ canonicalLog: "zdrcleaner",+ success: true,+ deleteErrors,+ updateErrors,+ timeMs: Date.now() - start,+ });+ } else {+ logger.debug("zdrcleaner batch completed", {+ canonicalLog: "zdrcleaner",+ success: true,+ deleteErrors,+ timeMs: Date.now() - start,+ });+ }+ } catch (error) {+ logger.error(`Error looping through cleanup batch`, {+ canonicalLog: "zdrcleaner",+ success: false,+ timeMs: Date.now() - start,+ error,+ });+ }+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 4bccf2336..aebdf1a0a 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -7,7 +7,6 @@ import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-cleimport { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { cclogController } from "../controllers/v0/admin/cclog";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";-import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";import {metricsController,@@ -47,11 +46,6 @@ adminRouter.get(adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/cclog`, wrap(cclogController));-adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/zdrcleaner`,- wrap(zdrcleanerController),-);-adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.tsnew file mode 100644index 000000000..5dde4ddec--- /dev/null+++ b/apps/api/src/services/zdr-worker.ts@@ -0,0 +1,28 @@+import "dotenv/config";+import "./sentry";+import { setSentryServiceTag } from "./sentry";+import { logger } from "../lib/logger";+import { zdrcleaner } from "../lib/zdrcleaner";++let isShuttingDown = false;++process.on("SIGINT", () => {+ logger.info("Received SIGINT. Shutting down gracefully...");+ isShuttingDown = true;+});++process.on("SIGTERM", () => {+ logger.info("Received SIGTERM. Shutting down gracefully...");+ isShuttingDown = true;+});++(async () => {+ setSentryServiceTag("zdr-worker");++ while (!isShuttingDown) {+ await zdrcleaner();+ }++ logger.info("zdr-worker exiting");+ process.exit(0);+})();