Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: remove firecrawl_jobs cleanup from ZDR cleaner (#2777)
firecrawl-spring[bot]2dd1a34407de
diff --git a/apps/api/src/controllers/v0/admin/zdrcleaner.ts b/apps/api/src/controllers/v0/admin/zdrcleaner.tsindex 7e1ad206d..504094501 100644--- a/apps/api/src/controllers/v0/admin/zdrcleaner.ts+++ b/apps/api/src/controllers/v0/admin/zdrcleaner.ts@@ -9,97 +9,6 @@ async function cleanUpJob(jobId: string) {await removeJobFromGCS(jobId);}-async function cleanUpFirecrawlJobs(- specificTeamId: string | null,- _logger: Logger,-) {- const logger = _logger.child({- ...(specificTeamId ? { teamId: specificTeamId } : {}),- method: "cleanUpFirecrawlJobs",- });-- const cleanedUp: number[] = [];-- try {- for (let i = 0; ; i++) {- let selector = supabase_service- .from("firecrawl_jobs")- .select("id, job_id");-- if (specificTeamId) {- selector = selector- .eq("team_id", specificTeamId)- .not("dr_clean_by", "is", null);- } else {- selector = selector- .lte("dr_clean_by", new Date().toISOString())- .gte(- "dr_clean_by",- new Date(Date.now() - 1000 * 60 * 60 * 24 * 7).toISOString(),- );- // Explanation for the gte: since the cleaner should run every 5 minutes, it is very unlikely that- // the cleaner will be down for 7 days without anyone noticing.- // Since the firecrawl_jobs table is incredibly large, even with the index on dr_clean_by,- // not giving the select a lower bound guarantees that the select will not run with an empty result- // in reasonable time.- // Therefore, we give it a lower bound which should never cause problems.- }-- const { data: jobs } = await selector- .range(i * 1000, (i + 1) * 1000)- .throwOnError();-- if (jobs?.length === 0) {- break;- }-- for (let i = 0; i < Math.ceil((jobs?.length ?? 0) / 50); i++) {- const theseJobs = (jobs ?? []).slice(i * 50, (i + 1) * 50);- await Promise.allSettled(- theseJobs.map(async job => {- try {- await cleanUpJob(job.job_id);- cleanedUp.push(job.id);- } catch (error) {- logger.error(`Error cleaning up job`, {- method: "cleanUpJob",- jobId: job.job_id,- scrapeId: job.job_id,- error,- });- throw error;- }- }) ?? [],- );- }-- if ((jobs ?? []).length < 1000) {- break;- }- }- } catch (error) {- logger.error(`Error looping through jobs`, {- error,- });- }-- if (cleanedUp.length > 0) {- try {- await supabase_service- .from("firecrawl_jobs")- .update({- dr_clean_by: null,- })- .in("id", cleanedUp)- .throwOnError();- } catch (error) {- logger.error(`Error setting cleanup value on team`, {- error,- });- }- }-}-async function cleanUpRequests(specificTeamId: string | null, _logger: Logger) {const logger = _logger.child({...(specificTeamId ? { teamId: specificTeamId } : {}),@@ -212,10 +121,7 @@ export async function zdrcleanerController(req: Request, res: Response) {const teamId = (req.query.teamId as string | undefined) ?? null;- // Clean up old firecrawl_jobs table (legacy)- await cleanUpFirecrawlJobs(teamId, logger);-- // Clean up new requests/scrapes tables+ // Clean up requests/scrapes tablesawait cleanUpRequests(teamId, logger);logger.info("ZDR Cleaner finished!");