Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): temporarily disable precrawl queue processing in admin controller and indexing worker
rafaelmmiller2cae465dff66
diff --git a/apps/api/src/controllers/v0/admin/precrawl.ts b/apps/api/src/controllers/v0/admin/precrawl.tsindex 05b7ac496..bfcc8ad50 100644--- a/apps/api/src/controllers/v0/admin/precrawl.ts+++ b/apps/api/src/controllers/v0/admin/precrawl.ts@@ -2,6 +2,6 @@ import type { Request, Response } from "express";import { getPrecrawlQueue } from "../../../services/queue-service";export async function triggerPrecrawl(_: Request, res: Response) {- await getPrecrawlQueue().add(new Date().toISOString(), {});+ // await getPrecrawlQueue().add(new Date().toISOString(), {});res.json({ ok: true });}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex d56c6400c..c85f101ff 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -342,7 +342,7 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;processBillingJobInternal,);const precrawlWorkerPromise = process.env.PRECRAWL_TEAM_ID- ? workerFun(getPrecrawlQueue(), processPrecrawlJobInternal)+ ? Promise.resolve() // workerFun(getPrecrawlQueue(), processPrecrawlJobInternal): (async () => {logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");})();