Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: precrawl worker (#1783)
Gergő Móricza57277eaa7f6
diff --git a/apps/api/src/controllers/v0/admin/precrawl.ts b/apps/api/src/controllers/v0/admin/precrawl.tsnew file mode 100644index 000000000..e3aec1781--- /dev/null+++ b/apps/api/src/controllers/v0/admin/precrawl.ts@@ -0,0 +1,7 @@+import type { Request, Response } from "express";+import { getPrecrawlQueue } from "../../../services/queue-service";++export async function triggerPrecrawl(_: Request, res: Response) {+ await getPrecrawlQueue().add(new Date().toISOString(), {});+ res.json({ ok: true });+}\ No newline at end of filediff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 2c38f501d..fc8a3f8dc 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -12,6 +12,7 @@ import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { cclogController } from "../controllers/v0/admin/cclog";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";+import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";export const adminRouter = express.Router();@@ -62,3 +63,8 @@ adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),);++adminRouter.get(+ `/admin/${process.env.BULL_AUTH_KEY}/precrawl`,+ wrap(triggerPrecrawl),+);diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 15b24d1af..bf55b0870 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -505,7 +505,10 @@ export async function scrapeURL(meta.logger.info("Rewriting URL");}- const shouldRecordFrequency = useIndex && meta.options.storeInCache && !meta.internalOptions.zeroDataRetention;+ const shouldRecordFrequency = useIndex+ && meta.options.storeInCache+ && !meta.internalOptions.zeroDataRetention+ && internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;if (shouldRecordFrequency) {(async () => {try {diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 52fa8c4fc..05f7722b4 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -6,12 +6,16 @@ import { logger as _logger, logger } from "../../lib/logger";import {redisConnection,getBillingQueue,+ getPrecrawlQueue,} from "../queue-service";import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";import systemMonitor from "../system-monitor";import { v4 as uuidv4 } from "uuid";-import { processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs } from "..";+import { index_supabase_service, processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs } from "..";import { processWebhookInsertJobs } from "../webhook";+import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toLegacyCrawlerOptions } from "../../controllers/v1/types";+import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";+import { _addScrapeJobToBullMQ } from "../queue-jobs";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -82,6 +86,112 @@ const processBillingJobInternal = async (token: string, job: Job) => {return err;};+const processPrecrawlJobInternal = async (token: string, job: Job) => {+ const logger = _logger.child({+ module: "index-worker",+ method: "processPrecrawlJobInternal",+ });++ const extendLockInterval = setInterval(async () => {+ logger.info(`🔄 Worker extending lock on precrawl job ${job.id}`);+ await job.extendLock(token, jobLockExtensionTime);+ }, jobLockExtendInterval);++ const teamId = process.env.PRECRAWL_TEAM_ID!;++ try {+ const budget = 10000;+ const { data, error } = await index_supabase_service.rpc("precrawl_get_top_domains", {+ i_newer_than: new Date(Date.now() - 1000 * 60 * 60 * 24 * 7).toISOString(),+ });++ if (error) {+ logger.error("Error getting top domains", { error });+ throw error;+ }++ const total_hits = data.reduce((a, x) => a + x.count, 0);+ for (const item of data) {+ const urlObj = new URL(item.example_url);+ urlObj.pathname = "/";+ urlObj.search = "";+ urlObj.hash = "";++ const url = urlObj.toString();+ const limit = Math.round(item.count / total_hits * budget);++ logger.info("Running pre-crawl", { url, limit, hits: item.count, budget });++ const crawlerOptions = {+ ...crawlRequestSchema.parse({ url, limit }),+ url: undefined,+ scrapeOptions: undefined,+ };+ const scrapeOptions = scrapeOptionsSchema.parse({});++ const sc: StoredCrawl = {+ originUrl: url,+ crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),+ scrapeOptions,+ internalOptions: {+ disableSmartWaitCache: true,+ teamId,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ zeroDataRetention: true,+ }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter+ team_id: teamId,+ createdAt: Date.now(),+ maxConcurrency: undefined,+ zeroDataRetention: false,+ };++ const crawlId = uuidv4();++ const crawler = crawlToCrawler(crawlId, sc, null);++ try {+ sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);+ const robotsCrawlDelay = crawler.getRobotsCrawlDelay();+ if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {+ sc.crawlerOptions.delay = robotsCrawlDelay;+ }+ } catch (e) {+ logger.debug("Failed to get robots.txt (this is probably fine!)", {+ error: e,+ });+ }++ await saveCrawl(crawlId, sc);++ await _addScrapeJobToBullMQ(+ {+ url: url,+ mode: "kickoff" as const,+ team_id: teamId,+ crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: "precrawl",+ integration: null,+ crawl_id: crawlId,+ webhook: undefined,+ v1: true,+ zeroDataRetention: false,+ },+ {},+ crypto.randomUUID(),+ 10,+ );+ }++ } catch (error) {+ logger.error("Error processing precrawl job", { error });+ await job.moveToFailed(error, token, false);+ } finally {+ clearInterval(extendLockInterval);+ }+};+let isShuttingDown = false;process.on("SIGINT", () => {@@ -200,6 +310,9 @@ const OMCE_INSERT_INTERVAL = 5000;// Start billing worker and batch processingstartBillingBatchProcessing();const billingWorkerPromise = workerFun(getBillingQueue(), processBillingJobInternal);+ const precrawlWorkerPromise = process.env.PRECRAWL_TEAM_ID+ ? workerFun(getPrecrawlQueue(), processPrecrawlJobInternal)+ : (async () => { logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker"); })();const indexInserterInterval = setInterval(async () => {if (isShuttingDown) {@@ -231,7 +344,7 @@ const OMCE_INSERT_INTERVAL = 5000;}, OMCE_INSERT_INTERVAL);// Wait for all workers to complete (which should only happen on shutdown)- await Promise.all([billingWorkerPromise]);+ await Promise.all([billingWorkerPromise, precrawlWorkerPromise]);clearInterval(indexInserterInterval);clearInterval(webhookInserterInterval);diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 77f62c73a..6068fde9e 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -11,6 +11,7 @@ let indexQueue: Queue;let deepResearchQueue: Queue;let generateLlmsTxtQueue: Queue;let billingQueue: Queue;+let precrawlQueue: Queue;export const redisConnection = new IORedis(process.env.REDIS_URL!, {maxRetriesPerRequest: null,@@ -23,6 +24,7 @@ export const indexQueueName = "{indexQueue}";export const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";+export const precrawlQueueName = "{precrawlQueue}";export function getScrapeQueue() {if (!scrapeQueue) {@@ -113,3 +115,21 @@ export function getBillingQueue() {}return billingQueue;}++export function getPrecrawlQueue() {+ if (!precrawlQueue) {+ precrawlQueue = new Queue(precrawlQueueName, {+ connection: redisConnection,+ defaultJobOptions: {+ removeOnComplete: {+ age: 24 * 60 * 60, // 1 day+ },+ removeOnFail: {+ age: 24 * 60 * 60, // 1 day+ },+ },+ });+ logger.info("Precrawl queue created");+ }+ return precrawlQueue;+}