Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: precrawl worker (#1783)

Gergő Móricza57277eaa7f6
diff --git a/apps/api/src/controllers/v0/admin/precrawl.ts b/apps/api/src/controllers/v0/admin/precrawl.ts
new file mode 100644
index 000000000..e3aec1781
--- /dev/null
+++ b/apps/api/src/controllers/v0/admin/precrawl.ts
@@ -0,0 +1,7 @@
+import type { Request, Response } from "express";
+import { getPrecrawlQueue } from "../../../services/queue-service";
+
+export async function triggerPrecrawl(_: Request, res: Response) {
+ await getPrecrawlQueue().add(new Date().toISOString(), {});
+ res.json({ ok: true });
+}
\ No newline at end of file
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 2c38f501d..fc8a3f8dc 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -12,6 +12,7 @@ import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
import { cclogController } from "../controllers/v0/admin/cclog";
import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner";
+import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
export const adminRouter = express.Router();
@@ -62,3 +63,8 @@ adminRouter.get(
`/admin/${process.env.BULL_AUTH_KEY}/index-queue-prometheus`,
wrap(indexQueuePrometheus),
);
+
+adminRouter.get(
+ `/admin/${process.env.BULL_AUTH_KEY}/precrawl`,
+ wrap(triggerPrecrawl),
+);
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 15b24d1af..bf55b0870 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -505,7 +505,10 @@ export async function scrapeURL(
meta.logger.info("Rewriting URL");
}
- const shouldRecordFrequency = useIndex && meta.options.storeInCache && !meta.internalOptions.zeroDataRetention;
+ const shouldRecordFrequency = useIndex
+ && meta.options.storeInCache
+ && !meta.internalOptions.zeroDataRetention
+ && internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;
if (shouldRecordFrequency) {
(async () => {
try {
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 52fa8c4fc..05f7722b4 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -6,12 +6,16 @@ import { logger as _logger, logger } from "../../lib/logger";
import {
redisConnection,
getBillingQueue,
+ getPrecrawlQueue,
} from "../queue-service";
import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";
import systemMonitor from "../system-monitor";
import { v4 as uuidv4 } from "uuid";
-import { processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs } from "..";
+import { index_supabase_service, processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs } from "..";
import { processWebhookInsertJobs } from "../webhook";
+import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toLegacyCrawlerOptions } from "../../controllers/v1/types";
+import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
+import { _addScrapeJobToBullMQ } from "../queue-jobs";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
@@ -82,6 +86,112 @@ const processBillingJobInternal = async (token: string, job: Job) => {
return err;
};
+const processPrecrawlJobInternal = async (token: string, job: Job) => {
+ const logger = _logger.child({
+ module: "index-worker",
+ method: "processPrecrawlJobInternal",
+ });
+
+ const extendLockInterval = setInterval(async () => {
+ logger.info(`🔄 Worker extending lock on precrawl job ${job.id}`);
+ await job.extendLock(token, jobLockExtensionTime);
+ }, jobLockExtendInterval);
+
+ const teamId = process.env.PRECRAWL_TEAM_ID!;
+
+ try {
+ const budget = 10000;
+ const { data, error } = await index_supabase_service.rpc("precrawl_get_top_domains", {
+ i_newer_than: new Date(Date.now() - 1000 * 60 * 60 * 24 * 7).toISOString(),
+ });
+
+ if (error) {
+ logger.error("Error getting top domains", { error });
+ throw error;
+ }
+
+ const total_hits = data.reduce((a, x) => a + x.count, 0);
+ for (const item of data) {
+ const urlObj = new URL(item.example_url);
+ urlObj.pathname = "/";
+ urlObj.search = "";
+ urlObj.hash = "";
+
+ const url = urlObj.toString();
+ const limit = Math.round(item.count / total_hits * budget);
+
+ logger.info("Running pre-crawl", { url, limit, hits: item.count, budget });
+
+ const crawlerOptions = {
+ ...crawlRequestSchema.parse({ url, limit }),
+ url: undefined,
+ scrapeOptions: undefined,
+ };
+ const scrapeOptions = scrapeOptionsSchema.parse({});
+
+ const sc: StoredCrawl = {
+ originUrl: url,
+ crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
+ scrapeOptions,
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
+ zeroDataRetention: true,
+ }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
+ team_id: teamId,
+ createdAt: Date.now(),
+ maxConcurrency: undefined,
+ zeroDataRetention: false,
+ };
+
+ const crawlId = uuidv4();
+
+ const crawler = crawlToCrawler(crawlId, sc, null);
+
+ try {
+ sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);
+ const robotsCrawlDelay = crawler.getRobotsCrawlDelay();
+ if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {
+ sc.crawlerOptions.delay = robotsCrawlDelay;
+ }
+ } catch (e) {
+ logger.debug("Failed to get robots.txt (this is probably fine!)", {
+ error: e,
+ });
+ }
+
+ await saveCrawl(crawlId, sc);
+
+ await _addScrapeJobToBullMQ(
+ {
+ url: url,
+ mode: "kickoff" as const,
+ team_id: teamId,
+ crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: "precrawl",
+ integration: null,
+ crawl_id: crawlId,
+ webhook: undefined,
+ v1: true,
+ zeroDataRetention: false,
+ },
+ {},
+ crypto.randomUUID(),
+ 10,
+ );
+ }
+
+ } catch (error) {
+ logger.error("Error processing precrawl job", { error });
+ await job.moveToFailed(error, token, false);
+ } finally {
+ clearInterval(extendLockInterval);
+ }
+};
+
let isShuttingDown = false;
process.on("SIGINT", () => {
@@ -200,6 +310,9 @@ const OMCE_INSERT_INTERVAL = 5000;
// Start billing worker and batch processing
startBillingBatchProcessing();
const billingWorkerPromise = workerFun(getBillingQueue(), processBillingJobInternal);
+ const precrawlWorkerPromise = process.env.PRECRAWL_TEAM_ID
+ ? workerFun(getPrecrawlQueue(), processPrecrawlJobInternal)
+ : (async () => { logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker"); })();
const indexInserterInterval = setInterval(async () => {
if (isShuttingDown) {
@@ -231,7 +344,7 @@ const OMCE_INSERT_INTERVAL = 5000;
}, OMCE_INSERT_INTERVAL);
// Wait for all workers to complete (which should only happen on shutdown)
- await Promise.all([billingWorkerPromise]);
+ await Promise.all([billingWorkerPromise, precrawlWorkerPromise]);
clearInterval(indexInserterInterval);
clearInterval(webhookInserterInterval);
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 77f62c73a..6068fde9e 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -11,6 +11,7 @@ let indexQueue: Queue;
let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
+let precrawlQueue: Queue;
export const redisConnection = new IORedis(process.env.REDIS_URL!, {
maxRetriesPerRequest: null,
@@ -23,6 +24,7 @@ export const indexQueueName = "{indexQueue}";
export const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
+export const precrawlQueueName = "{precrawlQueue}";
export function getScrapeQueue() {
if (!scrapeQueue) {
@@ -113,3 +115,21 @@ export function getBillingQueue() {
}
return billingQueue;
}
+
+export function getPrecrawlQueue() {
+ if (!precrawlQueue) {
+ precrawlQueue = new Queue(precrawlQueueName, {
+ connection: redisConnection,
+ defaultJobOptions: {
+ removeOnComplete: {
+ age: 24 * 60 * 60, // 1 day
+ },
+ removeOnFail: {
+ age: 24 * 60 * 60, // 1 day
+ },
+ },
+ });
+ logger.info("Precrawl queue created");
+ }
+ return precrawlQueue;
+}