Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(index): store request frequency for precrawling (#1782)
Gergő Móriczeab6909c0e9b
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex b19eee756..672d858a1 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -34,6 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/extract/extraction-service";+import { addIndexRFInsertJob, generateDomainSplits, hashURL, normalizeURLForIndex, useIndex } from "../../services/index";export type ScrapeUrlResponse = (| {@@ -504,6 +505,23 @@ export async function scrapeURL(meta.logger.info("Rewriting URL");}+ const shouldRecordFrequency = useIndex && meta.options.storeInCache && !meta.internalOptions.zeroDataRetention;+ if (shouldRecordFrequency) {+ (async () => {+ try {+ const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);+ const domainHash = hashURL(domainSplits.slice(-1)[0]);++ await addIndexRFInsertJob({+ domain_hash: domainHash,+ url: meta.url,+ });+ } catch (error) {+ meta.logger.warn("Failed to record frequency", { error });+ }+ })();+ }+try {while (true) {try {diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 281b3476f..a35be9908 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -237,6 +237,36 @@ export async function getIndexInsertQueueLength(): Promise<number> {return await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY) ?? 0;}+const INDEX_RF_INSERT_QUEUE_KEY = "index-rf-insert-queue";+const INDEX_RF_INSERT_BATCH_SIZE = 1000;++export async function addIndexRFInsertJob(data: any) {+ await redisEvictConnection.rpush(INDEX_RF_INSERT_QUEUE_KEY, JSON.stringify(data));+}++export async function getIndexRFInsertJobs(): Promise<any[]> {+ const jobs = (await redisEvictConnection.lpop(INDEX_RF_INSERT_QUEUE_KEY, INDEX_RF_INSERT_BATCH_SIZE)) ?? [];+ return jobs.map(x => JSON.parse(x));+}++export async function processIndexRFInsertJobs() {+ const jobs = await getIndexRFInsertJobs();+ if (jobs.length === 0) {+ return;+ }+ _logger.info(`Index RF inserter found jobs to insert`, { jobCount: jobs.length });+ try {+ await index_supabase_service.from("request_frequency").insert(jobs);+ _logger.info(`Index RF inserter inserted jobs`, { jobCount: jobs.length });+ } catch (error) {+ _logger.error(`Index RF inserter failed to insert jobs`, { error, jobCount: jobs.length });+ }+}++export async function getIndexRFInsertQueueLength(): Promise<number> {+ return await redisEvictConnection.llen(INDEX_RF_INSERT_QUEUE_KEY) ?? 0;+}+export async function queryIndexAtSplitLevel(url: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {return [];diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 7ff0b9f4c..737fd0999 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -10,7 +10,7 @@ import {import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";import systemMonitor from "../system-monitor";import { v4 as uuidv4 } from "uuid";-import { processIndexInsertJobs } from "..";+import { processIndexInsertJobs, processIndexRFInsertJobs } from "..";import { processWebhookInsertJobs } from "../webhook";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;@@ -215,9 +215,17 @@ const WEBHOOK_INSERT_INTERVAL = 15000;await processWebhookInsertJobs();}, WEBHOOK_INSERT_INTERVAL);+ const indexRFInserterInterval = setInterval(async () => {+ if (isShuttingDown) {+ return;+ }+ await processIndexRFInsertJobs();+ }, INDEX_INSERT_INTERVAL);+// Wait for all workers to complete (which should only happen on shutdown)await Promise.all([billingWorkerPromise]);clearInterval(indexInserterInterval);clearInterval(webhookInserterInterval);+ clearInterval(indexRFInserterInterval);})();