Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(index): store request frequency for precrawling (#1782)

Gergő Móriczeab6909c0e9b
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index b19eee756..672d858a1 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -34,6 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
import { loadMock, MockState } from "./lib/mock";
import { CostTracking } from "../../lib/extract/extraction-service";
+import { addIndexRFInsertJob, generateDomainSplits, hashURL, normalizeURLForIndex, useIndex } from "../../services/index";
export type ScrapeUrlResponse = (
| {
@@ -504,6 +505,23 @@ export async function scrapeURL(
meta.logger.info("Rewriting URL");
}
+ const shouldRecordFrequency = useIndex && meta.options.storeInCache && !meta.internalOptions.zeroDataRetention;
+ if (shouldRecordFrequency) {
+ (async () => {
+ try {
+ const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);
+ const domainHash = hashURL(domainSplits.slice(-1)[0]);
+
+ await addIndexRFInsertJob({
+ domain_hash: domainHash,
+ url: meta.url,
+ });
+ } catch (error) {
+ meta.logger.warn("Failed to record frequency", { error });
+ }
+ })();
+ }
+
try {
while (true) {
try {
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 281b3476f..a35be9908 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -237,6 +237,36 @@ export async function getIndexInsertQueueLength(): Promise<number> {
return await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY) ?? 0;
}
+const INDEX_RF_INSERT_QUEUE_KEY = "index-rf-insert-queue";
+const INDEX_RF_INSERT_BATCH_SIZE = 1000;
+
+export async function addIndexRFInsertJob(data: any) {
+ await redisEvictConnection.rpush(INDEX_RF_INSERT_QUEUE_KEY, JSON.stringify(data));
+}
+
+export async function getIndexRFInsertJobs(): Promise<any[]> {
+ const jobs = (await redisEvictConnection.lpop(INDEX_RF_INSERT_QUEUE_KEY, INDEX_RF_INSERT_BATCH_SIZE)) ?? [];
+ return jobs.map(x => JSON.parse(x));
+}
+
+export async function processIndexRFInsertJobs() {
+ const jobs = await getIndexRFInsertJobs();
+ if (jobs.length === 0) {
+ return;
+ }
+ _logger.info(`Index RF inserter found jobs to insert`, { jobCount: jobs.length });
+ try {
+ await index_supabase_service.from("request_frequency").insert(jobs);
+ _logger.info(`Index RF inserter inserted jobs`, { jobCount: jobs.length });
+ } catch (error) {
+ _logger.error(`Index RF inserter failed to insert jobs`, { error, jobCount: jobs.length });
+ }
+}
+
+export async function getIndexRFInsertQueueLength(): Promise<number> {
+ return await redisEvictConnection.llen(INDEX_RF_INSERT_QUEUE_KEY) ?? 0;
+}
+
export async function queryIndexAtSplitLevel(url: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {
if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
return [];
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 7ff0b9f4c..737fd0999 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -10,7 +10,7 @@ import {
import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";
import systemMonitor from "../system-monitor";
import { v4 as uuidv4 } from "uuid";
-import { processIndexInsertJobs } from "..";
+import { processIndexInsertJobs, processIndexRFInsertJobs } from "..";
import { processWebhookInsertJobs } from "../webhook";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
@@ -215,9 +215,17 @@ const WEBHOOK_INSERT_INTERVAL = 15000;
await processWebhookInsertJobs();
}, WEBHOOK_INSERT_INTERVAL);
+ const indexRFInserterInterval = setInterval(async () => {
+ if (isShuttingDown) {
+ return;
+ }
+ await processIndexRFInsertJobs();
+ }, INDEX_INSERT_INTERVAL);
+
// Wait for all workers to complete (which should only happen on shutdown)
await Promise.all([billingWorkerPromise]);
clearInterval(indexInserterInterval);
clearInterval(webhookInserterInterval);
+ clearInterval(indexRFInserterInterval);
})();