Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: indexer queue (#2830)
delong3cffd34478090
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 80656945e..59fc13ee7 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -90,6 +90,10 @@ const configSchema = z.object({FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),FIRE_ENGINE_AB_MODE: z.enum(["mirror", "split"]).default("mirror"),+ // Indexer+ INDEXER_RABBITMQ_URL: z.string().optional(),+ INDEXER_TRAFFIC_SHARE: z.coerce.number().default(0.0),+// ScrapeURLSCRAPEURL_AB_HOST: z.string().optional(),SCRAPEURL_AB_RATE: z.coerce.number().optional(),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 8b9c3ada5..bc4042b07 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -36,6 +36,7 @@ import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";import responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";+import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -138,8 +139,10 @@ async function startServer(port = DEFAULT_PORT) {logger.info("Server closed.");nuqShutdown().finally(() => {shutdownWebhookQueue().finally(() => {- logger.info("NUQ shutdown complete");- process.exit(0);+ shutdownIndexerQueue().finally(() => {+ logger.info("NUQ shutdown complete");+ process.exit(0);+ });});});});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex b5317556b..317c44ffc 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -17,6 +17,8 @@ import { sendDocumentToIndex } from "../engines/index/index";import { sendDocumentToSearchIndex } from "./sendToSearchIndex";import { hasFormatOfType } from "../../../lib/format-utils";import { brandingTransformer } from "../../../lib/branding/transformer";+import { indexerQueue } from "../../../services/indexing/indexer-queue";+import { config } from "../../../config";type Transformer = (meta: Meta,@@ -148,21 +150,73 @@ async function deriveLinksFromHTML(meta: Meta,document: Document,): Promise<Document> {- // Only derive if the formats has links- if (hasFormatOfType(meta.options.formats, "links")) {- if (document.html === undefined) {- throw new Error(- "html is undefined -- this transformer is being called out of order",- );+ if (document.html === undefined) {+ throw new Error(+ "html is undefined -- this transformer is being called out of order",+ );+ }++ const rate = config.INDEXER_TRAFFIC_SHARE+ ? Math.max(0, Math.min(1, Number(config.INDEXER_TRAFFIC_SHARE)))+ : 0;++ const shouldForwardTraffic =+ rate > 0 && Math.random() <= rate && !!config.INDEXER_RABBITMQ_URL;++ const forwardToIndexer =+ !!meta.internalOptions.teamId &&+ !meta.internalOptions.teamId?.includes("robots-txt") &&+ !meta.internalOptions.teamId?.includes("sitemap") &&+ shouldForwardTraffic;++ const requiresLinks = !!hasFormatOfType(meta.options.formats, "links");++ if (!forwardToIndexer && !requiresLinks) {+ return document;+ }++ document.links = await extractLinks(+ document.html,+ document.metadata.url ??+ document.metadata.sourceURL ??+ meta.rewrittenUrl ??+ meta.url,+ );++ if (forwardToIndexer) {+ try {+ let linksDeduped: Set<string> = new Set();+ if (!!document.links) {+ linksDeduped = new Set([...document.links]);+ }++ indexerQueue+ .sendToWorker({+ id: meta.id,+ type: "links",+ discovery_url:+ document.metadata.url ??+ document.metadata.sourceURL ??+ meta.rewrittenUrl ??+ meta.url,+ urls: [...linksDeduped],+ })+ .catch(error => {+ meta.logger.error("Failed to queue links for indexing", {+ error: (error as Error)?.message,+ url: meta.url,+ });+ });+ } catch (error) {+ meta.logger.error("Failed to queue links for indexing", {+ error: (error as Error)?.message,+ url: meta.url,+ });}+ }- document.links = await extractLinks(- document.html,- document.metadata.url ??- document.metadata.sourceURL ??- meta.rewrittenUrl ??- meta.url,- );+ if (!requiresLinks) {+ delete document.links;}return document;diff --git a/apps/api/src/services/indexing/indexer-queue.ts b/apps/api/src/services/indexing/indexer-queue.tsnew file mode 100644index 000000000..9e3866c95--- /dev/null+++ b/apps/api/src/services/indexing/indexer-queue.ts@@ -0,0 +1,209 @@+import amqp from "amqplib";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";++const logger = _logger.child({ module: "indexer-queue" });++const INDEX_EXCHANGE = "index.jobs";+const LINKS_QUEUE_NAME = "index.jobs.links";+const LINKS_RK = "job.links";++const CONNECT_TIMEOUT = 5000;+const DRAIN_TIMEOUT = 30000;++type IndexWorkerMessage = {+ id: string;+ type: "links";+ discovery_url: string;+ urls: string[];+};++class IndexerQueue {+ private connection: amqp.ChannelModel | null = null;+ private publishChannel: amqp.Channel | null = null;++ private connectPromise: Promise<void> | null = null;++ private closing: boolean = false;+ private noop: boolean | null = null;++ async connect(): Promise<void> {+ if (this.connectPromise) return this.connectPromise;+ if (this.closing) return;+ if (this.noop) return;+ if (this.connection && this.publishChannel) return;++ this.connectPromise = this._establishConnection();++ try {+ await this.connectPromise;+ } finally {+ this.connectPromise = null;+ }+ }++ private async _establishConnection(): Promise<void> {+ if (!config.INDEXER_RABBITMQ_URL) {+ logger.debug("Indexer disabled via config");+ this.noop = true;+ return;+ }++ try {+ logger.info("Connecting to index RabbitMQ");++ this.connection = await amqp.connect(config.INDEXER_RABBITMQ_URL);+ this.publishChannel = await this.connection.createChannel();++ // NOTE: this queue is already created by another service+ await this.publishChannel.checkQueue(LINKS_QUEUE_NAME);++ this._registerConnectionEvents();++ logger.info("Connected to index RabbitMQ");+ } catch (err) {+ this.connection = null;+ this.publishChannel = null;+ throw err;+ }+ }++ private _registerConnectionEvents() {+ if (this.noop) return;+ if (!this.connection || !this.publishChannel) return;++ this.connection.on("close", () => {+ if (this.closing) return;++ logger.warn("Index RabbitMQ connection closed", {+ module: "index-queue",+ });++ this.cleanup();+ setTimeout(+ () =>+ this.connect().catch(err =>+ logger.error("Reconnection failed", {+ err,+ }),+ ),+ CONNECT_TIMEOUT,+ );+ });++ this.connection.on("error", err => {+ logger.error("Index RabbitMQ connection error", {+ err,+ });+ });++ this.publishChannel.on("error", err => {+ logger.error("Index RabbitMQ channel error", {+ err,+ });+ });+ }++ async sendToWorker(message: IndexWorkerMessage): Promise<void> {+ await this.connect();++ if (this.noop) return;+ if (!this.publishChannel) throw new Error("Channel not available");++ const payload = {+ id: message.id,+ } as any;++ if (message.type === "links") {+ payload.urls = message.urls;+ payload.discovery_url = message.discovery_url;+ }++ const buffer = Buffer.from(JSON.stringify(payload));++ const canSendMore = this.publishChannel.publish(+ INDEX_EXCHANGE,+ LINKS_RK,+ buffer,+ {+ persistent: true,+ contentType: "application/json",+ messageId: message.id,+ },+ );++ if (!canSendMore) {+ logger.warn("Index message buffer full, waiting for drain");+ await this.waitForDrain();+ }+ }++ private async waitForDrain(): Promise<void> {+ if (this.noop) return;++ return new Promise((resolve, reject) => {+ if (!this.publishChannel) {+ return reject(new Error("Channel not available"));+ }++ const listeners: Record<string, any> = {};++ const cleanup = () => {+ if (!this.publishChannel) return;+ this.publishChannel.removeListener("drain", listeners.drain);+ this.publishChannel.removeListener("error", listeners.error);+ this.publishChannel.removeListener("close", listeners.close);+ clearTimeout(listeners.timeout);+ };++ listeners.drain = () => {+ cleanup();+ resolve();+ };+ listeners.error = (err: Error) => {+ cleanup();+ reject(err);+ };+ listeners.close = () => {+ cleanup();+ reject(new Error("Channel closed during drain"));+ };++ listeners.timeout = setTimeout(() => {+ cleanup();+ reject(new Error(`Drain timeout after ${DRAIN_TIMEOUT}ms`));+ }, DRAIN_TIMEOUT);++ this.publishChannel.on("drain", listeners.drain);+ this.publishChannel.on("error", listeners.error);+ this.publishChannel.on("close", listeners.close);+ });+ }++ async close(): Promise<void> {+ if (this.noop) return;+ this.closing = true;++ try {+ await this.publishChannel?.close();+ await this.connection?.close();+ } catch (err) {+ logger.warn("Error while closing RabbitMQ connection", { err });+ } finally {+ this.cleanup();+ logger.info("Indexer RabbitMQ closed");+ }+ }++ private cleanup() {+ this.connection = null;+ this.publishChannel = null;+ this.connectPromise = null;+ }+}++export const indexerQueue = new IndexerQueue();++export async function shutdownIndexerQueue(): Promise<void> {+ await indexerQueue.close();+}