Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: indexer queue (#2830)

delong3cffd34478090
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 80656945e..59fc13ee7 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -90,6 +90,10 @@ const configSchema = z.object({
FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),
FIRE_ENGINE_AB_MODE: z.enum(["mirror", "split"]).default("mirror"),
+ // Indexer
+ INDEXER_RABBITMQ_URL: z.string().optional(),
+ INDEXER_TRAFFIC_SHARE: z.coerce.number().default(0.0),
+
// ScrapeURL
SCRAPEURL_AB_HOST: z.string().optional(),
SCRAPEURL_AB_RATE: z.coerce.number().optional(),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 8b9c3ada5..bc4042b07 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -36,6 +36,7 @@ import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
+import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -138,8 +139,10 @@ async function startServer(port = DEFAULT_PORT) {
logger.info("Server closed.");
nuqShutdown().finally(() => {
shutdownWebhookQueue().finally(() => {
- logger.info("NUQ shutdown complete");
- process.exit(0);
+ shutdownIndexerQueue().finally(() => {
+ logger.info("NUQ shutdown complete");
+ process.exit(0);
+ });
});
});
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index b5317556b..317c44ffc 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -17,6 +17,8 @@ import { sendDocumentToIndex } from "../engines/index/index";
import { sendDocumentToSearchIndex } from "./sendToSearchIndex";
import { hasFormatOfType } from "../../../lib/format-utils";
import { brandingTransformer } from "../../../lib/branding/transformer";
+import { indexerQueue } from "../../../services/indexing/indexer-queue";
+import { config } from "../../../config";
type Transformer = (
meta: Meta,
@@ -148,21 +150,73 @@ async function deriveLinksFromHTML(
meta: Meta,
document: Document,
): Promise<Document> {
- // Only derive if the formats has links
- if (hasFormatOfType(meta.options.formats, "links")) {
- if (document.html === undefined) {
- throw new Error(
- "html is undefined -- this transformer is being called out of order",
- );
+ if (document.html === undefined) {
+ throw new Error(
+ "html is undefined -- this transformer is being called out of order",
+ );
+ }
+
+ const rate = config.INDEXER_TRAFFIC_SHARE
+ ? Math.max(0, Math.min(1, Number(config.INDEXER_TRAFFIC_SHARE)))
+ : 0;
+
+ const shouldForwardTraffic =
+ rate > 0 && Math.random() <= rate && !!config.INDEXER_RABBITMQ_URL;
+
+ const forwardToIndexer =
+ !!meta.internalOptions.teamId &&
+ !meta.internalOptions.teamId?.includes("robots-txt") &&
+ !meta.internalOptions.teamId?.includes("sitemap") &&
+ shouldForwardTraffic;
+
+ const requiresLinks = !!hasFormatOfType(meta.options.formats, "links");
+
+ if (!forwardToIndexer && !requiresLinks) {
+ return document;
+ }
+
+ document.links = await extractLinks(
+ document.html,
+ document.metadata.url ??
+ document.metadata.sourceURL ??
+ meta.rewrittenUrl ??
+ meta.url,
+ );
+
+ if (forwardToIndexer) {
+ try {
+ let linksDeduped: Set<string> = new Set();
+ if (!!document.links) {
+ linksDeduped = new Set([...document.links]);
+ }
+
+ indexerQueue
+ .sendToWorker({
+ id: meta.id,
+ type: "links",
+ discovery_url:
+ document.metadata.url ??
+ document.metadata.sourceURL ??
+ meta.rewrittenUrl ??
+ meta.url,
+ urls: [...linksDeduped],
+ })
+ .catch(error => {
+ meta.logger.error("Failed to queue links for indexing", {
+ error: (error as Error)?.message,
+ url: meta.url,
+ });
+ });
+ } catch (error) {
+ meta.logger.error("Failed to queue links for indexing", {
+ error: (error as Error)?.message,
+ url: meta.url,
+ });
}
+ }
- document.links = await extractLinks(
- document.html,
- document.metadata.url ??
- document.metadata.sourceURL ??
- meta.rewrittenUrl ??
- meta.url,
- );
+ if (!requiresLinks) {
+ delete document.links;
}
return document;
diff --git a/apps/api/src/services/indexing/indexer-queue.ts b/apps/api/src/services/indexing/indexer-queue.ts
new file mode 100644
index 000000000..9e3866c95
--- /dev/null
+++ b/apps/api/src/services/indexing/indexer-queue.ts
@@ -0,0 +1,209 @@
+import amqp from "amqplib";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+
+const logger = _logger.child({ module: "indexer-queue" });
+
+const INDEX_EXCHANGE = "index.jobs";
+const LINKS_QUEUE_NAME = "index.jobs.links";
+const LINKS_RK = "job.links";
+
+const CONNECT_TIMEOUT = 5000;
+const DRAIN_TIMEOUT = 30000;
+
+type IndexWorkerMessage = {
+ id: string;
+ type: "links";
+ discovery_url: string;
+ urls: string[];
+};
+
+class IndexerQueue {
+ private connection: amqp.ChannelModel | null = null;
+ private publishChannel: amqp.Channel | null = null;
+
+ private connectPromise: Promise<void> | null = null;
+
+ private closing: boolean = false;
+ private noop: boolean | null = null;
+
+ async connect(): Promise<void> {
+ if (this.connectPromise) return this.connectPromise;
+ if (this.closing) return;
+ if (this.noop) return;
+ if (this.connection && this.publishChannel) return;
+
+ this.connectPromise = this._establishConnection();
+
+ try {
+ await this.connectPromise;
+ } finally {
+ this.connectPromise = null;
+ }
+ }
+
+ private async _establishConnection(): Promise<void> {
+ if (!config.INDEXER_RABBITMQ_URL) {
+ logger.debug("Indexer disabled via config");
+ this.noop = true;
+ return;
+ }
+
+ try {
+ logger.info("Connecting to index RabbitMQ");
+
+ this.connection = await amqp.connect(config.INDEXER_RABBITMQ_URL);
+ this.publishChannel = await this.connection.createChannel();
+
+ // NOTE: this queue is already created by another service
+ await this.publishChannel.checkQueue(LINKS_QUEUE_NAME);
+
+ this._registerConnectionEvents();
+
+ logger.info("Connected to index RabbitMQ");
+ } catch (err) {
+ this.connection = null;
+ this.publishChannel = null;
+ throw err;
+ }
+ }
+
+ private _registerConnectionEvents() {
+ if (this.noop) return;
+ if (!this.connection || !this.publishChannel) return;
+
+ this.connection.on("close", () => {
+ if (this.closing) return;
+
+ logger.warn("Index RabbitMQ connection closed", {
+ module: "index-queue",
+ });
+
+ this.cleanup();
+ setTimeout(
+ () =>
+ this.connect().catch(err =>
+ logger.error("Reconnection failed", {
+ err,
+ }),
+ ),
+ CONNECT_TIMEOUT,
+ );
+ });
+
+ this.connection.on("error", err => {
+ logger.error("Index RabbitMQ connection error", {
+ err,
+ });
+ });
+
+ this.publishChannel.on("error", err => {
+ logger.error("Index RabbitMQ channel error", {
+ err,
+ });
+ });
+ }
+
+ async sendToWorker(message: IndexWorkerMessage): Promise<void> {
+ await this.connect();
+
+ if (this.noop) return;
+ if (!this.publishChannel) throw new Error("Channel not available");
+
+ const payload = {
+ id: message.id,
+ } as any;
+
+ if (message.type === "links") {
+ payload.urls = message.urls;
+ payload.discovery_url = message.discovery_url;
+ }
+
+ const buffer = Buffer.from(JSON.stringify(payload));
+
+ const canSendMore = this.publishChannel.publish(
+ INDEX_EXCHANGE,
+ LINKS_RK,
+ buffer,
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: message.id,
+ },
+ );
+
+ if (!canSendMore) {
+ logger.warn("Index message buffer full, waiting for drain");
+ await this.waitForDrain();
+ }
+ }
+
+ private async waitForDrain(): Promise<void> {
+ if (this.noop) return;
+
+ return new Promise((resolve, reject) => {
+ if (!this.publishChannel) {
+ return reject(new Error("Channel not available"));
+ }
+
+ const listeners: Record<string, any> = {};
+
+ const cleanup = () => {
+ if (!this.publishChannel) return;
+ this.publishChannel.removeListener("drain", listeners.drain);
+ this.publishChannel.removeListener("error", listeners.error);
+ this.publishChannel.removeListener("close", listeners.close);
+ clearTimeout(listeners.timeout);
+ };
+
+ listeners.drain = () => {
+ cleanup();
+ resolve();
+ };
+ listeners.error = (err: Error) => {
+ cleanup();
+ reject(err);
+ };
+ listeners.close = () => {
+ cleanup();
+ reject(new Error("Channel closed during drain"));
+ };
+
+ listeners.timeout = setTimeout(() => {
+ cleanup();
+ reject(new Error(`Drain timeout after ${DRAIN_TIMEOUT}ms`));
+ }, DRAIN_TIMEOUT);
+
+ this.publishChannel.on("drain", listeners.drain);
+ this.publishChannel.on("error", listeners.error);
+ this.publishChannel.on("close", listeners.close);
+ });
+ }
+
+ async close(): Promise<void> {
+ if (this.noop) return;
+ this.closing = true;
+
+ try {
+ await this.publishChannel?.close();
+ await this.connection?.close();
+ } catch (err) {
+ logger.warn("Error while closing RabbitMQ connection", { err });
+ } finally {
+ this.cleanup();
+ logger.info("Indexer RabbitMQ closed");
+ }
+ }
+
+ private cleanup() {
+ this.connection = null;
+ this.publishChannel = null;
+ this.connectPromise = null;
+ }
+}
+
+export const indexerQueue = new IndexerQueue();
+
+export async function shutdownIndexerQueue(): Promise<void> {
+ await indexerQueue.close();
+}