Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: webhook dispatcher (#2534)
tom88d3b5166fdd
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex cc5bee52e..1a408e5cb 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -132,6 +132,7 @@ const configSchema = z.object({SLACK_ADMIN_WEBHOOK_URL: z.string().optional(),DISABLE_WEBHOOK_DELIVERY: z.stringbool().optional(),ALLOW_LOCAL_WEBHOOKS: z.stringbool().optional(),+ WEBHOOK_USE_RABBITMQ: z.stringbool().optional(),// Firecrawl FeaturesFIRECRAWL_DEBUG_FILTER_LINKS: z.stringbool().optional(),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 4e9a6b5b2..dd95ac020 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -37,6 +37,7 @@ import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";import responseTime from "response-time";+import { shutdownWebhookQueue } from "./services/webhook";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -137,8 +138,10 @@ async function startServer(port = DEFAULT_PORT) {server.close(() => {logger.info("Server closed.");nuqShutdown().finally(() => {- logger.info("NUQ shutdown complete");- process.exit(0);+ shutdownWebhookQueue().finally(() => {+ logger.info("NUQ shutdown complete");+ process.exit(0);+ });});});};diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex f91ac0d54..fea161184 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -6,9 +6,16 @@ import {getSecureDispatcherNoCookies,isIPPrivate,} from "../../scraper/scrapeURL/engines/utils/safeFetch";-import { WebhookConfig, WebhookEvent, WebhookEventDataMap } from "./types";+import type {+ WebhookConfig,+ WebhookEvent,+ WebhookEventDataMap,+ WebhookQueueMessage,+} from "./types";import { redisEvictConnection } from "../redis";import { supabase_service } from "../supabase";+import { webhookQueue } from "./queue";+import { randomUUID } from "crypto";const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";const WEBHOOK_INSERT_BATCH_SIZE = 1000;@@ -45,6 +52,7 @@ export class WebhookSender {success: data.success,type: event,[this.context.v0 ? "jobId" : "id"]: this.context.jobId,+ webhookId: randomUUID(), // Unique ID for this webhook delivery (used for e.g. retries)data: "data" in data ? data.data : [],error: "error" in data ? data.error : undefined,metadata: this.config.metadata || undefined,@@ -84,6 +92,35 @@ export class WebhookSender {return;}+ if (config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL) {+ const queueMessage: WebhookQueueMessage = {+ webhook_url: this.config.url,+ payload,+ headers: this.config.headers,+ team_id: this.context.teamId,+ job_id: this.context.jobId,+ scrape_id: scrapeId ?? null,+ event: payload.type,+ timeout_ms: this.context.v0 ? 30000 : 10000,+ };++ try {+ await webhookQueue.publish(queueMessage);+ this.logger.info("Webhook queued for delivery", {+ webhookUrl: this.config.url,+ event: payload.type,+ });+ } catch (error) {+ this.logger.error("Failed to queue webhook", {+ error,+ webhookUrl: this.config.url,+ });+ throw error;+ }++ return;+ }+const payloadString = JSON.stringify(payload);const headers: Record<string, string> = {"Content-Type": "application/json",diff --git a/apps/api/src/services/webhook/index.ts b/apps/api/src/services/webhook/index.tsindex f3369ffe8..232ac135f 100644--- a/apps/api/src/services/webhook/index.ts+++ b/apps/api/src/services/webhook/index.ts@@ -30,3 +30,4 @@ export {processWebhookInsertJobs,} from "./delivery";export { WebhookEvent } from "./types";+export { shutdownWebhookQueue } from "./queue";diff --git a/apps/api/src/services/webhook/queue.ts b/apps/api/src/services/webhook/queue.tsnew file mode 100644index 000000000..16d8cf5ae--- /dev/null+++ b/apps/api/src/services/webhook/queue.ts@@ -0,0 +1,183 @@+import amqp from "amqplib";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { WebhookQueueMessage } from "./types";++const WEBHOOK_QUEUE_NAME = "webhooks";+const CONNECT_TIMEOUT = 5000;+const DRAIN_TIMEOUT = 30000;++class WebhookQueue {+ private connection: amqp.ChannelModel | null = null;+ private channel: amqp.Channel | null = null;++ private connectPromise: Promise<void> | null = null;++ async connect(): Promise<void> {+ if (this.connection && this.channel) return;+ if (this.connectPromise) return this.connectPromise;++ this.connectPromise = this._establishConnection();++ try {+ await this.connectPromise;+ } finally {+ this.connectPromise = null;+ }+ }++ private async _establishConnection(): Promise<void> {+ if (!config.NUQ_RABBITMQ_URL) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }++ try {+ _logger.info("Connecting to webhook RabbitMQ", {+ module: "webhook-queue",+ });++ this.connection = await amqp.connect(config.NUQ_RABBITMQ_URL);+ this.channel = await this.connection.createChannel();++ await this.channel.checkQueue(WEBHOOK_QUEUE_NAME);++ this._registerConnectionEvents();++ _logger.info("Connected to webhook RabbitMQ", {+ module: "webhook-queue",+ });+ } catch (err) {+ this.connection = null;+ this.channel = null;+ throw err;+ }+ }++ private _registerConnectionEvents() {+ if (!this.connection || !this.channel) return;++ this.connection.on("close", () => {+ _logger.warn("Webhook RabbitMQ connection closed", {+ module: "webhook-queue",+ });+ this.cleanup();+ setTimeout(+ () =>+ this.connect().catch(err =>+ _logger.error("Reconnection failed", {+ module: "webhook-queue",+ err,+ }),+ ),+ CONNECT_TIMEOUT,+ );+ });++ this.connection.on("error", err => {+ _logger.error("Webhook RabbitMQ connection error", {+ module: "webhook-queue",+ err,+ });+ });++ this.channel.on("error", err => {+ _logger.error("Webhook RabbitMQ channel error", {+ module: "webhook-queue",+ err,+ });+ });+ }++ async publish(message: WebhookQueueMessage): Promise<void> {+ await this.connect();++ if (!this.channel) throw new Error("Channel not available");++ const buffer = Buffer.from(JSON.stringify(message));++ const canSendMore = this.channel.sendToQueue(WEBHOOK_QUEUE_NAME, buffer, {+ persistent: true,+ contentType: "application/json",+ });++ if (!canSendMore) {+ _logger.warn("Webhook message buffer full, waiting for drain", {+ module: "webhook-queue",+ teamId: message.team_id,+ });++ await this.waitForDrain();+ }++ _logger.info("Webhook message published", {+ module: "webhook-queue",+ teamId: message.team_id,+ jobId: message.job_id,+ event: message.event,+ });+ }++ private async waitForDrain(): Promise<void> {+ return new Promise((resolve, reject) => {+ if (!this.channel) {+ return reject(new Error("Channel not available"));+ }++ const listeners: Record<string, any> = {};++ const cleanup = () => {+ if (!this.channel) return;+ this.channel.removeListener("drain", listeners.drain);+ this.channel.removeListener("error", listeners.error);+ this.channel.removeListener("close", listeners.close);+ clearTimeout(listeners.timeout);+ };++ listeners.drain = () => {+ cleanup();+ resolve();+ };+ listeners.error = (err: Error) => {+ cleanup();+ reject(err);+ };+ listeners.close = () => {+ cleanup();+ reject(new Error("Channel closed during drain"));+ };++ listeners.timeout = setTimeout(() => {+ cleanup();+ reject(new Error(`Drain timeout after ${DRAIN_TIMEOUT}ms`));+ }, DRAIN_TIMEOUT);++ this.channel.on("drain", listeners.drain);+ this.channel.on("error", listeners.error);+ this.channel.on("close", listeners.close);+ });+ }++ async close(): Promise<void> {+ try {+ await this.channel?.close();+ await this.connection?.close();+ } catch (err) {+ _logger.warn("Error while closing RabbitMQ connection", { err });+ } finally {+ this.cleanup();+ _logger.info("Webhook RabbitMQ closed", { module: "webhook-queue" });+ }+ }++ private cleanup() {+ this.connection = null;+ this.channel = null;+ this.connectPromise = null;+ }+}++export const webhookQueue = new WebhookQueue();++export async function shutdownWebhookQueue(): Promise<void> {+ await webhookQueue.close();+}diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.tsindex 92684cce0..a3fbdabde 100644--- a/apps/api/src/services/webhook/types.ts+++ b/apps/api/src/services/webhook/types.ts@@ -29,6 +29,26 @@ export type WebhookEventDataMap = {export type WebhookConfig = z.infer<typeof webhookSchema>;+export type WebhookQueueMessage = {+ webhook_url: string;+ payload: {+ success: boolean;+ type: string;+ webhookId: string;+ id?: string;+ jobId?: string;+ data: any[];+ error?: string;+ metadata?: Record<string, string>;+ };+ headers: Record<string, string>;+ team_id: string;+ job_id: string;+ scrape_id: string | null;+ event: WebhookEvent;+ timeout_ms: number;+};+interface WebhookDocument {content?: string;markdown: string;