Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: webhook dispatcher (#2534)

tom88d3b5166fdd
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index cc5bee52e..1a408e5cb 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -132,6 +132,7 @@ const configSchema = z.object({
SLACK_ADMIN_WEBHOOK_URL: z.string().optional(),
DISABLE_WEBHOOK_DELIVERY: z.stringbool().optional(),
ALLOW_LOCAL_WEBHOOKS: z.stringbool().optional(),
+ WEBHOOK_USE_RABBITMQ: z.stringbool().optional(),
// Firecrawl Features
FIRECRAWL_DEBUG_FILTER_LINKS: z.stringbool().optional(),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 4e9a6b5b2..dd95ac020 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -37,6 +37,7 @@ import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";
import responseTime from "response-time";
+import { shutdownWebhookQueue } from "./services/webhook";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -137,8 +138,10 @@ async function startServer(port = DEFAULT_PORT) {
server.close(() => {
logger.info("Server closed.");
nuqShutdown().finally(() => {
- logger.info("NUQ shutdown complete");
- process.exit(0);
+ shutdownWebhookQueue().finally(() => {
+ logger.info("NUQ shutdown complete");
+ process.exit(0);
+ });
});
});
};
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index f91ac0d54..fea161184 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -6,9 +6,16 @@ import {
getSecureDispatcherNoCookies,
isIPPrivate,
} from "../../scraper/scrapeURL/engines/utils/safeFetch";
-import { WebhookConfig, WebhookEvent, WebhookEventDataMap } from "./types";
+import type {
+ WebhookConfig,
+ WebhookEvent,
+ WebhookEventDataMap,
+ WebhookQueueMessage,
+} from "./types";
import { redisEvictConnection } from "../redis";
import { supabase_service } from "../supabase";
+import { webhookQueue } from "./queue";
+import { randomUUID } from "crypto";
const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";
const WEBHOOK_INSERT_BATCH_SIZE = 1000;
@@ -45,6 +52,7 @@ export class WebhookSender {
success: data.success,
type: event,
[this.context.v0 ? "jobId" : "id"]: this.context.jobId,
+ webhookId: randomUUID(), // Unique ID for this webhook delivery (used for e.g. retries)
data: "data" in data ? data.data : [],
error: "error" in data ? data.error : undefined,
metadata: this.config.metadata || undefined,
@@ -84,6 +92,35 @@ export class WebhookSender {
return;
}
+ if (config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL) {
+ const queueMessage: WebhookQueueMessage = {
+ webhook_url: this.config.url,
+ payload,
+ headers: this.config.headers,
+ team_id: this.context.teamId,
+ job_id: this.context.jobId,
+ scrape_id: scrapeId ?? null,
+ event: payload.type,
+ timeout_ms: this.context.v0 ? 30000 : 10000,
+ };
+
+ try {
+ await webhookQueue.publish(queueMessage);
+ this.logger.info("Webhook queued for delivery", {
+ webhookUrl: this.config.url,
+ event: payload.type,
+ });
+ } catch (error) {
+ this.logger.error("Failed to queue webhook", {
+ error,
+ webhookUrl: this.config.url,
+ });
+ throw error;
+ }
+
+ return;
+ }
+
const payloadString = JSON.stringify(payload);
const headers: Record<string, string> = {
"Content-Type": "application/json",
diff --git a/apps/api/src/services/webhook/index.ts b/apps/api/src/services/webhook/index.ts
index f3369ffe8..232ac135f 100644
--- a/apps/api/src/services/webhook/index.ts
+++ b/apps/api/src/services/webhook/index.ts
@@ -30,3 +30,4 @@ export {
processWebhookInsertJobs,
} from "./delivery";
export { WebhookEvent } from "./types";
+export { shutdownWebhookQueue } from "./queue";
diff --git a/apps/api/src/services/webhook/queue.ts b/apps/api/src/services/webhook/queue.ts
new file mode 100644
index 000000000..16d8cf5ae
--- /dev/null
+++ b/apps/api/src/services/webhook/queue.ts
@@ -0,0 +1,183 @@
+import amqp from "amqplib";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { WebhookQueueMessage } from "./types";
+
+const WEBHOOK_QUEUE_NAME = "webhooks";
+const CONNECT_TIMEOUT = 5000;
+const DRAIN_TIMEOUT = 30000;
+
+class WebhookQueue {
+ private connection: amqp.ChannelModel | null = null;
+ private channel: amqp.Channel | null = null;
+
+ private connectPromise: Promise<void> | null = null;
+
+ async connect(): Promise<void> {
+ if (this.connection && this.channel) return;
+ if (this.connectPromise) return this.connectPromise;
+
+ this.connectPromise = this._establishConnection();
+
+ try {
+ await this.connectPromise;
+ } finally {
+ this.connectPromise = null;
+ }
+ }
+
+ private async _establishConnection(): Promise<void> {
+ if (!config.NUQ_RABBITMQ_URL) {
+ throw new Error("NUQ_RABBITMQ_URL is not configured");
+ }
+
+ try {
+ _logger.info("Connecting to webhook RabbitMQ", {
+ module: "webhook-queue",
+ });
+
+ this.connection = await amqp.connect(config.NUQ_RABBITMQ_URL);
+ this.channel = await this.connection.createChannel();
+
+ await this.channel.checkQueue(WEBHOOK_QUEUE_NAME);
+
+ this._registerConnectionEvents();
+
+ _logger.info("Connected to webhook RabbitMQ", {
+ module: "webhook-queue",
+ });
+ } catch (err) {
+ this.connection = null;
+ this.channel = null;
+ throw err;
+ }
+ }
+
+ private _registerConnectionEvents() {
+ if (!this.connection || !this.channel) return;
+
+ this.connection.on("close", () => {
+ _logger.warn("Webhook RabbitMQ connection closed", {
+ module: "webhook-queue",
+ });
+ this.cleanup();
+ setTimeout(
+ () =>
+ this.connect().catch(err =>
+ _logger.error("Reconnection failed", {
+ module: "webhook-queue",
+ err,
+ }),
+ ),
+ CONNECT_TIMEOUT,
+ );
+ });
+
+ this.connection.on("error", err => {
+ _logger.error("Webhook RabbitMQ connection error", {
+ module: "webhook-queue",
+ err,
+ });
+ });
+
+ this.channel.on("error", err => {
+ _logger.error("Webhook RabbitMQ channel error", {
+ module: "webhook-queue",
+ err,
+ });
+ });
+ }
+
+ async publish(message: WebhookQueueMessage): Promise<void> {
+ await this.connect();
+
+ if (!this.channel) throw new Error("Channel not available");
+
+ const buffer = Buffer.from(JSON.stringify(message));
+
+ const canSendMore = this.channel.sendToQueue(WEBHOOK_QUEUE_NAME, buffer, {
+ persistent: true,
+ contentType: "application/json",
+ });
+
+ if (!canSendMore) {
+ _logger.warn("Webhook message buffer full, waiting for drain", {
+ module: "webhook-queue",
+ teamId: message.team_id,
+ });
+
+ await this.waitForDrain();
+ }
+
+ _logger.info("Webhook message published", {
+ module: "webhook-queue",
+ teamId: message.team_id,
+ jobId: message.job_id,
+ event: message.event,
+ });
+ }
+
+ private async waitForDrain(): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (!this.channel) {
+ return reject(new Error("Channel not available"));
+ }
+
+ const listeners: Record<string, any> = {};
+
+ const cleanup = () => {
+ if (!this.channel) return;
+ this.channel.removeListener("drain", listeners.drain);
+ this.channel.removeListener("error", listeners.error);
+ this.channel.removeListener("close", listeners.close);
+ clearTimeout(listeners.timeout);
+ };
+
+ listeners.drain = () => {
+ cleanup();
+ resolve();
+ };
+ listeners.error = (err: Error) => {
+ cleanup();
+ reject(err);
+ };
+ listeners.close = () => {
+ cleanup();
+ reject(new Error("Channel closed during drain"));
+ };
+
+ listeners.timeout = setTimeout(() => {
+ cleanup();
+ reject(new Error(`Drain timeout after ${DRAIN_TIMEOUT}ms`));
+ }, DRAIN_TIMEOUT);
+
+ this.channel.on("drain", listeners.drain);
+ this.channel.on("error", listeners.error);
+ this.channel.on("close", listeners.close);
+ });
+ }
+
+ async close(): Promise<void> {
+ try {
+ await this.channel?.close();
+ await this.connection?.close();
+ } catch (err) {
+ _logger.warn("Error while closing RabbitMQ connection", { err });
+ } finally {
+ this.cleanup();
+ _logger.info("Webhook RabbitMQ closed", { module: "webhook-queue" });
+ }
+ }
+
+ private cleanup() {
+ this.connection = null;
+ this.channel = null;
+ this.connectPromise = null;
+ }
+}
+
+export const webhookQueue = new WebhookQueue();
+
+export async function shutdownWebhookQueue(): Promise<void> {
+ await webhookQueue.close();
+}
diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.ts
index 92684cce0..a3fbdabde 100644
--- a/apps/api/src/services/webhook/types.ts
+++ b/apps/api/src/services/webhook/types.ts
@@ -29,6 +29,26 @@ export type WebhookEventDataMap = {
export type WebhookConfig = z.infer<typeof webhookSchema>;
+export type WebhookQueueMessage = {
+ webhook_url: string;
+ payload: {
+ success: boolean;
+ type: string;
+ webhookId: string;
+ id?: string;
+ jobId?: string;
+ data: any[];
+ error?: string;
+ metadata?: Record<string, string>;
+ };
+ headers: Record<string, string>;
+ team_id: string;
+ job_id: string;
+ scrape_id: string | null;
+ event: WebhookEvent;
+ timeout_ms: number;
+};
+
interface WebhookDocument {
content?: string;
markdown: string;