Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(webhook): refactor callWebhook and add logWebhook (FIR-2218) (#1629)
Gergő Móricz78580f65df62
diff --git a/apps/api/src/controllers/v0/admin/index-queue-prometheus.ts b/apps/api/src/controllers/v0/admin/index-queue-prometheus.tsindex d14768692..a9ec6b03d 100644--- a/apps/api/src/controllers/v0/admin/index-queue-prometheus.ts+++ b/apps/api/src/controllers/v0/admin/index-queue-prometheus.ts@@ -1,12 +1,15 @@import type { Request, Response } from "express";import { getIndexInsertQueueLength } from "../../../services";+import { getWebhookInsertQueueLength } from "../../../services/webhook";export async function indexQueuePrometheus(req: Request, res: Response) {const queueLength = await getIndexInsertQueueLength();+ const webhookQueueLength = await getWebhookInsertQueueLength();res.setHeader("Content-Type", "text/plain");res.send(`\# HELP firecrawl_index_queue_length The number of items in the index insert queue# TYPE firecrawl_index_queue_length gaugefirecrawl_index_queue_length ${queueLength}+firecrawl_webhook_queue_length ${webhookQueueLength}`);}\ No newline at end of filediff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 5016e2ab7..556551c0c 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -171,14 +171,14 @@ export async function batchScrapeController(logger.debug("Calling webhook with batch_scrape.started...", {webhook: req.body.webhook,});- await callWebhook(- req.auth.team_id,- id,- null,- req.body.webhook,- true,- "batch_scrape.started",- );+ await callWebhook({+ teamId: req.auth.team_id,+ crawlId: id,+ data: null,+ webhook: req.body.webhook,+ v1: true,+ eventType: "batch_scrape.started",+ });}const protocol = process.env.ENV === "local" ? req.protocol : "https";diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 677d33fab..06ad21f5d 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -15,6 +15,7 @@ import { processBillingBatch, queueBillingOperation, startBillingBatchProcessingimport systemMonitor from "../system-monitor";import { v4 as uuidv4 } from "uuid";import { processIndexInsertJobs } from "..";+import { processWebhookInsertJobs } from "../webhook";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -228,6 +229,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =};const INDEX_INSERT_INTERVAL = 15000;+const WEBHOOK_INSERT_INTERVAL = 15000;// Start the workers(async () => {@@ -246,8 +248,16 @@ const INDEX_INSERT_INTERVAL = 15000;await processIndexInsertJobs();}, INDEX_INSERT_INTERVAL);+ const webhookInserterInterval = setInterval(async () => {+ if (isShuttingDown) {+ return;+ }+ await processWebhookInsertJobs();+ }, WEBHOOK_INSERT_INTERVAL);+// Wait for both workers to complete (which should only happen on shutdown)await Promise.all([indexWorkerPromise, billingWorkerPromise]);clearInterval(indexInserterInterval);+ clearInterval(webhookInserterInterval);})();diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 4f178cbf0..793fa85c8 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -344,16 +344,16 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {// v0 web hooks, call when done with all the dataif (!job.data.v1) {- callWebhook(- job.data.team_id,- job.data.crawl_id,+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,data,- job.data.webhook,- job.data.v1,- job.data.crawlerOptions !== null+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null? "crawl.completed": "batch_scrape.completed",- );+ });}} else {const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);@@ -384,16 +384,16 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {// v1 web hooks, call when done with no data, but with event completedif (job.data.v1 && job.data.webhook) {- callWebhook(- job.data.team_id,- job.data.crawl_id,- [],- job.data.webhook,- job.data.v1,- job.data.crawlerOptions !== null+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: [],+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null? "crawl.completed": "batch_scrape.completed",- );+ });}}}@@ -988,14 +988,14 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {logger.debug("Calling webhook with crawl.started...", {webhook: job.data.webhook,});- await callWebhook(- job.data.team_id,- job.data.crawl_id,- null,- job.data.webhook,- true,- "crawl.started",- );+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ data: null,+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: "crawl.started",+ });}const sitemap = sc.crawlerOptions.ignoreSitemap@@ -1478,15 +1478,15 @@ async function processJob(job: Job & { id: string }, token: string) {logger.debug("Calling webhook with success...", {webhook: job.data.webhook,});- await callWebhook(- job.data.team_id,- job.data.crawl_id,+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id,+ scrapeId: job.id,data,- job.data.webhook,- job.data.v1,- job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- true,- );+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });}logger.debug("Declaring job as done...");@@ -1583,14 +1583,15 @@ async function processJob(job: Job & { id: string }, token: string) {};if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {- callWebhook(- job.data.team_id,- job.data.crawl_id ?? (job.id as string),+ callWebhook({+ teamId: job.data.team_id,+ crawlId: job.data.crawl_id ?? (job.id as string),+ scrapeId: job.id,data,- job.data.webhook,- job.data.v1,- job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",- );+ webhook: job.data.webhook,+ v1: job.data.v1,+ eventType: job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",+ });}const end = Date.now();diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex 48d864a89..ae0071070 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -1,38 +1,107 @@-import axios from "axios";-import { logger as _logger } from "../lib/logger";+import axios, { AxiosError } from "axios";+import { logger as _logger, logger } from "../lib/logger";import { supabase_rr_service, supabase_service } from "./supabase";import { WebhookEventType } from "../types";import { configDotenv } from "dotenv";import { z } from "zod";import { webhookSchema } from "../controllers/v1/types";+import { redisEvictConnection } from "./redis";+import { index_supabase_service } from ".";configDotenv();-export const callWebhook = async (- teamId: string,- id: string,- data: any | null,- specified?: z.infer<typeof webhookSchema>,- v1 = false,- eventType: WebhookEventType = "crawl.page",- awaitWebhook: boolean = false,-) => {+const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";+const WEBHOOK_INSERT_BATCH_SIZE = 1000;++async function addWebhookInsertJob(data: any) {+ await redisEvictConnection.rpush(WEBHOOK_INSERT_QUEUE_KEY, JSON.stringify(data));+}++export async function getWebhookInsertQueueLength(): Promise<number> {+ return await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY) ?? 0;+}++async function getWebhookInsertJobs(): Promise<any[]> {+ const jobs = (await redisEvictConnection.lpop(WEBHOOK_INSERT_QUEUE_KEY, WEBHOOK_INSERT_BATCH_SIZE)) ?? [];+ return jobs.map(x => JSON.parse(x));+}++export async function processWebhookInsertJobs() {+ const jobs = await getWebhookInsertJobs();+ if (jobs.length === 0) {+ return;+ }+ logger.info(`Webhook inserter found jobs to insert`, { jobCount: jobs.length });+ try {+ await supabase_service.from("webhook_logs").insert(jobs);+ logger.info(`Webhook inserter inserted jobs`, { jobCount: jobs.length });+ } catch (error) {+ logger.error(`Webhook inserter failed to insert jobs`, { error, jobCount: jobs.length });+ }+}++async function logWebhook(data: {+ success: boolean;+ error?: string;+ teamId: string;+ crawlId: string;+ scrapeId?: string;+ url: string;+ statusCode?: number;+ event: WebhookEventType+}) {+ try {+ await addWebhookInsertJob({+ success: data.success,+ error: data.error ?? null,+ team_id: data.teamId,+ crawl_id: data.crawlId,+ scrape_id: data.scrapeId ?? null,+ url: data.url,+ status_code: data.statusCode ?? null,+ event: data.event,+ });+ } catch (error) {+ _logger.error("Error logging webhook", { error, crawlId: data.crawlId, scrapeId: data.scrapeId, teamId: data.teamId, team_id: data.teamId, module: "webhook", method: "logWebhook" });+ }+}++export const callWebhook = async ({+ teamId,+ crawlId,+ scrapeId,+ data,+ webhook,+ v1,+ eventType,+ awaitWebhook = false,+}: {+ teamId: string;+ crawlId: string;+ scrapeId?: string;+ webhook?: z.infer<typeof webhookSchema>,+ v1: boolean,+ data: any | null;+ eventType: WebhookEventType,+ awaitWebhook?: boolean;+}) => {const logger = _logger.child({module: "webhook",method: "callWebhook",teamId, team_id: teamId,- crawlId: id,+ crawlId,+ scrapeId,eventType,awaitWebhook,- webhook: specified,+ webhook,isV1: v1,});- if (specified) {+ if (webhook) {let subType = eventType.split(".")[1];- if (!specified.events.includes(subType as any)) {+ if (!webhook.events.includes(subType as any)) {logger.debug("Webhook event type not in specified events", {subType,- specified,+ webhook,});return false;}@@ -41,11 +110,11 @@ export const callWebhook = async (try {const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}",- id,+ crawlId,);const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";let webhookUrl =- specified ??+ webhook ??(selfHostedUrl ? webhookSchema.parse({ url: selfHostedUrl }) : undefined);// Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set@@ -103,7 +172,7 @@ export const callWebhook = async (if (awaitWebhook) {try {- await axios.post(+ const res = await axios.post(webhookUrl.url,{success: !v1@@ -112,7 +181,7 @@ export const callWebhook = async (? data.success: true,type: eventType,- [v1 ? "id" : "jobId"]: id,+ [v1 ? "id" : "jobId"]: crawlId,data: dataToSend,error: !v1? data?.error || undefined@@ -129,6 +198,15 @@ export const callWebhook = async (timeout: v1 ? 10000 : 30000, // 10 seconds timeout (v1)},);+ logWebhook({+ success: res.status >= 200 && res.status < 300,+ teamId,+ crawlId,+ scrapeId,+ url: webhookUrl.url,+ event: eventType,+ statusCode: res.status,+ });} catch (error) {logger.error(`Failed to send webhook`,@@ -136,6 +214,16 @@ export const callWebhook = async (error,},);+ logWebhook({+ success: false,+ teamId,+ crawlId,+ scrapeId,+ url: webhookUrl.url,+ event: eventType,+ error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),+ statusCode: error instanceof AxiosError ? error.response?.status : undefined,+ });}} else {axios@@ -148,7 +236,7 @@ export const callWebhook = async (? data.success: true,type: eventType,- [v1 ? "id" : "jobId"]: id,+ [v1 ? "id" : "jobId"]: crawlId,data: dataToSend,error: !v1? data?.error || undefined@@ -164,6 +252,17 @@ export const callWebhook = async (},},)+ .then((res) => {+ logWebhook({+ success: res.status >= 200 && res.status < 300,+ teamId,+ crawlId,+ scrapeId,+ url: webhookUrl.url,+ event: eventType,+ statusCode: res.status,+ });+ }).catch((error) => {logger.error(`Failed to send webhook`,@@ -171,6 +270,16 @@ export const callWebhook = async (error,},);+ logWebhook({+ success: false,+ teamId,+ crawlId,+ scrapeId,+ url: webhookUrl.url,+ event: eventType,+ error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),+ statusCode: error instanceof AxiosError ? error.response?.status : undefined,+ });});}} catch (error) {