Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1/webhook): complex webhook object w/ headers (#899)
Gergő Móricz32be2cf78629
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ec78509a1..b2edd6e7c 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -220,11 +220,22 @@ const crawlerOptions = z.object({export type CrawlerOptions = z.infer<typeof crawlerOptions>;+export const webhookSchema = z.preprocess(x => {+ if (typeof x === "string") {+ return { url: x };+ } else {+ return x;+ }+}, z.object({+ url: z.string().url(),+ headers: z.record(z.string(), z.string()).default({}),+}).strict(strictMessage))+export const crawlRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}),- webhook: z.string().url().optional(),+ webhook: webhookSchema.optional(),limit: z.number().default(10000),}).strict(strictMessage);diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex 620b68329..1cc4db84a 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -1,15 +1,17 @@import axios from "axios";-import { logger } from "../../src/lib/logger";+import { logger } from "../lib/logger";import { supabase_service } from "./supabase";import { WebhookEventType } from "../types";import { configDotenv } from "dotenv";+import { z } from "zod";+import { webhookSchema } from "../controllers/v1/types";configDotenv();export const callWebhook = async (teamId: string,id: string,data: any | null,- specified?: string,+ specified?: z.infer<typeof webhookSchema>,v1 = false,eventType: WebhookEventType = "crawl.page",awaitWebhook: boolean = false@@ -20,7 +22,7 @@ export const callWebhook = async (id);const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- let webhookUrl = specified ?? selfHostedUrl;+ let webhookUrl = specified ?? (selfHostedUrl ? webhookSchema.parse({ url: selfHostedUrl }) : undefined);// Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set// and the USE_DB_AUTHENTICATION environment variable is set to true@@ -73,7 +75,7 @@ export const callWebhook = async (if (awaitWebhook) {try {await axios.post(- webhookUrl,+ webhookUrl.url,{success: !v1? data.success@@ -92,6 +94,7 @@ export const callWebhook = async ({headers: {"Content-Type": "application/json",+ ...webhookUrl.headers,},timeout: v1 ? 10000 : 30000, // 10 seconds timeout (v1)}@@ -104,7 +107,7 @@ export const callWebhook = async (} else {axios.post(- webhookUrl,+ webhookUrl.url,{success: !v1? data.success@@ -123,6 +126,7 @@ export const callWebhook = async ({headers: {"Content-Type": "application/json",+ ...webhookUrl.headers,},timeout: v1 ? 10000 : 30000, // 10 seconds timeout (v1)}diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 2da97bd1f..d78214072 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -1,4 +1,5 @@-import { AuthCreditUsageChunk, ScrapeOptions, Document as V1Document } from "./controllers/v1/types";+import { z } from "zod";+import { AuthCreditUsageChunk, ScrapeOptions, Document as V1Document, webhookSchema } from "./controllers/v1/types";import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";@@ -33,7 +34,7 @@ export interface WebScraperOptions {origin?: string;crawl_id?: string;sitemapped?: boolean;- webhook?: string;+ webhook?: z.infer<typeof webhookSchema>;v1?: boolean;is_scrape?: boolean;}diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 401b1c204..45e19197b 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -153,7 +153,10 @@ export interface CrawlParams {allowExternalLinks?: boolean;ignoreSitemap?: boolean;scrapeOptions?: CrawlScrapeOptions;- webhook?: string;+ webhook?: string | {+ url: string;+ headers?: Record<string, string>;+ };deduplicateSimilarURLs?: boolean;ignoreQueryParameters?: boolean;}