Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Implement webhook signatures (ENG-3018) (#2087)
tom6559ff10acd4
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex fcb564c50..6b8a900cc 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -65,6 +65,9 @@ BLOCK_MEDIA=# Set this to the URL of your webhook when using the self-hosted version of FireCrawlSELF_HOSTED_WEBHOOK_URL=+# Set this to the HMAC secret of your webhook when using the self-hosted version of FireCrawl+SELF_HOSTED_WEBHOOK_HMAC_SECRET=+# Resend API Key for transactional emailsRESEND_API_KEY=diff --git a/apps/api/package.json b/apps/api/package.jsonindex b92c3bdcf..f38137a27 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -48,6 +48,7 @@"jest": "^29.6.3","jest-fetch-mock": "^3.0.3","prettier": "^3.4.2",+ "supabase": "^2.39.2","supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",@@ -142,7 +143,6 @@"resend": "^3.4.0","robots-parser": "^3.0.1","stripe": "^16.1.0",- "supabase": "^1.77.9","systeminformation": "^5.22.11","tldts": "^6.1.75","tough-cookie": "^4.1.4",@@ -168,7 +168,8 @@},"pnpm": {"onlyBuiltDependencies": [- "@sentry-internal/node-cpu-profiler"+ "@sentry-internal/node-cpu-profiler",+ "supabase"],"overrides": {"brace-expansion@>=2.0.0 <=2.0.1": "2.0.2"diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex c05d3dbd8..71dd78b1e 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -269,9 +269,6 @@ importers:stripe:specifier: ^16.1.0version: 16.1.0- supabase:- specifier: ^1.77.9- version: 1.172.2systeminformation:specifier: ^5.22.11version: 5.27.7@@ -366,6 +363,9 @@ importers:prettier:specifier: ^3.4.2version: 3.4.2+ supabase:+ specifier: ^2.39.2+ version: 2.39.2supertest:specifier: ^6.3.3version: 6.3.4@@ -3162,9 +3162,9 @@ packages:resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}deprecated: pkg version number incorrect- resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}- engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-sdleLVfCjBtgO5cNjA2HVRvWBJAHs4zwenaCPMNJAJU0yNxpzj80IpjOIimkpkr+mhlA+how5poQtt53PygbHA==}+ engines: {node: ^18.17.0 || >=20.5.0}resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}@@ -3336,9 +3336,9 @@ packages:resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-FMabTRlc5t5zjdenF6mS0MBeFZm0XqHqeOkcskKFb/LYCcRQ5fVgLOHVc4Lq9CqABd9zhjwPjMBCJvMCziSVtA==}- engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-rtpaCbr164TPPh+zFdkWpCyZuKkjpAzODfaZCf/SVJZzJN+4bHQb/LP3Jzq5/+84um3XXY8r548XiWKSborwVw==}+ engines: {node: ^18.17.0 || >=20.5.0}resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}@@ -4172,10 +4172,6 @@ packages:resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}engines: {node: '>= 6'}- resolution: {integrity: sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==}- engines: {node: '>= 14'}-resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}engines: {node: '>= 14'}@@ -4829,8 +4825,8 @@ packages:resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}engines: {node: '>=16 || 14 >=14.17'}- resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==}+ resolution: {integrity: sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==}engines: {node: '>= 18'}@@ -4949,9 +4945,9 @@ packages:resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-dMxCf+zZ+3zeQZXKxmyuCKlIDPGuv8EF940xbkC4kQVDTtqoh6rJFO+JTKSA6/Rwi0getWmtuy4Itup0AMcaDQ==}- engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==}+ engines: {node: ^18.17.0 || >=20.5.0}resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==}@@ -5229,6 +5225,10 @@ packages:resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==}engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}+ resolution: {integrity: sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==}+ engines: {node: ^18.17.0 || >=20.5.0}+resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}@@ -5345,9 +5345,9 @@ packages:resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-yILWifhaSEEytfXI76kB9xEEiG1AiozaCJZ83A87ytjRiN+jVibXjedjCRNjoZviinhG+4UkalO3mWTd8u5O0Q==}- engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-SEbJV7tohp3DAAILbEMPXavBjAnMN0tVnh4+9G8ihV4Pq3HYF9h8QNez9zkJ1ILkv9G2BjdzwctznGZXgu/HGw==}+ engines: {node: ^18.17.0 || >=20.5.0}resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}@@ -5436,11 +5436,6 @@ packages:resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}engines: {node: '>= 4'}- resolution: {integrity: sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==}- engines: {node: '>=14.18'}- hasBin: true-resolution: {integrity: sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==}engines: {node: '>=10.0.0'}@@ -5699,8 +5694,8 @@ packages:resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==}- resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==}+ resolution: {integrity: sha512-/LDPMDIDmuDwj3UsKVw+wA+uHF7QhEF8xgJnKpnk1vqVdr+lA6xRSwWQzgaNuwPj5YPt6+78JKp+wzKziTsRVw==}engines: {npm: '>=8'}hasBin: true@@ -5751,8 +5746,8 @@ packages:os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]hasBin: true- resolution: {integrity: sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==}+ resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}engines: {node: '>=18'}@@ -6198,9 +6193,9 @@ packages:resolution: {integrity: sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==}engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0}- resolution: {integrity: sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==}- engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-GmqrO8WJ1NuzJ2DrziEI2o57jKAVIQNf8a18W3nCYU3H7PNWqCCVTeH6/NQE93CIllIgQS98rrmVkYgTX9fFJQ==}+ engines: {node: ^18.17.0 || >=20.5.0}resolution: {integrity: sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==}@@ -11009,12 +11004,13 @@ snapshots:dependencies:- cmd-shim: 6.0.3- npm-normalize-package-bin: 3.0.1- read-cmd-shim: 4.0.0- write-file-atomic: 5.0.1+ cmd-shim: 7.0.0+ npm-normalize-package-bin: 4.0.0+ proc-log: 5.0.0+ read-cmd-shim: 5.0.0+ write-file-atomic: 6.0.0dependencies:@@ -11222,7 +11218,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}@@ -12144,17 +12140,10 @@ snapshots:transitivePeerDependencies:- supports-color- dependencies:- agent-base: 7.1.3- debug: 4.4.1- transitivePeerDependencies:- - supports-color-dependencies:agent-base: 7.1.3- debug: 4.3.5+ debug: 4.4.1transitivePeerDependencies:- supports-color@@ -13019,10 +13008,9 @@ snapshots:dependencies:minipass: 7.1.2- rimraf: 5.0.7optionalDependencies:@@ -13113,7 +13101,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}dependencies:@@ -13421,6 +13409,8 @@ snapshots:ansi-styles: 5.2.0react-is: 18.3.1+ [email protected]: {}+@@ -13542,7 +13532,7 @@ snapshots:dependencies:loose-envify: 1.4.0- [email protected]: {}+ [email protected]: {}dependencies:@@ -13637,10 +13627,6 @@ snapshots:- dependencies:- glob: 10.4.2-@@ -13924,12 +13910,12 @@ snapshots:dependencies:- bin-links: 4.0.4- https-proxy-agent: 7.0.4+ bin-links: 5.0.0+ https-proxy-agent: 7.0.6node-fetch: 3.3.2- tar: 7.2.0+ tar: 7.4.3transitivePeerDependencies:- supports-color@@ -13983,12 +13969,12 @@ snapshots:dependencies:'@isaacs/fs-minipass': 4.0.1chownr: 3.0.0minipass: 7.1.2- minizlib: 3.0.1+ minizlib: 3.0.2mkdirp: 3.0.1yallist: 5.0.0@@ -14460,7 +14446,7 @@ snapshots:imurmurhash: 0.1.4signal-exit: 3.0.7dependencies:imurmurhash: 0.1.4signal-exit: 4.1.0diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f3e80fdc9..2e0534769 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -26,7 +26,7 @@ export enum IntegrationEnum {MAKE = "make",FLOWISE = "flowise",METAGPT = "metagpt",- RELEVANCEAI = 'relevanceai',+ RELEVANCEAI = "relevanceai",}export type Format =@@ -77,15 +77,16 @@ export const url = z.preprocess(} catch (_) {return false;}- }, "Invalid URL")+ }, "Invalid URL"),// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);const strictMessage ="Unrecognized key in body -- please review the v1 API documentation for request body changes";-export const agentExtractModelValue = 'fire-1'-export const isAgentExtractModelValid = (x: string | undefined) => x?.toLowerCase() === agentExtractModelValue;+export const agentExtractModelValue = "fire-1";+export const isAgentExtractModelValid = (x: string | undefined) =>+ x?.toLowerCase() === agentExtractModelValue;export const agentOptionsExtract = z.object({@@ -115,17 +116,15 @@ export const extractOptions = zmessage: "Invalid JSON schema.",},),- systemPrompt: z- .string()- .max(10000)- .default(""),+ systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),}).strict(strictMessage).transform((data) => ({...data,- systemPrompt: "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."+ systemPrompt:+ "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required.",}));export const extractOptionsWithAgent = z@@ -148,10 +147,7 @@ export const extractOptionsWithAgent = zmessage: "Invalid JSON schema.",},),- systemPrompt: z- .string()- .max(10000)- .default(""),+ systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),agent: z@@ -179,7 +175,7 @@ Key Instructions:* If the content requires user interaction or pagination to be fully accessible, set \`shouldUseSmartscrape\` to \`true\` in your response and provide a clear \`reasoning\` and \`prompt\` for the SmartScrape tool.* If the content is simply JavaScript rendered but doesn't require interaction, set \`shouldUseSmartscrape\` to \`false\`.5. **Output Format:** Your final output MUST be a single, valid JSON object conforming precisely to the schema. Do not include any explanatory text outside the JSON structure.`- : "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."+ : "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required.",}));export type ExtractOptions = z.infer<typeof extractOptions>;@@ -206,60 +202,73 @@ function calculateTotalWaitTime(return waitFor + actionWaitTime;}-export const actionSchema = z- .union([- z- .object({- type: z.literal("wait"),- milliseconds: z.number().int().positive().finite().optional(),- selector: z.string().optional(),- })- .refine(- (data) =>- (data.milliseconds !== undefined || data.selector !== undefined) &&- !(data.milliseconds !== undefined && data.selector !== undefined),- {- message:- "Either 'milliseconds' or 'selector' must be provided, but not both.",- },- ),- z.object({- type: z.literal("click"),- selector: z.string(),- all: z.boolean().default(false),- }),- z.object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- quality: z.number().min(1).max(100).optional(),- }),- z.object({- type: z.literal("write"),- text: z.string(),- }),- z.object({- type: z.literal("press"),- key: z.string(),- }),- z.object({- type: z.literal("scroll"),- direction: z.enum(["up", "down"]).optional().default("down"),+export const actionSchema = z.union([+ z+ .object({+ type: z.literal("wait"),+ milliseconds: z.number().int().positive().finite().optional(),selector: z.string().optional(),- }),- z.object({- type: z.literal("scrape"),- }),- z.object({- type: z.literal("executeJavascript"),- script: z.string(),- }),- z.object({- type: z.literal("pdf"),- landscape: z.boolean().default(false),- scale: z.number().default(1),- format: z.enum(['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'Letter', 'Legal', 'Tabloid', 'Ledger']).default("Letter"),- }),- ]);+ })+ .refine(+ (data) =>+ (data.milliseconds !== undefined || data.selector !== undefined) &&+ !(data.milliseconds !== undefined && data.selector !== undefined),+ {+ message:+ "Either 'milliseconds' or 'selector' must be provided, but not both.",+ },+ ),+ z.object({+ type: z.literal("click"),+ selector: z.string(),+ all: z.boolean().default(false),+ }),+ z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ quality: z.number().min(1).max(100).optional(),+ }),+ z.object({+ type: z.literal("write"),+ text: z.string(),+ }),+ z.object({+ type: z.literal("press"),+ key: z.string(),+ }),+ z.object({+ type: z.literal("scroll"),+ direction: z.enum(["up", "down"]).optional().default("down"),+ selector: z.string().optional(),+ }),+ z.object({+ type: z.literal("scrape"),+ }),+ z.object({+ type: z.literal("executeJavascript"),+ script: z.string(),+ }),+ z.object({+ type: z.literal("pdf"),+ landscape: z.boolean().default(false),+ scale: z.number().default(1),+ format: z+ .enum([+ "A0",+ "A1",+ "A2",+ "A3",+ "A4",+ "A5",+ "A6",+ "Letter",+ "Legal",+ "Tabloid",+ "Ledger",+ ])+ .default("Letter"),+ }),+]);export type Action = z.infer<typeof actionSchema>;@@ -278,7 +287,7 @@ export const actionsSchema = zfunction transformIframeSelector(selector: string): string {return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => {- const prefix = match.match(/^[\s,]/)?.[0] || '';+ const prefix = match.match(/^[\s,]/)?.[0] || "";return prefix + 'div[data-original-tag="iframe"]';});}@@ -310,11 +319,15 @@ const baseScrapeOptions = z"The changeTracking format requires the markdown format to be specified as well",),headers: z.record(z.string(), z.string()).optional(),- includeTags: z.string().array()- .transform(tags => tags.map(transformIframeSelector))+ includeTags: z+ .string()+ .array()+ .transform((tags) => tags.map(transformIframeSelector)).optional(),- excludeTags: z.string().array()- .transform(tags => tags.map(transformIframeSelector))+ excludeTags: z+ .string()+ .array()+ .transform((tags) => tags.map(transformIframeSelector)).optional(),onlyMainContent: z.boolean().default(true),timeout: z.number().int().positive().finite().safe().optional(),@@ -412,17 +425,21 @@ const baseScrapeOptions = z.strict(strictMessage);const fire1Refine = (obj) => {- if (obj.agent?.model?.toLowerCase() === "fire-1" && obj.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {+ if (+ obj.agent?.model?.toLowerCase() === "fire-1" &&+ obj.jsonOptions?.agent?.model?.toLowerCase() === "fire-1"+ ) {return false;}return true;-}+};const fire1RefineOpts = {- message: "You may only specify the FIRE-1 model in agent or jsonOptions.agent, but not both.",+ message:+ "You may only specify the FIRE-1 model in agent or jsonOptions.agent, but not both.",};const waitForRefine = (obj) => {if (obj.waitFor && obj.timeout) {- if (typeof obj.timeout !== 'number' || obj.timeout <= 0) {+ if (typeof obj.timeout !== "number" || obj.timeout <= 0) {return false;}return obj.waitFor <= obj.timeout / 2;@@ -460,7 +477,10 @@ const extractTransform = (obj: ScrapeOptions) => {obj = { ...obj, timeout: 60000 };}- if (obj.formats?.includes("changeTracking") && (obj.waitFor === undefined || obj.waitFor < 5000)) {+ if (+ obj.formats?.includes("changeTracking") &&+ (obj.waitFor === undefined || obj.waitFor < 5000)+ ) {obj = { ...obj, waitFor: 5000 };}@@ -472,7 +492,10 @@ const extractTransform = (obj: ScrapeOptions) => {obj = { ...obj, timeout: 300000 };}- if ((obj.proxy === "stealth" || obj.proxy === "auto") && obj.timeout === 30000) {+ if (+ (obj.proxy === "stealth" || obj.proxy === "auto") &&+ obj.timeout === 30000+ ) {obj = { ...obj, timeout: 120000 };}@@ -520,21 +543,21 @@ export const scrapeOptions = baseScrapeOptions.refine(extractRefine, extractRefineOpts).refine(fire1Refine, fire1RefineOpts).refine(waitForRefine, waitForRefineOpts)- .transform(obj => {+ .transform((obj) => {return extractTransform(obj) as typeof obj;});export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;export type ScrapeOptions = BaseScrapeOptions & {- extract?: z.infer<typeof extractOptionsWithAgent>,- jsonOptions?: z.infer<typeof extractOptionsWithAgent>,+ extract?: z.infer<typeof extractOptionsWithAgent>;+ jsonOptions?: z.infer<typeof extractOptionsWithAgent>;agent?: {- model: string,- prompt: string,- sessionId?: string,- waitBeforeClosingMs?: number,- },+ model: string;+ prompt: string;+ sessionId?: string;+ waitBeforeClosingMs?: number;+ };};const ajv = new Ajv();@@ -569,9 +592,14 @@ export const extractV1Options = zincludeSubdomains: z.boolean().default(true),allowExternalLinks: z.boolean().default(false),enableWebSearch: z.boolean().default(false),- scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(),+ scrapeOptions: baseScrapeOptions+ .default({ onlyMainContent: false })+ .optional(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),urlTrace: z.boolean().default(false),timeout: z.number().int().positive().finite().safe().default(60000),__experimental_streamSteps: z.boolean().default(false),@@ -634,7 +662,10 @@ export const scrapeRequestSchema = baseScrapeOptionsextract: extractOptionsWithAgent.optional(),jsonOptions: extractOptionsWithAgent.optional(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),timeout: z.number().int().positive().finite().safe().default(30000),zeroDataRetention: z.boolean().optional(),})@@ -649,6 +680,7 @@ export const scrapeRequestSchema = baseScrapeOptionsexport type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;+const BLACKLISTED_WEBHOOK_HEADERS = ["x-firecrawl-signature"];export const webhookSchema = z.preprocess((x) => {if (typeof x === "string") {@@ -666,14 +698,28 @@ export const webhookSchema = z.preprocess(.array(z.enum(["completed", "failed", "page", "started"])).default(["completed", "failed", "page", "started"]),})- .strict(strictMessage),+ .strict(strictMessage)+ .refine(+ (obj) => {+ const blacklistedLower = BLACKLISTED_WEBHOOK_HEADERS.map((h) =>+ h.toLowerCase(),+ );+ return !Object.keys(obj.headers).some((key) =>+ blacklistedLower.includes(key.toLowerCase()),+ );+ },+ `The following headers are not allowed: ${BLACKLISTED_WEBHOOK_HEADERS.join(", ")}`,+ ),);export const batchScrapeRequestSchema = baseScrapeOptions.extend({urls: url.array(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(false),@@ -690,7 +736,10 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions.extend({urls: z.string().array(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(false),@@ -743,7 +792,10 @@ export const crawlRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),scrapeOptions: baseScrapeOptions.default({}),webhook: webhookSchema.optional(),limit: z.number().default(10000),@@ -765,8 +817,8 @@ export const crawlRequestSchema = crawlerOptions},{message: "URL depth exceeds the specified maxDepth",- path: ["url"]- }+ path: ["url"],+ },).transform((x) => {if (x.crawlEntireDomain !== undefined) {@@ -801,7 +853,10 @@ export const mapRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),includeSubdomains: z.boolean().default(true),search: z.string().optional(),ignoreQueryParameters: z.boolean().default(true),@@ -871,7 +926,7 @@ export type Document = {};};json?: any;- }+ };metadata: {title?: string;description?: string;@@ -933,11 +988,11 @@ export type ErrorResponse = {export type ScrapeResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document;- scrape_id?: string;- };+ success: true;+ warning?: string;+ data: Document;+ scrape_id?: string;+ };export interface ScrapeResponseRequestTest {statusCode: number;@@ -988,28 +1043,28 @@ export interface ExtractResponseRequestTest {export type CrawlResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- };+ success: true;+ id: string;+ url: string;+ };export type BatchScrapeResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- invalidURLs?: string[];- };+ success: true;+ id: string;+ url: string;+ invalidURLs?: string[];+ };// Note: This type has been transitioned to v2/types.ts (see MapV2Response) while maintaining backwards compatibilityexport type MapResponse =| ErrorResponse| {- success: true;- links: string[];- scrape_id?: string;- };+ success: true;+ links: string[];+ scrape_id?: string;+ };export type CrawlStatusParams = {jobId: string;@@ -1022,48 +1077,48 @@ export type ConcurrencyCheckParams = {export type ConcurrencyCheckResponse =| ErrorResponse| {- success: true;- concurrency: number;- maxConcurrency: number;- };+ success: true;+ concurrency: number;+ maxConcurrency: number;+ };export type CrawlStatusResponse =| ErrorResponse| {- success: true;- status: "scraping" | "completed" | "failed" | "cancelled";- completed: number;- total: number;- creditsUsed: number;- expiresAt: string;- next?: string;- data: Document[];- };+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: string;+ next?: string;+ data: Document[];+ };export type OngoingCrawlsResponse =| ErrorResponse| {- success: true;- crawls: {- id: string;- teamId: string;- url: string;- created_at: string;- options: CrawlerOptions;- }[];- };+ success: true;+ crawls: {+ id: string;+ teamId: string;+ url: string;+ created_at: string;+ options: CrawlerOptions;+ }[];+ };export type CrawlErrorsResponse =| ErrorResponse| {- errors: {- id: string;- timestamp?: string;- url: string;- error: string;- }[];- robotsBlocked: string[];- };+ errors: {+ id: string;+ timestamp?: string;+ url: string;+ error: string;+ }[];+ robotsBlocked: string[];+ };type AuthObject = {team_id: string;@@ -1123,7 +1178,10 @@ export type TeamFlags = {crawlTtlHours?: number;} | null;-export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;+export type AuthCreditUsageChunkFromTeam = Omit<+ AuthCreditUsageChunk,+ "api_key"+>;export interface RequestWithMaybeACUC<ReqParams = {},@@ -1204,10 +1262,13 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {regexOnFullURL: x.regexOnFullURL,maxDiscoveryDepth: x.maxDiscoveryDepth,delay: x.delay,- }+ };}-export function fromLegacyCrawlerOptions(x: any, teamId: string): {+export function fromLegacyCrawlerOptions(+ x: any,+ teamId: string,+): {crawlOptions: CrawlerOptions;internalOptions: InternalOptions;} {@@ -1259,7 +1320,7 @@ export function fromLegacyScrapeOptions(? ("screenshot@fullPage" as const): null,extractorOptions !== undefined &&- extractorOptions.mode.includes("llm-extraction")+ extractorOptions.mode.includes("llm-extraction")? ("extract" as const): null,"links",@@ -1283,12 +1344,12 @@ export function fromLegacyScrapeOptions(removeBase64Images: pageOptions.removeBase64Images,extract:extractorOptions !== undefined &&- extractorOptions.mode.includes("llm-extraction")+ extractorOptions.mode.includes("llm-extraction")? {- systemPrompt: extractorOptions.extractionPrompt,- prompt: extractorOptions.userPrompt,- schema: extractorOptions.extractionSchema,- }+ systemPrompt: extractorOptions.extractionPrompt,+ prompt: extractorOptions.userPrompt,+ schema: extractorOptions.extractionSchema,+ }: undefined,mobile: pageOptions.mobile,fastMode: pageOptions.useFastMode,@@ -1315,7 +1376,10 @@ export function fromLegacyCombo(timeout,teamId,);- const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions, teamId);+ const { internalOptions: i2 } = fromLegacyCrawlerOptions(+ crawlerOptions,+ teamId,+ );return { scrapeOptions, internalOptions: Object.assign(i1, i2) };}@@ -1365,7 +1429,10 @@ export const searchRequestSchema = zcountry: z.string().optional().default("us"),location: z.string().optional(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),timeout: z.number().int().positive().finite().safe().default(60000),ignoreInvalidURLs: z.boolean().optional().default(false),__searchPreviewToken: z.string().optional(),@@ -1405,10 +1472,10 @@ export type SearchRequestInput = z.input<typeof searchRequestSchema>;export type SearchResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document[];- };+ success: true;+ warning?: string;+ data: Document[];+ };export type TokenUsage = {promptTokens: number;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex e670df22b..8ff6ae6e1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -9,7 +9,11 @@ import {Document as V0Document,WebSearchResult,} from "../../lib/entities";-import { agentOptionsExtract, AuthCreditUsageChunk, ScrapeOptions as V1ScrapeOptions } from "../v1/types";+import {+ agentOptionsExtract,+ AuthCreditUsageChunk,+ ScrapeOptions as V1ScrapeOptions,+} from "../v1/types";import type { InternalOptions } from "../../scraper/scrapeURL";import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";@@ -27,7 +31,7 @@ export enum IntegrationEnum {MAKE = "make",FLOWISE = "flowise",METAGPT = "metagpt",- RELEVANCEAI = 'relevanceai',+ RELEVANCEAI = "relevanceai",}export type Format =@@ -79,7 +83,7 @@ export const url = z.preprocess(} catch (_) {return false;}- }, "Invalid URL")+ }, "Invalid URL"),// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);@@ -108,64 +112,79 @@ function calculateTotalWaitTime(return waitFor + actionWaitTime;}-export const actionSchema = z- .union([- z+export const actionSchema = z.union([+ z+ .object({+ type: z.literal("wait"),+ milliseconds: z.number().int().positive().finite().optional(),+ selector: z.string().optional(),+ })+ .refine(+ (data) =>+ (data.milliseconds !== undefined || data.selector !== undefined) &&+ !(data.milliseconds !== undefined && data.selector !== undefined),+ {+ message:+ "Either 'milliseconds' or 'selector' must be provided, but not both.",+ },+ ),+ z.object({+ type: z.literal("click"),+ selector: z.string(),+ all: z.boolean().default(false),+ }),+ z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ quality: z.number().min(1).max(100).optional(),+ viewport: z.object({- type: z.literal("wait"),- milliseconds: z.number().int().positive().finite().optional(),- selector: z.string().optional(),- })- .refine(- (data) =>- (data.milliseconds !== undefined || data.selector !== undefined) &&- !(data.milliseconds !== undefined && data.selector !== undefined),- {- message:- "Either 'milliseconds' or 'selector' must be provided, but not both.",- },- ),- z.object({- type: z.literal("click"),- selector: z.string(),- all: z.boolean().default(false),- }),- z.object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- quality: z.number().min(1).max(100).optional(),- viewport: z.object({width: z.number().int().positive().finite().max(7680), // 8K resolution widthheight: z.number().int().positive().finite().max(4320), // 8K resolution height- }).optional(),- }),- z.object({- type: z.literal("write"),- text: z.string(),- }),- z.object({- type: z.literal("press"),- key: z.string(),- }),- z.object({- type: z.literal("scroll"),- direction: z.enum(["up", "down"]).optional().default("down"),- selector: z.string().optional(),- }),- z.object({- type: z.literal("scrape"),- }),- z.object({- type: z.literal("executeJavascript"),- script: z.string(),- }),- z.object({- type: z.literal("pdf"),- landscape: z.boolean().default(false),- scale: z.number().default(1),- format: z.enum(['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'Letter', 'Legal', 'Tabloid', 'Ledger']).default("Letter"),- }),- ]);+ })+ .optional(),+ }),+ z.object({+ type: z.literal("write"),+ text: z.string(),+ }),+ z.object({+ type: z.literal("press"),+ key: z.string(),+ }),+ z.object({+ type: z.literal("scroll"),+ direction: z.enum(["up", "down"]).optional().default("down"),+ selector: z.string().optional(),+ }),+ z.object({+ type: z.literal("scrape"),+ }),+ z.object({+ type: z.literal("executeJavascript"),+ script: z.string(),+ }),+ z.object({+ type: z.literal("pdf"),+ landscape: z.boolean().default(false),+ scale: z.number().default(1),+ format: z+ .enum([+ "A0",+ "A1",+ "A2",+ "A3",+ "A4",+ "A5",+ "A6",+ "Letter",+ "Legal",+ "Tabloid",+ "Ledger",+ ])+ .default("Letter"),+ }),+]);export type Action = z.infer<typeof actionSchema>;@@ -182,48 +201,69 @@ export const actionsSchema = z},);-export const jsonFormatWithOptions = z.object({- type: z.literal("json"),- schema: z.any().optional(),- prompt: z.string().max(10000).optional(),-}).strict();+export const jsonFormatWithOptions = z+ .object({+ type: z.literal("json"),+ schema: z.any().optional(),+ prompt: z.string().max(10000).optional(),+ })+ .strict();export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;-export const changeTrackingFormatWithOptions = z.object({- type: z.literal("changeTracking"),- prompt: z.string().optional(),- schema: z.any().optional(),- modes: z.enum(["json", "git-diff"]).array().optional().default([]),- tag: z.string().or(z.null()).default(null),-}).strict();+export const changeTrackingFormatWithOptions = z+ .object({+ type: z.literal("changeTracking"),+ prompt: z.string().optional(),+ schema: z.any().optional(),+ modes: z.enum(["json", "git-diff"]).array().optional().default([]),+ tag: z.string().or(z.null()).default(null),+ })+ .strict();-export type ChangeTrackingFormatWithOptions = z.output<typeof changeTrackingFormatWithOptions>;+export type ChangeTrackingFormatWithOptions = z.output<+ typeof changeTrackingFormatWithOptions+>;export const screenshotFormatWithOptions = z.object({type: z.literal("screenshot"),fullPage: z.boolean().default(false),quality: z.number().min(1).max(100).optional(),- viewport: z.object({- width: z.number().int().positive().finite().max(7680), // 8K resolution width- height: z.number().int().positive().finite().max(4320), // 8K resolution height- }).optional(),+ viewport: z+ .object({+ width: z.number().int().positive().finite().max(7680), // 8K resolution width+ height: z.number().int().positive().finite().max(4320), // 8K resolution height+ })+ .optional(),});-export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;--export const attributesFormatWithOptions = z.object({- type: z.literal("attributes"),- selectors: z.array(z.object({- selector: z.string().describe("CSS selector to find elements"),- attribute: z.string().describe("Attribute name to extract (e.g., 'data-vehicle-name' or 'id')")- })).describe("Extract specific attributes from elements"),-}).strict();+export type ScreenshotFormatWithOptions = z.output<+ typeof screenshotFormatWithOptions+>;-export type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;+export const attributesFormatWithOptions = z+ .object({+ type: z.literal("attributes"),+ selectors: z+ .array(+ z.object({+ selector: z.string().describe("CSS selector to find elements"),+ attribute: z+ .string()+ .describe(+ "Attribute name to extract (e.g., 'data-vehicle-name' or 'id')",+ ),+ }),+ )+ .describe("Extract specific attributes from elements"),+ })+ .strict();+export type AttributesFormatWithOptions = z.output<+ typeof attributesFormatWithOptions+>;-export type FormatObject =+export type FormatObject =| { type: "markdown" }| { type: "html" }| { type: "rawHtml" }@@ -233,27 +273,24 @@ export type FormatObject =| JsonFormatWithOptions| ChangeTrackingFormatWithOptions| ScreenshotFormatWithOptions- | AttributesFormatWithOptions+ | AttributesFormatWithOptions;-export const pdfParserWithOptions = z.object({- type: z.literal("pdf"),- maxPages: z.number().int().positive().finite().max(10000).optional(),-}).strict();+export const pdfParserWithOptions = z+ .object({+ type: z.literal("pdf"),+ maxPages: z.number().int().positive().finite().max(10000).optional(),+ })+ .strict();export const parsersSchema = z- .array(- z.union([- z.literal("pdf"),- pdfParserWithOptions,- ])- )+ .array(z.union([z.literal("pdf"), pdfParserWithOptions])).default(["pdf"]);export type Parsers = z.infer<typeof parsersSchema>;export function shouldParsePDF(parsers?: Parsers): boolean {if (!parsers) return true;- return parsers.some(parser => {+ return parsers.some((parser) => {if (parser === "pdf") return true;if (typeof parser === "object" && parser !== null && "type" in parser) {return (parser as any).type === "pdf";@@ -264,7 +301,7 @@ export function shouldParsePDF(parsers?: Parsers): boolean {export function getPDFMaxPages(parsers?: Parsers): number | undefined {if (!parsers) return undefined;- const pdfParser = parsers.find(parser => {+ const pdfParser = parsers.find((parser) => {if (typeof parser === "object" && parser !== null && "type" in parser) {return (parser as any).type === "pdf";}@@ -278,60 +315,59 @@ export function getPDFMaxPages(parsers?: Parsers): number | undefined {function transformIframeSelector(selector: string): string {return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => {- const prefix = match.match(/^[\s,]/)?.[0] || '';+ const prefix = match.match(/^[\s,]/)?.[0] || "";return prefix + 'div[data-original-tag="iframe"]';});}const baseScrapeOptions = z.object({- formats: z.preprocess(- (val) => {- if (!Array.isArray(val)) return val;- return val.map(format => {- if (typeof format === 'string') {- return { type: format };- }- return format;- });- },- z- .union([- z.object({ type: z.literal("markdown") }),- z.object({ type: z.literal("html") }),- z.object({ type: z.literal("rawHtml") }),- z.object({ type: z.literal("links") }),- z.object({ type: z.literal("images") }),- z.object({ type: z.literal("summary") }),- jsonFormatWithOptions,- changeTrackingFormatWithOptions,- screenshotFormatWithOptions,- attributesFormatWithOptions,- ])- .array()- .optional()- .default([{ type: "markdown" }])- )- .refine(- (x) => {- return x.filter(f => f.type === "screenshot").length <= 1;+ formats: z+ .preprocess(+ (val) => {+ if (!Array.isArray(val)) return val;+ return val.map((format) => {+ if (typeof format === "string") {+ return { type: format };+ }+ return format;+ });},- "You may only specify one screenshot format",+ z+ .union([+ z.object({ type: z.literal("markdown") }),+ z.object({ type: z.literal("html") }),+ z.object({ type: z.literal("rawHtml") }),+ z.object({ type: z.literal("links") }),+ z.object({ type: z.literal("images") }),+ z.object({ type: z.literal("summary") }),+ jsonFormatWithOptions,+ changeTrackingFormatWithOptions,+ screenshotFormatWithOptions,+ attributesFormatWithOptions,+ ])+ .array()+ .optional()+ .default([{ type: "markdown" }]),)- .refine(- (x) => {- const hasChangeTracking = x.find(f => f.type === "changeTracking");- const hasMarkdown = x.find(f => f.type === "markdown");- return !hasChangeTracking || hasMarkdown;- },- "The changeTracking format requires the markdown format to be specified as well",- ),+ .refine((x) => {+ return x.filter((f) => f.type === "screenshot").length <= 1;+ }, "You may only specify one screenshot format")+ .refine((x) => {+ const hasChangeTracking = x.find((f) => f.type === "changeTracking");+ const hasMarkdown = x.find((f) => f.type === "markdown");+ return !hasChangeTracking || hasMarkdown;+ }, "The changeTracking format requires the markdown format to be specified as well"),headers: z.record(z.string(), z.string()).optional(),- includeTags: z.string().array()- .transform(tags => tags.map(transformIframeSelector))+ includeTags: z+ .string()+ .array()+ .transform((tags) => tags.map(transformIframeSelector)).optional(),- excludeTags: z.string().array()- .transform(tags => tags.map(transformIframeSelector))+ excludeTags: z+ .string()+ .array()+ .transform((tags) => tags.map(transformIframeSelector)).optional(),onlyMainContent: z.boolean().default(true),timeout: z.number().int().positive().finite().safe().optional(),@@ -346,7 +382,7 @@ const baseScrapeOptions = zmobile: z.boolean().default(false),parsers: parsersSchema.optional(),actions: actionsSchema.optional(),-+location: z.object({country: z@@ -384,7 +420,7 @@ const baseScrapeOptions = zconst waitForRefine = (obj) => {if (obj.waitFor && obj.timeout) {- if (typeof obj.timeout !== 'number' || obj.timeout <= 0) {+ if (typeof obj.timeout !== "number" || obj.timeout <= 0) {return false;}return obj.waitFor <= obj.timeout / 2;@@ -398,13 +434,16 @@ const waitForRefineOpts = {const extractTransform = (obj) => {// Handle timeoutif (- (obj.formats.find(x => typeof x === "object" && x.type === "json")) &&+ obj.formats.find((x) => typeof x === "object" && x.type === "json") &&obj.timeout === 30000) {obj = { ...obj, timeout: 60000 };}- if (obj.formats?.includes("changeTracking") && (obj.waitFor === undefined || obj.waitFor < 5000)) {+ if (+ obj.formats?.includes("changeTracking") &&+ (obj.waitFor === undefined || obj.waitFor < 5000)+ ) {obj = { ...obj, waitFor: 5000 };}@@ -412,8 +451,10 @@ const extractTransform = (obj) => {obj = { ...obj, timeout: 60000 };}-- if ((obj.proxy === "stealth" || obj.proxy === "auto") && obj.timeout === 30000) {+ if (+ (obj.proxy === "stealth" || obj.proxy === "auto") &&+ obj.timeout === 30000+ ) {obj = { ...obj, timeout: 120000 };}@@ -473,9 +514,14 @@ export const extractOptions = zincludeSubdomains: z.boolean().default(true),allowExternalLinks: z.boolean().default(false),enableWebSearch: z.boolean().default(false),- scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(),+ scrapeOptions: baseScrapeOptions+ .default({ onlyMainContent: false })+ .optional(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),urlTrace: z.boolean().default(false),timeout: z.number().int().positive().finite().safe().optional(),agent: agentOptionsExtract.optional(),@@ -519,7 +565,10 @@ export const scrapeRequestSchema = baseScrapeOptions.extend({url,origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),zeroDataRetention: z.boolean().optional(),}).strict(strictMessage)@@ -529,6 +578,7 @@ export const scrapeRequestSchema = baseScrapeOptionsexport type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;+const BLACKLISTED_WEBHOOK_HEADERS = ["x-firecrawl-signature"];export const webhookSchema = z.preprocess((x) => {if (typeof x === "string") {@@ -546,14 +596,28 @@ export const webhookSchema = z.preprocess(.array(z.enum(["completed", "failed", "page", "started"])).default(["completed", "failed", "page", "started"]),})- .strict(strictMessage),+ .strict(strictMessage)+ .refine(+ (obj) => {+ const blacklistedLower = BLACKLISTED_WEBHOOK_HEADERS.map((h) =>+ h.toLowerCase(),+ );+ return !Object.keys(obj.headers).some((key) =>+ blacklistedLower.includes(key.toLowerCase()),+ );+ },+ `The following headers are not allowed: ${BLACKLISTED_WEBHOOK_HEADERS.join(", ")}`,+ ),);export const batchScrapeRequestSchema = baseScrapeOptions.extend({urls: url.array(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(true),@@ -568,7 +632,10 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions.extend({urls: z.string().array(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(true),@@ -616,7 +683,10 @@ export const crawlRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),scrapeOptions: baseScrapeOptions.default({}),webhook: webhookSchema.optional(),limit: z.number().default(10000),@@ -655,7 +725,10 @@ export const mapRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),includeSubdomains: z.boolean().default(true),ignoreQueryParameters: z.boolean().default(true),search: z.string().optional(),@@ -729,7 +802,7 @@ export type Document = {};};json?: any;- }+ };metadata: {title?: string;description?: string;@@ -791,11 +864,11 @@ export type ErrorResponse = {export type ScrapeResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document;- scrape_id?: string;- };+ success: true;+ warning?: string;+ data: Document;+ scrape_id?: string;+ };export interface ScrapeResponseRequestTest {statusCode: number;@@ -846,19 +919,19 @@ export interface ExtractResponseRequestTest {export type CrawlResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- };+ success: true;+ id: string;+ url: string;+ };export type BatchScrapeResponse =| ErrorResponse| {- success: true;- id: string;- url: string;- invalidURLs?: string[];- };+ success: true;+ id: string;+ url: string;+ invalidURLs?: string[];+ };// Map document interface (transitioned from v1)export interface MapDocument {@@ -871,9 +944,9 @@ export interface MapDocument {export type MapResponse =| ErrorResponse| {- success: true;- links?: MapDocument[];- };+ success: true;+ links?: MapDocument[];+ };export type CrawlStatusParams = {jobId: string;@@ -886,49 +959,49 @@ export type ConcurrencyCheckParams = {export type ConcurrencyCheckResponse =| ErrorResponse| {- success: true;- concurrency: number;- maxConcurrency: number;- };+ success: true;+ concurrency: number;+ maxConcurrency: number;+ };export type CrawlStatusResponse =| ErrorResponse| {- success: true;- status: "scraping" | "completed" | "failed" | "cancelled";- completed: number;- total: number;- creditsUsed: number;- expiresAt: string;- next?: string;- data: Document[];- };+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: string;+ next?: string;+ data: Document[];+ };export type OngoingCrawlsResponse =| ErrorResponse| {- success: true;- crawls: {- id: string;- teamId: string;- url: string;- created_at: string;- options: CrawlerOptions;- }[];- };+ success: true;+ crawls: {+ id: string;+ teamId: string;+ url: string;+ created_at: string;+ options: CrawlerOptions;+ }[];+ };export type CrawlErrorsResponse =| ErrorResponse| {- errors: {- id: string;- timestamp?: string;- url: string;- code?: ErrorCodes;- error: string;- }[];- robotsBlocked: string[];- };+ errors: {+ id: string;+ timestamp?: string;+ url: string;+ code?: ErrorCodes;+ error: string;+ }[];+ robotsBlocked: string[];+ };type AuthObject = {team_id: string;@@ -1017,10 +1090,13 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {regexOnFullURL: x.regexOnFullURL,maxDiscoveryDepth: x.maxDiscoveryDepth,delay: x.delay,- }+ };}-export function fromV0CrawlerOptions(x: any, teamId: string): {+export function fromV0CrawlerOptions(+ x: any,+ teamId: string,+): {crawlOptions: CrawlerOptions;internalOptions: InternalOptions;} {@@ -1066,15 +1142,15 @@ export function fromV0ScrapeOptions((pageOptions.includeRawHtml ?? false) ? ("rawHtml" as const) : null,(pageOptions.screenshot ?? false) ? ("screenshot" as const) : null,(pageOptions.fullPageScreenshot ?? false)- ? ({ type: "screenshot" as const, fullPage: true })+ ? { type: "screenshot" as const, fullPage: true }: null,extractorOptions !== undefined &&- extractorOptions.mode.includes("llm-extraction")- ? ({- type: "json" as const,- prompt: extractorOptions.userPrompt,- schema: extractorOptions.extractionSchema,- })+ extractorOptions.mode.includes("llm-extraction")+ ? {+ type: "json" as const,+ prompt: extractorOptions.userPrompt,+ schema: extractorOptions.extractionSchema,+ }: null,"links",].filter((x) => x !== null),@@ -1090,7 +1166,12 @@ export function fromV0ScrapeOptions(: pageOptions.removeTags,onlyMainContent: pageOptions.onlyMainContent ?? false,timeout: timeout,- parsers: pageOptions.parsePDF !== undefined ? (pageOptions.parsePDF ? ["pdf"] : []) : undefined,+ parsers:+ pageOptions.parsePDF !== undefined+ ? pageOptions.parsePDF+ ? ["pdf"]+ : []+ : undefined,actions: pageOptions.actions,location: pageOptions.geolocation,skipTlsVerification: pageOptions.skipTlsVerification,@@ -1102,9 +1183,12 @@ export function fromV0ScrapeOptions(atsv: pageOptions.atsv,v0DisableJsDom: pageOptions.disableJsDom,teamId,- ...(extractorOptions !== undefined && extractorOptions.mode.includes("llm-extraction") ? {- v1JSONSystemPrompt: extractorOptions.extractionPrompt- } : {})+ ...(extractorOptions !== undefined &&+ extractorOptions.mode.includes("llm-extraction")+ ? {+ v1JSONSystemPrompt: extractorOptions.extractionPrompt,+ }+ : {}),},// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks};@@ -1126,14 +1210,14 @@ export function fromV1ScrapeOptions(delete (spreadScrapeOptions as any).maxConcurrency;// v2 scrapeOptions schema is strict and does not include `agent`. We carry it via internalOptions below.delete (spreadScrapeOptions as any).agent;-+delete spreadScrapeOptions.__experimental_cache;delete spreadScrapeOptions.jsonOptions;delete spreadScrapeOptions.changeTrackingOptions;delete spreadScrapeOptions.extract;delete spreadScrapeOptions.geolocation;delete (spreadScrapeOptions as any).parsePDF;-+// Track the original format for v1 backward compatibility// Check json first since when user specifies "json", both "json" and "extract" are present// When user specifies "extract", only "extract" is present@@ -1143,60 +1227,72 @@ export function fromV1ScrapeOptions(} else if (v1ScrapeOptions.formats.includes("extract")) {v1OriginalFormat = "extract";}-+return {scrapeOptions: scrapeOptions.parse({...spreadScrapeOptions,- ...(v1ScrapeOptions.__experimental_cache ? {- maxAge: v1ScrapeOptions.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours- } : {}),+ ...(v1ScrapeOptions.__experimental_cache+ ? {+ maxAge: v1ScrapeOptions.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours+ }+ : {}),location: v1ScrapeOptions.location ?? v1ScrapeOptions.geolocation,- formats: v1ScrapeOptions.formats.map(x => {- // json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other- if (x === "extract") {- const opts = v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract;- const fmt: JsonFormatWithOptions = {- type: "json",- schema: opts?.schema,- prompt: opts?.prompt,- };- return fmt;- } else if (x === "json") {- // If jsonOptions are provided with json format, create JsonFormatWithOptions- const opts = v1ScrapeOptions.jsonOptions;- if (opts) {+ formats: v1ScrapeOptions.formats+ .map((x) => {+ // json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other+ if (x === "extract") {+ const opts = v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract;const fmt: JsonFormatWithOptions = {type: "json",- schema: opts.schema,- prompt: opts.prompt,+ schema: opts?.schema,+ prompt: opts?.prompt,};- return v1ScrapeOptions.formats.includes("extract") ? null : fmt;+ return fmt;+ } else if (x === "json") {+ // If jsonOptions are provided with json format, create JsonFormatWithOptions+ const opts = v1ScrapeOptions.jsonOptions;+ if (opts) {+ const fmt: JsonFormatWithOptions = {+ type: "json",+ schema: opts.schema,+ prompt: opts.prompt,+ };+ return v1ScrapeOptions.formats.includes("extract") ? null : fmt;+ }+ return null;+ } else if (x === "changeTracking") {+ const opts = v1ScrapeOptions.changeTrackingOptions;+ const fmt: ChangeTrackingFormatWithOptions = {+ type: "changeTracking",+ modes: opts?.modes ?? [],+ tag: opts?.tag ?? null,+ schema: opts?.schema,+ prompt: opts?.prompt,+ };+ return fmt;+ } else if (x === "screenshot@fullPage") {+ return { type: "screenshot" as const, fullPage: true };+ } else {+ return x;}- return null;- } else if (x === "changeTracking") {- const opts = v1ScrapeOptions.changeTrackingOptions;- const fmt: ChangeTrackingFormatWithOptions = {- type: "changeTracking",- modes: opts?.modes ?? [],- tag: opts?.tag ?? null,- schema: opts?.schema,- prompt: opts?.prompt,- };- return fmt;- } else if (x === "screenshot@fullPage") {- return { type: "screenshot" as const, fullPage: true };- } else {- return x;- }- }).filter(x => x !== null),- parsers: v1ScrapeOptions.parsePDF !== undefined ? (v1ScrapeOptions.parsePDF ? ["pdf"] : []) : undefined,+ })+ .filter((x) => x !== null),+ parsers:+ v1ScrapeOptions.parsePDF !== undefined+ ? v1ScrapeOptions.parsePDF+ ? ["pdf"]+ : []+ : undefined,}),internalOptions: {teamId,v1Agent: v1ScrapeOptions.agent,- v1JSONSystemPrompt: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.systemPrompt,- v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.agent,+ v1JSONSystemPrompt: (+ v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract+ )?.systemPrompt,+ v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)+ ?.agent,v1OriginalFormat,},};@@ -1251,35 +1347,47 @@ export function toLegacyDocument(// These allow fine-grained control over each search source type// Similar to how scrape formats work with jsonFormatWithOptions, etc.-export const webSearchSourceOptions = z.object({- type: z.literal("web"),- tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)- filter: z.string().optional(), // Search filter- lang: z.string().optional(), // Language override for this source- country: z.string().optional(), // Country override for this source- location: z.string().optional(), // Location override for this source-}).strict();+export const webSearchSourceOptions = z+ .object({+ type: z.literal("web"),+ tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)+ filter: z.string().optional(), // Search filter+ lang: z.string().optional(), // Language override for this source+ country: z.string().optional(), // Country override for this source+ location: z.string().optional(), // Location override for this source+ })+ .strict();-export const imagesSearchSourceOptions = z.object({- type: z.literal("images"),-}).strict();+export const imagesSearchSourceOptions = z+ .object({+ type: z.literal("images"),+ })+ .strict();-export const newsSearchSourceOptions = z.object({- type: z.literal("news"),-}).strict();+export const newsSearchSourceOptions = z+ .object({+ type: z.literal("news"),+ })+ .strict();export type WebSearchSourceOptions = z.infer<typeof webSearchSourceOptions>;-export type ImagesSearchSourceOptions = z.infer<typeof imagesSearchSourceOptions>;+export type ImagesSearchSourceOptions = z.infer<+ typeof imagesSearchSourceOptions+>;export type NewsSearchSourceOptions = z.infer<typeof newsSearchSourceOptions>;// Category source type definitions-export const githubCategoryOptions = z.object({- type: z.literal("github"),-}).strict();+export const githubCategoryOptions = z+ .object({+ type: z.literal("github"),+ })+ .strict();-export const researchCategoryOptions = z.object({- type: z.literal("research"),-}).strict();+export const researchCategoryOptions = z+ .object({+ type: z.literal("research"),+ })+ .strict();export type GithubCategoryOptions = z.infer<typeof githubCategoryOptions>;export type ResearchCategoryOptions = z.infer<typeof researchCategoryOptions>;@@ -1308,7 +1416,7 @@ export const searchRequestSchema = zwebSearchSourceOptions,imagesSearchSourceOptions,newsSearchSourceOptions,- ])+ ]),),]).optional()@@ -1318,56 +1426,52 @@ export const searchRequestSchema = z// Array of strings (simple format)z.array(z.enum(["github", "research"])),// Array of objects (advanced format)- z.array(- z.union([- githubCategoryOptions,- researchCategoryOptions,- ])- ),+ z.array(z.union([githubCategoryOptions, researchCategoryOptions])),]).optional(),lang: z.string().optional().default("en"),country: z.string().optional().default("us"),location: z.string().optional(),origin: z.string().optional().default("api"),- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),+ integration: z+ .nativeEnum(IntegrationEnum)+ .optional()+ .transform((val) => val || null),timeout: z.number().int().positive().finite().safe().default(60000),ignoreInvalidURLs: z.boolean().optional().default(false),asyncScraping: z.boolean().optional().default(false),__searchPreviewToken: z.string().optional(),scrapeOptions: baseScrapeOptions.extend({- formats: z.preprocess(- (val) => {- if (!Array.isArray(val)) return val;- return val.map(format => {- if (typeof format === 'string') {- return { type: format };- }- return format;- });- },- z- .union([- z.object({ type: z.literal("markdown") }),- z.object({ type: z.literal("html") }),- z.object({ type: z.literal("rawHtml") }),- z.object({ type: z.literal("links") }),- z.object({ type: z.literal("images") }),- z.object({ type: z.literal("summary") }),- jsonFormatWithOptions,- screenshotFormatWithOptions,- ])- .array()- .optional()- .default([])- )- .refine(- (x) => {- return x.filter(f => f.type === "screenshot").length <= 1;+ formats: z+ .preprocess(+ (val) => {+ if (!Array.isArray(val)) return val;+ return val.map((format) => {+ if (typeof format === "string") {+ return { type: format };+ }+ return format;+ });},- "You may only specify one screenshot format",- ),+ z+ .union([+ z.object({ type: z.literal("markdown") }),+ z.object({ type: z.literal("html") }),+ z.object({ type: z.literal("rawHtml") }),+ z.object({ type: z.literal("links") }),+ z.object({ type: z.literal("images") }),+ z.object({ type: z.literal("summary") }),+ jsonFormatWithOptions,+ screenshotFormatWithOptions,+ ])+ .array()+ .optional()+ .default([]),+ )+ .refine((x) => {+ return x.filter((f) => f.type === "screenshot").length <= 1;+ }, "You may only specify one screenshot format"),}).default({}),})@@ -1380,27 +1484,27 @@ export const searchRequestSchema = zlet sources = x.sources;if (sources && Array.isArray(sources) && sources.length > 0) {// Check if it's a string array by checking the first element- if (typeof sources[0] === 'string') {+ if (typeof sources[0] === "string") {// It's a string array, transform to object arraysources = (sources as string[]).map((s) => {switch (s) {- case 'web':+ case "web":return {- type: 'web' as const,+ type: "web" as const,tbs: x.tbs,filter: x.filter,lang: x.lang,country: x.country,location: x.location,};- case 'images':+ case "images":return {- type: 'images' as const,+ type: "images" as const,// Images don't inherit global params in the simple format};- case 'news':+ case "news":return {- type: 'news' as const,+ type: "news" as const,tbs: x.tbs,lang: x.lang,country: x.country,@@ -1413,22 +1517,22 @@ export const searchRequestSchema = z}// Otherwise it's already an object array, keep as is}-+// Transform string array categories to object formatlet categories = x.categories;if (categories && Array.isArray(categories) && categories.length > 0) {// Check if it's a string array by checking the first element- if (typeof categories[0] === 'string') {+ if (typeof categories[0] === "string") {// It's a string array, transform to object arraycategories = (categories as string[]).map((c) => {switch (c) {- case 'github':+ case "github":return {- type: 'github' as const+ type: "github" as const,};- case 'research':+ case "research":return {- type: 'research' as const,+ type: "research" as const,};default:return { type: c as any };@@ -1437,7 +1541,7 @@ export const searchRequestSchema = z}// Otherwise it's already an object array, keep as is}-+return {...x,sources,@@ -1452,28 +1556,28 @@ export type SearchRequestInput = z.input<typeof searchRequestSchema>;export type SearchResponse =| ErrorResponse| {- success: true;- warning?: string;- data: Document[];- creditsUsed: number;- }+ success: true;+ warning?: string;+ data: Document[];+ creditsUsed: number;+ }| {- success: true;- warning?: string;- data: import("../../lib/entities").SearchV2Response;- creditsUsed: number;- }+ success: true;+ warning?: string;+ data: import("../../lib/entities").SearchV2Response;+ creditsUsed: number;+ }| {- success: true;- warning?: string;- data: import("../../lib/entities").SearchV2Response;- scrapeIds: {- web?: string[];- news?: string[];- images?: string[];+ success: true;+ warning?: string;+ data: import("../../lib/entities").SearchV2Response;+ scrapeIds: {+ web?: string[];+ news?: string[];+ images?: string[];+ };+ creditsUsed: number;};- creditsUsed: number;- };export type TokenUsage = {promptTokens: number;diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex cdc4046a5..9a963b16d 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -6,12 +6,38 @@ import { configDotenv } from "dotenv";import { z } from "zod";import { webhookSchema } from "../controllers/v1/types";import { redisEvictConnection } from "./redis";-import { getSecureDispatcher, isIPPrivate } from "../scraper/scrapeURL/engines/utils/safeFetch";+import { createHmac } from "crypto";+import {+ getSecureDispatcher,+ isIPPrivate,+} from "../scraper/scrapeURL/engines/utils/safeFetch";configDotenv();const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";const WEBHOOK_INSERT_BATCH_SIZE = 1000;+interface WebhookLogData {+ success: boolean;+ error?: string;+ teamId: string;+ crawlId: string;+ scrapeId?: string;+ url: string;+ statusCode?: number;+ event: WebhookEventType;+}++interface CallWebhookParams {+ teamId: string;+ crawlId: string;+ scrapeId?: string;+ webhook?: z.infer<typeof webhookSchema>;+ v1: boolean;+ data: any | null;+ eventType: WebhookEventType;+ awaitWebhook?: boolean;+}+async function addWebhookInsertJob(data: any) {await redisEvictConnection.rpush(WEBHOOK_INSERT_QUEUE_KEY,@@ -51,16 +77,7 @@ export async function processWebhookInsertJobs() {}}-async function logWebhook(data: {- success: boolean;- error?: string;- teamId: string;- crawlId: string;- scrapeId?: string;- url: string;- statusCode?: number;- event: WebhookEventType;-}) {+async function logWebhook(data: WebhookLogData) {try {await addWebhookInsertJob({success: data.success,@@ -85,6 +102,12 @@ async function logWebhook(data: {}}+function generateHmacSignature(payload: string, secret: string): string {+ const hmac = createHmac("sha256", secret);+ hmac.update(payload);+ return hmac.digest("hex");+}+export const callWebhook = async ({teamId,crawlId,@@ -94,16 +117,7 @@ export const callWebhook = async ({v1,eventType,awaitWebhook = false,-}: {- teamId: string;- crawlId: string;- scrapeId?: string;- webhook?: z.infer<typeof webhookSchema>;- v1: boolean;- data: any | null;- eventType: WebhookEventType;- awaitWebhook?: boolean;-}) => {+}: CallWebhookParams) => {const logger = _logger.child({module: "webhook",method: "callWebhook",@@ -138,6 +152,8 @@ export const callWebhook = async ({webhook ??(selfHostedUrl ? webhookSchema.parse({ url: selfHostedUrl }) : undefined);+ // TODO: do these queries upstream so we don't have to query the DB multiple times per crawl+// Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set// and the USE_DB_AUTHENTICATION environment variable is set to trueif (!webhookUrl && useDbAuthentication) {@@ -160,6 +176,25 @@ export const callWebhook = async ({webhookUrl = webhooksData[0].url;}+ let hmacSecret: string | undefined =+ process.env.SELF_HOSTED_WEBHOOK_HMAC_SECRET;+ if (useDbAuthentication) {+ const { data: teamData, error: teamError } = await supabase_rr_service+ .from("teams")+ .select("hmac_secret")+ .eq("id", teamId)+ .limit(1)+ .single();++ if (teamError) {+ logger.error(`Error fetching team HMAC secret`, {+ error: teamError,+ });+ }++ if (teamData?.hmac_secret) hmacSecret = teamData.hmac_secret;+ }+logger.debug("Calling webhook...", {webhookUrl,});@@ -198,36 +233,49 @@ export const callWebhook = async ({}}- if (awaitWebhook) {+ const payload = {+ success: !v1+ ? data.success+ : eventType === "crawl.page"+ ? data.success+ : true,+ type: eventType,+ [v1 ? "id" : "jobId"]: crawlId,+ data: dataToSend,+ error: !v1+ ? data?.error || undefined+ : eventType === "crawl.page"+ ? data?.error || undefined+ : undefined,+ metadata: webhookUrl.metadata || undefined,+ };++ const payloadString = JSON.stringify(payload);++ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ ...webhookUrl.headers,+ };++ if (hmacSecret) {+ const signature = generateHmacSignature(payloadString, hmacSecret);+ headers["X-Firecrawl-Signature"] = `sha256=${signature}`;+ }++ const executeWebhookRequest = async () => {try {const res = await undici.fetch(webhookUrl.url, {method: "POST",- headers: {- "Content-Type": "application/json",- ...webhookUrl.headers,- },- body: JSON.stringify({- success: !v1- ? data.success- : eventType === "crawl.page"- ? data.success- : true,- type: eventType,- [v1 ? "id" : "jobId"]: crawlId,- data: dataToSend,- error: !v1- ? data?.error || undefined- : eventType === "crawl.page"- ? data?.error || undefined- : undefined,- metadata: webhookUrl.metadata || undefined,- }),+ headers,+ body: payloadString,dispatcher: getSecureDispatcher(),signal: AbortSignal.timeout(v1 ? 10000 : 30000), // 10 seconds timeout (v1)});+if (!res.ok) {throw { status: res.status };}+logWebhook({success: res.status >= 200 && res.status < 300,teamId,@@ -237,10 +285,13 @@ export const callWebhook = async ({event: eventType,statusCode: res.status,});++ return res;} catch (error) {logger.error(`Failed to send webhook`, {error,});+logWebhook({success: false,teamId,@@ -259,70 +310,17 @@ export const callWebhook = async ({? (error as any).status: undefined,});++ throw error;}+ };++ if (awaitWebhook) {+ await executeWebhookRequest();} else {- undici- .fetch(webhookUrl.url, {- method: "POST",- headers: {- "Content-Type": "application/json",- ...webhookUrl.headers,- },- body: JSON.stringify({- success: !v1- ? data.success- : eventType === "crawl.page"- ? data.success- : true,- type: eventType,- [v1 ? "id" : "jobId"]: crawlId,- data: dataToSend,- error: !v1- ? data?.error || undefined- : eventType === "crawl.page"- ? data?.error || undefined- : undefined,- metadata: webhookUrl.metadata || undefined,- }),- dispatcher: getSecureDispatcher(),- })- .then((res) => {- if (!res.ok) {- throw { status: res.status };- }- logWebhook({- success: res.status >= 200 && res.status < 300,- teamId,- crawlId,- scrapeId,- url: webhookUrl.url,- event: eventType,- statusCode: res.status,- });- })- .catch((error) => {- logger.error(`Failed to send webhook`, {- error,- });- logWebhook({- success: false,- teamId,- crawlId,- scrapeId,- url: webhookUrl.url,- event: eventType,- error:- error instanceof Error- ? error.message- : typeof error === "string"- ? error- : undefined,- statusCode:- typeof (error as any)?.status === "number"- ? (error as any).status- : undefined,- });- });+ executeWebhookRequest().catch(() => {+ // already logged in executeWebhookRequest+ });}} catch (error) {logger.warn(`Error sending webhook`, {