Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: prevent empty cookie header in webhook callbacks (#2504)
Himanshu Guptaccbd8d6ae8f5
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex d5a7e82d1..a430e50bb 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -18,12 +18,12 @@ export function isIPPrivate(address: string): boolean {return addr.range() !== "unicast";}-function makeSecureDispatcher(skipTlsVerification: boolean) {+function createBaseAgent(skipTlsVerification: boolean) {const agentOpts: undici.Agent.Options = {maxRedirections: 5000,};- const baseAgent = process.env.PROXY_SERVER+ return process.env.PROXY_SERVER? new undici.ProxyAgent({uri: process.env.PROXY_SERVER.includes("://")? process.env.PROXY_SERVER@@ -42,11 +42,9 @@ function makeSecureDispatcher(skipTlsVerification: boolean) {},...agentOpts,});+}- const cookieJar = new CookieJar();-- const agent = baseAgent.compose(cookie({ jar: cookieJar }));-+function attachSecurityCheck(agent: undici.Dispatcher) {agent.on("connect", (_, targets) => {const client: undici.Client = targets.slice(-1)[0] as undici.Client;const socketSymbol = Object.getOwnPropertySymbols(client).find(@@ -62,12 +60,37 @@ function makeSecureDispatcher(skipTlsVerification: boolean) {socket.destroy(new InsecureConnectionError());}});+}+// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows)+function makeSecureDispatcher(skipTlsVerification: boolean) {+ const baseAgent = createBaseAgent(skipTlsVerification);+ const cookieJar = new CookieJar();+ const agent = baseAgent.compose(cookie({ jar: cookieJar }));+ attachSecurityCheck(agent);+ return agent;+}++// Dispatcher WITHOUT cookie handling (for webhooks - avoids empty cookie header bug)+function makeSecureDispatcherNoCookies(skipTlsVerification: boolean) {+ const agent = createBaseAgent(skipTlsVerification);+ attachSecurityCheck(agent);return agent;}const secureDispatcher = makeSecureDispatcher(false);const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);+const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false);+const secureDispatcherNoCookiesSkipTlsVerification =+ makeSecureDispatcherNoCookies(true);export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;++// Use this for webhook delivery to avoid sending empty cookie headers+export const getSecureDispatcherNoCookies = (+ skipTlsVerification: boolean = false,+) =>+ skipTlsVerification+ ? secureDispatcherNoCookiesSkipTlsVerification+ : secureDispatcherNoCookies;diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex 2f6b23662..4fe61e6f6 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -2,7 +2,7 @@ import undici from "undici";import { createHmac } from "crypto";import { logger as _logger, logger } from "../../lib/logger";import {- getSecureDispatcher,+ getSecureDispatcherNoCookies,isIPPrivate,} from "../../scraper/scrapeURL/engines/utils/safeFetch";import { WebhookConfig, WebhookEvent, WebhookEventDataMap } from "./types";@@ -113,7 +113,7 @@ export class WebhookSender {method: "POST",headers,body: payloadString,- dispatcher: getSecureDispatcher(),+ dispatcher: getSecureDispatcherNoCookies(),signal: abortController.signal,});