Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: prevent empty cookie header in webhook callbacks (#2504)

Himanshu Guptaccbd8d6ae8f5
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index d5a7e82d1..a430e50bb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -18,12 +18,12 @@ export function isIPPrivate(address: string): boolean {
return addr.range() !== "unicast";
}
-function makeSecureDispatcher(skipTlsVerification: boolean) {
+function createBaseAgent(skipTlsVerification: boolean) {
const agentOpts: undici.Agent.Options = {
maxRedirections: 5000,
};
- const baseAgent = process.env.PROXY_SERVER
+ return process.env.PROXY_SERVER
? new undici.ProxyAgent({
uri: process.env.PROXY_SERVER.includes("://")
? process.env.PROXY_SERVER
@@ -42,11 +42,9 @@ function makeSecureDispatcher(skipTlsVerification: boolean) {
},
...agentOpts,
});
+}
- const cookieJar = new CookieJar();
-
- const agent = baseAgent.compose(cookie({ jar: cookieJar }));
-
+function attachSecurityCheck(agent: undici.Dispatcher) {
agent.on("connect", (_, targets) => {
const client: undici.Client = targets.slice(-1)[0] as undici.Client;
const socketSymbol = Object.getOwnPropertySymbols(client).find(
@@ -62,12 +60,37 @@ function makeSecureDispatcher(skipTlsVerification: boolean) {
socket.destroy(new InsecureConnectionError());
}
});
+}
+// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows)
+function makeSecureDispatcher(skipTlsVerification: boolean) {
+ const baseAgent = createBaseAgent(skipTlsVerification);
+ const cookieJar = new CookieJar();
+ const agent = baseAgent.compose(cookie({ jar: cookieJar }));
+ attachSecurityCheck(agent);
+ return agent;
+}
+
+// Dispatcher WITHOUT cookie handling (for webhooks - avoids empty cookie header bug)
+function makeSecureDispatcherNoCookies(skipTlsVerification: boolean) {
+ const agent = createBaseAgent(skipTlsVerification);
+ attachSecurityCheck(agent);
return agent;
}
const secureDispatcher = makeSecureDispatcher(false);
const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
+const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false);
+const secureDispatcherNoCookiesSkipTlsVerification =
+ makeSecureDispatcherNoCookies(true);
export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>
skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;
+
+// Use this for webhook delivery to avoid sending empty cookie headers
+export const getSecureDispatcherNoCookies = (
+ skipTlsVerification: boolean = false,
+) =>
+ skipTlsVerification
+ ? secureDispatcherNoCookiesSkipTlsVerification
+ : secureDispatcherNoCookies;
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index 2f6b23662..4fe61e6f6 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -2,7 +2,7 @@ import undici from "undici";
import { createHmac } from "crypto";
import { logger as _logger, logger } from "../../lib/logger";
import {
- getSecureDispatcher,
+ getSecureDispatcherNoCookies,
isIPPrivate,
} from "../../scraper/scrapeURL/engines/utils/safeFetch";
import { WebhookConfig, WebhookEvent, WebhookEventDataMap } from "./types";
@@ -113,7 +113,7 @@ export class WebhookSender {
method: "POST",
headers,
body: payloadString,
- dispatcher: getSecureDispatcher(),
+ dispatcher: getSecureDispatcherNoCookies(),
signal: abortController.signal,
});