Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: keep webhook secrets out of logs and wait longer to connect (#18801)

Tobias Wochingercdc82e75aa64
diff --git a/packages/shared/src/server/outbound-url/connection.ts b/packages/shared/src/server/outbound-url/connection.ts
index 77aa30c11b..f5fd359a4c 100644
--- a/packages/shared/src/server/outbound-url/connection.ts
+++ b/packages/shared/src/server/outbound-url/connection.ts
@@ -25,6 +25,8 @@ export type RequestInitWithDispatcher = RequestInit & { dispatcher?: unknown };
export interface OutboundUrlConnectionValidationOptions {
whitelist?: OutboundUrlValidationWhitelist;
logContext?: string;
+ /** addressAttemptTimeoutMs bounds each connect attempt; omitted callers keep Node's default. */
+ addressAttemptTimeoutMs?: number;
}
const secureOutboundDispatchersByPolicy = new Map<string, UndiciAgent>();
@@ -166,6 +168,7 @@ function getSecureOutboundDispatcher(
validationOptions: OutboundUrlConnectionValidationOptions,
): UndiciAgent {
const policyKey = getConnectionValidationPolicyKey(validationOptions);
+ const { addressAttemptTimeoutMs } = validationOptions;
return getOrCreatePolicyResource(
secureOutboundDispatchersByPolicy,
policyKey,
@@ -176,6 +179,13 @@ function getSecureOutboundDispatcher(
// This lookup runs at socket connection time, so DNS rebinding between
// the earlier URL validation and the actual HTTP connection is blocked.
lookup: createSecureOutboundLookup(validationOptions),
+ /** autoSelectFamilyAttemptTimeout applies only while autoSelectFamily is on. */
+ ...(addressAttemptTimeoutMs != null
+ ? {
+ autoSelectFamily: true,
+ autoSelectFamilyAttemptTimeout: addressAttemptTimeoutMs,
+ }
+ : {}),
},
}),
);
@@ -229,15 +239,19 @@ function validateConnectionTimeLookupResult(
function getConnectionValidationPolicyKey({
whitelist,
logContext,
+ addressAttemptTimeoutMs,
}: OutboundUrlConnectionValidationOptions): string {
// Keep separate Agent pools per whitelist. A single shared Agent could reuse
// a socket opened under a permissive policy for a later stricter request to
// the same origin without performing another DNS lookup. The lookup closure
// also captures logContext, so include it to avoid mislabeling connection-time
- // block warnings when different flows share the same whitelist.
+ // block warnings when different flows share the same whitelist. The address
+ // attempt timeout is fixed at Agent construction, so it belongs in the key
+ // too.
return JSON.stringify({
whitelist: normalizeWhitelist(whitelist),
logContext: logContext ?? "Outbound URL",
+ addressAttemptTimeoutMs: addressAttemptTimeoutMs ?? null,
});
}
diff --git a/packages/shared/src/server/outbound-url/fetch.test.ts b/packages/shared/src/server/outbound-url/fetch.test.ts
index 47816a06fe..ad1ce9adc4 100644
--- a/packages/shared/src/server/outbound-url/fetch.test.ts
+++ b/packages/shared/src/server/outbound-url/fetch.test.ts
@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
-import { redactUrlCredentials } from "./fetch";
+import {
+ CircularRedirectError,
+ MaxRedirectsExceededError,
+ redactUrlCredentials,
+ urlForLog,
+} from "./fetch";
describe("redactUrlCredentials", () => {
it("strips a user:password userinfo", () => {
@@ -43,3 +48,71 @@ describe("redactUrlCredentials", () => {
);
});
});
+
+describe("urlForLog", () => {
+ it("keeps the origin and path and drops a query-string secret", () => {
+ expect(
+ urlForLog(
+ "https://prod.westus.logic.azure.com/workflows/abc/triggers/manual/paths/invoke?api-version=2016-06-01&sig=super-secret-sig",
+ ),
+ ).toBe(
+ "https://prod.westus.logic.azure.com/workflows/abc/triggers/manual/paths/invoke",
+ );
+ });
+
+ it("drops userinfo and a fragment along with the query", () => {
+ expect(
+ urlForLog("https://exporter:[email protected]/batch/?x=1#frag"),
+ ).toBe("https://host.example/batch/");
+ });
+
+ it("keeps a credential-free path", () => {
+ expect(urlForLog("https://webhook-error.example.com/test")).toBe(
+ "https://webhook-error.example.com/test",
+ );
+ });
+
+ it("strips a query from a string that is not a URL", () => {
+ expect(urlForLog("not a url?sig=super-secret-sig")).toBe("not a url");
+ });
+
+ it("keeps the host of a non-special scheme", () => {
+ expect(urlForLog("gopher://169.254.169.254/x?sig=super-secret-sig")).toBe(
+ "gopher://169.254.169.254/x",
+ );
+ expect(urlForLog("file:///etc/passwd")).toBe("file:///etc/passwd");
+ });
+});
+
+const signedRedirectUrl =
+ "https://exporter:[email protected]/hook?sig=super-secret-sig";
+
+const expectSignedUrlRedacted = (
+ error: Error & { redirectChain: string[] },
+) => {
+ expect(error.message).toContain("https://host.example/hook");
+ expect(error.message).not.toContain("super-secret-sig");
+ expect(error.message).not.toContain("hunter2");
+ expect(error.stack).not.toContain("super-secret-sig");
+ expect(error.stack).not.toContain("hunter2");
+ expect(error.redirectChain).toEqual([
+ "https://host.example/hook",
+ "https://host.example/hook",
+ ]);
+ expect(JSON.stringify(error)).not.toContain("super-secret-sig");
+ expect(JSON.stringify(error)).not.toContain("hunter2");
+};
+
+describe("redirect error reporting", () => {
+ it("keeps a signed URL out of MaxRedirectsExceededError", () => {
+ expectSignedUrlRedacted(
+ new MaxRedirectsExceededError(5, [signedRedirectUrl, signedRedirectUrl]),
+ );
+ });
+
+ it("keeps a signed URL out of CircularRedirectError", () => {
+ expectSignedUrlRedacted(
+ new CircularRedirectError([signedRedirectUrl, signedRedirectUrl]),
+ );
+ });
+});
diff --git a/packages/shared/src/server/outbound-url/fetch.ts b/packages/shared/src/server/outbound-url/fetch.ts
index 9c13e9a252..72bda8a4cf 100644
--- a/packages/shared/src/server/outbound-url/fetch.ts
+++ b/packages/shared/src/server/outbound-url/fetch.ts
@@ -20,6 +20,23 @@ const URL_CREDENTIALS = /:\/\/[^/?#\s@]*@/g;
export const redactUrlCredentials = (text: string): string =>
text.replace(URL_CREDENTIALS, "://***@");
+/**
+ * Scheme, host, and path for logs and persisted errors. Query strings and
+ * fragments are dropped because webhook destinations put shared secrets there
+ * (`sig`, SAS tokens). Userinfo is dropped too. Built from protocol and host
+ * because `URL.origin` is the string "null" for non-special schemes.
+ */
+export const urlForLog = (urlString: string): string => {
+ try {
+ const url = new URL(urlString);
+ return `${url.protocol}//${url.host}${url.pathname}`;
+ } catch {
+ const withoutFragment = urlString.split("#")[0] ?? urlString;
+ const withoutQuery = withoutFragment.split("?")[0] ?? withoutFragment;
+ return redactUrlCredentials(withoutQuery);
+ }
+};
+
/**
* Custom error for redirect validation failures. `cause` is required so the
* inner error's typed `code` survives for classification, and goes through the
@@ -27,17 +44,23 @@ export const redactUrlCredentials = (text: string): string =>
* enumerable field, and the rejected target may carry credentials.
*/
export class RedirectValidationError extends Error {
+ readonly redirectUrl: string;
+ readonly redirectDepth: number;
+
constructor(
message: string,
- public redirectUrl: string,
- public redirectDepth: number,
+ redirectUrl: string,
+ redirectDepth: number,
cause: unknown,
) {
+ const loggedUrl = urlForLog(redirectUrl);
super(
- `Redirect validation failed at depth ${redirectDepth} for url ${redirectUrl}: ${message}`,
+ `Redirect validation failed at depth ${redirectDepth} for url ${loggedUrl}: ${message}`,
{ cause },
);
this.name = "RedirectValidationError";
+ this.redirectUrl = loggedUrl;
+ this.redirectDepth = redirectDepth;
}
}
@@ -45,14 +68,17 @@ export class RedirectValidationError extends Error {
* Custom error for exceeding maximum redirect depth
*/
export class MaxRedirectsExceededError extends Error {
- constructor(
- public maxRedirects: number,
- public redirectChain: string[],
- ) {
+ readonly maxRedirects: number;
+ readonly redirectChain: string[];
+
+ constructor(maxRedirects: number, redirectChain: string[]) {
+ const chainForLog = redirectChain.map(urlForLog);
super(
- `Maximum redirects (${maxRedirects}) exceeded. Chain: ${redirectChain.join(" → ")}`,
+ `Maximum redirects (${maxRedirects}) exceeded. Chain: ${chainForLog.join(" → ")}`,
);
this.name = "MaxRedirectsExceededError";
+ this.maxRedirects = maxRedirects;
+ this.redirectChain = chainForLog;
}
}
@@ -60,9 +86,13 @@ export class MaxRedirectsExceededError extends Error {
* Custom error for circular redirect detection
*/
export class CircularRedirectError extends Error {
- constructor(public redirectChain: string[]) {
- super(`Circular redirect detected: ${redirectChain.join(" → ")}`);
+ readonly redirectChain: string[];
+
+ constructor(redirectChain: string[]) {
+ const chainForLog = redirectChain.map(urlForLog);
+ super(`Circular redirect detected: ${chainForLog.join(" → ")}`);
this.name = "CircularRedirectError";
+ this.redirectChain = chainForLog;
}
}
@@ -89,6 +119,7 @@ interface RedirectValidationOptions {
validateUrl: RedirectUrlValidator;
whitelist?: OutboundUrlValidationWhitelist;
logContext?: string;
+ addressAttemptTimeoutMs?: number;
}
/**
@@ -161,7 +192,7 @@ export async function fetchWithSecureRedirects(
while (redirectDepth <= maxRedirects) {
logger.debug("Fetching URL with manual redirect handling", {
- url: currentUrl,
+ url: urlForLog(currentUrl),
redirectDepth,
maxRedirects,
});
@@ -178,7 +209,7 @@ export async function fetchWithSecureRedirects(
if (!isRedirect) {
// This is the final response, return it
logger.debug("Received non-redirect response", {
- url: currentUrl,
+ url: urlForLog(currentUrl),
status: response.status,
redirectDepth,
});
@@ -195,7 +226,7 @@ export async function fetchWithSecureRedirects(
if (!location) {
throw new Error(
- `Redirect response (${response.status}) missing Location header at ${currentUrl}`,
+ `Redirect response (${response.status}) missing Location header at ${urlForLog(currentUrl)}`,
);
}
@@ -206,13 +237,13 @@ export async function fetchWithSecureRedirects(
redirectUrl = resolvedUrl.toString();
} catch (error) {
throw new Error(
- `Invalid redirect URL "${location}" at ${currentUrl}: ${error instanceof Error ? error.message : "Unknown error"}`,
+ `Invalid redirect URL "${urlForLog(location)}" at ${urlForLog(currentUrl)}: ${error instanceof Error ? error.message : "Unknown error"}`,
);
}
logger.debug("Redirect detected", {
- from: currentUrl,
- to: redirectUrl,
+ from: urlForLog(currentUrl),
+ to: urlForLog(redirectUrl),
status: response.status,
redirectDepth,
});
@@ -244,8 +275,8 @@ export async function fetchWithSecureRedirects(
);
} catch (error) {
logger.warn("Redirect validation failed", {
- from: redactUrlCredentials(currentUrl),
- to: redactUrlCredentials(redirectUrl),
+ from: urlForLog(currentUrl),
+ to: urlForLog(redirectUrl),
redirectDepth,
error:
error instanceof Error
@@ -311,6 +342,8 @@ function addConnectionTimeValidation(
return addSecureOutboundConnectionValidation(options, {
whitelist: redirectOptions.redirectValidation.whitelist,
logContext: redirectOptions.redirectValidation.logContext,
+ addressAttemptTimeoutMs:
+ redirectOptions.redirectValidation.addressAttemptTimeoutMs,
});
}
diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
index efe440debe..c52b04f31d 100644
--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
@@ -461,15 +461,15 @@ function captureRethrow(error: unknown): unknown {
}
describe("terminal outbound failure reporting", () => {
- // The rejected redirect target is embedded verbatim in the validation error —
- // in its message, its stack, and its enumerable `redirectUrl` field — so a
- // credentialed Location header would otherwise reach the log line and
- // BullMQ's persisted failedReason.
+ // The Location URL is copied into the error message, the stack, and the
+ // enumerable `redirectUrl` field. A query-string secret or userinfo must
+ // not survive on any of those, because the webhook executor logs the
+ // original error and BullMQ persists failedReason.
const credentialedRedirect = () =>
Object.assign(new TypeError("fetch failed"), {
cause: new RedirectValidationError(
"Blocked IP address detected: 169.254.169.254",
- "http://exporter:[email protected]/",
+ "http://exporter:[email protected]/latest?sig=super-secret-sig",
0,
new OutboundUrlValidationError(
"blocked-ip",
@@ -484,19 +484,27 @@ describe("terminal outbound failure reporting", () => {
expect(isUnrecoverableError(thrown)).toBe(true);
const message = (thrown as Error).message;
expect(message).not.toContain("hunter2");
- expect(message).toContain("http://***@169.254.169.254/");
+ expect(message).not.toContain("super-secret-sig");
+ expect(message).toContain("http://169.254.169.254/latest");
// The diagnostic reason must survive redaction, otherwise the operator
// cannot tell a blocked IP from a rejected scheme.
expect(message).toContain("Blocked IP address detected: 169.254.169.254");
});
it("keeps credentials out of every field a logger or tracer can serialize", () => {
- const thrown = captureRethrow(credentialedRedirect());
+ const source = credentialedRedirect();
+ // The webhook executor logs this error directly, before any queue-level
+ // sanitizer runs, so the original object has to be clean too.
+ expect(serializableSurface(source)).not.toContain("hunter2");
+ expect(serializableSurface(source)).not.toContain("super-secret-sig");
+
+ const thrown = captureRethrow(source);
// Redacting only the message is not enough: the queue's generic `failed`
// handler passes the whole error to the logger and to traceException, so a
// raw error retained as an enumerable `cause` leaks the URL anyway.
expect(serializableSurface(thrown)).not.toContain("hunter2");
+ expect(serializableSurface(thrown)).not.toContain("super-secret-sig");
expect(serializableSurface(thrown)).not.toContain(
"exporter:[email protected]",
);
diff --git a/worker/src/__tests__/webhooks.test.ts b/worker/src/__tests__/webhooks.test.ts
index d79abb72ab..ebbcbde6a9 100644
--- a/worker/src/__tests__/webhooks.test.ts
+++ b/worker/src/__tests__/webhooks.test.ts
@@ -20,6 +20,7 @@ import {
WebhookInput,
createOrgProjectAndApiKey,
getActionByIdWithSecrets,
+ logger,
redis,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
@@ -556,6 +557,108 @@ describe("Webhook Integration Tests", () => {
});
});
+ it("sends a query-string secret and keeps it out of logs and the execution error", async () => {
+ const signedUrl =
+ "https://webhook-error.example.com/test?api-version=2016-06-01&sig=super-secret-sig";
+ const action = await prisma.action.findUnique({
+ where: { id: actionId },
+ });
+ if (!action) {
+ throw new Error("Action not found");
+ }
+
+ await prisma.action.update({
+ where: { id: actionId },
+ data: {
+ projectId,
+ type: "WEBHOOK",
+ config: {
+ ...(action.config as WebhookActionConfigWithSecrets),
+ url: signedUrl,
+ },
+ },
+ });
+
+ await prisma.automationExecution.create({
+ data: {
+ id: executionId,
+ projectId,
+ triggerId,
+ automationId,
+ actionId,
+ status: ActionExecutionStatus.PENDING,
+ sourceId: v4(),
+ input: {
+ promptName: "test-prompt",
+ promptVersion: 1,
+ action: "created",
+ type: "prompt-version",
+ },
+ },
+ });
+
+ const info = vi.spyOn(logger, "info");
+ const warn = vi.spyOn(logger, "warn");
+ const error = vi.spyOn(logger, "error");
+
+ try {
+ await executeWebhook(
+ {
+ projectId,
+ automationId,
+ executionId,
+ payload: {
+ prompt: PromptDomainSchema.parse(
+ await prisma.prompt.findUnique({ where: { id: promptId } }),
+ ),
+ action: "created",
+ type: "prompt-version",
+ user: {
+ id: "user-123",
+ name: "Test User",
+ email: "[email protected]",
+ },
+ },
+ },
+ { skipValidation: true },
+ );
+
+ const requests = webhookServer.getReceivedRequests();
+ expect(requests.length).toBeGreaterThan(0);
+ expect(requests.every((request) => request.url === signedUrl)).toBe(
+ true,
+ );
+
+ const execution = await prisma.automationExecution.findUnique({
+ where: { id: executionId },
+ });
+ expect(execution?.error).toContain(
+ "https://webhook-error.example.com/test",
+ );
+ expect(execution?.error).not.toContain("super-secret-sig");
+
+ const logged = [info, warn, error]
+ .flatMap((spy) => spy.mock.calls)
+ .map((call) =>
+ call
+ .map((arg) => {
+ if (arg instanceof Error) {
+ return `${arg.name} ${arg.message} ${arg.stack ?? ""} ${JSON.stringify(arg)}`;
+ }
+ return typeof arg === "string" ? arg : JSON.stringify(arg);
+ })
+ .join(" "),
+ )
+ .join("\n");
+ expect(logged).toContain("https://webhook-error.example.com/test");
+ expect(logged).not.toContain("super-secret-sig");
+ } finally {
+ info.mockRestore();
+ warn.mockRestore();
+ error.mockRestore();
+ }
+ });
+
it("should disable trigger after 5 consecutive failures and store lastFailingExecutionId", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
diff --git a/worker/src/queues/webhooks.ts b/worker/src/queues/webhooks.ts
index 7cd487924b..a30561277b 100644
--- a/worker/src/queues/webhooks.ts
+++ b/worker/src/queues/webhooks.ts
@@ -17,6 +17,7 @@ import {
validateWebhookURL,
whitelistFromEnv,
fetchWithSecureRedirects,
+ urlForLog,
WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
buildWebhookRequestHeaders,
} from "@langfuse/shared/src/server";
@@ -43,6 +44,9 @@ import { backOff } from "exponential-backoff";
import { env } from "../env";
import { SlackMessageBuilder } from "../features/slack/slackMessageBuilder";
+/** WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS lets a retransmitted SYN finish before Node closes the socket. */
+const WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS = 2_000;
+
// GitHub repository_dispatch client_payload: max 10 top-level properties and <64KB.
// https://docs.github.com/en/rest/repos/repos#create-a-repository-dispatch-event
const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;
@@ -208,6 +212,7 @@ async function executeHttpAction({
actionConfig,
additionalSensitiveHeaders,
payloadType,
+ addressAttemptTimeoutMs,
}: {
url: string;
payload: string;
@@ -220,6 +225,7 @@ async function executeHttpAction({
actionConfig: ActionDomainWithSecrets;
additionalSensitiveHeaders?: string[];
payloadType: WebhookInput["payload"]["type"];
+ addressAttemptTimeoutMs?: number;
}): Promise<{ httpStatus: number; responseBody: string }> {
// monitor-alert does not write AutomationExecution rows; auto-disable rides a Redis counter instead.
const isMonitorAlert = payloadType === "monitor-alert";
@@ -231,7 +237,7 @@ async function executeHttpAction({
await backOff(
async () => {
logger.info(
- `Sending HTTP request to ${url} for action ${automation.action.id}`,
+ `Sending HTTP request to ${urlForLog(url)} for action ${automation.action.id}`,
);
// Create AbortController for timeout
@@ -260,6 +266,7 @@ async function executeHttpAction({
validateUrl: validateWebhookURL,
whitelist,
logContext: WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
+ addressAttemptTimeoutMs,
},
additionalSensitiveHeaders,
};
@@ -281,10 +288,10 @@ async function executeHttpAction({
if (redirectResult.redirectChain.length > 0) {
logger.info("Webhook followed redirects", {
actionId: automation.action.id,
- initialUrl: url,
- finalUrl: redirectResult.finalUrl,
+ initialUrl: urlForLog(url),
+ finalUrl: urlForLog(redirectResult.finalUrl),
redirectCount: redirectResult.redirectChain.length,
- redirectChain: redirectResult.redirectChain,
+ redirectChain: redirectResult.redirectChain.map(urlForLog),
});
}
@@ -293,19 +300,19 @@ async function executeHttpAction({
if (!res.ok) {
logger.warn(
- `Webhook does not return 2xx status: failed with status ${res.status} for url ${url} and project ${projectId}. Body: ${responseBody}`,
+ `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}. Body: ${responseBody}`,
);
throw new Error(
- `Webhook does not return 2xx status: failed with status ${res.status} for url ${url} and project ${projectId}`,
+ `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}`,
);
}
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
logger.warn(
- `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${url} and project ${projectId}`,
+ `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
);
throw new Error(
- `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${url} and project ${projectId}`,
+ `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
);
}
throw error;
@@ -560,6 +567,7 @@ async function executeWebhookAction({
actionConfig,
additionalSensitiveHeaders,
payloadType: input.payload.type,
+ addressAttemptTimeoutMs: WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS,
});
}