🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: keep webhook secrets out of logs and wait longer to connect (#18801)
Tobias Wochingercdc82e75aa64
diff --git a/packages/shared/src/server/outbound-url/connection.ts b/packages/shared/src/server/outbound-url/connection.tsindex 77aa30c11b..f5fd359a4c 100644--- a/packages/shared/src/server/outbound-url/connection.ts+++ b/packages/shared/src/server/outbound-url/connection.ts@@ -25,6 +25,8 @@ export type RequestInitWithDispatcher = RequestInit & { dispatcher?: unknown };export interface OutboundUrlConnectionValidationOptions {whitelist?: OutboundUrlValidationWhitelist;logContext?: string;+ /** addressAttemptTimeoutMs bounds each connect attempt; omitted callers keep Node's default. */+ addressAttemptTimeoutMs?: number;}const secureOutboundDispatchersByPolicy = new Map<string, UndiciAgent>();@@ -166,6 +168,7 @@ function getSecureOutboundDispatcher(validationOptions: OutboundUrlConnectionValidationOptions,): UndiciAgent {const policyKey = getConnectionValidationPolicyKey(validationOptions);+ const { addressAttemptTimeoutMs } = validationOptions;return getOrCreatePolicyResource(secureOutboundDispatchersByPolicy,policyKey,@@ -176,6 +179,13 @@ function getSecureOutboundDispatcher(// This lookup runs at socket connection time, so DNS rebinding between// the earlier URL validation and the actual HTTP connection is blocked.lookup: createSecureOutboundLookup(validationOptions),+ /** autoSelectFamilyAttemptTimeout applies only while autoSelectFamily is on. */+ ...(addressAttemptTimeoutMs != null+ ? {+ autoSelectFamily: true,+ autoSelectFamilyAttemptTimeout: addressAttemptTimeoutMs,+ }+ : {}),},}),);@@ -229,15 +239,19 @@ function validateConnectionTimeLookupResult(function getConnectionValidationPolicyKey({whitelist,logContext,+ addressAttemptTimeoutMs,}: OutboundUrlConnectionValidationOptions): string {// Keep separate Agent pools per whitelist. A single shared Agent could reuse// a socket opened under a permissive policy for a later stricter request to// the same origin without performing another DNS lookup. The lookup closure// also captures logContext, so include it to avoid mislabeling connection-time- // block warnings when different flows share the same whitelist.+ // block warnings when different flows share the same whitelist. The address+ // attempt timeout is fixed at Agent construction, so it belongs in the key+ // too.return JSON.stringify({whitelist: normalizeWhitelist(whitelist),logContext: logContext ?? "Outbound URL",+ addressAttemptTimeoutMs: addressAttemptTimeoutMs ?? null,});}diff --git a/packages/shared/src/server/outbound-url/fetch.test.ts b/packages/shared/src/server/outbound-url/fetch.test.tsindex 47816a06fe..ad1ce9adc4 100644--- a/packages/shared/src/server/outbound-url/fetch.test.ts+++ b/packages/shared/src/server/outbound-url/fetch.test.ts@@ -1,5 +1,10 @@import { describe, expect, it } from "vitest";-import { redactUrlCredentials } from "./fetch";+import {+ CircularRedirectError,+ MaxRedirectsExceededError,+ redactUrlCredentials,+ urlForLog,+} from "./fetch";describe("redactUrlCredentials", () => {it("strips a user:password userinfo", () => {@@ -43,3 +48,71 @@ describe("redactUrlCredentials", () => {);});});++describe("urlForLog", () => {+ it("keeps the origin and path and drops a query-string secret", () => {+ expect(+ urlForLog(+ "https://prod.westus.logic.azure.com/workflows/abc/triggers/manual/paths/invoke?api-version=2016-06-01&sig=super-secret-sig",+ ),+ ).toBe(+ "https://prod.westus.logic.azure.com/workflows/abc/triggers/manual/paths/invoke",+ );+ });++ it("drops userinfo and a fragment along with the query", () => {+ expect(+ urlForLog("https://exporter:[email protected]/batch/?x=1#frag"),+ ).toBe("https://host.example/batch/");+ });++ it("keeps a credential-free path", () => {+ expect(urlForLog("https://webhook-error.example.com/test")).toBe(+ "https://webhook-error.example.com/test",+ );+ });++ it("strips a query from a string that is not a URL", () => {+ expect(urlForLog("not a url?sig=super-secret-sig")).toBe("not a url");+ });++ it("keeps the host of a non-special scheme", () => {+ expect(urlForLog("gopher://169.254.169.254/x?sig=super-secret-sig")).toBe(+ "gopher://169.254.169.254/x",+ );+ expect(urlForLog("file:///etc/passwd")).toBe("file:///etc/passwd");+ });+});++const signedRedirectUrl =+ "https://exporter:[email protected]/hook?sig=super-secret-sig";++const expectSignedUrlRedacted = (+ error: Error & { redirectChain: string[] },+) => {+ expect(error.message).toContain("https://host.example/hook");+ expect(error.message).not.toContain("super-secret-sig");+ expect(error.message).not.toContain("hunter2");+ expect(error.stack).not.toContain("super-secret-sig");+ expect(error.stack).not.toContain("hunter2");+ expect(error.redirectChain).toEqual([+ "https://host.example/hook",+ "https://host.example/hook",+ ]);+ expect(JSON.stringify(error)).not.toContain("super-secret-sig");+ expect(JSON.stringify(error)).not.toContain("hunter2");+};++describe("redirect error reporting", () => {+ it("keeps a signed URL out of MaxRedirectsExceededError", () => {+ expectSignedUrlRedacted(+ new MaxRedirectsExceededError(5, [signedRedirectUrl, signedRedirectUrl]),+ );+ });++ it("keeps a signed URL out of CircularRedirectError", () => {+ expectSignedUrlRedacted(+ new CircularRedirectError([signedRedirectUrl, signedRedirectUrl]),+ );+ });+});diff --git a/packages/shared/src/server/outbound-url/fetch.ts b/packages/shared/src/server/outbound-url/fetch.tsindex 9c13e9a252..72bda8a4cf 100644--- a/packages/shared/src/server/outbound-url/fetch.ts+++ b/packages/shared/src/server/outbound-url/fetch.ts@@ -20,6 +20,23 @@ const URL_CREDENTIALS = /:\/\/[^/?#\s@]*@/g;export const redactUrlCredentials = (text: string): string =>text.replace(URL_CREDENTIALS, "://***@");+/**+ * Scheme, host, and path for logs and persisted errors. Query strings and+ * fragments are dropped because webhook destinations put shared secrets there+ * (`sig`, SAS tokens). Userinfo is dropped too. Built from protocol and host+ * because `URL.origin` is the string "null" for non-special schemes.+ */+export const urlForLog = (urlString: string): string => {+ try {+ const url = new URL(urlString);+ return `${url.protocol}//${url.host}${url.pathname}`;+ } catch {+ const withoutFragment = urlString.split("#")[0] ?? urlString;+ const withoutQuery = withoutFragment.split("?")[0] ?? withoutFragment;+ return redactUrlCredentials(withoutQuery);+ }+};+/*** Custom error for redirect validation failures. `cause` is required so the* inner error's typed `code` survives for classification, and goes through the@@ -27,17 +44,23 @@ export const redactUrlCredentials = (text: string): string =>* enumerable field, and the rejected target may carry credentials.*/export class RedirectValidationError extends Error {+ readonly redirectUrl: string;+ readonly redirectDepth: number;+constructor(message: string,- public redirectUrl: string,- public redirectDepth: number,+ redirectUrl: string,+ redirectDepth: number,cause: unknown,) {+ const loggedUrl = urlForLog(redirectUrl);super(- `Redirect validation failed at depth ${redirectDepth} for url ${redirectUrl}: ${message}`,+ `Redirect validation failed at depth ${redirectDepth} for url ${loggedUrl}: ${message}`,{ cause },);this.name = "RedirectValidationError";+ this.redirectUrl = loggedUrl;+ this.redirectDepth = redirectDepth;}}@@ -45,14 +68,17 @@ export class RedirectValidationError extends Error {* Custom error for exceeding maximum redirect depth*/export class MaxRedirectsExceededError extends Error {- constructor(- public maxRedirects: number,- public redirectChain: string[],- ) {+ readonly maxRedirects: number;+ readonly redirectChain: string[];++ constructor(maxRedirects: number, redirectChain: string[]) {+ const chainForLog = redirectChain.map(urlForLog);super(- `Maximum redirects (${maxRedirects}) exceeded. Chain: ${redirectChain.join(" → ")}`,+ `Maximum redirects (${maxRedirects}) exceeded. Chain: ${chainForLog.join(" → ")}`,);this.name = "MaxRedirectsExceededError";+ this.maxRedirects = maxRedirects;+ this.redirectChain = chainForLog;}}@@ -60,9 +86,13 @@ export class MaxRedirectsExceededError extends Error {* Custom error for circular redirect detection*/export class CircularRedirectError extends Error {- constructor(public redirectChain: string[]) {- super(`Circular redirect detected: ${redirectChain.join(" → ")}`);+ readonly redirectChain: string[];++ constructor(redirectChain: string[]) {+ const chainForLog = redirectChain.map(urlForLog);+ super(`Circular redirect detected: ${chainForLog.join(" → ")}`);this.name = "CircularRedirectError";+ this.redirectChain = chainForLog;}}@@ -89,6 +119,7 @@ interface RedirectValidationOptions {validateUrl: RedirectUrlValidator;whitelist?: OutboundUrlValidationWhitelist;logContext?: string;+ addressAttemptTimeoutMs?: number;}/**@@ -161,7 +192,7 @@ export async function fetchWithSecureRedirects(while (redirectDepth <= maxRedirects) {logger.debug("Fetching URL with manual redirect handling", {- url: currentUrl,+ url: urlForLog(currentUrl),redirectDepth,maxRedirects,});@@ -178,7 +209,7 @@ export async function fetchWithSecureRedirects(if (!isRedirect) {// This is the final response, return itlogger.debug("Received non-redirect response", {- url: currentUrl,+ url: urlForLog(currentUrl),status: response.status,redirectDepth,});@@ -195,7 +226,7 @@ export async function fetchWithSecureRedirects(if (!location) {throw new Error(- `Redirect response (${response.status}) missing Location header at ${currentUrl}`,+ `Redirect response (${response.status}) missing Location header at ${urlForLog(currentUrl)}`,);}@@ -206,13 +237,13 @@ export async function fetchWithSecureRedirects(redirectUrl = resolvedUrl.toString();} catch (error) {throw new Error(- `Invalid redirect URL "${location}" at ${currentUrl}: ${error instanceof Error ? error.message : "Unknown error"}`,+ `Invalid redirect URL "${urlForLog(location)}" at ${urlForLog(currentUrl)}: ${error instanceof Error ? error.message : "Unknown error"}`,);}logger.debug("Redirect detected", {- from: currentUrl,- to: redirectUrl,+ from: urlForLog(currentUrl),+ to: urlForLog(redirectUrl),status: response.status,redirectDepth,});@@ -244,8 +275,8 @@ export async function fetchWithSecureRedirects();} catch (error) {logger.warn("Redirect validation failed", {- from: redactUrlCredentials(currentUrl),- to: redactUrlCredentials(redirectUrl),+ from: urlForLog(currentUrl),+ to: urlForLog(redirectUrl),redirectDepth,error:error instanceof Error@@ -311,6 +342,8 @@ function addConnectionTimeValidation(return addSecureOutboundConnectionValidation(options, {whitelist: redirectOptions.redirectValidation.whitelist,logContext: redirectOptions.redirectValidation.logContext,+ addressAttemptTimeoutMs:+ redirectOptions.redirectValidation.addressAttemptTimeoutMs,});}diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.tsindex efe440debe..c52b04f31d 100644--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts@@ -461,15 +461,15 @@ function captureRethrow(error: unknown): unknown {}describe("terminal outbound failure reporting", () => {- // The rejected redirect target is embedded verbatim in the validation error —- // in its message, its stack, and its enumerable `redirectUrl` field — so a- // credentialed Location header would otherwise reach the log line and- // BullMQ's persisted failedReason.+ // The Location URL is copied into the error message, the stack, and the+ // enumerable `redirectUrl` field. A query-string secret or userinfo must+ // not survive on any of those, because the webhook executor logs the+ // original error and BullMQ persists failedReason.const credentialedRedirect = () =>Object.assign(new TypeError("fetch failed"), {cause: new RedirectValidationError("Blocked IP address detected: 169.254.169.254",- "http://exporter:[email protected]/",+ "http://exporter:[email protected]/latest?sig=super-secret-sig",0,new OutboundUrlValidationError("blocked-ip",@@ -484,19 +484,27 @@ describe("terminal outbound failure reporting", () => {expect(isUnrecoverableError(thrown)).toBe(true);const message = (thrown as Error).message;expect(message).not.toContain("hunter2");- expect(message).toContain("http://***@169.254.169.254/");+ expect(message).not.toContain("super-secret-sig");+ expect(message).toContain("http://169.254.169.254/latest");// The diagnostic reason must survive redaction, otherwise the operator// cannot tell a blocked IP from a rejected scheme.expect(message).toContain("Blocked IP address detected: 169.254.169.254");});it("keeps credentials out of every field a logger or tracer can serialize", () => {- const thrown = captureRethrow(credentialedRedirect());+ const source = credentialedRedirect();+ // The webhook executor logs this error directly, before any queue-level+ // sanitizer runs, so the original object has to be clean too.+ expect(serializableSurface(source)).not.toContain("hunter2");+ expect(serializableSurface(source)).not.toContain("super-secret-sig");++ const thrown = captureRethrow(source);// Redacting only the message is not enough: the queue's generic `failed`// handler passes the whole error to the logger and to traceException, so a// raw error retained as an enumerable `cause` leaks the URL anyway.expect(serializableSurface(thrown)).not.toContain("hunter2");+ expect(serializableSurface(thrown)).not.toContain("super-secret-sig");expect(serializableSurface(thrown)).not.toContain("exporter:[email protected]",);diff --git a/worker/src/__tests__/webhooks.test.ts b/worker/src/__tests__/webhooks.test.tsindex d79abb72ab..ebbcbde6a9 100644--- a/worker/src/__tests__/webhooks.test.ts+++ b/worker/src/__tests__/webhooks.test.ts@@ -20,6 +20,7 @@ import {WebhookInput,createOrgProjectAndApiKey,getActionByIdWithSecrets,+ logger,redis,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";@@ -556,6 +557,108 @@ describe("Webhook Integration Tests", () => {});});+ it("sends a query-string secret and keeps it out of logs and the execution error", async () => {+ const signedUrl =+ "https://webhook-error.example.com/test?api-version=2016-06-01&sig=super-secret-sig";+ const action = await prisma.action.findUnique({+ where: { id: actionId },+ });+ if (!action) {+ throw new Error("Action not found");+ }++ await prisma.action.update({+ where: { id: actionId },+ data: {+ projectId,+ type: "WEBHOOK",+ config: {+ ...(action.config as WebhookActionConfigWithSecrets),+ url: signedUrl,+ },+ },+ });++ await prisma.automationExecution.create({+ data: {+ id: executionId,+ projectId,+ triggerId,+ automationId,+ actionId,+ status: ActionExecutionStatus.PENDING,+ sourceId: v4(),+ input: {+ promptName: "test-prompt",+ promptVersion: 1,+ action: "created",+ type: "prompt-version",+ },+ },+ });++ const info = vi.spyOn(logger, "info");+ const warn = vi.spyOn(logger, "warn");+ const error = vi.spyOn(logger, "error");++ try {+ await executeWebhook(+ {+ projectId,+ automationId,+ executionId,+ payload: {+ prompt: PromptDomainSchema.parse(+ await prisma.prompt.findUnique({ where: { id: promptId } }),+ ),+ action: "created",+ type: "prompt-version",+ user: {+ id: "user-123",+ name: "Test User",+ email: "[email protected]",+ },+ },+ },+ { skipValidation: true },+ );++ const requests = webhookServer.getReceivedRequests();+ expect(requests.length).toBeGreaterThan(0);+ expect(requests.every((request) => request.url === signedUrl)).toBe(+ true,+ );++ const execution = await prisma.automationExecution.findUnique({+ where: { id: executionId },+ });+ expect(execution?.error).toContain(+ "https://webhook-error.example.com/test",+ );+ expect(execution?.error).not.toContain("super-secret-sig");++ const logged = [info, warn, error]+ .flatMap((spy) => spy.mock.calls)+ .map((call) =>+ call+ .map((arg) => {+ if (arg instanceof Error) {+ return `${arg.name} ${arg.message} ${arg.stack ?? ""} ${JSON.stringify(arg)}`;+ }+ return typeof arg === "string" ? arg : JSON.stringify(arg);+ })+ .join(" "),+ )+ .join("\n");+ expect(logged).toContain("https://webhook-error.example.com/test");+ expect(logged).not.toContain("super-secret-sig");+ } finally {+ info.mockRestore();+ warn.mockRestore();+ error.mockRestore();+ }+ });+it("should disable trigger after 5 consecutive failures and store lastFailingExecutionId", async () => {const fullPrompt = await prisma.prompt.findUnique({where: { id: promptId },diff --git a/worker/src/queues/webhooks.ts b/worker/src/queues/webhooks.tsindex 7cd487924b..a30561277b 100644--- a/worker/src/queues/webhooks.ts+++ b/worker/src/queues/webhooks.ts@@ -17,6 +17,7 @@ import {validateWebhookURL,whitelistFromEnv,fetchWithSecureRedirects,+ urlForLog,WEBHOOK_URL_VALIDATION_LOG_CONTEXT,buildWebhookRequestHeaders,} from "@langfuse/shared/src/server";@@ -43,6 +44,9 @@ import { backOff } from "exponential-backoff";import { env } from "../env";import { SlackMessageBuilder } from "../features/slack/slackMessageBuilder";+/** WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS lets a retransmitted SYN finish before Node closes the socket. */+const WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS = 2_000;+// GitHub repository_dispatch client_payload: max 10 top-level properties and <64KB.// https://docs.github.com/en/rest/repos/repos#create-a-repository-dispatch-eventconst GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;@@ -208,6 +212,7 @@ async function executeHttpAction({actionConfig,additionalSensitiveHeaders,payloadType,+ addressAttemptTimeoutMs,}: {url: string;payload: string;@@ -220,6 +225,7 @@ async function executeHttpAction({actionConfig: ActionDomainWithSecrets;additionalSensitiveHeaders?: string[];payloadType: WebhookInput["payload"]["type"];+ addressAttemptTimeoutMs?: number;}): Promise<{ httpStatus: number; responseBody: string }> {// monitor-alert does not write AutomationExecution rows; auto-disable rides a Redis counter instead.const isMonitorAlert = payloadType === "monitor-alert";@@ -231,7 +237,7 @@ async function executeHttpAction({await backOff(async () => {logger.info(- `Sending HTTP request to ${url} for action ${automation.action.id}`,+ `Sending HTTP request to ${urlForLog(url)} for action ${automation.action.id}`,);// Create AbortController for timeout@@ -260,6 +266,7 @@ async function executeHttpAction({validateUrl: validateWebhookURL,whitelist,logContext: WEBHOOK_URL_VALIDATION_LOG_CONTEXT,+ addressAttemptTimeoutMs,},additionalSensitiveHeaders,};@@ -281,10 +288,10 @@ async function executeHttpAction({if (redirectResult.redirectChain.length > 0) {logger.info("Webhook followed redirects", {actionId: automation.action.id,- initialUrl: url,- finalUrl: redirectResult.finalUrl,+ initialUrl: urlForLog(url),+ finalUrl: urlForLog(redirectResult.finalUrl),redirectCount: redirectResult.redirectChain.length,- redirectChain: redirectResult.redirectChain,+ redirectChain: redirectResult.redirectChain.map(urlForLog),});}@@ -293,19 +300,19 @@ async function executeHttpAction({if (!res.ok) {logger.warn(- `Webhook does not return 2xx status: failed with status ${res.status} for url ${url} and project ${projectId}. Body: ${responseBody}`,+ `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}. Body: ${responseBody}`,);throw new Error(- `Webhook does not return 2xx status: failed with status ${res.status} for url ${url} and project ${projectId}`,+ `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}`,);}} catch (error) {if (error instanceof Error && error.name === "AbortError") {logger.warn(- `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${url} and project ${projectId}`,+ `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,);throw new Error(- `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${url} and project ${projectId}`,+ `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,);}throw error;@@ -560,6 +567,7 @@ async function executeWebhookAction({actionConfig,additionalSensitiveHeaders,payloadType: input.payload.type,+ addressAttemptTimeoutMs: WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS,});}