🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker,shared): auto-disable analytics exports on a redirect-hop SSRF block (#16259)
Niklas Semmler6fe7e94553f3
diff --git a/packages/shared/src/server/outbound-url/fetch.test.ts b/packages/shared/src/server/outbound-url/fetch.test.tsnew file mode 100644index 0000000000..47816a06fe--- /dev/null+++ b/packages/shared/src/server/outbound-url/fetch.test.ts@@ -0,0 +1,45 @@+import { describe, expect, it } from "vitest";+import { redactUrlCredentials } from "./fetch";++describe("redactUrlCredentials", () => {+ it("strips a user:password userinfo", () => {+ expect(+ redactUrlCredentials("http://exporter:[email protected]/"),+ ).toBe("http://***@169.254.169.254/");+ });++ it("strips a username-only userinfo", () => {+ expect(+ redactUrlCredentials("https://[email protected]/batch/"),+ ).toBe("https://***@host.example/batch/");+ });++ // The redirect reporting paths embed URLs in a longer message, so a single+ // replacement is not enough.+ it("redacts every occurrence in a message", () => {+ expect(+ redactUrlCredentials(+ "failed for url http://a:[email protected]/ from https://c:[email protected]/",+ ),+ ).toBe(+ "failed for url http://***@10.0.0.1/ from https://***@host.example/",+ );+ });++ it("leaves a credential-free URL untouched", () => {+ expect(redactUrlCredentials("https://host.example/batch/?x=1#y")).toBe(+ "https://host.example/batch/?x=1#y",+ );+ });++ // Non-vacuity: an @ that is not userinfo must survive, or redaction would be+ // mangling ordinary paths and prose.+ it("does not touch an @ in the path or in bare text", () => {+ expect(redactUrlCredentials("https://host.example/@handle")).toBe(+ "https://host.example/@handle",+ );+ expect(redactUrlCredentials("contact [email protected]")).toBe(+ "contact [email protected]",+ );+ });+});diff --git a/packages/shared/src/server/outbound-url/fetch.ts b/packages/shared/src/server/outbound-url/fetch.tsindex ba957207e7..9c13e9a252 100644--- a/packages/shared/src/server/outbound-url/fetch.ts+++ b/packages/shared/src/server/outbound-url/fetch.ts@@ -10,17 +10,32 @@ const SENSITIVE_REDIRECT_HEADERS = new Set([WebhookSignatureHeader,]);+// A Location header may carry userinfo, so a rejected target can hold a+// password. Strip it before the URL reaches a log line or a retained error.+// Anchored on `://` rather than on a scheme pattern: a leading `[a-z][a-z0-9+.-]*`+// rescans and backtracks at every offset of an attacker-supplied string, which+// is quadratic, and the scheme itself contributes nothing to the match.+const URL_CREDENTIALS = /:\/\/[^/?#\s@]*@/g;++export const redactUrlCredentials = (text: string): string =>+ text.replace(URL_CREDENTIALS, "://***@");+/**- * Custom error for redirect validation failures+ * Custom error for redirect validation failures. `cause` is required so the+ * inner error's typed `code` survives for classification, and goes through the+ * options bag so it stays non-enumerable — a structured logger serializes every+ * enumerable field, and the rejected target may carry credentials.*/export class RedirectValidationError extends Error {constructor(message: string,public redirectUrl: string,public redirectDepth: number,+ cause: unknown,) {super(`Redirect validation failed at depth ${redirectDepth} for url ${redirectUrl}: ${message}`,+ { cause },);this.name = "RedirectValidationError";}@@ -229,16 +244,20 @@ export async function fetchWithSecureRedirects();} catch (error) {logger.warn("Redirect validation failed", {- from: currentUrl,- to: redirectUrl,+ from: redactUrlCredentials(currentUrl),+ to: redactUrlCredentials(redirectUrl),redirectDepth,- error: error instanceof Error ? error.message : "Unknown error",+ error:+ error instanceof Error+ ? redactUrlCredentials(error.message)+ : "Unknown error",});throw new RedirectValidationError(error instanceof Error ? error.message : "Validation failed",redirectUrl,redirectDepth,+ error,);}}diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.tsindex 16294867fe..54e2cc859b 100644--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts@@ -152,9 +152,11 @@ import {} from "../features/posthog/handlePostHogIntegrationProjectJob";import {CircularRedirectError,+ fetchWithSecureRedirects,MaxRedirectsExceededError,OutboundUrlValidationError,RedirectValidationError,+ validateWebhookURL,} from "@langfuse/shared/src/server";import { rethrowIfOutboundValidationFailure } from "../features/analyticsIntegrationEgress";import { MixpanelClient } from "../features/mixpanel/mixpanelClient";@@ -223,6 +225,104 @@ describe("PostHog integration project job — SSRF connect-time pinning", () =>}, 40_000);});+/**+ * A blocked redirect target is the same customer-config fault as a blocked+ * configured host, so it must disable too: the typed `code` has to survive the+ * RedirectValidationError re-wrap for the classifier to see it.+ */+describe("PostHog integration project job — SSRF block on a redirect hop", () => {+ const exporterHost = "https://exporter.analytics.example";+ const metadataUrl = "http://169.254.169.254/latest/meta-data/";++ beforeEach(() => {+ h.posthogIntegrationUpdate.mockClear();+ h.posthogIntegrationUpdateMany.mockClear();+ h.recordIncrement.mockClear();+ h.dispatchProjectNotification.mockClear();+ h.db.integration = h.integration();+ });++ const redirectOnce = () =>+ vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValue(+ new Response(null, { status: 302, headers: { Location: metadataUrl } }),+ );++ // Stub the pre-check for the harness host only (as the tests above do); the+ // redirect target goes through the real validator, so the classified fault is+ // a genuine OutboundUrlValidationError and not a shape invented here.+ async function useRealValidatorForRedirectTargets() {+ const actual = await vi.importActual<+ typeof import("@langfuse/shared/src/server")+ >("@langfuse/shared/src/server");+ const exporterOrigin = new URL(exporterHost).origin;+ vi.mocked(validateWebhookURL).mockImplementation(async (url, whitelist) => {+ // Origin equality, not a prefix: exporter.analytics.example.evil.test+ // starts with the harness host but is a different origin.+ if (new URL(url).origin === exporterOrigin) return;+ await actual.validateWebhookURL(url, whitelist);+ });+ }++ it("disables the integration when a redirect target is a blocked host", async () => {+ await useRealValidatorForRedirectTargets();+ h.db.integration.posthogHostName = exporterHost;+ // Stubbed at the socket boundary: only redirect validation can reject here.+ const fetchSpy = redirectOnce();++ let thrown: unknown;+ try {+ await handlePostHogIntegrationProjectJob(makeJob());+ } catch (error) {+ thrown = error;+ }++ // Non-vacuous: the 302 was served, so the block came from the redirect hop.+ expect(fetchSpy).toHaveBeenCalled();+ // Must not throw: that would fire the Failures monitor every cycle.+ expect(thrown).toBeUndefined();+ expect(h.recordIncrement).toHaveBeenCalledWith(+ POSTHOG_INTEGRATION_DISABLED_METRIC,+ 1,+ { reason: "ssrf_blocked_endpoint" },+ );+ expect(h.posthogIntegrationUpdateMany).toHaveBeenCalledWith(+ expect.objectContaining({ data: { enabled: false } }),+ );+ expect(h.dispatchProjectNotification).toHaveBeenCalled();+ }, 40_000);++ // The contract the disable rests on: the inner typed error stays reachable,+ // and stays off the enumerable surface a logger serializes.+ it("keeps the inner validation error reachable as a non-enumerable cause", async () => {+ const actual = await vi.importActual<+ typeof import("@langfuse/shared/src/server")+ >("@langfuse/shared/src/server");+ redirectOnce();++ const thrown = await fetchWithSecureRedirects(+ `${exporterHost}/batch/`,+ { method: "POST" },+ {+ maxRedirects: 3,+ redirectValidation: { validateUrl: actual.validateWebhookURL },+ },+ ).then(+ () => undefined,+ (error: unknown) => error,+ );++ expect((thrown as Error | undefined)?.name).toBe("RedirectValidationError");+ const cause = (thrown as { cause?: unknown } | undefined)?.cause as+ | { name?: string; code?: string }+ | undefined;+ expect(cause?.name).toBe("OutboundUrlValidationError");+ expect(cause?.code).toBe("blocked-hostname");+ expect(Object.keys(thrown as object)).not.toContain("cause");+ });+});+describe("Mixpanel sender — SSRF connect-time pinning", () => {it("rejects a validated host that connects to a blocked IP, before egress and terminally", async () => {let requestCount = 0;@@ -352,6 +452,10 @@ describe("terminal outbound failure reporting", () => {"Blocked IP address detected: 169.254.169.254","http://exporter:[email protected]/",0,+ new OutboundUrlValidationError(+ "blocked-ip",+ "Blocked IP address detected: 169.254.169.254",+ ),),});diff --git a/worker/src/errors/findOutboundUrlValidationError.ts b/worker/src/errors/findOutboundUrlValidationError.tsindex 86b243b232..6aff02ed3b 100644--- a/worker/src/errors/findOutboundUrlValidationError.ts+++ b/worker/src/errors/findOutboundUrlValidationError.ts@@ -33,11 +33,9 @@ const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([REDIRECT_LOOP_ERROR_NAME,]);-// A resolver hiccup is not a policy block: the host may be legitimate and the-// next attempt may succeed, so it must stay retryable. RedirectValidationError-// re-wraps the inner failure by message only — no code, no cause — so a DNS-// failure on a redirect hop is recognisable only by its message text. The-// substring is imported from the throw site so the two stay in sync.+// A resolver hiccup may succeed on the next attempt, so it stays retryable. A+// redirect hop restates the reason in its own message, so match that too;+// substring imported from the throw site to stay in sync.const isTransientDnsFailure = (error: Error): boolean =>(error as { code?: unknown }).code === "dns-lookup-failed" ||error.message.includes(DNS_LOOKUP_FAILED_MESSAGE_PREFIX);diff --git a/worker/src/features/analyticsIntegrationEgress.ts b/worker/src/features/analyticsIntegrationEgress.tsindex bc7d87294e..f72b546545 100644--- a/worker/src/features/analyticsIntegrationEgress.ts+++ b/worker/src/features/analyticsIntegrationEgress.ts@@ -1,5 +1,6 @@import {logger,+ redactUrlCredentials,validateWebhookURL,whitelistFromEnv,type RedirectOptions,@@ -34,13 +35,6 @@ export const buildAnalyticsRedirectOptions = (},});-// A rejected redirect target is reported with its URL embedded verbatim, and a-// Location header may carry userinfo, so the message can hold a password.-const URL_CREDENTIALS = /([a-z][a-z0-9+.-]*:\/\/)[^/?#\s@]*@/gi;--const redactUrlCredentials = (text: string): string =>- text.replace(URL_CREDENTIALS, "$1***@");-/*** A serialization-safe stand-in for the validation error. The original holds the* offending URL three times over — in its message, in its stack, and, for adiff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.tsindex 3d31819875..2657a26d7b 100644--- a/worker/src/features/integrations/customerFaultClassification.test.ts+++ b/worker/src/features/integrations/customerFaultClassification.test.ts@@ -1,5 +1,8 @@import { describe, it, expect } from "vitest";-import { OutboundUrlValidationError } from "@langfuse/shared/src/server";+import {+ OutboundUrlValidationError,+ RedirectValidationError,+} from "@langfuse/shared/src/server";import {classifyCustomerFault,isCustomerFaultError,@@ -216,6 +219,39 @@ describe("isCustomerFaultError", () => {expect(isCustomerFaultError(wrapped(err))).toBe(false);});+ it("classifies an SSRF block raised on a redirect hop as customer_fault", () => {+ // The production shape: the configured host answers 3xx with a blocked+ // target. The typed code reaches here only via the redirect `cause`.+ const redirectBlock = new RedirectValidationError(+ "Blocked hostname detected",+ "http://169.254.169.254/latest/meta-data/",+ 0,+ new OutboundUrlValidationError(+ "blocked-hostname",+ "Blocked hostname detected",+ ),+ );+ expect(classifyCustomerFault(wrapped(redirectBlock))).toBe(+ "ssrf_blocked_endpoint",+ );+ });++ it("keeps a dns-lookup-failed on a redirect hop out of customer_fault", () => {+ // Same wrapper, transient inner code: must not become a permanent disable.+ const redirectDnsFailure = new RedirectValidationError(+ "DNS lookup failed for host.example",+ "https://host.example/batch/",+ 0,+ new OutboundUrlValidationError(+ "dns-lookup-failed",+ "DNS lookup failed for host.example",+ ),+ );+ expect(+ classifyCustomerFault(wrapped(redirectDnsFailure)),+ ).toBeUndefined();+ });+it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {// Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying// the same deterministic code, with no `cause` chained.diff --git a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.tsindex 079f9b9e06..2696d02fde 100644--- a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts+++ b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts@@ -444,13 +444,10 @@ export const handlePostHogIntegrationProjectJob = async ().slice(0, 1000);if (reason === undefined) {- // A connect-time SSRF block is a permanent misconfiguration of the- // integration host, not a transient failure, so skip the remaining BullMQ- // attempts for this job. Only reached for blocks the classifier above does- // not own (e.g. a rejected redirect target): a blocked hostname/IP is a- // customer fault and disables the integration instead. The integration- // stays enabled here, so the schedule re-enqueues the project next cycle,- // which is what lets a fixed endpoint recover on its own.+ // Defensive fallback: every code today either classifies above and+ // disables (including one raised on a redirect hop, reached through the+ // redirect error's `cause`) or is transient. An unrecognised block stays+ // terminal, but leaves the integration enabled to recover next run.rethrowIfOutboundValidationFailure(error, {logSubject: `[POSTHOG] Outbound send for project ${projectId}`,jobSubject: `PostHog integration for project ${projectId}`,