Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker,shared): auto-disable analytics exports on a redirect-hop SSRF block (#16259)

Niklas Semmler6fe7e94553f3
diff --git a/packages/shared/src/server/outbound-url/fetch.test.ts b/packages/shared/src/server/outbound-url/fetch.test.ts
new file mode 100644
index 0000000000..47816a06fe
--- /dev/null
+++ b/packages/shared/src/server/outbound-url/fetch.test.ts
@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+import { redactUrlCredentials } from "./fetch";
+
+describe("redactUrlCredentials", () => {
+ it("strips a user:password userinfo", () => {
+ expect(
+ redactUrlCredentials("http://exporter:[email protected]/"),
+ ).toBe("http://***@169.254.169.254/");
+ });
+
+ it("strips a username-only userinfo", () => {
+ expect(
+ redactUrlCredentials("https://[email protected]/batch/"),
+ ).toBe("https://***@host.example/batch/");
+ });
+
+ // The redirect reporting paths embed URLs in a longer message, so a single
+ // replacement is not enough.
+ it("redacts every occurrence in a message", () => {
+ expect(
+ redactUrlCredentials(
+ "failed for url http://a:[email protected]/ from https://c:[email protected]/",
+ ),
+ ).toBe(
+ "failed for url http://***@10.0.0.1/ from https://***@host.example/",
+ );
+ });
+
+ it("leaves a credential-free URL untouched", () => {
+ expect(redactUrlCredentials("https://host.example/batch/?x=1#y")).toBe(
+ "https://host.example/batch/?x=1#y",
+ );
+ });
+
+ // Non-vacuity: an @ that is not userinfo must survive, or redaction would be
+ // mangling ordinary paths and prose.
+ it("does not touch an @ in the path or in bare text", () => {
+ expect(redactUrlCredentials("https://host.example/@handle")).toBe(
+ "https://host.example/@handle",
+ );
+ expect(redactUrlCredentials("contact [email protected]")).toBe(
+ "contact [email protected]",
+ );
+ });
+});
diff --git a/packages/shared/src/server/outbound-url/fetch.ts b/packages/shared/src/server/outbound-url/fetch.ts
index ba957207e7..9c13e9a252 100644
--- a/packages/shared/src/server/outbound-url/fetch.ts
+++ b/packages/shared/src/server/outbound-url/fetch.ts
@@ -10,17 +10,32 @@ const SENSITIVE_REDIRECT_HEADERS = new Set([
WebhookSignatureHeader,
]);
+// A Location header may carry userinfo, so a rejected target can hold a
+// password. Strip it before the URL reaches a log line or a retained error.
+// Anchored on `://` rather than on a scheme pattern: a leading `[a-z][a-z0-9+.-]*`
+// rescans and backtracks at every offset of an attacker-supplied string, which
+// is quadratic, and the scheme itself contributes nothing to the match.
+const URL_CREDENTIALS = /:\/\/[^/?#\s@]*@/g;
+
+export const redactUrlCredentials = (text: string): string =>
+ text.replace(URL_CREDENTIALS, "://***@");
+
/**
- * Custom error for redirect validation failures
+ * Custom error for redirect validation failures. `cause` is required so the
+ * inner error's typed `code` survives for classification, and goes through the
+ * options bag so it stays non-enumerable — a structured logger serializes every
+ * enumerable field, and the rejected target may carry credentials.
*/
export class RedirectValidationError extends Error {
constructor(
message: string,
public redirectUrl: string,
public redirectDepth: number,
+ cause: unknown,
) {
super(
`Redirect validation failed at depth ${redirectDepth} for url ${redirectUrl}: ${message}`,
+ { cause },
);
this.name = "RedirectValidationError";
}
@@ -229,16 +244,20 @@ export async function fetchWithSecureRedirects(
);
} catch (error) {
logger.warn("Redirect validation failed", {
- from: currentUrl,
- to: redirectUrl,
+ from: redactUrlCredentials(currentUrl),
+ to: redactUrlCredentials(redirectUrl),
redirectDepth,
- error: error instanceof Error ? error.message : "Unknown error",
+ error:
+ error instanceof Error
+ ? redactUrlCredentials(error.message)
+ : "Unknown error",
});
throw new RedirectValidationError(
error instanceof Error ? error.message : "Validation failed",
redirectUrl,
redirectDepth,
+ error,
);
}
}
diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
index 16294867fe..54e2cc859b 100644
--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
@@ -152,9 +152,11 @@ import {
} from "../features/posthog/handlePostHogIntegrationProjectJob";
import {
CircularRedirectError,
+ fetchWithSecureRedirects,
MaxRedirectsExceededError,
OutboundUrlValidationError,
RedirectValidationError,
+ validateWebhookURL,
} from "@langfuse/shared/src/server";
import { rethrowIfOutboundValidationFailure } from "../features/analyticsIntegrationEgress";
import { MixpanelClient } from "../features/mixpanel/mixpanelClient";
@@ -223,6 +225,104 @@ describe("PostHog integration project job — SSRF connect-time pinning", () =>
}, 40_000);
});
+/**
+ * A blocked redirect target is the same customer-config fault as a blocked
+ * configured host, so it must disable too: the typed `code` has to survive the
+ * RedirectValidationError re-wrap for the classifier to see it.
+ */
+describe("PostHog integration project job — SSRF block on a redirect hop", () => {
+ const exporterHost = "https://exporter.analytics.example";
+ const metadataUrl = "http://169.254.169.254/latest/meta-data/";
+
+ beforeEach(() => {
+ h.posthogIntegrationUpdate.mockClear();
+ h.posthogIntegrationUpdateMany.mockClear();
+ h.recordIncrement.mockClear();
+ h.dispatchProjectNotification.mockClear();
+ h.db.integration = h.integration();
+ });
+
+ const redirectOnce = () =>
+ vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValue(
+ new Response(null, { status: 302, headers: { Location: metadataUrl } }),
+ );
+
+ // Stub the pre-check for the harness host only (as the tests above do); the
+ // redirect target goes through the real validator, so the classified fault is
+ // a genuine OutboundUrlValidationError and not a shape invented here.
+ async function useRealValidatorForRedirectTargets() {
+ const actual = await vi.importActual<
+ typeof import("@langfuse/shared/src/server")
+ >("@langfuse/shared/src/server");
+ const exporterOrigin = new URL(exporterHost).origin;
+ vi.mocked(validateWebhookURL).mockImplementation(async (url, whitelist) => {
+ // Origin equality, not a prefix: exporter.analytics.example.evil.test
+ // starts with the harness host but is a different origin.
+ if (new URL(url).origin === exporterOrigin) return;
+ await actual.validateWebhookURL(url, whitelist);
+ });
+ }
+
+ it("disables the integration when a redirect target is a blocked host", async () => {
+ await useRealValidatorForRedirectTargets();
+ h.db.integration.posthogHostName = exporterHost;
+ // Stubbed at the socket boundary: only redirect validation can reject here.
+ const fetchSpy = redirectOnce();
+
+ let thrown: unknown;
+ try {
+ await handlePostHogIntegrationProjectJob(makeJob());
+ } catch (error) {
+ thrown = error;
+ }
+
+ // Non-vacuous: the 302 was served, so the block came from the redirect hop.
+ expect(fetchSpy).toHaveBeenCalled();
+ // Must not throw: that would fire the Failures monitor every cycle.
+ expect(thrown).toBeUndefined();
+ expect(h.recordIncrement).toHaveBeenCalledWith(
+ POSTHOG_INTEGRATION_DISABLED_METRIC,
+ 1,
+ { reason: "ssrf_blocked_endpoint" },
+ );
+ expect(h.posthogIntegrationUpdateMany).toHaveBeenCalledWith(
+ expect.objectContaining({ data: { enabled: false } }),
+ );
+ expect(h.dispatchProjectNotification).toHaveBeenCalled();
+ }, 40_000);
+
+ // The contract the disable rests on: the inner typed error stays reachable,
+ // and stays off the enumerable surface a logger serializes.
+ it("keeps the inner validation error reachable as a non-enumerable cause", async () => {
+ const actual = await vi.importActual<
+ typeof import("@langfuse/shared/src/server")
+ >("@langfuse/shared/src/server");
+ redirectOnce();
+
+ const thrown = await fetchWithSecureRedirects(
+ `${exporterHost}/batch/`,
+ { method: "POST" },
+ {
+ maxRedirects: 3,
+ redirectValidation: { validateUrl: actual.validateWebhookURL },
+ },
+ ).then(
+ () => undefined,
+ (error: unknown) => error,
+ );
+
+ expect((thrown as Error | undefined)?.name).toBe("RedirectValidationError");
+ const cause = (thrown as { cause?: unknown } | undefined)?.cause as
+ | { name?: string; code?: string }
+ | undefined;
+ expect(cause?.name).toBe("OutboundUrlValidationError");
+ expect(cause?.code).toBe("blocked-hostname");
+ expect(Object.keys(thrown as object)).not.toContain("cause");
+ });
+});
+
describe("Mixpanel sender — SSRF connect-time pinning", () => {
it("rejects a validated host that connects to a blocked IP, before egress and terminally", async () => {
let requestCount = 0;
@@ -352,6 +452,10 @@ describe("terminal outbound failure reporting", () => {
"Blocked IP address detected: 169.254.169.254",
"http://exporter:[email protected]/",
0,
+ new OutboundUrlValidationError(
+ "blocked-ip",
+ "Blocked IP address detected: 169.254.169.254",
+ ),
),
});
diff --git a/worker/src/errors/findOutboundUrlValidationError.ts b/worker/src/errors/findOutboundUrlValidationError.ts
index 86b243b232..6aff02ed3b 100644
--- a/worker/src/errors/findOutboundUrlValidationError.ts
+++ b/worker/src/errors/findOutboundUrlValidationError.ts
@@ -33,11 +33,9 @@ const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([
REDIRECT_LOOP_ERROR_NAME,
]);
-// A resolver hiccup is not a policy block: the host may be legitimate and the
-// next attempt may succeed, so it must stay retryable. RedirectValidationError
-// re-wraps the inner failure by message only — no code, no cause — so a DNS
-// failure on a redirect hop is recognisable only by its message text. The
-// substring is imported from the throw site so the two stay in sync.
+// A resolver hiccup may succeed on the next attempt, so it stays retryable. A
+// redirect hop restates the reason in its own message, so match that too;
+// substring imported from the throw site to stay in sync.
const isTransientDnsFailure = (error: Error): boolean =>
(error as { code?: unknown }).code === "dns-lookup-failed" ||
error.message.includes(DNS_LOOKUP_FAILED_MESSAGE_PREFIX);
diff --git a/worker/src/features/analyticsIntegrationEgress.ts b/worker/src/features/analyticsIntegrationEgress.ts
index bc7d87294e..f72b546545 100644
--- a/worker/src/features/analyticsIntegrationEgress.ts
+++ b/worker/src/features/analyticsIntegrationEgress.ts
@@ -1,5 +1,6 @@
import {
logger,
+ redactUrlCredentials,
validateWebhookURL,
whitelistFromEnv,
type RedirectOptions,
@@ -34,13 +35,6 @@ export const buildAnalyticsRedirectOptions = (
},
});
-// A rejected redirect target is reported with its URL embedded verbatim, and a
-// Location header may carry userinfo, so the message can hold a password.
-const URL_CREDENTIALS = /([a-z][a-z0-9+.-]*:\/\/)[^/?#\s@]*@/gi;
-
-const redactUrlCredentials = (text: string): string =>
- text.replace(URL_CREDENTIALS, "$1***@");
-
/**
* A serialization-safe stand-in for the validation error. The original holds the
* offending URL three times over — in its message, in its stack, and, for a
diff --git a/worker/src/features/integrations/customerFaultClassification.test.ts b/worker/src/features/integrations/customerFaultClassification.test.ts
index 3d31819875..2657a26d7b 100644
--- a/worker/src/features/integrations/customerFaultClassification.test.ts
+++ b/worker/src/features/integrations/customerFaultClassification.test.ts
@@ -1,5 +1,8 @@
import { describe, it, expect } from "vitest";
-import { OutboundUrlValidationError } from "@langfuse/shared/src/server";
+import {
+ OutboundUrlValidationError,
+ RedirectValidationError,
+} from "@langfuse/shared/src/server";
import {
classifyCustomerFault,
isCustomerFaultError,
@@ -216,6 +219,39 @@ describe("isCustomerFaultError", () => {
expect(isCustomerFaultError(wrapped(err))).toBe(false);
});
+ it("classifies an SSRF block raised on a redirect hop as customer_fault", () => {
+ // The production shape: the configured host answers 3xx with a blocked
+ // target. The typed code reaches here only via the redirect `cause`.
+ const redirectBlock = new RedirectValidationError(
+ "Blocked hostname detected",
+ "http://169.254.169.254/latest/meta-data/",
+ 0,
+ new OutboundUrlValidationError(
+ "blocked-hostname",
+ "Blocked hostname detected",
+ ),
+ );
+ expect(classifyCustomerFault(wrapped(redirectBlock))).toBe(
+ "ssrf_blocked_endpoint",
+ );
+ });
+
+ it("keeps a dns-lookup-failed on a redirect hop out of customer_fault", () => {
+ // Same wrapper, transient inner code: must not become a permanent disable.
+ const redirectDnsFailure = new RedirectValidationError(
+ "DNS lookup failed for host.example",
+ "https://host.example/batch/",
+ 0,
+ new OutboundUrlValidationError(
+ "dns-lookup-failed",
+ "DNS lookup failed for host.example",
+ ),
+ );
+ expect(
+ classifyCustomerFault(wrapped(redirectDnsFailure)),
+ ).toBeUndefined();
+ });
+
it("classifies a validation rejection re-wrapped with guidance (code preserved) as customer_fault", () => {
// Mirrors validateBlobStorageEndpoint's catch: a new typed error carrying
// the same deterministic code, with no `cause` chained.
diff --git a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
index 079f9b9e06..2696d02fde 100644
--- a/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
+++ b/worker/src/features/posthog/handlePostHogIntegrationProjectJob.ts
@@ -444,13 +444,10 @@ export const handlePostHogIntegrationProjectJob = async (
).slice(0, 1000);
if (reason === undefined) {
- // A connect-time SSRF block is a permanent misconfiguration of the
- // integration host, not a transient failure, so skip the remaining BullMQ
- // attempts for this job. Only reached for blocks the classifier above does
- // not own (e.g. a rejected redirect target): a blocked hostname/IP is a
- // customer fault and disables the integration instead. The integration
- // stays enabled here, so the schedule re-enqueues the project next cycle,
- // which is what lets a fixed endpoint recover on its own.
+ // Defensive fallback: every code today either classifies above and
+ // disables (including one raised on a redirect hop, reached through the
+ // redirect error's `cause`) or is transient. An unrecognised block stays
+ // terminal, but leaves the integration enabled to recover next run.
rethrowIfOutboundValidationFailure(error, {
logSubject: `[POSTHOG] Outbound send for project ${projectId}`,
jobSubject: `PostHog integration for project ${projectId}`,