πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): stop retrying analytics export redirect loops, fix SSRF wording (#16258)
Niklas Semmler421c7a95f9ce
diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.tsindex acad98a6da..16294867fe 100644--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts@@ -151,6 +151,8 @@ import {POSTHOG_INTEGRATION_DISABLED_METRIC,} from "../features/posthog/handlePostHogIntegrationProjectJob";import {+ CircularRedirectError,+ MaxRedirectsExceededError,OutboundUrlValidationError,RedirectValidationError,} from "@langfuse/shared/src/server";@@ -290,6 +292,23 @@ describe("outbound validation classification", () => {),).toBeDefined();});++ // A redirect loop or exhausted budget can never succeed on retry β the job+ // would re-read the same events from ClickHouse and hit the same chain every+ // attempt β so both must classify as terminal on the very first attempt.+ it("treats an exhausted redirect budget and a redirect loop as terminal", () => {+ expect(+ findOutboundUrlValidationError(+ new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),+ ),+ ).toBeDefined();++ expect(+ findOutboundUrlValidationError(+ new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),+ ),+ ).toBeDefined();+ });});/**@@ -389,4 +408,44 @@ describe("terminal outbound failure reporting", () => {),).toThrow("Blocked IP address detected: 127.0.0.1");});++ // Neither a redirect loop nor an exhausted budget was ever checked for+ // safety β outbound-url/fetch.ts detects both before validating the hop that+ // would have closed the loop or exceeded the budget β so the message must+ // not claim a security block, and must not claim the destination was safe+ // either.+ it("labels an exhausted redirect budget distinctly from an SSRF block, and calls the destination unverified", () => {+ const thrown = captureRethrow(+ new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),+ );++ expect(isUnrecoverableError(thrown)).toBe(true);+ const message = (thrown as Error).message;+ expect(message).toContain("too many redirects");+ expect(message).toContain("final destination not verified");+ expect(message).not.toContain("blocked by SSRF protection");+ });++ it("labels a redirect loop distinctly from an SSRF block, and calls the destination unverified", () => {+ const thrown = captureRethrow(+ new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),+ );++ expect(isUnrecoverableError(thrown)).toBe(true);+ const message = (thrown as Error).message;+ expect(message).toContain("redirect loop");+ expect(message).toContain("final destination not verified");+ expect(message).not.toContain("blocked by SSRF protection");+ });++ it("keeps calling an actual policy block a security block", () => {+ const thrown = captureRethrow(+ new OutboundUrlValidationError(+ "blocked-ip",+ "Blocked IP address detected: 127.0.0.1",+ ),+ );++ expect((thrown as Error).message).toContain("blocked by SSRF protection");+ });});diff --git a/worker/src/errors/findOutboundUrlValidationError.ts b/worker/src/errors/findOutboundUrlValidationError.tsindex 030ad9c48c..86b243b232 100644--- a/worker/src/errors/findOutboundUrlValidationError.ts+++ b/worker/src/errors/findOutboundUrlValidationError.ts@@ -12,11 +12,27 @@*/import { DNS_LOOKUP_FAILED_MESSAGE_PREFIX } from "@langfuse/shared/src/server";-const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([+// Host/IP policy blocks β the SSRF signal proper. RedirectValidationError+// belongs here: it means a redirect TARGET failed host validation.+const POLICY_BLOCK_ERROR_NAMES = new Set(["OutboundUrlValidationError","RedirectValidationError",]);+// Redirect budget exhaustion and loops. Also permanent β retrying re-reads the+// same events and hits the same chain β but the SSRF status is genuinely+// UNKNOWN: outbound-url/fetch.ts detects both before validating the hop that+// would have closed the loop or exceeded the budget, so that target was never+// checked. Reporting must say so instead of claiming a security block.+const REDIRECT_BUDGET_ERROR_NAME = "MaxRedirectsExceededError";+const REDIRECT_LOOP_ERROR_NAME = "CircularRedirectError";++const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([+ ...POLICY_BLOCK_ERROR_NAMES,+ REDIRECT_BUDGET_ERROR_NAME,+ REDIRECT_LOOP_ERROR_NAME,+]);+// A resolver hiccup is not a policy block: the host may be legitimate and the// next attempt may succeed, so it must stay retryable. RedirectValidationError// re-wraps the inner failure by message only β no code, no cause β so a DNS@@ -55,3 +71,22 @@ export function findOutboundUrlValidationError(return undefined;}++/**+ * How to describe a terminal outbound failure in a log line and in the error+ * that reaches BullMQ's `failedReason`.+ *+ * A host/IP policy block is stated as one. A redirect budget or loop fault is+ * NOT: it only asserts that the chain stopped before its final target was+ * validated, so it must not claim the destination was blocked (nothing checked+ * it) or fine (it might be the metadata service).+ */+export function describeOutboundFailure(error: Error): string {+ if (error.name === REDIRECT_BUDGET_ERROR_NAME) {+ return "too many redirects; final destination not verified";+ }+ if (error.name === REDIRECT_LOOP_ERROR_NAME) {+ return "redirect loop; final destination not verified";+ }+ return "blocked by SSRF protection";+}diff --git a/worker/src/features/analyticsIntegrationEgress.ts b/worker/src/features/analyticsIntegrationEgress.tsindex b30ce9b0ee..bc7d87294e 100644--- a/worker/src/features/analyticsIntegrationEgress.ts+++ b/worker/src/features/analyticsIntegrationEgress.ts@@ -5,7 +5,10 @@ import {type RedirectOptions,} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../errors/UnrecoverableError";-import { findOutboundUrlValidationError } from "../errors/findOutboundUrlValidationError";+import {+ describeOutboundFailure,+ findOutboundUrlValidationError,+} from "../errors/findOutboundUrlValidationError";/*** Redirect budget for the analytics integration exporters. Both senders@@ -55,9 +58,10 @@ const sanitizedCause = (validationError: Error): Error => {};/**- * Rethrow a connect-time SSRF block as terminal, so BullMQ stops re-attempting- * a send that cannot succeed; no-op for anything else, which the caller keeps- * handling as retryable.+ * Rethrow a permanent outbound-URL failure β a connect-time SSRF block, or a+ * redirect chain that exhausted its budget or looped β as terminal, so BullMQ+ * stops re-attempting a send that cannot succeed; no-op for anything else,+ * which the caller keeps handling as retryable.** Nothing the thrown error carries may hold a credential: the message, the* retained cause and that cause's own stack all outlive the job, via the log@@ -71,12 +75,13 @@ export function rethrowIfOutboundValidationFailure(if (!validationError) return;const reason = redactUrlCredentials(validationError.message);- logger.error(`${labels.logSubject} blocked by SSRF protection: ${reason}`, {+ const description = describeOutboundFailure(validationError);+ logger.error(`${labels.logSubject} ${description}: ${reason}`, {errorName: validationError.name,});const unrecoverable = new UnrecoverableError(- `${labels.jobSubject} blocked by SSRF protection: ${reason}`,+ `${labels.jobSubject} ${description}: ${reason}`,);// Non-enumerable, because a structured logger handed this error copies every// enumerable field into the log line β which is how the raw URL escaped a