Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): stop retrying analytics export redirect loops, fix SSRF wording (#16258)

Niklas Semmler421c7a95f9ce
diff --git a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
index acad98a6da..16294867fe 100644
--- a/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
+++ b/worker/src/__tests__/analyticsIntegrationSsrfPinning.test.ts
@@ -151,6 +151,8 @@ import {
POSTHOG_INTEGRATION_DISABLED_METRIC,
} from "../features/posthog/handlePostHogIntegrationProjectJob";
import {
+ CircularRedirectError,
+ MaxRedirectsExceededError,
OutboundUrlValidationError,
RedirectValidationError,
} from "@langfuse/shared/src/server";
@@ -290,6 +292,23 @@ describe("outbound validation classification", () => {
),
).toBeDefined();
});
+
+ // A redirect loop or exhausted budget can never succeed on retry β€” the job
+ // would re-read the same events from ClickHouse and hit the same chain every
+ // attempt β€” so both must classify as terminal on the very first attempt.
+ it("treats an exhausted redirect budget and a redirect loop as terminal", () => {
+ expect(
+ findOutboundUrlValidationError(
+ new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),
+ ),
+ ).toBeDefined();
+
+ expect(
+ findOutboundUrlValidationError(
+ new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),
+ ),
+ ).toBeDefined();
+ });
});
/**
@@ -389,4 +408,44 @@ describe("terminal outbound failure reporting", () => {
),
).toThrow("Blocked IP address detected: 127.0.0.1");
});
+
+ // Neither a redirect loop nor an exhausted budget was ever checked for
+ // safety β€” outbound-url/fetch.ts detects both before validating the hop that
+ // would have closed the loop or exceeded the budget β€” so the message must
+ // not claim a security block, and must not claim the destination was safe
+ // either.
+ it("labels an exhausted redirect budget distinctly from an SSRF block, and calls the destination unverified", () => {
+ const thrown = captureRethrow(
+ new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),
+ );
+
+ expect(isUnrecoverableError(thrown)).toBe(true);
+ const message = (thrown as Error).message;
+ expect(message).toContain("too many redirects");
+ expect(message).toContain("final destination not verified");
+ expect(message).not.toContain("blocked by SSRF protection");
+ });
+
+ it("labels a redirect loop distinctly from an SSRF block, and calls the destination unverified", () => {
+ const thrown = captureRethrow(
+ new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),
+ );
+
+ expect(isUnrecoverableError(thrown)).toBe(true);
+ const message = (thrown as Error).message;
+ expect(message).toContain("redirect loop");
+ expect(message).toContain("final destination not verified");
+ expect(message).not.toContain("blocked by SSRF protection");
+ });
+
+ it("keeps calling an actual policy block a security block", () => {
+ const thrown = captureRethrow(
+ new OutboundUrlValidationError(
+ "blocked-ip",
+ "Blocked IP address detected: 127.0.0.1",
+ ),
+ );
+
+ expect((thrown as Error).message).toContain("blocked by SSRF protection");
+ });
});
diff --git a/worker/src/errors/findOutboundUrlValidationError.ts b/worker/src/errors/findOutboundUrlValidationError.ts
index 030ad9c48c..86b243b232 100644
--- a/worker/src/errors/findOutboundUrlValidationError.ts
+++ b/worker/src/errors/findOutboundUrlValidationError.ts
@@ -12,11 +12,27 @@
*/
import { DNS_LOOKUP_FAILED_MESSAGE_PREFIX } from "@langfuse/shared/src/server";
-const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([
+// Host/IP policy blocks β€” the SSRF signal proper. RedirectValidationError
+// belongs here: it means a redirect TARGET failed host validation.
+const POLICY_BLOCK_ERROR_NAMES = new Set([
"OutboundUrlValidationError",
"RedirectValidationError",
]);
+// Redirect budget exhaustion and loops. Also permanent β€” retrying re-reads the
+// same events and hits the same chain β€” but the SSRF status is genuinely
+// UNKNOWN: outbound-url/fetch.ts detects both before validating the hop that
+// would have closed the loop or exceeded the budget, so that target was never
+// checked. Reporting must say so instead of claiming a security block.
+const REDIRECT_BUDGET_ERROR_NAME = "MaxRedirectsExceededError";
+const REDIRECT_LOOP_ERROR_NAME = "CircularRedirectError";
+
+const OUTBOUND_VALIDATION_ERROR_NAMES = new Set([
+ ...POLICY_BLOCK_ERROR_NAMES,
+ REDIRECT_BUDGET_ERROR_NAME,
+ REDIRECT_LOOP_ERROR_NAME,
+]);
+
// A resolver hiccup is not a policy block: the host may be legitimate and the
// next attempt may succeed, so it must stay retryable. RedirectValidationError
// re-wraps the inner failure by message only β€” no code, no cause β€” so a DNS
@@ -55,3 +71,22 @@ export function findOutboundUrlValidationError(
return undefined;
}
+
+/**
+ * How to describe a terminal outbound failure in a log line and in the error
+ * that reaches BullMQ's `failedReason`.
+ *
+ * A host/IP policy block is stated as one. A redirect budget or loop fault is
+ * NOT: it only asserts that the chain stopped before its final target was
+ * validated, so it must not claim the destination was blocked (nothing checked
+ * it) or fine (it might be the metadata service).
+ */
+export function describeOutboundFailure(error: Error): string {
+ if (error.name === REDIRECT_BUDGET_ERROR_NAME) {
+ return "too many redirects; final destination not verified";
+ }
+ if (error.name === REDIRECT_LOOP_ERROR_NAME) {
+ return "redirect loop; final destination not verified";
+ }
+ return "blocked by SSRF protection";
+}
diff --git a/worker/src/features/analyticsIntegrationEgress.ts b/worker/src/features/analyticsIntegrationEgress.ts
index b30ce9b0ee..bc7d87294e 100644
--- a/worker/src/features/analyticsIntegrationEgress.ts
+++ b/worker/src/features/analyticsIntegrationEgress.ts
@@ -5,7 +5,10 @@ import {
type RedirectOptions,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../errors/UnrecoverableError";
-import { findOutboundUrlValidationError } from "../errors/findOutboundUrlValidationError";
+import {
+ describeOutboundFailure,
+ findOutboundUrlValidationError,
+} from "../errors/findOutboundUrlValidationError";
/**
* Redirect budget for the analytics integration exporters. Both senders
@@ -55,9 +58,10 @@ const sanitizedCause = (validationError: Error): Error => {
};
/**
- * Rethrow a connect-time SSRF block as terminal, so BullMQ stops re-attempting
- * a send that cannot succeed; no-op for anything else, which the caller keeps
- * handling as retryable.
+ * Rethrow a permanent outbound-URL failure β€” a connect-time SSRF block, or a
+ * redirect chain that exhausted its budget or looped β€” as terminal, so BullMQ
+ * stops re-attempting a send that cannot succeed; no-op for anything else,
+ * which the caller keeps handling as retryable.
*
* Nothing the thrown error carries may hold a credential: the message, the
* retained cause and that cause's own stack all outlive the job, via the log
@@ -71,12 +75,13 @@ export function rethrowIfOutboundValidationFailure(
if (!validationError) return;
const reason = redactUrlCredentials(validationError.message);
- logger.error(`${labels.logSubject} blocked by SSRF protection: ${reason}`, {
+ const description = describeOutboundFailure(validationError);
+ logger.error(`${labels.logSubject} ${description}: ${reason}`, {
errorName: validationError.name,
});
const unrecoverable = new UnrecoverableError(
- `${labels.jobSubject} blocked by SSRF protection: ${reason}`,
+ `${labels.jobSubject} ${description}: ${reason}`,
);
// Non-enumerable, because a structured logger handed this error copies every
// enumerable field into the log line β€” which is how the raw URL escaped a