🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/**
* Connect-time SSRF pinning for the PostHog and Mixpanel analytics exporters.
*
* Both sender tests neutralise the string pre-check (that models "the host
* validated as public") and point the send at a `localhost` NAME, whose
* connect-time lookup re-resolves to 127.0.0.1 (that models "it rebinds at
* connect"). Pointing them at a statically-blocked host instead would pass even
* without connect-time pinning, which is why they are shaped this way.
*/
import { createServer, type Server } from "node:http";
import { AddressInfo } from "node:net";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { isUnrecoverableError } from "../errors/UnrecoverableError";
import { findOutboundUrlValidationError } from "../errors/findOutboundUrlValidationError";
// The senders read their allowlist from the environment, and vitest loads
// ../.env. A developer who allowlists localhost for local webhook testing would
// otherwise turn the block assertions below into mystery failures.
vi.hoisted(() => {
delete process.env.LANGFUSE_WEBHOOK_WHITELISTED_HOST;
delete process.env.LANGFUSE_WEBHOOK_WHITELISTED_IPS;
delete process.env.LANGFUSE_WEBHOOK_WHITELISTED_IP_SEGMENTS;
});
const openServers: Server[] = [];
async function startLoopbackServer(
onRequest: () => void,
): Promise<{ nameUrl: string; port: number }> {
const server = createServer((_req, res) => {
onRequest();
res.end("{}");
});
openServers.push(server);
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = (server.address() as AddressInfo).port;
return { nameUrl: `http://localhost:${port}`, port };
}
afterEach(async () => {
await Promise.all(
openServers
.splice(0)
.map((s) => new Promise<void>((r) => s.close(() => r()))),
);
vi.restoreAllMocks();
});
const h = vi.hoisted(() => {
const posthogIntegrationUpdate = vi.fn();
// The atomic disable claim (enabled true->false) the PostHog handler makes
// once it classifies the fault as the customer's.
const posthogIntegrationUpdateMany = vi.fn(async () => ({ count: 1 }));
const recordIncrement = vi.fn();
const dispatchProjectNotification = vi.fn(async () => {});
// Only scores yields, so the first flush attempts the blocked send and the
// handler stops — bounding runtime.
const oneRow = () =>
(async function* () {
yield { langfuse_id: "row-1" };
})();
const noRows = () => (async function* () {})();
const integration = () => ({
projectId: "project-1",
enabled: true,
exportSource: "TRACES_OBSERVATIONS",
posthogHostName: "http://placeholder.invalid",
encryptedPosthogApiKey: "enc",
lastSyncAt: new Date("2024-01-01"),
project: { name: "Test Project", createdAt: new Date("2023-01-01") },
});
const db = { integration: integration() as Record<string, unknown> };
return {
posthogIntegrationUpdate,
posthogIntegrationUpdateMany,
recordIncrement,
dispatchProjectNotification,
oneRow,
noRows,
integration,
db,
};
});
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
posthogIntegration: {
findFirst: vi.fn(async () => h.db.integration),
update: h.posthogIntegrationUpdate,
updateMany: h.posthogIntegrationUpdateMany,
},
},
}));
// Keep the REAL secure-outbound egress helpers; override only the data-stream
// sources, the logger, and validateWebhookURL.
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...actual,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
getCurrentSpan: vi.fn(() => undefined),
recordIncrement: h.recordIncrement,
dispatchProjectNotification: h.dispatchProjectNotification,
validateWebhookURL: vi.fn(async () => {}),
getTracesForAnalyticsIntegrations: vi.fn(() => h.noRows()),
getGenerationsForAnalyticsIntegrations: vi.fn(() => h.noRows()),
getScoresForAnalyticsIntegrations: vi.fn(() => h.oneRow()),
getEventsForAnalyticsIntegrations: vi.fn(() => h.noRows()),
};
});
// Transformers must return a payload the real posthog-node capture() accepts.
vi.mock("../features/posthog/transformers", () => {
const payload = () => ({
distinctId: "p",
event: "langfuse",
properties: {},
});
return {
transformTraceForPostHog: vi.fn(payload),
transformGenerationForPostHog: vi.fn(payload),
transformScoreForPostHog: vi.fn(payload),
transformEventForPostHog: vi.fn(payload),
};
});
vi.mock("@langfuse/shared/encryption", () => ({
decrypt: vi.fn(() => "phc_decrypted"),
}));
vi.mock("../env", () => ({
env: {
LANGFUSE_MIGRATION_V4_WRITE_MODE: "legacy",
LANGFUSE_POSTHOG_FLUSH_DELAY_MS: 0,
LANGFUSE_MIXPANEL_FLUSH_DELAY_MS: 0,
// Keep this non-zero so connect-time validation wins the race with the
// abort while still bounding the test when the protected request stalls.
LANGFUSE_MIXPANEL_TIMEOUT_MS: 1_000,
},
v4WritesToLegacyTables: (e: { LANGFUSE_MIGRATION_V4_WRITE_MODE: string }) =>
e.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "events_only",
v4WritesToEventsTable: (e: { LANGFUSE_MIGRATION_V4_WRITE_MODE: string }) =>
e.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy",
}));
vi.mock("posthog-node", async (importOriginal) => {
const actual = await importOriginal<typeof import("posthog-node")>();
return {
...actual,
PostHog: class extends actual.PostHog {
constructor(
apiKey: string,
options: ConstructorParameters<typeof actual.PostHog>[1],
) {
super(apiKey, {
...options,
requestTimeout: 1_000,
fetchRetryCount: 0,
});
}
},
};
});
// Imported after mocks are registered.
import {
handlePostHogIntegrationProjectJob,
POSTHOG_INTEGRATION_DISABLED_METRIC,
} from "../features/posthog/handlePostHogIntegrationProjectJob";
import {
CircularRedirectError,
fetchWithSecureRedirects,
MaxRedirectsExceededError,
OutboundUrlValidationError,
RedirectValidationError,
validateWebhookURL,
} from "@langfuse/shared/src/server";
import { rethrowIfOutboundValidationFailure } from "../features/analyticsIntegrationEgress";
import { MixpanelClient } from "../features/mixpanel/mixpanelClient";
import type { MixpanelEvent } from "../features/mixpanel/transformers";
function makeJob() {
return {
data: { id: "job-1", payload: { projectId: "project-1" } },
attemptsMade: 0,
} as unknown as Parameters<typeof handlePostHogIntegrationProjectJob>[0];
}
function causeChainIncludes(error: unknown, needle: string): boolean {
let current: any = error;
for (let depth = 0; depth < 6 && current; depth++) {
if (typeof current.message === "string" && current.message.includes(needle))
return true;
current = current.cause;
}
return false;
}
describe("PostHog integration project job — SSRF connect-time pinning", () => {
beforeEach(() => {
h.posthogIntegrationUpdate.mockClear();
h.posthogIntegrationUpdateMany.mockClear();
h.recordIncrement.mockClear();
h.dispatchProjectNotification.mockClear();
h.db.integration = h.integration();
});
// A blocked resolved IP is a customer-config fault, so the handler's shared
// classifier owns the terminal decision: disable the integration, notify, and
// RESOLVE. What this test adds over the classifier's own suite is the source
// of the fault — a host that PASSED the string pre-check and only failed when
// the socket resolved, which is reachable solely through connect-time pinning.
it("blocks a validated host that resolves to a blocked IP before egress, and disables the integration", async () => {
let requestCount = 0;
const { nameUrl } = await startLoopbackServer(() => {
requestCount++;
});
h.db.integration.posthogHostName = nameUrl;
let thrown: unknown;
try {
await handlePostHogIntegrationProjectJob(makeJob());
} catch (error) {
thrown = error;
}
expect(requestCount).toBe(0);
expect(thrown).toBeUndefined();
// Non-vacuous: only an OutboundUrlValidationError carrying `blocked-ip` (or
// `blocked-hostname`) produces this reason, and the pre-check is stubbed to
// pass, so the tag can only have come from the connect-time lookup.
expect(h.recordIncrement).toHaveBeenCalledWith(
POSTHOG_INTEGRATION_DISABLED_METRIC,
1,
{ reason: "ssrf_blocked_endpoint" },
);
expect(h.posthogIntegrationUpdateMany).toHaveBeenCalledWith(
expect.objectContaining({ data: { enabled: false } }),
);
expect(h.dispatchProjectNotification).toHaveBeenCalled();
}, 40_000);
});
/**
* A blocked redirect target is the same customer-config fault as a blocked
* configured host, so it must disable too: the typed `code` has to survive the
* RedirectValidationError re-wrap for the classifier to see it.
*/
describe("PostHog integration project job — SSRF block on a redirect hop", () => {
const exporterHost = "https://exporter.analytics.example";
const metadataUrl = "http://169.254.169.254/latest/meta-data/";
beforeEach(() => {
h.posthogIntegrationUpdate.mockClear();
h.posthogIntegrationUpdateMany.mockClear();
h.recordIncrement.mockClear();
h.dispatchProjectNotification.mockClear();
h.db.integration = h.integration();
});
const redirectOnce = () =>
vi
.spyOn(globalThis, "fetch")
.mockResolvedValue(
new Response(null, { status: 302, headers: { Location: metadataUrl } }),
);
// Stub the pre-check for the harness host only (as the tests above do); the
// redirect target goes through the real validator, so the classified fault is
// a genuine OutboundUrlValidationError and not a shape invented here.
async function useRealValidatorForRedirectTargets() {
const actual = await vi.importActual<
typeof import("@langfuse/shared/src/server")
>("@langfuse/shared/src/server");
const exporterOrigin = new URL(exporterHost).origin;
vi.mocked(validateWebhookURL).mockImplementation(async (url, whitelist) => {
// Origin equality, not a prefix: exporter.analytics.example.evil.test
// starts with the harness host but is a different origin.
if (new URL(url).origin === exporterOrigin) return;
await actual.validateWebhookURL(url, whitelist);
});
}
it("disables the integration when a redirect target is a blocked host", async () => {
await useRealValidatorForRedirectTargets();
h.db.integration.posthogHostName = exporterHost;
// Stubbed at the socket boundary: only redirect validation can reject here.
const fetchSpy = redirectOnce();
let thrown: unknown;
try {
await handlePostHogIntegrationProjectJob(makeJob());
} catch (error) {
thrown = error;
}
// Non-vacuous: the 302 was served, so the block came from the redirect hop.
expect(fetchSpy).toHaveBeenCalled();
// Must not throw: that would fire the Failures monitor every cycle.
expect(thrown).toBeUndefined();
expect(h.recordIncrement).toHaveBeenCalledWith(
POSTHOG_INTEGRATION_DISABLED_METRIC,
1,
{ reason: "ssrf_blocked_endpoint" },
);
expect(h.posthogIntegrationUpdateMany).toHaveBeenCalledWith(
expect.objectContaining({ data: { enabled: false } }),
);
expect(h.dispatchProjectNotification).toHaveBeenCalled();
}, 40_000);
// The contract the disable rests on: the inner typed error stays reachable,
// and stays off the enumerable surface a logger serializes.
it("keeps the inner validation error reachable as a non-enumerable cause", async () => {
const actual = await vi.importActual<
typeof import("@langfuse/shared/src/server")
>("@langfuse/shared/src/server");
redirectOnce();
const thrown = await fetchWithSecureRedirects(
`${exporterHost}/batch/`,
{ method: "POST" },
{
maxRedirects: 3,
redirectValidation: { validateUrl: actual.validateWebhookURL },
},
).then(
() => undefined,
(error: unknown) => error,
);
expect((thrown as Error | undefined)?.name).toBe("RedirectValidationError");
const cause = (thrown as { cause?: unknown } | undefined)?.cause as
| { name?: string; code?: string }
| undefined;
expect(cause?.name).toBe("OutboundUrlValidationError");
expect(cause?.code).toBe("blocked-hostname");
expect(Object.keys(thrown as object)).not.toContain("cause");
});
});
describe("Mixpanel sender — SSRF connect-time pinning", () => {
it("rejects a validated host that connects to a blocked IP, before egress and terminally", async () => {
let requestCount = 0;
const { nameUrl, port } = await startLoopbackServer(() => {
requestCount++;
});
// Negative control: the pre-fix path (bare global fetch) reaches the server,
// so a later count of 0 proves the pinning blocked the send rather than the
// harness being unreachable.
await fetch(`http://localhost:${port}/import?strict=1`, {
method: "POST",
body: "[]",
});
expect(requestCount).toBe(1);
const client = new MixpanelClient({
projectToken: "t",
region: "api",
baseUrl: nameUrl,
});
client.addEvent({
event: "trace",
properties: { token: "t", distinct_id: "1", $insert_id: 1 },
} as unknown as MixpanelEvent);
let thrown: unknown;
try {
await client.flush();
} catch (error) {
thrown = error;
}
expect(requestCount).toBe(1);
expect(thrown).toBeDefined();
expect(isUnrecoverableError(thrown)).toBe(true);
expect(causeChainIncludes(thrown, "Blocked IP address detected")).toBe(
true,
);
}, 5_000);
});
describe("outbound validation classification", () => {
it("treats a DNS lookup failure as retryable and a blocked IP as terminal", () => {
const wrap = (cause: Error) =>
Object.assign(new TypeError("fetch failed"), { cause });
expect(
findOutboundUrlValidationError(
wrap(
new OutboundUrlValidationError(
"dns-lookup-failed",
"DNS lookup failed for exporter.example.com",
),
),
),
).toBeUndefined();
expect(
findOutboundUrlValidationError(
wrap(
new OutboundUrlValidationError(
"blocked-ip",
"Blocked IP address detected: 127.0.0.1",
),
),
),
).toBeDefined();
});
// A redirect loop or exhausted budget can never succeed on retry — the job
// would re-read the same events from ClickHouse and hit the same chain every
// attempt — so both must classify as terminal on the very first attempt.
it("treats an exhausted redirect budget and a redirect loop as terminal", () => {
expect(
findOutboundUrlValidationError(
new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),
),
).toBeDefined();
expect(
findOutboundUrlValidationError(
new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),
),
).toBeDefined();
});
});
/**
* Everything a structured logger or tracer can pull off a thrown error: its
* enumerable own fields (winston copies those into the JSON log line), its
* message, its stack, and the same again for each `cause` a chain-walking
* reporter follows. A credential must appear in none of them.
*/
function serializableSurface(error: unknown): string {
const parts: string[] = [];
let current: any = error;
for (let depth = 0; depth < 6 && current; depth++) {
parts.push(JSON.stringify({ ...current }));
if (typeof current.message === "string") parts.push(current.message);
if (typeof current.stack === "string") parts.push(current.stack);
current = current.cause;
}
return parts.join("\n");
}
function captureRethrow(error: unknown): unknown {
try {
rethrowIfOutboundValidationFailure(error, {
logSubject: "Mixpanel outbound send",
jobSubject: "Mixpanel export",
});
} catch (caught) {
return caught;
}
return undefined;
}
describe("terminal outbound failure reporting", () => {
// The Location URL is copied into the error message, the stack, and the
// enumerable `redirectUrl` field. A query-string secret or userinfo must
// not survive on any of those, because the webhook executor logs the
// original error and BullMQ persists failedReason.
const credentialedRedirect = () =>
Object.assign(new TypeError("fetch failed"), {
cause: new RedirectValidationError(
"Blocked IP address detected: 169.254.169.254",
"http://exporter:[email protected]/latest?sig=super-secret-sig",
0,
new OutboundUrlValidationError(
"blocked-ip",
"Blocked IP address detected: 169.254.169.254",
),
),
});
it("redacts credentials from a rejected redirect target", () => {
const thrown = captureRethrow(credentialedRedirect());
expect(isUnrecoverableError(thrown)).toBe(true);
const message = (thrown as Error).message;
expect(message).not.toContain("hunter2");
expect(message).not.toContain("super-secret-sig");
expect(message).toContain("http://169.254.169.254/latest");
// The diagnostic reason must survive redaction, otherwise the operator
// cannot tell a blocked IP from a rejected scheme.
expect(message).toContain("Blocked IP address detected: 169.254.169.254");
});
it("keeps credentials out of every field a logger or tracer can serialize", () => {
const source = credentialedRedirect();
// The webhook executor logs this error directly, before any queue-level
// sanitizer runs, so the original object has to be clean too.
expect(serializableSurface(source)).not.toContain("hunter2");
expect(serializableSurface(source)).not.toContain("super-secret-sig");
const thrown = captureRethrow(source);
// Redacting only the message is not enough: the queue's generic `failed`
// handler passes the whole error to the logger and to traceException, so a
// raw error retained as an enumerable `cause` leaks the URL anyway.
expect(serializableSurface(thrown)).not.toContain("hunter2");
expect(serializableSurface(thrown)).not.toContain("super-secret-sig");
expect(serializableSurface(thrown)).not.toContain(
"exporter:[email protected]",
);
// Non-vacuous: the redacted reason really is reachable on that surface.
expect(serializableSurface(thrown)).toContain(
"Blocked IP address detected: 169.254.169.254",
);
});
it("does not expose the retained cause as an enumerable field", () => {
const thrown = captureRethrow(credentialedRedirect());
expect((thrown as Error).cause).toBeDefined();
expect(Object.keys(thrown as object)).not.toContain("cause");
});
it("leaves a credential-free message untouched and stays a no-op for other errors", () => {
expect(() =>
rethrowIfOutboundValidationFailure(new Error("connection reset"), {
logSubject: "Mixpanel outbound send",
jobSubject: "Mixpanel export",
}),
).not.toThrow();
expect(() =>
rethrowIfOutboundValidationFailure(
new OutboundUrlValidationError(
"blocked-ip",
"Blocked IP address detected: 127.0.0.1",
),
{ logSubject: "Mixpanel outbound send", jobSubject: "Mixpanel export" },
),
).toThrow("Blocked IP address detected: 127.0.0.1");
});
// Neither a redirect loop nor an exhausted budget was ever checked for
// safety — outbound-url/fetch.ts detects both before validating the hop that
// would have closed the loop or exceeded the budget — so the message must
// not claim a security block, and must not claim the destination was safe
// either.
it("labels an exhausted redirect budget distinctly from an SSRF block, and calls the destination unverified", () => {
const thrown = captureRethrow(
new MaxRedirectsExceededError(10, ["http://a/", "http://b/"]),
);
expect(isUnrecoverableError(thrown)).toBe(true);
const message = (thrown as Error).message;
expect(message).toContain("too many redirects");
expect(message).toContain("final destination not verified");
expect(message).not.toContain("blocked by SSRF protection");
});
it("labels a redirect loop distinctly from an SSRF block, and calls the destination unverified", () => {
const thrown = captureRethrow(
new CircularRedirectError(["http://a/", "http://b/", "http://a/"]),
);
expect(isUnrecoverableError(thrown)).toBe(true);
const message = (thrown as Error).message;
expect(message).toContain("redirect loop");
expect(message).toContain("final destination not verified");
expect(message).not.toContain("blocked by SSRF protection");
});
it("keeps calling an actual policy block a security block", () => {
const thrown = captureRethrow(
new OutboundUrlValidationError(
"blocked-ip",
"Blocked IP address detected: 127.0.0.1",
),
);
expect((thrown as Error).message).toContain("blocked by SSRF protection");
});
});