🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
describe,
it,
expect,
beforeEach,
beforeAll,
afterAll,
afterEach,
vi,
} from "vitest";
import { setupServer } from "msw/node";
import { http, HttpResponse, delay } from "msw";
import type { SharedEnv } from "@langfuse/shared/src/env";
import { prepareOtelBatch } from "../features/otel-ingestion/prepareOtelBatch";
import * as masking from "@langfuse/shared/src/server/ee/ingestionMasking";
import {
applyIngestionMasking,
isIngestionMaskingEnabled,
} from "@langfuse/shared/src/server/ee/ingestionMasking";
vi.mock(
"@langfuse/shared/src/server/ee/ingestionMasking",
async (importOriginal) => {
const actual = await importOriginal<typeof masking>();
return {
...actual,
applyIngestionMasking: vi.fn(actual.applyIngestionMasking),
isIngestionMaskingEnabled: vi.fn(actual.isIngestionMaskingEnabled),
};
},
);
// Sample OTEL span data for testing
const sampleSpanData = [
{
resource: {
attributes: [
{ key: "service.name", value: { stringValue: "test-service" } },
],
},
scopeSpans: [
{
scope: { name: "test-scope" },
spans: [
{
traceId: "abc123",
spanId: "def456",
name: "test-span",
attributes: [
{ key: "sensitive.data", value: { stringValue: "secret-value" } },
],
},
],
},
],
},
];
// Masked version of the sample data
const maskedSpanData = [
{
resource: {
attributes: [
{ key: "service.name", value: { stringValue: "test-service" } },
],
},
scopeSpans: [
{
scope: { name: "test-scope" },
spans: [
{
traceId: "abc123",
spanId: "def456",
name: "test-span",
attributes: [
{
key: "sensitive.data",
value: { stringValue: "***REDACTED***" },
},
],
},
],
},
],
},
];
// Mock webhook server for testing HTTP requests
class MaskingCallbackTestServer {
private server;
private receivedRequests: Array<{
url: string;
method: string;
headers: Record<string, string>;
body: unknown;
}> = [];
constructor() {
this.server = setupServer(
// Success endpoint - returns masked data
http.post("https://masking.example.com/success", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: await request.json(),
});
return HttpResponse.json(maskedSpanData, { status: 200 });
}),
// Echo endpoint - returns the same data it receives
http.post("https://masking.example.com/echo", async ({ request }) => {
const body = await request.json();
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body,
});
return HttpResponse.json(body, { status: 200 });
}),
// Error endpoint - returns 500
http.post("https://masking.example.com/error", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: await request.json(),
});
return HttpResponse.json(
{ error: "Internal Server Error" },
{ status: 500 },
);
}),
// Timeout endpoint - delays response
http.post("https://masking.example.com/timeout", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: await request.json(),
});
// Delay longer than the timeout
await delay(5000);
return HttpResponse.json(maskedSpanData, { status: 200 });
}),
);
}
setup() {
this.server.listen();
}
reset() {
this.receivedRequests = [];
this.server.resetHandlers();
}
teardown() {
this.server.close();
}
getReceivedRequests() {
return this.receivedRequests;
}
getLastRequest() {
return this.receivedRequests[this.receivedRequests.length - 1];
}
}
const maskingServer = new MaskingCallbackTestServer();
const VALID_EE_LICENSE_KEY = "langfuse_ee_test-license-key";
// Default test env with masking disabled
const defaultTestEnv: SharedEnv = {
LANGFUSE_INGESTION_MASKING_CALLBACK_URL: undefined,
LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: 500,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "false",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 1,
LANGFUSE_INGESTION_MASKING_PROPAGATED_HEADERS: [],
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
LANGFUSE_EE_LICENSE_KEY: undefined,
} as SharedEnv;
function createTestEnv(overrides: Partial<SharedEnv> = {}): SharedEnv {
return { ...defaultTestEnv, ...overrides } as SharedEnv;
}
describe("Ingestion Masking", () => {
beforeAll(() => {
maskingServer.setup();
});
beforeEach(() => {
maskingServer.reset();
});
afterEach(() => {
maskingServer.reset();
});
afterAll(() => {
maskingServer.teardown();
});
describe("isIngestionMaskingEnabled", () => {
it("should return false when callback URL is not configured", () => {
const testEnv = createTestEnv({
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
expect(isIngestionMaskingEnabled(testEnv)).toBe(false);
});
it("should return false when EE license is not available", () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
});
expect(isIngestionMaskingEnabled(testEnv)).toBe(false);
});
it("should return true when callback URL and EE license are configured", () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
expect(isIngestionMaskingEnabled(testEnv)).toBe(true);
});
it("should return true when callback URL is configured and running in cloud region", () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "US",
});
expect(isIngestionMaskingEnabled(testEnv)).toBe(true);
});
});
describe("applyIngestionMasking", () => {
it.each([
"success",
"bom-success",
"double-bom-success",
"triple-bom-fail-open",
"fail-open",
"fail-closed",
] as const)(
"compacts only the accepted raw masking input (%s)",
async (outcome) => {
const originalMediaBody = Buffer.alloc(4096, 0x6f);
const maskedMediaBody = Buffer.alloc(4096, 0x6d);
const original = {
input: `data:image/png;base64,${originalMediaBody.toString("base64")}`,
};
const originalBytes = Buffer.from(
JSON.stringify({ ...original, note: "🔥" }, null, 2),
);
const masked = {
input: `data:image/png;base64,${maskedMediaBody.toString("base64")}`,
};
const accepted =
outcome === "success" ||
outcome === "bom-success" ||
outcome === "double-bom-success";
const responseBody = accepted
? (outcome === "bom-success"
? "\uFEFF"
: outcome === "double-bom-success"
? "\uFEFF\uFEFF"
: "") + JSON.stringify(masked)
: outcome === "triple-bom-fail-open"
? "\uFEFF\uFEFF\uFEFF" + JSON.stringify(masked)
: "{";
const overrides = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/raw",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED:
outcome === "fail-closed" ? "true" : "false",
});
const configuredMasking = vi.mocked(masking.applyIngestionMasking);
const applyMasking = configuredMasking.getMockImplementation()!;
configuredMasking.mockImplementation((params, _env, transport) =>
applyMasking(params, overrides, transport),
);
const fetch = vi
.spyOn(globalThis, "fetch")
.mockResolvedValueOnce(new Response(responseBody));
if (accepted) {
await expect(new Response(responseBody).json()).resolves.toEqual(
masked,
);
} else if (outcome === "triple-bom-fail-open") {
await expect(new Response(responseBody).json()).rejects.toThrow();
}
let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;
try {
vi.mocked(masking.isIngestionMaskingEnabled).mockReturnValueOnce(
true,
);
prepared = await prepareOtelBatch({
bytes: originalBytes,
projectId: "test-project",
extractMedia: true,
});
// Masking sees the original inline content, before reference substitution.
const body = fetch.mock.calls[0]?.[1]?.body;
expect(body).toBeInstanceOf(Uint8Array);
expect(Buffer.from(body as Uint8Array)).toEqual(originalBytes);
if (outcome === "fail-closed") {
expect(prepared).toEqual({ error: expect.any(String) });
return;
}
const batch = prepared?.batch;
expect(batch).toBeDefined();
expect(batch!.media).toHaveLength(1);
expect(await batch!.mediaBody(0)).toEqual(
accepted ? maskedMediaBody : originalMediaBody,
);
expect(
JSON.parse(batch!.takeJsonBuffer().toString("utf8")).input,
).toBe(batch!.media[0].reference);
} finally {
await prepared?.batch?.dispose();
fetch.mockRestore();
configuredMasking.mockImplementation(applyMasking);
}
},
);
it.each(["request", "response"] as const)(
"sanitizes invalid UTF-8 in the masking %s before media discovery",
async (side) => {
const mediaBody = Buffer.alloc(4096, 0x68);
const mediaUri = `data:image/png;base64,${mediaBody.toString("base64")}`;
const malformed = Buffer.concat([
Buffer.from('[{"note":"'),
Buffer.from([0xff, 0xe2, 0x82]),
Buffer.from(`","input":"${mediaUri}"}]`),
]);
const sanitized = Buffer.from(malformed.toString("utf8"));
const originalBytes = side === "request" ? malformed : sanitized;
const responseBytes = side === "response" ? malformed : sanitized;
const overrides = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/raw",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
});
const configuredMasking = vi.mocked(masking.applyIngestionMasking);
const applyMasking = configuredMasking.getMockImplementation()!;
configuredMasking.mockImplementation((params, _env, transport) =>
applyMasking(params, overrides, transport),
);
const fetch = vi
.spyOn(globalThis, "fetch")
.mockResolvedValueOnce(new Response(new Uint8Array(responseBytes)));
let prepared: Awaited<ReturnType<typeof prepareOtelBatch>> | undefined;
try {
vi.mocked(masking.isIngestionMaskingEnabled).mockReturnValueOnce(
true,
);
prepared = await prepareOtelBatch({
bytes: originalBytes,
projectId: "test-project",
extractMedia: true,
});
const body = fetch.mock.calls[0]?.[1]?.body;
expect(body).toBeInstanceOf(Uint8Array);
expect(Buffer.from(body as Uint8Array)).toEqual(sanitized);
expect(fetch).toHaveBeenCalledTimes(1);
const batch = prepared.batch!;
expect(
JSON.parse(batch.takeJsonBuffer().toString("utf8"))[0].note,
).toBe("��");
expect(batch.media).toHaveLength(1);
expect(await batch.mediaBody(0)).toEqual(mediaBody);
} finally {
await prepared?.batch?.dispose();
fetch.mockRestore();
configuredMasking.mockImplementation(applyMasking);
}
},
);
it.each(["retry-success", "fail-open", "fail-closed"] as const)(
"validates raw masking responses inside retries (%s)",
async (outcome) => {
const data = Buffer.from(JSON.stringify(sampleSpanData));
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 1,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED:
outcome === "fail-closed" ? "true" : "false",
});
let validations = 0;
const result = await applyIngestionMasking(
{ data, projectId: "test-project" },
testEnv,
{
body: (bytes) => new Uint8Array(bytes),
read: async (response) => {
const bytes = Buffer.from(await response.arrayBuffer());
validations++;
if (outcome !== "retry-success" || validations === 1) {
throw new Error("invalid masked JSON");
}
return bytes;
},
},
);
expect(validations).toBe(2);
expect(maskingServer.getReceivedRequests()).toHaveLength(2);
for (const request of maskingServer.getReceivedRequests()) {
expect(request.headers["content-type"]).toBe("application/json");
expect(request.body).toEqual(sampleSpanData);
}
expect(result.success).toBe(outcome !== "fail-closed");
expect(result.masked).toBe(outcome === "retry-success");
if (outcome === "retry-success") {
expect(JSON.parse(result.data.toString())).toEqual(maskedSpanData);
} else {
expect(result.data).toBe(data);
if (outcome === "fail-closed") {
expect(result.error).toContain("invalid masked JSON");
}
}
},
);
it("should return original data immediately when masking is not configured", async () => {
const testEnv = createTestEnv();
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(false);
expect(result.data).toEqual(sampleSpanData);
// Verify no HTTP call was made
expect(maskingServer.getReceivedRequests()).toHaveLength(0);
});
it("should return original data when EE license is not available", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(false);
expect(result.data).toEqual(sampleSpanData);
// Verify no HTTP call was made
expect(maskingServer.getReceivedRequests()).toHaveLength(0);
});
it("should return masked data on successful callback", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(true);
expect(result.data).toEqual(maskedSpanData);
// Verify HTTP call was made
const requests = maskingServer.getReceivedRequests();
expect(requests).toHaveLength(1);
expect(requests[0].body).toEqual(sampleSpanData);
});
it("should include X-Langfuse-Org-Id and X-Langfuse-Project-Id headers", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project-123",
orgId: "test-org-456",
},
testEnv,
);
const request = maskingServer.getLastRequest();
expect(request?.headers["x-langfuse-org-id"]).toBe("test-org-456");
expect(request?.headers["x-langfuse-project-id"]).toBe(
"test-project-123",
);
});
it("should propagate custom headers when configured", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/success",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
propagatedHeaders: {
"x-custom-header": "custom-value",
"x-another-header": "another-value",
},
},
testEnv,
);
const request = maskingServer.getLastRequest();
expect(request?.headers["x-custom-header"]).toBe("custom-value");
expect(request?.headers["x-another-header"]).toBe("another-value");
});
it("should return original data on HTTP 500 with fail-open (default)", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/error",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "false",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(false);
expect(result.data).toEqual(sampleSpanData);
});
it("should return failure on HTTP 500 with fail-closed", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/error",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "true",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(false);
expect(result.masked).toBe(false);
expect(result.error).toContain("500");
});
it("should retry on failure", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/error",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "false",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 2,
});
await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
// Should have made 3 requests (1 initial + 2 retries)
const requests = maskingServer.getReceivedRequests();
expect(requests).toHaveLength(3);
});
it("should handle timeout with fail-open", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/timeout",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: 100, // Short timeout
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "false",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(false);
expect(result.data).toEqual(sampleSpanData);
}, 10000);
it("should handle timeout with fail-closed", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/timeout",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: 100, // Short timeout
LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED: "true",
LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 0,
});
const result = await applyIngestionMasking(
{
data: sampleSpanData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(false);
expect(result.masked).toBe(false);
expect(result.error).toContain("timeout");
}, 10000);
it("should work with generic data types", async () => {
const testEnv = createTestEnv({
LANGFUSE_INGESTION_MASKING_CALLBACK_URL:
"https://masking.example.com/echo",
LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,
});
const customData = { key: "value", nested: { data: [1, 2, 3] } };
const result = await applyIngestionMasking(
{
data: customData,
projectId: "test-project",
orgId: "test-org",
},
testEnv,
);
expect(result.success).toBe(true);
expect(result.masked).toBe(true);
expect(result.data).toEqual(customData);
});
});
});