🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(ingestion): separate masking payload transport (#18218)
Valery Meleshkin1cfcf5af37ff
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex c670099c1a..ca434dbb4e 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -57,6 +57,10 @@unclustered installs): `clickhouse/migrations/canonical/*`- ClickHouse SQL identifier and string quoting:`src/server/clickhouse/clickhouseIdentifiers.ts`+- Ingestion masking: `src/server/ee/ingestionMasking` keeps HTTP retries and failure+ policy independent of in-memory representation. The callback always exchanges JSON;+ the optional transport supplies reusable JSON text or UTF-8 bytes and validates+ responses inside retries. Default callers retain JSON serialization.- Seeder and support scripts: `scripts/seeder/*`, `clickhouse/scripts/*`## Export Entry Pointsdiff --git a/packages/shared/src/server/ee/ingestionMasking/applyIngestionMasking.ts b/packages/shared/src/server/ee/ingestionMasking/applyIngestionMasking.tsindex ef6098b191..eade80d532 100644--- a/packages/shared/src/server/ee/ingestionMasking/applyIngestionMasking.ts+++ b/packages/shared/src/server/ee/ingestionMasking/applyIngestionMasking.ts@@ -10,6 +10,7 @@ import type {IngestionMaskingConfig,ApplyIngestionMaskingParams,MaskingResult,+ IngestionMaskingTransport,} from "./types";/**@@ -66,8 +67,17 @@ async function makeCallbackRequest<T>(params: {orgId?: string;propagatedHeaders?: Record<string, string>;attempt: number;+ transport: IngestionMaskingTransport<T>;}): Promise<{ success: boolean; data: T; error?: string }> {- const { config, data, projectId, orgId, propagatedHeaders, attempt } = params;+ const {+ config,+ data,+ projectId,+ orgId,+ propagatedHeaders,+ attempt,+ transport,+ } = params;const startTime = Date.now();const controller = new AbortController();@@ -84,7 +94,7 @@ async function makeCallbackRequest<T>(params: {const response = await fetch(config.callbackUrl, {method: "POST",headers,- body: JSON.stringify(data),+ body: transport.body(data),signal: controller.signal,});@@ -107,7 +117,7 @@ async function makeCallbackRequest<T>(params: {};}- const maskedData = (await response.json()) as T;+ const maskedData = await transport.read(response);return { success: true, data: maskedData };} catch (error) {const duration = Date.now() - startTime;@@ -145,6 +155,10 @@ async function makeCallbackRequest<T>(params: {export async function applyIngestionMasking<T>(params: ApplyIngestionMaskingParams<T>,envOverride?: SharedEnv,+ transport: IngestionMaskingTransport<T> = {+ body: (data) => JSON.stringify(data),+ read: async (response) => (await response.json()) as T,+ },): Promise<MaskingResult<T>> {const { data, projectId, orgId, propagatedHeaders } = params;@@ -164,6 +178,7 @@ export async function applyIngestionMasking<T>(orgId,propagatedHeaders,attempt,+ transport,});if (result.success) {diff --git a/packages/shared/src/server/ee/ingestionMasking/types.ts b/packages/shared/src/server/ee/ingestionMasking/types.tsindex a8aad972aa..ee3811d2f0 100644--- a/packages/shared/src/server/ee/ingestionMasking/types.ts+++ b/packages/shared/src/server/ee/ingestionMasking/types.ts@@ -29,6 +29,13 @@ export interface ApplyIngestionMaskingParams<T> {propagatedHeaders?: Record<string, string>;}+/** JSON callback encoding and decoding; response validation runs inside masking retries. */+export interface IngestionMaskingTransport<T> {+ /** JSON text or UTF-8 JSON bytes, reusable across retries. */+ body(data: T): string | Uint8Array<ArrayBuffer>;+ read(response: Response): Promise<T>;+}+/*** Result of the masking operation.*/diff --git a/worker/src/__tests__/ingestionMasking.test.ts b/worker/src/__tests__/ingestionMasking.test.tsindex 406b7f4f25..25631fbc0b 100644--- a/worker/src/__tests__/ingestionMasking.test.ts+++ b/worker/src/__tests__/ingestionMasking.test.ts@@ -232,6 +232,53 @@ describe("Ingestion Masking", () => {});describe("applyIngestionMasking", () => {+ it.each(["retry-success", "fail-open", "fail-closed"] as const)(+ "validates raw masking responses inside retries (%s)",+ async (outcome) => {+ const data = Buffer.from(JSON.stringify(sampleSpanData));+ const testEnv = createTestEnv({+ LANGFUSE_INGESTION_MASKING_CALLBACK_URL:+ "https://masking.example.com/success",+ LANGFUSE_EE_LICENSE_KEY: VALID_EE_LICENSE_KEY,+ LANGFUSE_INGESTION_MASKING_MAX_RETRIES: 1,+ LANGFUSE_INGESTION_MASKING_CALLBACK_FAIL_CLOSED:+ outcome === "fail-closed" ? "true" : "false",+ });+ let validations = 0;+ const result = await applyIngestionMasking(+ { data, projectId: "test-project" },+ testEnv,+ {+ body: (bytes) => new Uint8Array(bytes),+ read: async (response) => {+ const bytes = Buffer.from(await response.arrayBuffer());+ validations++;+ if (outcome !== "retry-success" || validations === 1) {+ throw new Error("invalid masked JSON");+ }+ return bytes;+ },+ },+ );+ expect(validations).toBe(2);+ expect(maskingServer.getReceivedRequests()).toHaveLength(2);+ for (const request of maskingServer.getReceivedRequests()) {+ expect(request.headers["content-type"]).toBe("application/json");+ expect(request.body).toEqual(sampleSpanData);+ }+ expect(result.success).toBe(outcome !== "fail-closed");+ expect(result.masked).toBe(outcome === "retry-success");+ if (outcome === "retry-success") {+ expect(JSON.parse(result.data.toString())).toEqual(maskedSpanData);+ } else {+ expect(result.data).toBe(data);+ if (outcome === "fail-closed") {+ expect(result.error).toContain("invalid masked JSON");+ }+ }+ },+ );+it("should return original data immediately when masking is not configured", async () => {const testEnv = createTestEnv();