Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): add isolated OTEL replay integration harness (#17741)

Valery Meleshkineb92bd185235
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 5512b0e3c4..7d374e2dce 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -39,6 +39,9 @@
in `src/initialize.ts`, the health probe call in `src/api/index.ts`. Native code
records its own metrics and logs; see `../packages/native/AGENTS.md`.
- Tests: `src/__tests__/*`, `src/queues/__tests__/*`
+- Direct-event replay: `pnpm --filter worker run test:otel-replay` exercises the
+ production OTEL event phase with isolated ClickHouse tables. Setup and scope:
+ `src/features/otel-ingestion/README.md`.
## Shared Package Imports
diff --git a/worker/package.json b/worker/package.json
index b24617823a..cf520cdffd 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -28,7 +28,8 @@
"refill-billing-event": "dotenv -e ../.env -- tsx src/scripts/refill-billing-event.ts",
"refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts",
"retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts",
- "test:native-codec": "LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE=1 vitest run nativeCodec.integration"
+ "test:native-codec": "LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE=1 vitest run nativeCodec.integration",
+ "test:otel-replay": "vitest run otelReplay"
},
"author": "[email protected]",
"dependencies": {
diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.ts
new file mode 100644
index 0000000000..71b35574e1
--- /dev/null
+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts
@@ -0,0 +1,129 @@
+import { randomUUID } from "node:crypto";
+
+import {
+ clickhouseClient,
+ type ClickhouseClientType,
+ type ResourceSpan,
+ OtelIngestionProcessor,
+} from "@langfuse/shared/src/server";
+
+import { ClickhouseWriter, TableName } from "../../services/ClickhouseWriter";
+import { IngestionService } from "../../services/IngestionService";
+import { processOtelEvents } from "../../features/otel-ingestion/processOtelEvents";
+
+type OtelReplayStoredRow = Record<string, unknown>;
+
+type RunOtelReplayParams = {
+ resourceSpans: ResourceSpan[];
+ projectId?: string;
+ fileKey?: string;
+};
+
+type OtelReplayResult = {
+ storedRows: OtelReplayStoredRow[];
+};
+
+/**
+ * Runs the production OTEL event phase through the production writer and reads
+ * the rows persisted in an isolated Memory table.
+ */
+export async function runOtelReplay(
+ params: RunOtelReplayParams,
+): Promise<OtelReplayResult> {
+ const projectId = params.projectId ?? "otel-replay-test";
+ const fileKey = params.fileKey ?? "otel-replay/test.json";
+ const client = clickhouseClient();
+ const suffix = randomUUID().replaceAll("-", "");
+ const tableName = `otel_replay_events_${suffix}`;
+ let writer: ClickhouseWriter | undefined;
+ let tableCreated = false;
+ let writerShutdown = false;
+
+ try {
+ await client.command({
+ query: `CREATE TABLE ${tableName} AS events_full ENGINE = Memory`,
+ });
+ tableCreated = true;
+
+ writer = ClickhouseWriter.getInstance({
+ insert: async (params: Parameters<ClickhouseClientType["insert"]>[0]) =>
+ client.insert({
+ ...params,
+ table:
+ params.table === TableName.EventsFull ? tableName : params.table,
+ }),
+ } as ClickhouseClientType);
+
+ const processor = new OtelIngestionProcessor({
+ projectId,
+ publicKey: "",
+ sdkName: "otel-replay",
+ sdkVersion: "test",
+ fileKey,
+ });
+ const ingestionService = new IngestionService(
+ {} as never,
+ {} as never,
+ writer,
+ {} as never,
+ );
+
+ await processOtelEvents({
+ processor,
+ resourceSpans: params.resourceSpans,
+ ingestionService,
+ projectId,
+ fileKey,
+ shouldWriteToEventsTable: true,
+ });
+
+ await writer.shutdown();
+ writerShutdown = true;
+
+ const pendingRows = writer.queue[TableName.EventsFull].length;
+ if (pendingRows > 0) {
+ throw new Error(
+ `ClickHouse replay writer retained ${pendingRows} EventsFull rows after shutdown`,
+ );
+ }
+
+ const result = await client.query({
+ query: `
+ SELECT *
+ FROM ${tableName}
+ ORDER BY trace_id, span_id
+ SETTINGS asterisk_include_materialized_columns = 1,
+ asterisk_include_alias_columns = 1
+ `,
+ format: "JSONEachRow",
+ });
+
+ return {
+ storedRows: (await result.json()) as OtelReplayStoredRow[],
+ };
+ } catch (error) {
+ throw new Error(
+ `OTEL replay harness failed for ClickHouse table ${tableName}: ${formatUnknownError(error)}`,
+ { cause: error },
+ );
+ } finally {
+ try {
+ if (writer && !writerShutdown) {
+ await writer.shutdown();
+ }
+ } finally {
+ if (writer) {
+ // Each corpus replay needs a fresh singleton and interval timer.
+ ClickhouseWriter["instance"] = null;
+ ClickhouseWriter["client"] = null;
+ }
+ if (tableCreated) {
+ await client.command({ query: `DROP TABLE IF EXISTS ${tableName}` });
+ }
+ }
+ }
+}
+
+function formatUnknownError(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
diff --git a/worker/src/__tests__/helpers/otelReplaySetup.ts b/worker/src/__tests__/helpers/otelReplaySetup.ts
new file mode 100644
index 0000000000..6c436b741d
--- /dev/null
+++ b/worker/src/__tests__/helpers/otelReplaySetup.ts
@@ -0,0 +1,122 @@
+import { vi } from "vitest";
+import { env } from "../../env";
+
+const otelReplayMocks = vi.hoisted(() => ({
+ findModel: vi.fn(),
+ uploadMediaForTrace: vi.fn(),
+ getPrompt: vi.fn(),
+ fetchObservationEvalRules: vi.fn(),
+ createObservationEvalSchedulerDeps: vi.fn(),
+ scheduleObservationEvals: vi.fn(),
+}));
+
+export { otelReplayMocks };
+
+vi.mock("../../env", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../../env")>();
+ return {
+ ...actual,
+ env: {
+ ...actual.env,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "false",
+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: "false",
+ LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",
+ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",
+ LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: "false",
+ LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES: 2 * 1024 * 1024,
+ },
+ };
+});
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ findModel: otelReplayMocks.findModel,
+ PromptService: class {
+ getPrompt(...args: unknown[]) {
+ return otelReplayMocks.getPrompt(...args);
+ }
+ },
+ uploadMediaForTrace: otelReplayMocks.uploadMediaForTrace,
+}));
+
+vi.mock(
+ "../../features/evaluation/observationEval",
+ async (importOriginal) => ({
+ ...(await importOriginal<
+ typeof import("../../features/evaluation/observationEval")
+ >()),
+ fetchObservationEvalRules: otelReplayMocks.fetchObservationEvalRules,
+ createObservationEvalSchedulerDeps:
+ otelReplayMocks.createObservationEvalSchedulerDeps,
+ scheduleObservationEvals: otelReplayMocks.scheduleObservationEvals,
+ }),
+);
+
+export function configureDefaultOtelReplayMocks(): void {
+ for (const mock of Object.values(otelReplayMocks)) {
+ mock.mockReset();
+ }
+
+ otelReplayMocks.findModel.mockResolvedValue({
+ model: null,
+ pricingTiers: [],
+ });
+ otelReplayMocks.uploadMediaForTrace.mockResolvedValue({
+ mediaId: "otel-replay-media",
+ outcome: "uploaded",
+ });
+ otelReplayMocks.getPrompt.mockResolvedValue(null);
+ otelReplayMocks.fetchObservationEvalRules.mockResolvedValue([]);
+ otelReplayMocks.createObservationEvalSchedulerDeps.mockReturnValue({});
+ otelReplayMocks.scheduleObservationEvals.mockResolvedValue(undefined);
+}
+
+type ReplayEnvironmentOptions = {
+ mediaUploadEnabled?: boolean;
+ overflowEnabled?: boolean;
+ overflowSizeLimitBytes?: number;
+};
+
+/**
+ * Disable unrelated production side effects while an isolated replay runs.
+ * The returned function restores the worker environment object exactly.
+ */
+export function configureOtelReplayEnvironment(
+ options: ReplayEnvironmentOptions = {},
+): () => void {
+ const original = {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED:
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED,
+ LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS:
+ env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,
+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
+ LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
+ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: env.LANGFUSE_S3_MEDIA_UPLOAD_PREFIX,
+ LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:
+ env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED,
+ LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:
+ env.LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES,
+ };
+
+ Object.assign(env, {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "false",
+ LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS: 3,
+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: options.mediaUploadEnabled
+ ? "true"
+ : "false",
+ LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: "otel-replay-test",
+ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: "otel-replay/",
+ LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: options.overflowEnabled
+ ? "true"
+ : "false",
+ LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:
+ options.overflowSizeLimitBytes ?? 2 * 1024 * 1024,
+ });
+
+ return () => Object.assign(env, original);
+}
+
+configureDefaultOtelReplayMocks();
diff --git a/worker/src/__tests__/otelReplay.integration.test.ts b/worker/src/__tests__/otelReplay.integration.test.ts
new file mode 100644
index 0000000000..485d2ea2cb
--- /dev/null
+++ b/worker/src/__tests__/otelReplay.integration.test.ts
@@ -0,0 +1,262 @@
+import "./helpers/otelReplaySetup";
+
+import { Decimal } from "decimal.js";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import type { ResourceSpan } from "@langfuse/shared/src/server";
+import { runOtelReplay } from "./helpers/otelReplayHarness";
+import {
+ configureDefaultOtelReplayMocks,
+ configureOtelReplayEnvironment,
+ otelReplayMocks,
+} from "./helpers/otelReplaySetup";
+
+const PROJECT_ID = "otel-replay-integration";
+const FILE_KEY = "otel-replay/integration.json";
+const TRACE_ID = "11111111111111111111111111111111";
+const FIRST_SPAN_ID = "2222222222222222";
+const SECOND_SPAN_ID = "3333333333333333";
+
+function createBufferId(hex: string): Buffer {
+ return Buffer.from(hex, "hex");
+}
+
+function stringAttribute(key: string, value: string) {
+ return { key, value: { stringValue: value } };
+}
+
+function intAttribute(key: string, value: number) {
+ return {
+ key,
+ value: { intValue: { low: value, high: 0, unsigned: false } },
+ };
+}
+
+function buildResourceSpans(): ResourceSpan[] {
+ const imageDataUri = `data:image/png;base64,${Buffer.from("small-image").toString("base64")}`;
+ const commonAttributes = [
+ stringAttribute("langfuse.observation.type", "generation"),
+ stringAttribute("langfuse.observation.model.name", "gpt-4o-mini"),
+ stringAttribute("langfuse.observation.prompt.name", "replay-prompt"),
+ intAttribute("langfuse.observation.prompt.version", 3),
+ stringAttribute("langfuse.trace.name", "replay-trace"),
+ stringAttribute("user.id", "replay-user"),
+ stringAttribute("session.id", "replay-session"),
+ stringAttribute("langfuse.observation.metadata.source", "integration"),
+ { key: "gen_ai.request.temperature", value: { doubleValue: 0.25 } },
+ ];
+
+ return [
+ {
+ resource: {
+ attributes: [
+ stringAttribute("service.name", "otel-replay-service"),
+ stringAttribute("service.version", "1.2.3"),
+ stringAttribute("telemetry.sdk.language", "typescript"),
+ stringAttribute("telemetry.sdk.name", "otel-replay"),
+ stringAttribute("telemetry.sdk.version", "0.1.0"),
+ stringAttribute("langfuse.environment", "integration"),
+ stringAttribute("langfuse.release", "replay-release"),
+ ],
+ },
+ scopeSpans: [
+ {
+ scope: { name: "langfuse-sdk", version: "3.0.0" },
+ spans: [
+ {
+ traceId: createBufferId(TRACE_ID),
+ spanId: createBufferId(FIRST_SPAN_ID),
+ name: "replay-generation-with-media",
+ kind: 1,
+ startTimeUnixNano: "1714488530686000000",
+ endTimeUnixNano: "1714488530687000000",
+ attributes: [
+ ...commonAttributes,
+ stringAttribute("langfuse.observation.input", imageDataUri),
+ stringAttribute(
+ "langfuse.observation.output",
+ "short replay answer",
+ ),
+ ],
+ status: { code: 0 },
+ },
+ {
+ traceId: createBufferId(TRACE_ID),
+ spanId: createBufferId(SECOND_SPAN_ID),
+ parentSpanId: createBufferId(FIRST_SPAN_ID),
+ name: "replay-generation-with-overflow",
+ kind: 1,
+ startTimeUnixNano: "1714488531686000000",
+ endTimeUnixNano: "1714488531687000000",
+ attributes: [
+ ...commonAttributes,
+ stringAttribute("langfuse.observation.input", "plain input"),
+ stringAttribute(
+ "langfuse.observation.output",
+ "x".repeat(1024),
+ ),
+ stringAttribute(
+ "langfuse.observation.cost_details",
+ JSON.stringify({
+ input: 1_500_000,
+ output: -1_500_000,
+ total: 1_500_000,
+ }),
+ ),
+ ],
+ status: { code: 0 },
+ },
+ ],
+ },
+ ],
+ },
+ ];
+}
+
+// Vitest retries can overlap unfinished ClickHouse I/O and shared replay mocks.
+describe(
+ "OTEL replay persists enriched observations after media and overflow handling",
+ { retry: 0, timeout: 120_000 },
+ () => {
+ let restoreEnvironment: (() => void) | undefined;
+
+ beforeEach(() => {
+ configureDefaultOtelReplayMocks();
+ restoreEnvironment = configureOtelReplayEnvironment({
+ mediaUploadEnabled: true,
+ overflowEnabled: true,
+ overflowSizeLimitBytes: 256,
+ });
+
+ const model = {
+ id: "otel-replay-model",
+ modelName: "gpt-4o-mini",
+ tokenizerId: "openai",
+ tokenizerConfig: { tokenizerModel: "gpt-4o" },
+ };
+ otelReplayMocks.findModel.mockResolvedValue({
+ model,
+ pricingTiers: [
+ {
+ id: "otel-replay-tier",
+ name: "Replay default",
+ isDefault: true,
+ priority: 0,
+ conditions: [],
+ prices: [
+ { usageType: "input", price: new Decimal("0.01") },
+ { usageType: "output", price: new Decimal("0.02") },
+ ],
+ },
+ ],
+ });
+ otelReplayMocks.getPrompt.mockResolvedValue({ id: "replay-prompt-id" });
+ otelReplayMocks.fetchObservationEvalRules.mockResolvedValue([{}]);
+ otelReplayMocks.createObservationEvalSchedulerDeps.mockReturnValue({});
+ otelReplayMocks.scheduleObservationEvals.mockResolvedValue(undefined);
+ otelReplayMocks.uploadMediaForTrace.mockImplementation(
+ async (params: { field: string }) => ({
+ mediaId:
+ params.field === "input" ? "image-media-id" : "overflow-media-id",
+ outcome: "uploaded" as const,
+ }),
+ );
+ });
+
+ afterEach(() => {
+ restoreEnvironment?.();
+ restoreEnvironment = undefined;
+ // The test owns this mutable mock setup; restore its deterministic
+ // defaults so another worker suite in the same process is unaffected.
+ configureDefaultOtelReplayMocks();
+ });
+
+ it("writes both enriched observations and preserves accounting through readback", async () => {
+ const { storedRows } = await runOtelReplay({
+ resourceSpans: buildResourceSpans(),
+ projectId: PROJECT_ID,
+ fileKey: FILE_KEY,
+ });
+
+ expect(storedRows).toHaveLength(2);
+ expect(otelReplayMocks.scheduleObservationEvals).toHaveBeenCalledTimes(2);
+
+ const rowsBySpanId = new Map(
+ storedRows.map((row) => [String(row.span_id), row]),
+ );
+ const firstRow = rowsBySpanId.get(FIRST_SPAN_ID);
+ const secondRow = rowsBySpanId.get(SECOND_SPAN_ID);
+ expect(firstRow).toBeDefined();
+ expect(secondRow).toBeDefined();
+
+ expect(firstRow).toMatchObject({
+ project_id: PROJECT_ID,
+ trace_id: TRACE_ID,
+ span_id: FIRST_SPAN_ID,
+ parent_span_id: "",
+ name: "replay-generation-with-media",
+ type: "GENERATION",
+ environment: "integration",
+ release: "replay-release",
+ trace_name: "replay-trace",
+ user_id: "replay-user",
+ session_id: "replay-session",
+ prompt_id: "replay-prompt-id",
+ prompt_name: "replay-prompt",
+ prompt_version: 3,
+ model_id: "otel-replay-model",
+ provided_model_name: "gpt-4o-mini",
+ usage_pricing_tier_id: "otel-replay-tier",
+ usage_pricing_tier_name: "Replay default",
+ ingestion_sdk_name: "otel-replay",
+ ingestion_sdk_version: "test",
+ blob_storage_file_path: FILE_KEY,
+ });
+ expect(String(firstRow?.input)).toContain(
+ "@@@langfuseMedia:type=image/png|id=image-media-id|source=",
+ );
+ expect(firstRow?.output).toBe("short replay answer");
+ expect(firstRow?.metadata_names).toContain("source");
+ expect(firstRow?.metadata_values).toContain("integration");
+ expect(
+ Number((firstRow?.usage_details as Record<string, unknown>).input),
+ ).toBeGreaterThan(0);
+ expect(
+ Number((firstRow?.usage_details as Record<string, unknown>).output),
+ ).toBeGreaterThan(0);
+ expect(
+ Number((firstRow?.cost_details as Record<string, unknown>).total),
+ ).toBeGreaterThan(0);
+
+ expect(secondRow).toMatchObject({
+ project_id: PROJECT_ID,
+ trace_id: TRACE_ID,
+ span_id: SECOND_SPAN_ID,
+ parent_span_id: FIRST_SPAN_ID,
+ name: "replay-generation-with-overflow",
+ type: "GENERATION",
+ prompt_id: "replay-prompt-id",
+ model_id: "otel-replay-model",
+ provided_model_name: "gpt-4o-mini",
+ usage_details: {},
+ provided_cost_details: {
+ input: 999_999.999999,
+ output: -999_999.999999,
+ total: 999_999.999999,
+ },
+ cost_details: {
+ input: 999_999.999999,
+ output: -999_999.999999,
+ total: 999_999.999999,
+ },
+ });
+ expect(secondRow?.input).toBe("plain input");
+ expect(secondRow?.output).toBe(
+ "@@@langfuseMedia:type=text/plain|id=overflow-media-id|source=field_size_limit@@@",
+ );
+
+ for (const row of storedRows) {
+ expect(Number(row.event_bytes)).toBeGreaterThan(0);
+ }
+ });
+ },
+);
diff --git a/worker/src/features/otel-ingestion/README.md b/worker/src/features/otel-ingestion/README.md
new file mode 100644
index 0000000000..e80466409c
--- /dev/null
+++ b/worker/src/features/otel-ingestion/README.md
@@ -0,0 +1,47 @@
+# OTEL replay tests
+
+Run the production direct-event processing phase against an isolated ClickHouse
+table:
+
+```sh
+pnpm --filter worker run test:otel-replay
+```
+
+Use the normal worker test environment and a migrated ClickHouse database. The
+test client uses `CLICKHOUSE_URL`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`, and
+`CLICKHOUSE_DB`; Vitest loads the repository's `.env` without overriding supplied
+environment variables. The configured user needs to read the `events_full`
+schema and create, insert into, select from, and drop the test tables. An
+unavailable ClickHouse server fails the run.
+
+The tests start with decoded OTLP resource spans and call `processOtelEvents`,
+the same function used by the queue. They exercise event normalization, media
+transformation, enrichment, overflow handling, and the real ClickHouse writer.
+External service responses and uploads are supplied by test doubles. Each run
+creates a uniquely named Memory table from `events_full` and uses the production
+writer singleton. An insert-only adapter forwards the normal insert parameters
+and substitutes the temporary table only for `EventsFull`. The helper shuts
+down the writer before reading persisted rows. Cleanup clears the test's
+singleton references so each sequential replay starts with the normal writer
+settings and a fresh interval timer, then drops the table.
+
+The Memory table exercises ClickHouse types, column defaults, and serialization.
+Production MergeTree deduplication and materialized views are outside this run.
+
+This is an integration test of the direct-event phase. HTTP/protobuf decoding,
+S3 input loading, masking, routing, and legacy writes remain outside this test
+boundary. The existing `test:native-codec` suite compares the TypeScript writer
+with Rust Native encoding inside ClickHouse. A future complete Rust processor
+can consume the same replay inputs and use that table comparison.
+
+## Remote execution
+
+The same command can run from a checkout on an authorized remote host, including
+a bastion with an appropriate runtime and database access, or locally through
+an existing tunnel. Install the repository dependencies and build the shared
+package as for local worker tests; provide the remote database connection in the
+environment. No code needs to execute on the bastion when using a tunnel.
+
+The current corpus is synthetic. Fetching production S3 batches and replaying a
+time window are separate extensions; this command does not discover production
+credentials, enqueue production jobs, or download customer data.