πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ingestion): auto-direct-write OTel traffic for post-cutoff orgs (#15813)
Steffen Schmitzb9e4146b8932
diff --git a/.env.dev.example b/.env.dev.exampleindex 418e864f51..2cd6a20701 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -225,6 +225,10 @@ LANGFUSE_MIGRATION_V4_WRITE_MODE=dualLANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN=true# OTel ingestion behaviour: `dual_write` (SDK-version dispatch) or `direct`.LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR=dual_write+# Cloud-only rollout boundary (ISO date) for automatic direct OTel writes:+# organizations created on/after this date direct-write plain-OTel traffic.+# Unset disables it; see worker/src/env.ts for the full semantics.+# LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF=# Keep the WIP historic backfill dormant for local dev / CI. The v4 code default# is `true`, but the general test suite should not run the backfill underneath it# (it has a dedicated e2e spec). Set to `true` to exercise the backfill locally.diff --git a/worker/src/env.ts b/worker/src/env.tsindex abc2ffff6d..530796ce12 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -534,6 +534,21 @@ const EnvSchema = z.object({LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR: z.enum(["dual_write", "direct"]).default("direct"),+ // Cloud-only rollout boundary for automatic direct OTel event writes.+ // Organizations created on or after this date are past the point where the v4+ // preview is force-enabled and cannot be switched off, so the events table is+ // the only surface they read; their OTLP traffic therefore takes the direct+ // write even without an `x-langfuse-ingestion-version: 4` header. Applies to+ // non-Langfuse-SDK exports only β an older SDK keeps its established+ // dual-write shape. An ISO date (YYYY-MM-DD) read as midnight UTC; a plain+ // date is enough precision and easier to reason about than a timestamp.+ // Unset disables the rule, which is the right default when self-hosting:+ // those deployments move the whole deployment at once via+ // LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR=direct instead of rolling a+ // tenant cohort forward.+ LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF: z.iso+ .date()+ .optional(),LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN: z.enum(["true", "false"]).default("true"),diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex 636d7eb202..7a91be20b8 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -1,12 +1,248 @@import { describe, it, expect } from "vitest";import {+ batchContainsLangfuseScope,checkHeaderBasedDirectWrite,checkSdkVersionRequirements,getSdkInfoFromResourceSpans,+ isLangfuseSdkTraffic,+ isOrgPastOtelDirectWriteCutoff,+ resolveOtelWritePath,shouldProcessLegacyOtelMedia,type SdkInfo,} from "../otelIngestionQueue";+describe("isOrgPastOtelDirectWriteCutoff", () => {+ const cutoff = "2026-08-06";+ const inScope = {+ orgCreatedAt: new Date("2026-08-07T09:00:00.000Z"),+ cutoff,+ };++ it("includes organizations created after the cutoff", () => {+ expect(isOrgPastOtelDirectWriteCutoff(inScope)).toBe(true);+ });++ it("includes organizations created exactly at midnight UTC on the cutoff date", () => {+ expect(+ isOrgPastOtelDirectWriteCutoff({+ ...inScope,+ orgCreatedAt: new Date("2026-08-06T00:00:00.000Z"),+ }),+ ).toBe(true);+ });++ it("excludes organizations created just before the cutoff date", () => {+ expect(+ isOrgPastOtelDirectWriteCutoff({+ ...inScope,+ orgCreatedAt: new Date("2026-08-05T23:59:59.999Z"),+ }),+ ).toBe(false);+ });++ it("is disabled when the cutoff is unset", () => {+ expect(+ isOrgPastOtelDirectWriteCutoff({ ...inScope, cutoff: undefined }),+ ).toBe(false);+ });++ // An unresolvable organization must not read as "created at epoch" (in scope+ // for no cutoff) nor as "created now" (in scope for every cutoff).+ it.each([+ { orgCreatedAt: null, label: "null" },+ { orgCreatedAt: undefined, label: "undefined" },+ { orgCreatedAt: new Date("nonsense"), label: "an invalid Date" },+ ])(+ "keeps the pre-cutoff behaviour when the org date is $label",+ ({ orgCreatedAt }) => {+ expect(isOrgPastOtelDirectWriteCutoff({ ...inScope, orgCreatedAt })).toBe(+ false,+ );+ },+ );+});++describe("isLangfuseSdkTraffic", () => {+ it.each([+ { params: { sdkName: "python" }, expected: true, label: "sdk-name header" },+ {+ params: { sdkName: "javascript" },+ expected: true,+ label: "javascript sdk-name header",+ },+ {+ params: { scopeName: "langfuse-sdk" },+ expected: true,+ label: "langfuse instrumentation scope",+ },+ {+ params: { scopeName: "LangFuse-SDK" },+ expected: true,+ label: "scope match is case-insensitive",+ },+ // The header is normalized to this sentinel when absent, so it must not+ // read as a Langfuse SDK β that would exclude all plain OTel traffic.+ {+ params: { sdkName: "unknown" },+ expected: false,+ label: "unknown sdk-name sentinel",+ },+ {+ params: { sdkName: "unknown", scopeName: "openlit" },+ expected: false,+ label: "third-party scope with no sdk-name",+ },+ { params: {}, expected: false, label: "no signal at all" },+ {+ params: { scopeName: null },+ expected: false,+ label: "null scope name",+ },+ ])("$label β $expected", ({ params, expected }) => {+ expect(isLangfuseSdkTraffic(params)).toBe(expected);+ });+});++describe("batchContainsLangfuseScope", () => {+ const scoped = (...names: string[]) => ({+ scopeSpans: names.map((name) => ({ scope: { name }, spans: [] })),+ });++ it("finds a Langfuse scope that is not the batch's first scope", () => {+ expect(+ batchContainsLangfuseScope([scoped("openlit", "langfuse-sdk")]),+ ).toBe(true);+ });++ it("finds a Langfuse scope on a later resource", () => {+ expect(+ batchContainsLangfuseScope([scoped("openlit"), scoped("langfuse-sdk")]),+ ).toBe(true);+ });++ it("keeps scanning past a malformed resource to a later Langfuse scope", () => {+ expect(+ batchContainsLangfuseScope([+ { scopeSpans: {} } as never,+ scoped("langfuse-sdk"),+ ]),+ ).toBe(true);+ });++ it("rejects batches with only third-party scopes", () => {+ expect(+ batchContainsLangfuseScope([scoped("openlit", "openinference")]),+ ).toBe(false);+ });++ it.each([+ { input: [], label: "an empty batch" },+ { input: [{}], label: "a resource without scopeSpans" },+ {+ // Malformed OTLP reaches this code path; scanning must not throw.+ input: [{ scopeSpans: {} } as never],+ label: "a non-array scopeSpans shape",+ },+ // The payload is JSON.parse output with no shape guarantee at all β a+ // non-array top level must read as scope-less, not crash the job.+ { input: {} as never, label: "a non-array payload" },+ { input: "nonsense" as never, label: "a string payload" },+ ])("rejects $label", ({ input }) => {+ expect(batchContainsLangfuseScope(input)).toBe(false);+ });+});++describe("resolveOtelWritePath", () => {+ const none = {+ headerBasedDirectWrite: false,+ envForcesDirect: false,+ scopeBasedDirectWrite: false,+ orgCutoffForcesDirect: false,+ langfuseSdkTraffic: false,+ };++ it("falls back to the dual write when no signal qualifies", () => {+ expect(resolveOtelWritePath(none)).toEqual({+ useDirectEventWrite: false,+ writePath: "dual",+ });+ });++ it.each([+ { signal: "headerBasedDirectWrite", writePath: "direct_header" },+ { signal: "envForcesDirect", writePath: "direct_env" },+ { signal: "scopeBasedDirectWrite", writePath: "direct_scope" },+ { signal: "orgCutoffForcesDirect", writePath: "direct_org_cutoff" },+ ] as const)("$signal alone routes to $writePath", ({ signal, writePath }) => {+ expect(resolveOtelWritePath({ ...none, [signal]: true })).toEqual({+ useDirectEventWrite: true,+ writePath,+ });+ });++ // The org cutoff must not absorb batches another signal already routed+ // directly, otherwise the metric overstates what the rollout moved.+ it.each([+ { signal: "headerBasedDirectWrite", writePath: "direct_header" },+ { signal: "envForcesDirect", writePath: "direct_env" },+ { signal: "scopeBasedDirectWrite", writePath: "direct_scope" },+ ] as const)(+ "attributes to $writePath rather than the org cutoff when both apply",+ ({ signal, writePath }) => {+ expect(+ resolveOtelWritePath({+ ...none,+ [signal]: true,+ orgCutoffForcesDirect: true,+ }),+ ).toEqual({ useDirectEventWrite: true, writePath });+ },+ );++ // An older Langfuse SDK keeps the dual write: its trace-level attributes+ // surface on the synthetic root observation that only the dual path+ // materializes, so flipping it would change data those users already read.+ it("leaves Langfuse SDK traffic on the dual write despite the org cutoff", () => {+ expect(+ resolveOtelWritePath({+ ...none,+ orgCutoffForcesDirect: true,+ langfuseSdkTraffic: true,+ }),+ ).toEqual({ useDirectEventWrite: false, writePath: "dual" });+ });++ it("applies the org cutoff to plain OTel traffic", () => {+ expect(+ resolveOtelWritePath({+ ...none,+ orgCutoffForcesDirect: true,+ langfuseSdkTraffic: false,+ }),+ ).toEqual({ useDirectEventWrite: true, writePath: "direct_org_cutoff" });+ });++ // The exclusion is scoped to the cutoff rule β it must not hold back a batch+ // that a header, the env override, or a recognized scope already qualified.+ it.each([+ { signal: "headerBasedDirectWrite", writePath: "direct_header" },+ { signal: "envForcesDirect", writePath: "direct_env" },+ { signal: "scopeBasedDirectWrite", writePath: "direct_scope" },+ ] as const)(+ "still routes $writePath for Langfuse SDK traffic",+ ({ signal, writePath }) => {+ expect(+ resolveOtelWritePath({+ ...none,+ [signal]: true,+ orgCutoffForcesDirect: true,+ langfuseSdkTraffic: true,+ }),+ ).toEqual({ useDirectEventWrite: true, writePath });+ },+ );+});+describe("shouldProcessLegacyOtelMedia", () => {it("processes media whenever legacy tables are written", () => {expect(diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex 4d0da1994b..95ef4dce6c 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -22,6 +22,7 @@ import {ResourceSpan,type IngestionAttribution,UNKNOWN_INGESTION_SDK_VALUE,+ LocalCache,} from "@langfuse/shared/src/server";import {applyIngestionMasking,@@ -113,6 +114,222 @@ export function checkHeaderBasedDirectWrite(params: {return false;}+export type OtelWritePath =+ | "dual"+ | "direct_header"+ | "direct_env"+ | "direct_scope"+ | "direct_org_cutoff";++/**+ * Whether a batch was exported by a Langfuse SDK rather than a plain OTel+ * pipeline.+ *+ * Either signal is sufficient: the SDKs send `x-langfuse-sdk-name` (normalized+ * to UNKNOWN_INGESTION_SDK_VALUE when absent), and their spans carry a+ * "langfuse"-prefixed instrumentation scope. A third party that sets the+ * header anyway is treated as SDK traffic, which is the conservative direction+ * β it keeps the established write path.+ */+export function isLangfuseSdkTraffic(params: {+ sdkName?: string;+ scopeName?: string | null;+}): boolean {+ const { sdkName, scopeName } = params;++ if (sdkName && sdkName !== UNKNOWN_INGESTION_SDK_VALUE) {+ return true;+ }++ return Boolean(+ scopeName && String(scopeName).toLowerCase().includes("langfuse"),+ );+}++/**+ * Whether any instrumentation scope in the batch identifies a Langfuse SDK.+ *+ * The scope-based version check reads only the first scope (all spans a+ * Langfuse SDK exports share it), but the org-cutoff exclusion must be+ * conservative: a collector-forwarded batch can mix SDK spans with third-party+ * scopes in any order, and one Langfuse scope anywhere is enough to keep the+ * whole batch on its established write path. Scans scope metadata only β no+ * span bodies are touched.+ */+export function batchContainsLangfuseScope(+ resourceSpans: ResourceSpan[],+): boolean {+ // Malformed OTLP shapes reach this code (non-array payloads, non-array+ // scopeSpans and the like); every other resourceSpans consumer guards the+ // same way. Guard per resource so a malformed one reads as scope-less rather+ // than aborting the scan β it must not hide a Langfuse scope on a later+ // resource, nor fail the batch.+ if (!Array.isArray(resourceSpans)) {+ return false;+ }+ return resourceSpans.some(+ (resourceSpan) =>+ Array.isArray(resourceSpan?.scopeSpans) &&+ resourceSpan.scopeSpans.some((scopeSpan) =>+ isLangfuseSdkTraffic({ scopeName: scopeSpan?.scope?.name ?? null }),+ ),+ );+}++/**+ * Whether an organization's signup date puts it past the direct-write cutoff.+ *+ * Organizations created on or after the cutoff are past the point where the v4+ * preview is force-enabled and cannot be switched off, so the events table is+ * the only surface they read. Without `x-langfuse-ingestion-version: 4` their+ * OTLP traffic would take the dual-write path and appear with up to ~10 minutes+ * of delay β a broken first experience for an exporter that is otherwise set up+ * correctly.+ *+ * Deliberately date-based: there is no per-organization or per-project v4+ * column, only a per-user preview flag, so signup date is the only cohort+ * signal available. It mirrors the UI-side rollout boundary but keys on the+ * single organization owning the API key rather than the oldest organization a+ * user belongs to β ingestion has no user context.+ *+ * The rule is Cloud-only, gated at the call site+ * (isProjectOrgPastOtelDirectWriteCutoff): self-hosted deployments move the+ * whole deployment at once via LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR.+ */+export function isOrgPastOtelDirectWriteCutoff(params: {+ /** Organization signup date; nullish when it could not be resolved. */+ orgCreatedAt: Date | null | undefined;+ /** ISO date (YYYY-MM-DD), read as midnight UTC. Unset disables the rule. */+ cutoff: string | undefined;+}): boolean {+ const { orgCreatedAt, cutoff } = params;++ if (!cutoff || !orgCreatedAt) {+ return false;+ }++ // An invalid date on either side compares as NaN, which is false for every+ // comparison β unparsable input keeps the pre-cutoff behaviour by itself.+ return orgCreatedAt.getTime() >= Date.parse(`${cutoff}T00:00:00.000Z`);+}++// An organization's signup date never changes, and the project β organization+// mapping changes only through the rare `projects.transfer` mutation β a stale+// entry then self-corrects within the TTL, which is acceptable for a routing+// hint whose destinations both land the data. So the cache only needs to bound+// memory and let deleted projects fall out. A miss costs one indexed Postgres+// read, and only for batches that no other signal already routed (see below).+const ORG_CREATED_AT_CACHE_TTL_MS = 60 * 60 * 1000;+const orgCreatedAtCache = new LocalCache<{ createdAt: Date }>({+ namespace: "otel_org_created_at",+ enabled: true,+ ttlMs: ORG_CREATED_AT_CACHE_TTL_MS,+ max: 10_000,+});++/**+ * Whether the org-created cutoff rule can apply in this deployment at all:+ * it needs a configured date and only runs on Langfuse Cloud. Work that only+ * feeds the rule (the batch scope scan, the signup-date lookup) is skipped+ * while this is false β the shipped default and every self-hosted deployment.+ */+function isOtelDirectWriteOrgCutoffConfigured(): boolean {+ return Boolean(+ env.LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF &&+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ );+}++/**+ * Resolve whether the project's organization is past the direct-write cutoff.+ *+ * Keyed on project rather than the payload's optional `orgId` so replayed jobs+ * enqueued before that field existed resolve too. Returns false when the+ * project cannot be found, which keeps the batch on its existing path.+ */+export async function isProjectOrgPastOtelDirectWriteCutoff(+ projectId: string,+): Promise<boolean> {+ if (!isOtelDirectWriteOrgCutoffConfigured()) {+ return false;+ }+ const cutoff = env.LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF;++ try {+ const { value } = await orgCreatedAtCache.getOrLoad(projectId, async () => {+ const project = await prisma.project.findUnique({+ where: { id: projectId },+ select: { organization: { select: { createdAt: true } } },+ });++ return {+ value: project+ ? { createdAt: project.organization.createdAt }+ : undefined,+ source: "postgres",+ };+ });++ return isOrgPastOtelDirectWriteCutoff({+ orgCreatedAt: value?.createdAt,+ cutoff,+ });+ } catch (error) {+ // A lookup failure must not fail the batch: fall back to the pre-cutoff+ // path, which is still a correct destination in dual write mode.+ logger.warn(+ `Failed to resolve organization signup date for project ${projectId}; keeping the pre-cutoff OTel write path`,+ error,+ );+ return false;+ }+}++/**+ * Resolve which persistence path a batch takes, and which signal decided it.+ *+ * Precedence is header > env > scope > org cutoff. The org cutoff sits last so+ * the `direct_org_cutoff` label counts only the traffic that rule genuinely+ * moves off the dual path, rather than absorbing batches that a header or a+ * recognized SDK scope would have routed directly anyway.+ *+ * The org cutoff additionally excludes Langfuse SDK traffic. An older SDK has+ * an established dual-write shape β its trace-level attributes surface on the+ * synthetic root observation that only the dual path materializes β so flipping+ * it would change data those users already read. A plain OTel exporter has no+ * such prior expectation, and is the case the cutoff exists for.+ */+export function resolveOtelWritePath(params: {+ headerBasedDirectWrite: boolean;+ envForcesDirect: boolean;+ scopeBasedDirectWrite: boolean;+ orgCutoffForcesDirect: boolean;+ langfuseSdkTraffic: boolean;+}): { useDirectEventWrite: boolean; writePath: OtelWritePath } {+ const {+ headerBasedDirectWrite,+ envForcesDirect,+ scopeBasedDirectWrite,+ orgCutoffForcesDirect,+ langfuseSdkTraffic,+ } = params;++ if (headerBasedDirectWrite) {+ return { useDirectEventWrite: true, writePath: "direct_header" };+ }+ if (envForcesDirect) {+ return { useDirectEventWrite: true, writePath: "direct_env" };+ }+ if (scopeBasedDirectWrite) {+ return { useDirectEventWrite: true, writePath: "direct_scope" };+ }+ if (orgCutoffForcesDirect && !langfuseSdkTraffic) {+ return { useDirectEventWrite: true, writePath: "direct_org_cutoff" };+ }++ return { useDirectEventWrite: false, writePath: "dual" };+}+function extractBaseSdkVersion(sdkVersion: string): string {const version = sdkVersion.trim();@@ -180,7 +397,7 @@ export function checkSdkVersionRequirements(const { scopeName, scopeVersion, telemetrySdkLanguage } = sdkInfo;// Must be a Langfuse SDK- if (!scopeName || !String(scopeName).toLowerCase().includes("langfuse")) {+ if (!isLangfuseSdkTraffic({ scopeName })) {return false;}@@ -397,6 +614,9 @@ export const otelIngestionQueueProcessorBuilder = (//// Priority 2 (fallback): Per-span OTEL scope inspection (legacy).// - scope.name contains "langfuse", sdk-experiment environment, Python >= 3.9.0 or JS >= 4.4.0+ //+ // Priority 3 (last resort): the owning organization signed up past the+ // Cloud direct-write cutoff. Non-Langfuse-SDK exports only (see below).const headerBasedDirectWrite = checkHeaderBasedDirectWrite({sdkName: job.data.payload.sdkName,sdkVersion: job.data.payload.sdkVersion,@@ -407,9 +627,16 @@ export const otelIngestionQueueProcessorBuilder = (// LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR=direct forces every batch// onto the direct events_full path regardless of SDK headers/scopes.const envForcesDirect = v4ForceDirectOtelWrite(env);- let useDirectEventWrite = envForcesDirect || headerBasedDirectWrite;- if (!useDirectEventWrite) {+ // Evaluated whenever the higher-precedence header/env signals did not+ // already qualify, so the write_path label can attribute the decision+ // exactly and `direct_org_cutoff` counts only the traffic that rule+ // genuinely moves off the dual path.+ let scopeBasedDirectWrite = false;+ let langfuseSdkTraffic = isLangfuseSdkTraffic({+ sdkName: job.data.payload.sdkName,+ });+ if (!envForcesDirect && !headerBasedDirectWrite) {const hasExperimentEnvironment = observations.some((o) => {const body = o.body as { environment?: string };return body.environment === "sdk-experiment";@@ -422,22 +649,41 @@ export const otelIngestionQueueProcessorBuilder = (scopeVersion: null,telemetrySdkLanguage: null,};- useDirectEventWrite = checkSdkVersionRequirements(+ scopeBasedDirectWrite = checkSdkVersionRequirements(sdkInfo,hasExperimentEnvironment,);+ // An older SDK that predates the header still identifies itself through+ // its instrumentation scope, so both signals have to be consulted+ // before the org cutoff may flip a batch. Checked across every scope in+ // the batch, not just the first β collector-forwarded batches can mix+ // SDK spans with third-party scopes in any order. The scan only feeds+ // the cutoff rule, so it is skipped while that rule cannot apply.+ langfuseSdkTraffic =+ langfuseSdkTraffic ||+ (isOtelDirectWriteOrgCutoffConfigured() &&+ batchContainsLangfuseScope(parsedSpans));}- let writePath: "dual" | "direct_header" | "direct_env" | "direct_scope";- if (!useDirectEventWrite) {- writePath = "dual";- } else if (headerBasedDirectWrite) {- writePath = "direct_header";- } else if (envForcesDirect) {- writePath = "direct_env";- } else {- writePath = "direct_scope";- }+ // Resolved only when it can still change the outcome β every+ // higher-precedence signal has declined and this is not SDK traffic β so+ // the Postgres lookup stays off the path for batches already routed by a+ // header. resolveOtelWritePath remains authoritative for the rule; this+ // guard is purely an optimization.+ const orgCutoffForcesDirect =+ !headerBasedDirectWrite &&+ !envForcesDirect &&+ !scopeBasedDirectWrite &&+ !langfuseSdkTraffic &&+ (await isProjectOrgPastOtelDirectWriteCutoff(projectId));++ const { useDirectEventWrite, writePath } = resolveOtelWritePath({+ headerBasedDirectWrite,+ envForcesDirect,+ scopeBasedDirectWrite,+ orgCutoffForcesDirect,+ langfuseSdkTraffic,+ });span?.setAttribute("langfuse.ingestion.otel.write_path", writePath);recordIncrement("langfuse.ingestion.otel.write_path", 1, {