🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): recover OpenRouter error provenance (#15995)
Hassieb Pakzad3643b747e80b
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex a39c9315d8..d97a0d6b83 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -24,6 +24,7 @@ import {normalizeToolMetadataForObservation,normalizeEnvironment,DEFAULT_TRACE_ENVIRONMENT,+ LangfuseInternalTraceEnvironment,} from "../";import { LangfuseOtelSpanAttributes } from "./attributes";@@ -2253,9 +2254,48 @@ export class OtelIngestionProcessor {}}+ const openRouterEnvironment =+ this.extractOpenRouterFailedRequestEnvironment(+ attributes,+ resourceAttributes,+ );+ if (openRouterEnvironment) {+ return normalizeEnvironment(openRouterEnvironment);+ }+return DEFAULT_TRACE_ENVIRONMENT;}+ private extractOpenRouterFailedRequestEnvironment(+ attributes: Record<string, unknown>,+ resourceAttributes: Record<string, unknown>,+ ): LangfuseInternalTraceEnvironment.LLMJudge | undefined {+ if (+ resourceAttributes["service.name"] !== "openrouter" ||+ attributes["openrouter.source"] !== "openrouter"+ ) {+ return undefined;+ }++ const completion = this.parseJsonPayload(attributes["gen_ai.completion"]);+ if (+ !OtelIngestionProcessor.isPlainObject(completion) ||+ completion.completion !== null ||+ !OtelIngestionProcessor.isPlainObject(completion.rawRequest) ||+ !OtelIngestionProcessor.isPlainObject(completion.rawRequest.trace)+ ) {+ return undefined;+ }++ // OpenRouter Broadcast can omit custom trace metadata when a request fails,+ // but echoes the original request here. Recover only our reserved evaluator+ // marker so failed judge calls cannot be scheduled as fresh evaluations.+ return completion.rawRequest.trace.environment ===+ LangfuseInternalTraceEnvironment.LLMJudge+ ? LangfuseInternalTraceEnvironment.LLMJudge+ : undefined;+ }+private extractName(spanName: string,attributes: Record<string, unknown>,diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 43aa561691..ce77e36b2d 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -69,9 +69,11 @@ async function convertOtelSpanToIngestionEvent(resourceSpan: any,seenTraces: Set<string>,publicKey?: string,+ config: Partial<OtelIngestionProcessorConfig> = {},): Promise<TestIngestionEvent[]> {const processor = createTestOtelProcessor({publicKey: publicKey ?? "",+ ...config,});// For tests, we bypass Redis initialization and directly set the seen traces@@ -1271,6 +1273,112 @@ describe("OTel Resource Span Mapping", () => {},};+ const createOpenRouterBroadcastSpan = ({+ serviceName = "openrouter",+ source = "openrouter",+ completion = null,+ rawRequestEnvironment = "langfuse-llm-as-a-judge",+ explicitEnvironment,+ }: {+ serviceName?: string;+ source?: string;+ completion?: unknown;+ rawRequestEnvironment?: string;+ explicitEnvironment?: string;+ } = {}) => ({+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: serviceName },+ },+ ],+ },+ scopeSpans: [+ {+ scope: { name: "openrouter" },+ spans: [+ {+ ...defaultSpanProps,+ attributes: [+ {+ key: "openrouter.source",+ value: { stringValue: source },+ },+ ...(explicitEnvironment+ ? [+ {+ key: "langfuse.environment",+ value: { stringValue: explicitEnvironment },+ },+ ]+ : []),+ {+ key: "gen_ai.completion",+ value: {+ stringValue: JSON.stringify({+ completion,+ rawRequest: {+ trace: { environment: rawRequestEnvironment },+ },+ }),+ },+ },+ ],+ },+ ],+ },+ ],+ });++ it("recovers the internal evaluator environment from failed OpenRouter Broadcast requests", async () => {+ const events = await convertOtelSpanToIngestionEvent(+ createOpenRouterBroadcastSpan(),+ new Set(),+ undefined,+ { isLangfuseInternal: true },+ );++ expect(events.map((event) => event.body.environment)).toEqual([+ "llm-as-a-judge",+ "llm-as-a-judge",+ ]);+ });++ it("prefers an explicit environment over the OpenRouter failed-request fallback", async () => {+ const events = await convertOtelSpanToIngestionEvent(+ createOpenRouterBroadcastSpan({+ explicitEnvironment: "customer-production",+ }),+ new Set(),+ );++ expect(events.map((event) => event.body.environment)).toEqual([+ "customer-production",+ "customer-production",+ ]);+ });++ it.each([+ ["a non-OpenRouter service", { serviceName: "application" }],+ ["a non-OpenRouter source", { source: "application" }],+ ["a successful completion", { completion: {} }],+ ["a customer environment", { rawRequestEnvironment: "production" }],+ ])(+ "does not recover evaluator provenance from %s",+ async (_name, options) => {+ const events = await convertOtelSpanToIngestionEvent(+ createOpenRouterBroadcastSpan(options),+ new Set(),+ );++ expect(events.map((event) => event.body.environment)).toEqual([+ "default",+ "default",+ ]);+ },+ );+it.each([["missing attribute", undefined, false],["boolean true", { boolValue: true }, true],