Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): recover OpenRouter error provenance (#15995)

Hassieb Pakzad3643b747e80b
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index a39c9315d8..d97a0d6b83 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -24,6 +24,7 @@ import {
normalizeToolMetadataForObservation,
normalizeEnvironment,
DEFAULT_TRACE_ENVIRONMENT,
+ LangfuseInternalTraceEnvironment,
} from "../";
import { LangfuseOtelSpanAttributes } from "./attributes";
@@ -2253,9 +2254,48 @@ export class OtelIngestionProcessor {
}
}
+ const openRouterEnvironment =
+ this.extractOpenRouterFailedRequestEnvironment(
+ attributes,
+ resourceAttributes,
+ );
+ if (openRouterEnvironment) {
+ return normalizeEnvironment(openRouterEnvironment);
+ }
+
return DEFAULT_TRACE_ENVIRONMENT;
}
+ private extractOpenRouterFailedRequestEnvironment(
+ attributes: Record<string, unknown>,
+ resourceAttributes: Record<string, unknown>,
+ ): LangfuseInternalTraceEnvironment.LLMJudge | undefined {
+ if (
+ resourceAttributes["service.name"] !== "openrouter" ||
+ attributes["openrouter.source"] !== "openrouter"
+ ) {
+ return undefined;
+ }
+
+ const completion = this.parseJsonPayload(attributes["gen_ai.completion"]);
+ if (
+ !OtelIngestionProcessor.isPlainObject(completion) ||
+ completion.completion !== null ||
+ !OtelIngestionProcessor.isPlainObject(completion.rawRequest) ||
+ !OtelIngestionProcessor.isPlainObject(completion.rawRequest.trace)
+ ) {
+ return undefined;
+ }
+
+ // OpenRouter Broadcast can omit custom trace metadata when a request fails,
+ // but echoes the original request here. Recover only our reserved evaluator
+ // marker so failed judge calls cannot be scheduled as fresh evaluations.
+ return completion.rawRequest.trace.environment ===
+ LangfuseInternalTraceEnvironment.LLMJudge
+ ? LangfuseInternalTraceEnvironment.LLMJudge
+ : undefined;
+ }
+
private extractName(
spanName: string,
attributes: Record<string, unknown>,
diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
index 43aa561691..ce77e36b2d 100644
--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
@@ -69,9 +69,11 @@ async function convertOtelSpanToIngestionEvent(
resourceSpan: any,
seenTraces: Set<string>,
publicKey?: string,
+ config: Partial<OtelIngestionProcessorConfig> = {},
): Promise<TestIngestionEvent[]> {
const processor = createTestOtelProcessor({
publicKey: publicKey ?? "",
+ ...config,
});
// For tests, we bypass Redis initialization and directly set the seen traces
@@ -1271,6 +1273,112 @@ describe("OTel Resource Span Mapping", () => {
},
};
+ const createOpenRouterBroadcastSpan = ({
+ serviceName = "openrouter",
+ source = "openrouter",
+ completion = null,
+ rawRequestEnvironment = "langfuse-llm-as-a-judge",
+ explicitEnvironment,
+ }: {
+ serviceName?: string;
+ source?: string;
+ completion?: unknown;
+ rawRequestEnvironment?: string;
+ explicitEnvironment?: string;
+ } = {}) => ({
+ resource: {
+ attributes: [
+ {
+ key: "service.name",
+ value: { stringValue: serviceName },
+ },
+ ],
+ },
+ scopeSpans: [
+ {
+ scope: { name: "openrouter" },
+ spans: [
+ {
+ ...defaultSpanProps,
+ attributes: [
+ {
+ key: "openrouter.source",
+ value: { stringValue: source },
+ },
+ ...(explicitEnvironment
+ ? [
+ {
+ key: "langfuse.environment",
+ value: { stringValue: explicitEnvironment },
+ },
+ ]
+ : []),
+ {
+ key: "gen_ai.completion",
+ value: {
+ stringValue: JSON.stringify({
+ completion,
+ rawRequest: {
+ trace: { environment: rawRequestEnvironment },
+ },
+ }),
+ },
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ });
+
+ it("recovers the internal evaluator environment from failed OpenRouter Broadcast requests", async () => {
+ const events = await convertOtelSpanToIngestionEvent(
+ createOpenRouterBroadcastSpan(),
+ new Set(),
+ undefined,
+ { isLangfuseInternal: true },
+ );
+
+ expect(events.map((event) => event.body.environment)).toEqual([
+ "llm-as-a-judge",
+ "llm-as-a-judge",
+ ]);
+ });
+
+ it("prefers an explicit environment over the OpenRouter failed-request fallback", async () => {
+ const events = await convertOtelSpanToIngestionEvent(
+ createOpenRouterBroadcastSpan({
+ explicitEnvironment: "customer-production",
+ }),
+ new Set(),
+ );
+
+ expect(events.map((event) => event.body.environment)).toEqual([
+ "customer-production",
+ "customer-production",
+ ]);
+ });
+
+ it.each([
+ ["a non-OpenRouter service", { serviceName: "application" }],
+ ["a non-OpenRouter source", { source: "application" }],
+ ["a successful completion", { completion: {} }],
+ ["a customer environment", { rawRequestEnvironment: "production" }],
+ ])(
+ "does not recover evaluator provenance from %s",
+ async (_name, options) => {
+ const events = await convertOtelSpanToIngestionEvent(
+ createOpenRouterBroadcastSpan(options),
+ new Set(),
+ );
+
+ expect(events.map((event) => event.body.environment)).toEqual([
+ "default",
+ "default",
+ ]);
+ },
+ );
+
it.each([
["missing attribute", undefined, false],
["boolean true", { boolValue: true }, true],