🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): publish one generation for evaluator executions (#18415)
Tobias Wochingerdbf2316e1cb5
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.tsindex 3b266de579..95eeb7775e 100644--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts@@ -151,7 +151,30 @@ export function createAiSdkTelemetryCapture(params: {? buildEvaluationAttributes(traceSinkParams.evaluationContext): undefined;- const rootSpan: Span = tracer.startSpan(+ const experimentContext = traceSinkParams.eventsWriter?.experimentContext;+ // Customers pay per observation and only use an evaluator trace to inspect+ // input, output, and cost, so the model call is written on this root span+ // instead of adding another one. Prompt experiments still need the child span.+ const writeGenerationOnRoot =+ evaluationAttributes !== undefined && experimentContext === undefined;+ const childSpanMetadata = traceSinkParams.metadata+ ? Object.fromEntries(+ Object.entries(traceSinkParams.metadata).filter(+ ([key]) => key !== "structured_output_schema",+ ),+ )+ : undefined;+ const promptAttributes = traceSinkParams.prompt+ ? {+ // Link the LLM generation to the resolved Langfuse prompt.+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:+ traceSinkParams.prompt.name,+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:+ traceSinkParams.prompt.version,+ }+ : undefined;++ const rootSpan = tracer.startSpan(traceSinkParams.traceName,{attributes: {@@ -164,6 +187,7 @@ export function createAiSdkTelemetryCapture(params: {}: {}),...(evaluationAttributes ?? {}),+ ...(writeGenerationOnRoot ? (promptAttributes ?? {}) : {}),...(traceSinkParams.metadata? {[LangfuseOtelSpanAttributes.TRACE_METADATA]: JSON.stringify(@@ -171,6 +195,12 @@ export function createAiSdkTelemetryCapture(params: {),}: {}),+ ...(writeGenerationOnRoot && childSpanMetadata+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]:+ JSON.stringify(childSpanMetadata),+ }+ : {}),...(serializedInput !== undefined? {[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: serializedInput,@@ -183,7 +213,6 @@ export function createAiSdkTelemetryCapture(params: {);const activeContext = trace.setSpan(ROOT_CONTEXT, rootSpan);- const experimentContext = traceSinkParams.eventsWriter?.experimentContext;const experimentAttributes = experimentContext? buildExperimentAttributes(experimentContext,@@ -194,27 +223,10 @@ export function createAiSdkTelemetryCapture(params: {rootSpan.setAttributes(experimentAttributes);}- const promptAttributes = traceSinkParams.prompt- ? {- // Link the LLM generation spans to the resolved Langfuse prompt.- [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:- traceSinkParams.prompt.name,- [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:- traceSinkParams.prompt.version,- }- : undefined;-- const childSpanMetadata = traceSinkParams.metadata- ? Object.fromEntries(- Object.entries(traceSinkParams.metadata).filter(- ([key]) => key !== "structured_output_schema",- ),- )- : undefined;-- const otelIntegration = createGenerationSpanTelemetry({+ const generationTelemetry = createGenerationSpanTelemetry({tracer,recordedInput: generationInput,+ existingSpan: writeGenerationOnRoot ? rootSpan : undefined,attributes: {// Experiment linkage goes on every span so every materialized event// remains associated with the run item root.@@ -309,7 +321,10 @@ export function createAiSdkTelemetryCapture(params: {};return {- telemetry: { isEnabled: true, integrations: [otelIntegration] },+ telemetry: {+ isEnabled: true,+ integrations: [generationTelemetry],+ },run: (fn) => context.with(activeContext, fn),setRootOutput,setRootError,@@ -332,12 +347,42 @@ function createGenerationSpanTelemetry(params: {* may contain short-lived signed media URLs that must not enter traces.*/recordedInput?: unknown;+ /**+ * Evaluator executions write the model call onto the trace root instead of+ * adding a child span. The caller ends that span.+ */+ existingSpan?: Span;}): Telemetry {- const { tracer, attributes, recordedInput } = params;+ const { tracer, attributes, recordedInput, existingSpan } = params;const openSpans = new Map<string, Span>();+ const generationAttributes = (+ event: Parameters<NonNullable<Telemetry["onLanguageModelCallStart"]>>[0],+ ): Attributes => ({+ "gen_ai.operation.name": "chat",+ "gen_ai.provider.name": event.provider,+ "gen_ai.request.model": event.modelId,+ ...definedNumberAttributes({+ "gen_ai.request.max_tokens": event.maxOutputTokens,+ "gen_ai.request.temperature": event.temperature,+ "gen_ai.request.top_p": event.topP,+ }),+ ...(recordedInput !== undefined || event.messages !== undefined+ ? {+ "gen_ai.input.messages": safeJsonStringify(+ recordedInput ?? event.messages,+ ),+ }+ : {}),+ ...(event.tools && event.tools.length > 0+ ? { "gen_ai.tool.definitions": safeJsonStringify(event.tools) }+ : {}),+ ...attributes,+ });+const endAllOpenSpans = (error?: unknown): void => {for (const span of openSpans.values()) {+ if (span === existingSpan) continue;if (error !== undefined) {span.setAttribute("error.type", getErrorType(error));span.setStatus({@@ -354,6 +399,12 @@ function createGenerationSpanTelemetry(params: {return {onLanguageModelCallStart(event) {+ if (existingSpan) {+ existingSpan.setAttributes(generationAttributes(event));+ openSpans.set(event.callId, existingSpan);+ return;+ }+// Defensive: a lingering span for this call id means its end event never// fired (e.g. a retried attempt) — close it before starting the next.openSpans.get(event.callId)?.end();@@ -362,27 +413,7 @@ function createGenerationSpanTelemetry(params: {`chat ${event.modelId}`,{kind: SpanKind.CLIENT,- attributes: {- "gen_ai.operation.name": "chat",- "gen_ai.provider.name": event.provider,- "gen_ai.request.model": event.modelId,- ...definedNumberAttributes({- "gen_ai.request.max_tokens": event.maxOutputTokens,- "gen_ai.request.temperature": event.temperature,- "gen_ai.request.top_p": event.topP,- }),- ...(recordedInput !== undefined || event.messages !== undefined- ? {- "gen_ai.input.messages": safeJsonStringify(- recordedInput ?? event.messages,- ),- }- : {}),- ...(event.tools && event.tools.length > 0- ? { "gen_ai.tool.definitions": safeJsonStringify(event.tools) }- : {}),- ...attributes,- },+ attributes: generationAttributes(event),},context.active(),);@@ -395,8 +426,11 @@ function createGenerationSpanTelemetry(params: {if (!span) return;- openSpans.delete(event.callId);+ if (span !== existingSpan) openSpans.delete(event.callId);+ const outputMessages = safeJsonStringify([+ { role: "assistant", content: event.content },+ ]);span.setAttributes({"gen_ai.response.finish_reasons": [event.finishReason],...(event.responseId ? { "gen_ai.response.id": event.responseId } : {}),@@ -408,14 +442,21 @@ function createGenerationSpanTelemetry(params: {"gen_ai.usage.cache_creation.input_tokens":event.usage.inputTokenDetails?.cacheWriteTokens,}),- "gen_ai.output.messages": safeJsonStringify([- { role: "assistant", content: event.content },- ]),+ "gen_ai.output.messages": outputMessages,+ // The root span already has observation input. Without observation+ // output, ingestion ignores this model response when a later step fails.+ ...(span === existingSpan+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: outputMessages,+ }+ : {}),});- span.end();+ if (span !== existingSpan) span.end();},onError(event) {+ // The root span's error is recorded by setRootError. Marking it here too+ // would store the same exception twice.endAllOpenSpans(getTelemetryError(event));},diff --git a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.tsindex 6c324bc7a6..182a34810c 100644--- a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts+++ b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts@@ -24,7 +24,11 @@ import {type ModelParams,type TraceSinkParams,} from "../types";-import { generateLLMText, mapLegacyLLMCompletionParams } from "../llmText";+import {+ createLLMOutput,+ generateLLMText,+ mapLegacyLLMCompletionParams,+} from "../llmText";const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);@@ -195,23 +199,13 @@ describe("AI SDK telemetry integration", () => {rs.scopeSpans.flatMap((ss: any) => ss.spans),);- // Exactly two spans: the internal root and one generation per model call- // (the minimal telemetry integration emits no operation/step spans).- expect(spans).toHaveLength(2);- for (const span of spans) {- expect(span.traceId.toLowerCase()).toBe(VALID_TRACE_ID);- }-- const rootSpans = spans.filter((span: any) => !span.parentSpanId);- expect(rootSpans).toHaveLength(1);- expect(rootSpans[0].name).toBe("Execute evaluator: helpfulness");+ // The evaluator trace is the root span, with the model call written on it.+ expect(spans).toHaveLength(1);+ expect(spans[0].traceId.toLowerCase()).toBe(VALID_TRACE_ID);+ expect(spans[0].parentSpanId ?? "").toBe("");+ expect(spans[0].name).toBe("Execute evaluator: helpfulness");- const generationSpan = spans.find((span: any) => span.parentSpanId);- expect(generationSpan.name).toBe("chat gpt-4o");- expect(generationSpan.parentSpanId).toBe(rootSpans[0].spanId);- // OTLP SpanKind 3 = CLIENT. The detached trace owns the actual GenAI- // client span; worker eval.* spans remain INTERNAL orchestration spans.- expect(generationSpan.kind).toBe(3);+ const generationSpan = spans[0];expect(Object.fromEntries(generationSpan.attributes.map((attribute: any) => [@@ -285,7 +279,7 @@ describe("AI SDK telemetry integration", () => {const generation = costedObservations[0];expect(generation.type).toBe("generation-create");expect(generation.body).toMatchObject({- name: "chat gpt-4o",+ name: "Execute evaluator: helpfulness",model: "gpt-4o",usageDetails: { input: 3, output: 5 },environment: "langfuse-llm-judge",@@ -310,7 +304,7 @@ describe("AI SDK telemetry integration", () => {});const eventInputs = processor.processToEvent(resourceSpans);- expect(eventInputs).toHaveLength(2);+ expect(eventInputs).toHaveLength(1);for (const eventInput of eventInputs) {expect(eventInput).toMatchObject({evaluationContext: {@@ -462,7 +456,7 @@ describe("AI SDK telemetry integration", () => {const spans = resourceSpans.flatMap((resourceSpan: any) =>resourceSpan.scopeSpans.flatMap((scopeSpan: any) => scopeSpan.spans),);- const generationSpan = spans.find((span: any) => span.parentSpanId);+ const generationSpan = spans.find((span: any) => !span.parentSpanId);const attributes = Object.fromEntries(generationSpan.attributes.map((attribute: any) => [attribute.key,@@ -472,6 +466,46 @@ describe("AI SDK telemetry integration", () => {expect(generationSpan.status).toMatchObject({ code: 2 });expect(attributes).toMatchObject({ "error.type": "AI_APICallError" });+ expect(+ (generationSpan.events ?? []).filter(+ (event: { name?: string }) => event.name === "exception",+ ),+ ).toHaveLength(1);+ });++ it("keeps the model response when structured output parsing fails", async () => {+ await expect(+ generateLLMText({+ ...mapLegacyLLMCompletionParams({+ messages,+ modelParams,+ connection: { secretKey: encrypt("sk-test") },+ }),+ output: createLLMOutput({+ type: "object",+ properties: { score: { type: "string" } },+ required: ["score"],+ }),+ timeout: 10_000,+ trace: traceSinkParams,+ }),+ ).rejects.toThrow();++ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];+ const { OtelIngestionProcessor } = await vi.importActual<+ typeof import("../../otel/OtelIngestionProcessor")+ >("../../otel/OtelIngestionProcessor");+ const eventInputs = new OtelIngestionProcessor({+ projectId: "project-1",+ publicKey: "",+ sdkName: "langfuse-internal-ai-sdk",+ sdkVersion: "unknown",+ isLangfuseInternal: true,+ }).processToEvent(resourceSpans);+ const rootEvent = eventInputs.find((input: any) => !input.parentSpanId);++ expect(rootEvent).toMatchObject({ level: "ERROR" });+ expect(String(rootEvent.output)).toContain("Hello there");});it("records unsupported media URL errors without downloading or calling the model", async () => {@@ -552,8 +586,6 @@ describe("AI SDK telemetry integration", () => {expect(rootSpan.status).toMatchObject({ code: 2, message: errorMessage });expect(attributes).toMatchObject({ "error.type": "LLMValidationError" });- // Convert the failed root span through the production ingestion mapper to- // prove the evaluator execution trace exposes the validation error.const { OtelIngestionProcessor } = await vi.importActual<typeof import("../../otel/OtelIngestionProcessor")>("../../otel/OtelIngestionProcessor");@@ -572,4 +604,76 @@ describe("AI SDK telemetry integration", () => {environment: "langfuse-llm-judge",});});++ it("keeps a retried evaluator call on the root span", async () => {+ let calls = 0;+ vi.mocked(createOpenAI).mockReturnValue({+ chat: () =>+ new MockLanguageModelV4({+ provider: "openai",+ modelId: "gpt-4o",+ doGenerate: async () => {+ calls += 1;+ if (calls === 1) {+ throw new APICallError({+ message: "Rate limit exceeded",+ url: "https://api.openai.com/v1/chat/completions",+ requestBodyValues: {},+ statusCode: 429,+ isRetryable: true,+ });+ }+ return {+ content: [{ type: "text", text: "Hello there" }],+ finishReason: { unified: "stop", raw: "stop" },+ usage: {+ inputTokens: {+ total: 4,+ noCache: 4,+ cacheRead: undefined,+ cacheWrite: undefined,+ },+ outputTokens: { total: 6, text: 6, reasoning: undefined },+ },+ warnings: [],+ };+ },+ }),+ } as never);++ const result = await generateLLMText({+ ...mapLegacyLLMCompletionParams({+ messages,+ modelParams,+ connection: { secretKey: encrypt("sk-test") },+ }),+ maxRetries: 1,+ timeout: 10_000,+ trace: traceSinkParams,+ });++ expect(result.text).toBe("Hello there");+ expect(calls).toBe(2);+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);++ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];+ const spans = resourceSpans.flatMap((resourceSpan: any) =>+ resourceSpan.scopeSpans.flatMap((scopeSpan: any) => scopeSpan.spans),+ );+ expect(spans).toHaveLength(1);++ const attributes = Object.fromEntries(+ spans[0].attributes.map((attribute: any) => [+ attribute.key,+ attribute.value.stringValue ??+ attribute.value.intValue ??+ attribute.value,+ ]),+ );+ expect(+ JSON.parse(attributes["langfuse.observation.metadata"]),+ ).toMatchObject({ job_execution_id: "job-1" });+ expect(attributes["gen_ai.usage.input_tokens"]).toBe(4);+ expect(attributes["gen_ai.usage.output_tokens"]).toBe(6);+ });});