Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): publish one generation for evaluator executions (#18415)

Tobias Wochingerdbf2316e1cb5
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
index 3b266de579..95eeb7775e 100644
--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts
+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
@@ -151,7 +151,30 @@ export function createAiSdkTelemetryCapture(params: {
? buildEvaluationAttributes(traceSinkParams.evaluationContext)
: undefined;
- const rootSpan: Span = tracer.startSpan(
+ const experimentContext = traceSinkParams.eventsWriter?.experimentContext;
+ // Customers pay per observation and only use an evaluator trace to inspect
+ // input, output, and cost, so the model call is written on this root span
+ // instead of adding another one. Prompt experiments still need the child span.
+ const writeGenerationOnRoot =
+ evaluationAttributes !== undefined && experimentContext === undefined;
+ const childSpanMetadata = traceSinkParams.metadata
+ ? Object.fromEntries(
+ Object.entries(traceSinkParams.metadata).filter(
+ ([key]) => key !== "structured_output_schema",
+ ),
+ )
+ : undefined;
+ const promptAttributes = traceSinkParams.prompt
+ ? {
+ // Link the LLM generation to the resolved Langfuse prompt.
+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:
+ traceSinkParams.prompt.name,
+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:
+ traceSinkParams.prompt.version,
+ }
+ : undefined;
+
+ const rootSpan = tracer.startSpan(
traceSinkParams.traceName,
{
attributes: {
@@ -164,6 +187,7 @@ export function createAiSdkTelemetryCapture(params: {
}
: {}),
...(evaluationAttributes ?? {}),
+ ...(writeGenerationOnRoot ? (promptAttributes ?? {}) : {}),
...(traceSinkParams.metadata
? {
[LangfuseOtelSpanAttributes.TRACE_METADATA]: JSON.stringify(
@@ -171,6 +195,12 @@ export function createAiSdkTelemetryCapture(params: {
),
}
: {}),
+ ...(writeGenerationOnRoot && childSpanMetadata
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]:
+ JSON.stringify(childSpanMetadata),
+ }
+ : {}),
...(serializedInput !== undefined
? {
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: serializedInput,
@@ -183,7 +213,6 @@ export function createAiSdkTelemetryCapture(params: {
);
const activeContext = trace.setSpan(ROOT_CONTEXT, rootSpan);
- const experimentContext = traceSinkParams.eventsWriter?.experimentContext;
const experimentAttributes = experimentContext
? buildExperimentAttributes(
experimentContext,
@@ -194,27 +223,10 @@ export function createAiSdkTelemetryCapture(params: {
rootSpan.setAttributes(experimentAttributes);
}
- const promptAttributes = traceSinkParams.prompt
- ? {
- // Link the LLM generation spans to the resolved Langfuse prompt.
- [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:
- traceSinkParams.prompt.name,
- [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:
- traceSinkParams.prompt.version,
- }
- : undefined;
-
- const childSpanMetadata = traceSinkParams.metadata
- ? Object.fromEntries(
- Object.entries(traceSinkParams.metadata).filter(
- ([key]) => key !== "structured_output_schema",
- ),
- )
- : undefined;
-
- const otelIntegration = createGenerationSpanTelemetry({
+ const generationTelemetry = createGenerationSpanTelemetry({
tracer,
recordedInput: generationInput,
+ existingSpan: writeGenerationOnRoot ? rootSpan : undefined,
attributes: {
// Experiment linkage goes on every span so every materialized event
// remains associated with the run item root.
@@ -309,7 +321,10 @@ export function createAiSdkTelemetryCapture(params: {
};
return {
- telemetry: { isEnabled: true, integrations: [otelIntegration] },
+ telemetry: {
+ isEnabled: true,
+ integrations: [generationTelemetry],
+ },
run: (fn) => context.with(activeContext, fn),
setRootOutput,
setRootError,
@@ -332,12 +347,42 @@ function createGenerationSpanTelemetry(params: {
* may contain short-lived signed media URLs that must not enter traces.
*/
recordedInput?: unknown;
+ /**
+ * Evaluator executions write the model call onto the trace root instead of
+ * adding a child span. The caller ends that span.
+ */
+ existingSpan?: Span;
}): Telemetry {
- const { tracer, attributes, recordedInput } = params;
+ const { tracer, attributes, recordedInput, existingSpan } = params;
const openSpans = new Map<string, Span>();
+ const generationAttributes = (
+ event: Parameters<NonNullable<Telemetry["onLanguageModelCallStart"]>>[0],
+ ): Attributes => ({
+ "gen_ai.operation.name": "chat",
+ "gen_ai.provider.name": event.provider,
+ "gen_ai.request.model": event.modelId,
+ ...definedNumberAttributes({
+ "gen_ai.request.max_tokens": event.maxOutputTokens,
+ "gen_ai.request.temperature": event.temperature,
+ "gen_ai.request.top_p": event.topP,
+ }),
+ ...(recordedInput !== undefined || event.messages !== undefined
+ ? {
+ "gen_ai.input.messages": safeJsonStringify(
+ recordedInput ?? event.messages,
+ ),
+ }
+ : {}),
+ ...(event.tools && event.tools.length > 0
+ ? { "gen_ai.tool.definitions": safeJsonStringify(event.tools) }
+ : {}),
+ ...attributes,
+ });
+
const endAllOpenSpans = (error?: unknown): void => {
for (const span of openSpans.values()) {
+ if (span === existingSpan) continue;
if (error !== undefined) {
span.setAttribute("error.type", getErrorType(error));
span.setStatus({
@@ -354,6 +399,12 @@ function createGenerationSpanTelemetry(params: {
return {
onLanguageModelCallStart(event) {
+ if (existingSpan) {
+ existingSpan.setAttributes(generationAttributes(event));
+ openSpans.set(event.callId, existingSpan);
+ return;
+ }
+
// Defensive: a lingering span for this call id means its end event never
// fired (e.g. a retried attempt) — close it before starting the next.
openSpans.get(event.callId)?.end();
@@ -362,27 +413,7 @@ function createGenerationSpanTelemetry(params: {
`chat ${event.modelId}`,
{
kind: SpanKind.CLIENT,
- attributes: {
- "gen_ai.operation.name": "chat",
- "gen_ai.provider.name": event.provider,
- "gen_ai.request.model": event.modelId,
- ...definedNumberAttributes({
- "gen_ai.request.max_tokens": event.maxOutputTokens,
- "gen_ai.request.temperature": event.temperature,
- "gen_ai.request.top_p": event.topP,
- }),
- ...(recordedInput !== undefined || event.messages !== undefined
- ? {
- "gen_ai.input.messages": safeJsonStringify(
- recordedInput ?? event.messages,
- ),
- }
- : {}),
- ...(event.tools && event.tools.length > 0
- ? { "gen_ai.tool.definitions": safeJsonStringify(event.tools) }
- : {}),
- ...attributes,
- },
+ attributes: generationAttributes(event),
},
context.active(),
);
@@ -395,8 +426,11 @@ function createGenerationSpanTelemetry(params: {
if (!span) return;
- openSpans.delete(event.callId);
+ if (span !== existingSpan) openSpans.delete(event.callId);
+ const outputMessages = safeJsonStringify([
+ { role: "assistant", content: event.content },
+ ]);
span.setAttributes({
"gen_ai.response.finish_reasons": [event.finishReason],
...(event.responseId ? { "gen_ai.response.id": event.responseId } : {}),
@@ -408,14 +442,21 @@ function createGenerationSpanTelemetry(params: {
"gen_ai.usage.cache_creation.input_tokens":
event.usage.inputTokenDetails?.cacheWriteTokens,
}),
- "gen_ai.output.messages": safeJsonStringify([
- { role: "assistant", content: event.content },
- ]),
+ "gen_ai.output.messages": outputMessages,
+ // The root span already has observation input. Without observation
+ // output, ingestion ignores this model response when a later step fails.
+ ...(span === existingSpan
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: outputMessages,
+ }
+ : {}),
});
- span.end();
+ if (span !== existingSpan) span.end();
},
onError(event) {
+ // The root span's error is recorded by setRootError. Marking it here too
+ // would store the same exception twice.
endAllOpenSpans(getTelemetryError(event));
},
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
index 6c324bc7a6..182a34810c 100644
--- a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
+++ b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
@@ -24,7 +24,11 @@ import {
type ModelParams,
type TraceSinkParams,
} from "../types";
-import { generateLLMText, mapLegacyLLMCompletionParams } from "../llmText";
+import {
+ createLLMOutput,
+ generateLLMText,
+ mapLegacyLLMCompletionParams,
+} from "../llmText";
const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);
@@ -195,23 +199,13 @@ describe("AI SDK telemetry integration", () => {
rs.scopeSpans.flatMap((ss: any) => ss.spans),
);
- // Exactly two spans: the internal root and one generation per model call
- // (the minimal telemetry integration emits no operation/step spans).
- expect(spans).toHaveLength(2);
- for (const span of spans) {
- expect(span.traceId.toLowerCase()).toBe(VALID_TRACE_ID);
- }
-
- const rootSpans = spans.filter((span: any) => !span.parentSpanId);
- expect(rootSpans).toHaveLength(1);
- expect(rootSpans[0].name).toBe("Execute evaluator: helpfulness");
+ // The evaluator trace is the root span, with the model call written on it.
+ expect(spans).toHaveLength(1);
+ expect(spans[0].traceId.toLowerCase()).toBe(VALID_TRACE_ID);
+ expect(spans[0].parentSpanId ?? "").toBe("");
+ expect(spans[0].name).toBe("Execute evaluator: helpfulness");
- const generationSpan = spans.find((span: any) => span.parentSpanId);
- expect(generationSpan.name).toBe("chat gpt-4o");
- expect(generationSpan.parentSpanId).toBe(rootSpans[0].spanId);
- // OTLP SpanKind 3 = CLIENT. The detached trace owns the actual GenAI
- // client span; worker eval.* spans remain INTERNAL orchestration spans.
- expect(generationSpan.kind).toBe(3);
+ const generationSpan = spans[0];
expect(
Object.fromEntries(
generationSpan.attributes.map((attribute: any) => [
@@ -285,7 +279,7 @@ describe("AI SDK telemetry integration", () => {
const generation = costedObservations[0];
expect(generation.type).toBe("generation-create");
expect(generation.body).toMatchObject({
- name: "chat gpt-4o",
+ name: "Execute evaluator: helpfulness",
model: "gpt-4o",
usageDetails: { input: 3, output: 5 },
environment: "langfuse-llm-judge",
@@ -310,7 +304,7 @@ describe("AI SDK telemetry integration", () => {
});
const eventInputs = processor.processToEvent(resourceSpans);
- expect(eventInputs).toHaveLength(2);
+ expect(eventInputs).toHaveLength(1);
for (const eventInput of eventInputs) {
expect(eventInput).toMatchObject({
evaluationContext: {
@@ -462,7 +456,7 @@ describe("AI SDK telemetry integration", () => {
const spans = resourceSpans.flatMap((resourceSpan: any) =>
resourceSpan.scopeSpans.flatMap((scopeSpan: any) => scopeSpan.spans),
);
- const generationSpan = spans.find((span: any) => span.parentSpanId);
+ const generationSpan = spans.find((span: any) => !span.parentSpanId);
const attributes = Object.fromEntries(
generationSpan.attributes.map((attribute: any) => [
attribute.key,
@@ -472,6 +466,46 @@ describe("AI SDK telemetry integration", () => {
expect(generationSpan.status).toMatchObject({ code: 2 });
expect(attributes).toMatchObject({ "error.type": "AI_APICallError" });
+ expect(
+ (generationSpan.events ?? []).filter(
+ (event: { name?: string }) => event.name === "exception",
+ ),
+ ).toHaveLength(1);
+ });
+
+ it("keeps the model response when structured output parsing fails", async () => {
+ await expect(
+ generateLLMText({
+ ...mapLegacyLLMCompletionParams({
+ messages,
+ modelParams,
+ connection: { secretKey: encrypt("sk-test") },
+ }),
+ output: createLLMOutput({
+ type: "object",
+ properties: { score: { type: "string" } },
+ required: ["score"],
+ }),
+ timeout: 10_000,
+ trace: traceSinkParams,
+ }),
+ ).rejects.toThrow();
+
+ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];
+ const { OtelIngestionProcessor } = await vi.importActual<
+ typeof import("../../otel/OtelIngestionProcessor")
+ >("../../otel/OtelIngestionProcessor");
+ const eventInputs = new OtelIngestionProcessor({
+ projectId: "project-1",
+ publicKey: "",
+ sdkName: "langfuse-internal-ai-sdk",
+ sdkVersion: "unknown",
+ isLangfuseInternal: true,
+ }).processToEvent(resourceSpans);
+ const rootEvent = eventInputs.find((input: any) => !input.parentSpanId);
+
+ expect(rootEvent).toMatchObject({ level: "ERROR" });
+ expect(String(rootEvent.output)).toContain("Hello there");
});
it("records unsupported media URL errors without downloading or calling the model", async () => {
@@ -552,8 +586,6 @@ describe("AI SDK telemetry integration", () => {
expect(rootSpan.status).toMatchObject({ code: 2, message: errorMessage });
expect(attributes).toMatchObject({ "error.type": "LLMValidationError" });
- // Convert the failed root span through the production ingestion mapper to
- // prove the evaluator execution trace exposes the validation error.
const { OtelIngestionProcessor } = await vi.importActual<
typeof import("../../otel/OtelIngestionProcessor")
>("../../otel/OtelIngestionProcessor");
@@ -572,4 +604,76 @@ describe("AI SDK telemetry integration", () => {
environment: "langfuse-llm-judge",
});
});
+
+ it("keeps a retried evaluator call on the root span", async () => {
+ let calls = 0;
+ vi.mocked(createOpenAI).mockReturnValue({
+ chat: () =>
+ new MockLanguageModelV4({
+ provider: "openai",
+ modelId: "gpt-4o",
+ doGenerate: async () => {
+ calls += 1;
+ if (calls === 1) {
+ throw new APICallError({
+ message: "Rate limit exceeded",
+ url: "https://api.openai.com/v1/chat/completions",
+ requestBodyValues: {},
+ statusCode: 429,
+ isRetryable: true,
+ });
+ }
+ return {
+ content: [{ type: "text", text: "Hello there" }],
+ finishReason: { unified: "stop", raw: "stop" },
+ usage: {
+ inputTokens: {
+ total: 4,
+ noCache: 4,
+ cacheRead: undefined,
+ cacheWrite: undefined,
+ },
+ outputTokens: { total: 6, text: 6, reasoning: undefined },
+ },
+ warnings: [],
+ };
+ },
+ }),
+ } as never);
+
+ const result = await generateLLMText({
+ ...mapLegacyLLMCompletionParams({
+ messages,
+ modelParams,
+ connection: { secretKey: encrypt("sk-test") },
+ }),
+ maxRetries: 1,
+ timeout: 10_000,
+ trace: traceSinkParams,
+ });
+
+ expect(result.text).toBe("Hello there");
+ expect(calls).toBe(2);
+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);
+
+ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];
+ const spans = resourceSpans.flatMap((resourceSpan: any) =>
+ resourceSpan.scopeSpans.flatMap((scopeSpan: any) => scopeSpan.spans),
+ );
+ expect(spans).toHaveLength(1);
+
+ const attributes = Object.fromEntries(
+ spans[0].attributes.map((attribute: any) => [
+ attribute.key,
+ attribute.value.stringValue ??
+ attribute.value.intValue ??
+ attribute.value,
+ ]),
+ );
+ expect(
+ JSON.parse(attributes["langfuse.observation.metadata"]),
+ ).toMatchObject({ job_execution_id: "job-1" });
+ expect(attributes["gen_ai.usage.input_tokens"]).toBe(4);
+ expect(attributes["gen_ai.usage.output_tokens"]).toBe(6);
+ });
});