The Open Framework for Building Agents
feat(evals): report Datadog experiment runtime links (#4568)
Matan Kushnerc25caf3c11b7
diff --git a/.changeset/report-datadog-runtime-trace-links.md b/.changeset/report-datadog-runtime-trace-links.mdnew file mode 100644index 000000000..329bc2c3c--- /dev/null+++ b/.changeset/report-datadog-runtime-trace-links.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Datadog Experiment reports now link each eval to its sampled runtime traces using Datadog-indexed IDs. Linked traces include classified structural spans and, when capture is enabled, turn-level task input and output; zero-duration Experiment spans are normalized to a positive duration. Canonical runtime spans remain unchanged.diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdxindex c7b4f7025..4eed6be64 100644--- a/docs/evals/reporters.mdx+++ b/docs/evals/reporters.mdx@@ -57,7 +57,9 @@ export default defineEvalConfig({});```-The reporter creates one Datadog Experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. Without input recording, the external Experiment uses a placeholder dataset and spans are submitted as evals finish. With `recordInputs` (the default), the reporter instead syncs a versioned Datadog dataset after the evals finish, starts the Experiment against that dataset version, passes each dataset record id to its corresponding experiment span, and prints both Dataset and Experiment URLs. Buffering is necessary because an imperative eval's first `t.send(...)` input is only known after that eval executes. `dd-trace` generates dataset record, experiment trace, and experiment span identifiers; the reporter does not mutate or link the agent runtime's OpenTelemetry spans.+The reporter creates one Datadog Experiment, submits one synthetic experiment span per completed eve eval, associates that eval's assertion scores as experiment metrics, and prints the Datadog Experiment URL after the run completes. Without input recording, the external Experiment uses a placeholder dataset and spans are submitted as evals finish. With `recordInputs` (the default), the reporter instead syncs a versioned Datadog dataset after the evals finish, starts the Experiment against that dataset version, passes each dataset record id to its corresponding experiment span, and prints both Dataset and Experiment URLs. Buffering is necessary because an imperative eval's first `t.send(...)` input is only known after that eval executes.++`dd-trace` generates dataset record, experiment trace, and experiment span identifiers. For sampled eve sessions, the reporter also adds identifier-only `experimentRuntimeTraceLinks` metadata, holding the IDs Datadog assigns when it indexes the runtime traces, so an Experiment row can link to the agent trace that produced it. Runtime spans keep their stored IDs and parentage.Every run reuses the dataset named by `datasetName` (default `<projectName> evals`), so Datadog can compare experiments across runs. Records are matched by eval description, with `#2`, `#3` suffixes for duplicates, so give each case a unique description.diff --git a/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.test.ts b/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.test.tsnew file mode 100644index 000000000..3514156bf--- /dev/null+++ b/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.test.ts@@ -0,0 +1,61 @@+import { describe, expect, it } from "vitest";++import { resolveRuntimeTraceLinks } from "#evals/runner/reporters/datadog-runtime-trace-links.js";+import type { EveEvalTraceContext } from "#evals/types.js";++function traceContext(overrides: Partial<EveEvalTraceContext> = {}): EveEvalTraceContext {+ return {+ traceId: "0123456789abcdef0123456789abcdef",+ spanId: "0123456789abcdef",+ traceFlags: 1,+ sessionId: "session-1",+ primary: true,+ ...overrides,+ };+}++describe("resolveRuntimeTraceLinks", () => {+ it("stores the Datadog-indexed IDs for a real W3C runtime context", () => {+ expect(+ resolveRuntimeTraceLinks([+ traceContext({+ traceId: "010280a6f337b4e3117b3db95c1ad3fe",+ spanId: "140edab97d7fb4ef",+ sessionId: "wrun_01M3533VV7F1M7FF5J8NQ3W7G8",+ }),+ traceContext({+ traceId: "fedcba9876543210fedcba9876543210",+ spanId: "fedcba9876543210",+ sessionId: "secondary",+ primary: false,+ }),+ ]),+ ).toEqual([+ {+ relation: "experiment_runtime",+ traceId: "e4eea2b9661c5e3890ed96de9715238e",+ spanId: "1445333020641834223",+ sessionId: "wrun_01M3533VV7F1M7FF5J8NQ3W7G8",+ primary: true,+ },+ {+ relation: "experiment_runtime",+ traceId: "d8464e48436c5124a7b8b5b463a20f61",+ spanId: "18364758544493064720",+ sessionId: "secondary",+ primary: false,+ },+ ]);+ });++ it("omits unsampled, malformed, and zero runtime contexts", () => {+ expect(+ resolveRuntimeTraceLinks([+ traceContext({ traceFlags: 0 }),+ traceContext({ traceId: "invalid", spanId: "invalid" }),+ traceContext({ traceId: "0".repeat(32) }),+ traceContext({ spanId: "0".repeat(16) }),+ ]),+ ).toEqual([]);+ });+});diff --git a/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.ts b/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.tsnew file mode 100644index 000000000..e7b10d5b6--- /dev/null+++ b/packages/eve/src/evals/runner/reporters/datadog-runtime-trace-links.ts@@ -0,0 +1,60 @@+import { createHash } from "node:crypto";++import type { EveEvalTraceContext } from "#evals/types.js";++const DATADOG_LLMOBS_TRACE_ID_NAMESPACE = Buffer.from("f47ac10b58cc4372a5670e02b2c3d479", "hex");+const W3C_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/iu;+const W3C_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/iu;++/** Identifies one runtime trace by the IDs Datadog's APM-to-LLMObs indexer assigns. */+export interface ExperimentRuntimeTraceLink {+ readonly relation: "experiment_runtime";+ readonly traceId: string;+ readonly spanId: string;+ readonly sessionId: string;+ readonly primary: boolean;+}++/** Converts sampled runtime trace contexts to Datadog-indexed links; skips the rest. */+export function resolveRuntimeTraceLinks(+ traceContexts: readonly EveEvalTraceContext[],+): ExperimentRuntimeTraceLink[] {+ return traceContexts.flatMap((traceContext) => {+ if ((traceContext.traceFlags & 1) === 0) return [];+ const traceId = toDatadogLlmobsTraceId(traceContext.traceId);+ const spanId = toDatadogLlmobsSpanId(traceContext.spanId);+ if (traceId === undefined || spanId === undefined) return [];+ return [+ {+ relation: "experiment_runtime",+ traceId,+ spanId,+ sessionId: traceContext.sessionId,+ primary: traceContext.primary,+ },+ ];+ });+}++function toDatadogLlmobsTraceId(traceId: string): string | undefined {+ const canonicalTraceId = traceId.toLowerCase();+ if (!W3C_TRACE_ID_PATTERN.test(canonicalTraceId) || /^0+$/u.test(canonicalTraceId)) {+ return undefined;+ }++ // Datadog indexes OTel traces by their low 64 bits, then derives a UUIDv5-style LLMObs ID.+ const canonicalApmTraceId = canonicalTraceId.slice(-16).padStart(32, "0");+ const hash = createHash("sha1")+ .update(DATADOG_LLMOBS_TRACE_ID_NAMESPACE)+ .update(canonicalApmTraceId)+ .digest()+ .subarray(0, 16);+ hash[6] = (hash[6]! & 0x0f) | 0x50;+ hash[8] = (hash[8]! & 0x3f) | 0x80;+ return hash.toString("hex");+}++function toDatadogLlmobsSpanId(spanId: string): string | undefined {+ if (!W3C_SPAN_ID_PATTERN.test(spanId) || /^0+$/u.test(spanId)) return undefined;+ return BigInt(`0x${spanId}`).toString(10);+}diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.tsindex 778630e90..d9d3f4bf9 100644--- a/packages/eve/src/evals/runner/reporters/datadog.test.ts+++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts@@ -12,6 +12,14 @@ const RECORDING_DISABLED = {recordErrors: false,} satisfies DatadogReporterConfig;+const SAMPLED_TRACE_CONTEXT = {+ traceId: "0123456789abcdef0123456789abcdef",+ spanId: "0123456789abcdef",+ traceFlags: 1,+ sessionId: "session-123",+ primary: true,+};+function makeTarget(kind: "local" | "remote" = "local"): EveEvalTarget {return {capabilities: { devRoutes: kind === "local" },@@ -283,8 +291,55 @@ describe("Datadog", () => {expect.objectContaining({ label: "eve_reasoning_block_count", value: 0 }),]);expect(submittedSpan).not.toHaveProperty("metadata.eveAssertionScores");+ expect(submittedSpan).not.toHaveProperty("metadata.experimentRuntimeTraceLinks");});+ it("uses a positive duration for evals completed within the same millisecond", async () => {+ const { config, experiment } = makeConfig(RECORDING_DISABLED);+ const reporter = Datadog(config);+ const evaluation = makeEval();+ const result = makeEvalResult({+ startedAt: "2026-01-01T00:00:00.000Z",+ completedAt: "2026-01-01T00:00:00.000Z",+ });++ await reporter.onRunStart([evaluation], makeTarget());+ await reporter.onEvalComplete(result);++ expect(experiment.submitSpan).toHaveBeenCalledWith(expect.objectContaining({ durationMs: 1 }));+ });++ it.each(["failed", "waiting"] as const)(+ "records runtime trace links for %s evals",+ async (status) => {+ const { config, experiment } = makeConfig(RECORDING_DISABLED);+ const reporter = Datadog(config);+ const result = makeEvalResult({+ result: {+ ...makeEvalResult().result,+ status,+ traceContexts: [SAMPLED_TRACE_CONTEXT],+ },+ });++ await reporter.onRunStart([makeEval()], makeTarget());+ await reporter.onEvalComplete(result);++ expect(experiment.submitSpan).toHaveBeenCalledWith(+ expect.objectContaining({+ metadata: expect.objectContaining({+ experimentRuntimeTraceLinks: [+ expect.objectContaining({+ relation: "experiment_runtime",+ primary: true,+ }),+ ],+ }),+ }),+ );+ },+ );+it("redacts target URL secrets and omits execution errors when recordErrors is disabled", async () => {const { client, config, experiment } = makeConfig({ recordInputs: false, recordErrors: false });const reporter = Datadog(config);@@ -429,10 +484,15 @@ describe("Datadog", () => {]);});- it("records inputs, outputs, and expected outputs in a shared dataset by default", async () => {+ it("records inputs, outputs, expected outputs, and runtime links in a shared dataset by default", async () => {const { client, config, datasets, experiment, lines } = makeConfig({ experimentName: "run-1" });const reporter = Datadog(config);- const result = makeEvalResult();+ const result = makeEvalResult({+ result: {+ ...makeEvalResult().result,+ traceContexts: [SAMPLED_TRACE_CONTEXT],+ },+ });await reporter.onRunStart([makeEval()], makeTarget());await reporter.onEvalComplete(result);@@ -470,6 +530,14 @@ describe("Datadog", () => {output: "actual output",expectedOutput: "helpful onboarding answer",datasetRecordId: "record-1",+ metadata: expect.objectContaining({+ experimentRuntimeTraceLinks: [+ expect.objectContaining({+ traceId: "c6a8d65cb7d45f2cbbcd2b8e57bdd074",+ spanId: "81985529216486895",+ }),+ ],+ }),}),);expect(lines.join("\n")).toContain(diff --git a/packages/eve/src/evals/runner/reporters/datadog.ts b/packages/eve/src/evals/runner/reporters/datadog.tsindex 427c7684c..10c024cca 100644--- a/packages/eve/src/evals/runner/reporters/datadog.ts+++ b/packages/eve/src/evals/runner/reporters/datadog.ts@@ -1,4 +1,5 @@import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js";+import { resolveRuntimeTraceLinks } from "#evals/runner/reporters/datadog-runtime-trace-links.js";import type { EvalReporter } from "#evals/runner/reporters/types.js";import {composeAssertionScoreMetadata,@@ -476,6 +477,10 @@ function resolveResultMetadata(eveSkillLoads: result.result.derived.skillLoads.map((load) => load.skill),eveParked: result.result.derived.parked,});+ const runtimeTraceLinks = resolveRuntimeTraceLinks(result.result.traceContexts);+ if (runtimeTraceLinks.length > 0) {+ metadata.experimentRuntimeTraceLinks = runtimeTraceLinks;+ }if (recordAssertionDetails) {const failedAssertions = result.assertions.filter((assertion) => !assertion.passed)@@ -540,5 +545,9 @@ function elapsedMs(startedAt: string, completedAt: string): number | undefined {const start = Date.parse(startedAt);const completed = Date.parse(completedAt);if (!Number.isFinite(start) || !Number.isFinite(completed)) return undefined;- return Math.max(0, completed - start);++ // ISO timestamps have millisecond precision, so fast failures can start and+ // finish in the same millisecond. Datadog Experiment spans require a+ // positive duration; use the smallest representable duration in these units.+ return Math.max(1, completed - start);}diff --git a/packages/eve/src/tracing/agent-approval-instrumentation.ts b/packages/eve/src/tracing/agent-approval-instrumentation.tsindex 071b70fe9..3af78fe66 100644--- a/packages/eve/src/tracing/agent-approval-instrumentation.ts+++ b/packages/eve/src/tracing/agent-approval-instrumentation.ts@@ -110,7 +110,8 @@ export function createAgentApprovalInstrumentation(input: {"agent.step.attempt": state.attemptIndex,"agent.step.index": state.stepIndex,"agent.turn.id": state.turnId,- ...agentSpanNamingAttributes("agent.approval"),+ "gen_ai.operation.name": "workflow",+ ...agentSpanNamingAttributes("agent.approval", "workflow"),...agentTraceIdentityAttributes({rootSessionId: state.rootSessionId,traceSessionId: state.traceSessionId,diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex f4ca74c7e..eae892f18 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -1485,6 +1485,8 @@ describe("createAgentOtelInstrumentation", () => {expect(step.attributes).toMatchObject({"agent.framework.name": "eve","agent.model.id": "claude-test",+ "gen_ai.operation.name": "workflow",+ "operation.name": "workflow","agent.usage.cache_read_tokens": 4,"agent.usage.cache_write_tokens": 2,"agent.usage.input_tokens": 10,@@ -2266,7 +2268,8 @@ describe("createAgentOtelInstrumentation", () => {"agent.approval.request": expect.stringContaining("Approve weather?"),"agent.approval.request_id": "approval-1","agent.approval.response": expect.stringContaining("approve"),- "operation.name": "agent.approval",+ "gen_ai.operation.name": "workflow",+ "operation.name": "workflow","resource.name": "agent.approval","gen_ai.conversation.id": "session-1","agent.step.index": 0,@@ -2751,6 +2754,12 @@ describe("createAgentOtelInstrumentation", () => {const spans = runtime.exporter.getFinishedSpans();const model = byName(spans, "chat claude-test")[0]!;const tool = byName(spans, "execute_tool weather")[0]!;+ const turn = byName(spans, "invoke_agent weather")[0]!;+ expect(turn.attributes).toMatchObject({+ "gen_ai.input.messages":+ '[{"parts":[{"content":"real user text","type":"text"}],"role":"user"}]',+ "gen_ai.output.messages": expect.stringContaining("Checking the weather."),+ });expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");@@ -2815,6 +2824,12 @@ describe("createAgentOtelInstrumentation", () => {expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty("gen_ai.output.messages",);+ expect(byName(spans, "invoke_agent weather")[0]?.attributes).not.toHaveProperty(+ "gen_ai.input.messages",+ );+ expect(byName(spans, "invoke_agent weather")[0]?.attributes).not.toHaveProperty(+ "gen_ai.output.messages",+ );expect(byName(spans, "execute_tool weather")[0]?.attributes).not.toHaveProperty("gen_ai.tool.call.arguments",);diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 2d69f94e4..5880462d7 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -40,6 +40,10 @@ import {setGenAiUsage,} from "#tracing/agent-otel-usage.js";import { createAgentOtelSessionContext } from "#tracing/agent-otel-session-context.js";+import {+ rememberTurnInputMessages,+ rememberTurnOutputMessages,+} from "#tracing/agent-otel-turn-state.js";import type { TraceCapturePolicy } from "#tracing/otel-declaration.js";import { isSampledTrace, resolveTracePolicyDecision } from "#tracing/sampled-trace.js";import {@@ -358,7 +362,9 @@ export function createAgentOtelInstrumentation(}};- const onModelCallStarted = (event: InstrumentationModelCallStartedEvent): void => {+ const onModelCallStarted = (+ event: InstrumentationModelCallStartedEvent,+ ): void | PromiseLike<void> => {const attempt = steps.get(event.scope);if (attempt === undefined) return;attempt.span.setAttribute("agent.model.id", event.model.modelId);@@ -382,9 +388,12 @@ export function createAgentOtelInstrumentation(},attempt.context,);+ let inputMessagesAttribute: string | undefined;if (recordInputs && event.input !== undefined) {- const genAiMessages = genAiInputMessagesAttribute(event.input.messages);- if (genAiMessages !== undefined) span.setAttribute("gen_ai.input.messages", genAiMessages);+ inputMessagesAttribute = genAiInputMessagesAttribute(event.input.messages);+ if (inputMessagesAttribute !== undefined) {+ span.setAttribute("gen_ai.input.messages", inputMessagesAttribute);+ }const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);if (genAiSystem !== undefined) {span.setAttribute("gen_ai.system_instructions", genAiSystem);@@ -397,6 +406,7 @@ export function createAgentOtelInstrumentation(const state = { context: trace.setSpan(attempt.context, span), span };getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);+ return rememberTurnInputMessages(input.stateStore, event.scope, inputMessagesAttribute);};const onModelCallTerminal = async (@@ -429,6 +439,7 @@ export function createAgentOtelInstrumentation(const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);if (outputMessages !== undefined) {state.span.setAttribute("gen_ai.output.messages", outputMessages);+ await rememberTurnOutputMessages(input.stateStore, event.scope, outputMessages);}const reasoning = textContentAttribute(contentdiff --git a/packages/eve/src/tracing/agent-otel-runtime-context.ts b/packages/eve/src/tracing/agent-otel-runtime-context.tsindex fd6b7a928..40055a224 100644--- a/packages/eve/src/tracing/agent-otel-runtime-context.ts+++ b/packages/eve/src/tracing/agent-otel-runtime-context.ts@@ -50,7 +50,9 @@ export function agentActivationAttributes(input: {"agent.turn.id": input.turnId,"agent.turn.sequence": input.turn.sequence,"gen_ai.agent.name": input.agentName,+ "gen_ai.input.messages": input.turn.inputMessagesAttribute,"gen_ai.operation.name": "invoke_agent",+ "gen_ai.output.messages": input.turn.outputMessagesAttribute,...agentSpanNamingAttributes(agentInvocationSpanName(input.agentName), "invoke_agent"),...agentTraceIdentityAttributes({rootSessionId: input.turn.rootSessionId,@@ -113,7 +115,8 @@ export function agentStepAttributes(input: {...(channelClassification["agent.channel.kind"] === "unknown"? undefined: channelClassification),- ...agentSpanNamingAttributes("agent.step"),+ "gen_ai.operation.name": "workflow",+ ...agentSpanNamingAttributes("agent.step", "workflow"),...agentTraceIdentityAttributes({rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,traceSessionId: traceSessionIdOf(event.scope),diff --git a/packages/eve/src/tracing/agent-otel-turn-state.test.ts b/packages/eve/src/tracing/agent-otel-turn-state.test.tsnew file mode 100644index 000000000..b293d33bd--- /dev/null+++ b/packages/eve/src/tracing/agent-otel-turn-state.test.ts@@ -0,0 +1,46 @@+import { describe, expect, it } from "vitest";++import {+ rememberTurnInputMessages,+ rememberTurnOutputMessages,+} from "#tracing/agent-otel-turn-state.js";+import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";++describe("agent OTel turn state", () => {+ it("keeps the first model input and latest model output across a multi-call turn", async () => {+ const store = new InMemoryAgentTraceStateStore();+ store.setTurn("session-1", "turn-1", {+ context: spanContext(),+ rootSessionId: "session-1",+ traceSessionId: "session-1",+ sequence: 0,+ startTimeMs: 1,+ });+ const scope = {+ attemptId: "attempt-1",+ attemptIndex: 0,+ functionId: "agent",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };++ await rememberTurnInputMessages(store, scope, "first input");+ await rememberTurnInputMessages(store, scope, "second input");+ await rememberTurnOutputMessages(store, scope, "first output");+ await rememberTurnOutputMessages(store, scope, "final output");++ expect(store.getTurn("session-1", "turn-1")).toMatchObject({+ inputMessagesAttribute: "first input",+ outputMessagesAttribute: "final output",+ });+ });+});++function spanContext() {+ return {+ spanId: "2222222222222222",+ traceFlags: 1,+ traceId: "11111111111111111111111111111111",+ };+}diff --git a/packages/eve/src/tracing/agent-otel-turn-state.ts b/packages/eve/src/tracing/agent-otel-turn-state.tsnew file mode 100644index 000000000..e799fea1c--- /dev/null+++ b/packages/eve/src/tracing/agent-otel-turn-state.ts@@ -0,0 +1,28 @@+import type { InstrumentationAttemptScope } from "#instrumentation/lifecycle.js";+import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";++/** Keeps the first model input so the turn-level invoke_agent span represents its task. */+export function rememberTurnInputMessages(+ store: AgentTraceStateStore,+ scope: InstrumentationAttemptScope,+ inputMessagesAttribute: string | undefined,+): void | PromiseLike<void> {+ if (inputMessagesAttribute === undefined) return;+ return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({+ ...turn,+ inputMessagesAttribute: turn.inputMessagesAttribute ?? inputMessagesAttribute,+ }));+}++/** Keeps the latest model output so the completed invoke_agent span represents its result. */+export function rememberTurnOutputMessages(+ store: AgentTraceStateStore,+ scope: InstrumentationAttemptScope,+ outputMessagesAttribute: string | undefined,+): void | PromiseLike<void> {+ if (outputMessagesAttribute === undefined) return;+ return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({+ ...turn,+ outputMessagesAttribute,+ }));+}diff --git a/packages/eve/src/tracing/agent-trace-context-codec.ts b/packages/eve/src/tracing/agent-trace-context-codec.tsindex 1dac78e93..3f823b836 100644--- a/packages/eve/src/tracing/agent-trace-context-codec.ts+++ b/packages/eve/src/tracing/agent-trace-context-codec.ts@@ -8,6 +8,7 @@ import type {import { normalizeChannelAudience } from "#shared/channel-audience.js";import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";import { boundedTraceError } from "#tracing/bounded-error.js";+import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";import { boundedPrincipalId } from "#tracing/telemetry-budget.js";import { isInstrumentationPrincipalType } from "#instrumentation/lifecycle.js";@@ -112,7 +113,9 @@ function deserializeTurn(value: unknown): AgentTurnTraceState | undefined {context: value.context,currentPrincipal: deserializePrincipalSummary(value.currentPrincipal),initiatorPrincipal: deserializePrincipalSummary(value.initiatorPrincipal),+ inputMessagesAttribute: deserializeContentAttribute(value.inputMessagesAttribute),modelUsage: deserializeModelUsage(value.modelUsage),+ outputMessagesAttribute: deserializeContentAttribute(value.outputMessagesAttribute),parentLineage: deserializeParentLineage(value.parentLineage),rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",traceSessionId: decodeTraceSessionId(value),@@ -211,6 +214,10 @@ function deserializeAction(value: unknown): AgentActionTraceState | undefined {};}+function deserializeContentAttribute(value: unknown): string | undefined {+ return typeof value === "string" && value.length <= CONTENT_ATTRIBUTE_LIMIT ? value : undefined;+}+function deserializeModelUsage(value: unknown,): NonNullable<AgentTurnTraceState["modelUsage"]> | undefined {diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.tsindex 364ff5576..c24cd6cb1 100644--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts@@ -5,6 +5,7 @@ import { SessionTraceSeedKey } from "#context/keys.js";import { deserializeContext, serializeContext } from "#context/serialize.js";import { INSTRUMENTATION_PRINCIPAL_TYPES } from "#instrumentation/lifecycle.js";import { deserializeAgentTraceContextState } from "#tracing/agent-trace-context-codec.js";+import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";import {ContextAgentTraceStateStore,preserveSerializedAgentTraceState,@@ -36,7 +37,9 @@ describe("ContextAgentTraceStateStore", () => {context: spanContext("1", "3"),currentPrincipal: { id: "user-123", type: "user" },initiatorPrincipal: { type: "none" },+ inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',modelUsage: { inputTokens: 12, outputTokens: 4 },+ outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',caller: { ...spanContext("4", "2"), isRemote: true },rootSessionId: "session-1",traceSessionId: "session-1",@@ -67,7 +70,9 @@ describe("ContextAgentTraceStateStore", () => {},currentPrincipal: { id: "user-123", type: "user" },initiatorPrincipal: { type: "none" },+ inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',modelUsage: { inputTokens: 12, outputTokens: 4 },+ outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',caller: { ...spanContext("4", "2"), isRemote: true },startTimeMs: 1_700_000_000_000,subagentName: "researcher",@@ -84,6 +89,25 @@ describe("ContextAgentTraceStateStore", () => {});});+ it("rejects oversized persisted turn content attributes", () => {+ const state = deserializeAgentTraceContextState({+ turns: {+ turn: {+ context: spanContext("1", "2"),+ inputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),+ outputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),+ rootSessionId: "session-1",+ sequence: 0,+ startTimeMs: 1,+ },+ },+ });++ expect(state.turns.turn).toBeDefined();+ expect(state.turns.turn?.inputMessagesAttribute).toBeUndefined();+ expect(state.turns.turn?.outputMessagesAttribute).toBeUndefined();+ });+it.each([...INSTRUMENTATION_PRINCIPAL_TYPES.map((type) => ({input: { type, id: "user-123" },diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsindex 82189757e..74278d1a3 100644--- a/packages/eve/src/tracing/agent-trace-state.ts+++ b/packages/eve/src/tracing/agent-trace-state.ts@@ -33,7 +33,11 @@ export interface AgentTurnTraceState {readonly currentPrincipal?: InstrumentationPrincipalSummary;readonly initiatorPrincipal?: InstrumentationPrincipalSummary;readonly parentLineage?: InstrumentationParentLineage;+ /** First model input projected onto the turn-level invoke_agent span. */+ readonly inputMessagesAttribute?: string;readonly modelUsage?: { readonly inputTokens?: number; readonly outputTokens?: number };+ /** Latest model output projected onto the turn-level invoke_agent span. */+ readonly outputMessagesAttribute?: string;readonly rootSessionId: string;readonly sequence: number;readonly startTimeMs: number;diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.tsindex 81fb692cf..78f79f98c 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts@@ -229,7 +229,19 @@ describe("local instrumentation runtime", () => {? "execute_tool": exported.name.startsWith("chat ")? "chat"- : exported.name,+ : "workflow",+ },+ },+ {+ key: "gen_ai.operation.name",+ value: {+ stringValue: exported.name.startsWith("invoke_agent ")+ ? "invoke_agent"+ : exported.name.startsWith("execute_tool ")+ ? "execute_tool"+ : exported.name.startsWith("chat ")+ ? "chat"+ : "workflow",},},]),