The Open Framework for Building Agents
Record tool definitions on GenAI chat spans (#3695)
roboeve[bot]880dd8b95445
diff --git a/.changeset/quiet-tools-explain.md b/.changeset/quiet-tools-explain.mdnew file mode 100644index 000000000..7c5c56953--- /dev/null+++ b/.changeset/quiet-tools-explain.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Record offered tool definitions on GenAI chat spans and expose them to lifecycle handlers when input capture is enabled.diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.tsindex 91fbe118f..9d4e1b8f4 100644--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts@@ -62,12 +62,15 @@ describe("createAiSdkHookBridge", () => {it("publishes normalized model lifecycle to every provider", async () => {const calls: string[] = [];+ const startedEvents: InstrumentationModelCallStartedEvent[] = [];+ const tools = [{ inputSchema: { type: "object" }, name: "get_weather" }];const provider = (name: string): InstrumentationProviderDefinition => {const states = new Map<string, string>();return {events: {"model.call.started"(event) {calls.push(`${name}:started:${event.idempotencyKey}`);+ startedEvents.push(event);states.set(event.idempotencyKey, `${name}-state`);},"model.call.completed"(event) {@@ -77,13 +80,14 @@ describe("createAiSdkHookBridge", () => {},},name,+ tracePolicy: contentTracePolicy,};};const hooks = createInstrumentationHooks([provider("a"), provider("b")]);const bridge = createAiSdkHookBridge(scope, hooks);await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [- { callId: "call-1", messages: [], modelId: "model", provider: "test", tools: undefined },+ { callId: "call-1", messages: [], modelId: "model", provider: "test", tools },]);await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [{@@ -103,6 +107,10 @@ describe("createAiSdkHookBridge", () => {`a:completed:${id}:a-state`,`b:completed:${id}:b-state`,]);+ expect(startedEvents).toHaveLength(2);+ expect(startedEvents[0]?.input?.tools).toEqual(tools);+ expect(startedEvents[0]?.input?.tools).not.toBe(tools);+ expect(Object.isFrozen(startedEvents[0]?.input?.tools)).toBe(true);});it("gives provider retries distinct identities and terminalizes failed calls", async () => {diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.tsindex 9b295e5ba..80c8006f9 100644--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts@@ -208,6 +208,7 @@ function toModelCallStarted(? Object.freeze({instructions: source.instructions,messages: Object.freeze([...source.messages]),+ tools: source.tools === undefined ? undefined : Object.freeze([...source.tools]),}): undefined,model: Object.freeze({ modelId: source.modelId, provider: source.provider }),diff --git a/packages/eve/src/instrumentation/lifecycle.ts b/packages/eve/src/instrumentation/lifecycle.tsindex 4292104dd..4e997975d 100644--- a/packages/eve/src/instrumentation/lifecycle.ts+++ b/packages/eve/src/instrumentation/lifecycle.ts@@ -51,6 +51,7 @@ export interface InstrumentationUsage {export interface InstrumentationModelInput {readonly instructions?: unknown;readonly messages: readonly unknown[];+ readonly tools?: readonly unknown[];}/**diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.tsindex 93f2ab53a..74dd467a0 100644--- a/packages/eve/src/tracing/agent-otel-content.test.ts+++ b/packages/eve/src/tracing/agent-otel-content.test.ts@@ -6,6 +6,7 @@ import {genAiInputMessagesAttribute,genAiOutputMessagesAttribute,genAiSystemInstructionsAttribute,+ genAiToolDefinitionsAttribute,toolResultsContentAttribute,} from "#tracing/agent-otel-content.js";@@ -195,6 +196,47 @@ describe("GenAI message attributes", () => {});});+describe("genAiToolDefinitionsAttribute", () => {+ it("maps AI SDK input schemas to GenAI parameters", () => {+ expect(+ genAiToolDefinitionsAttribute([+ {+ description: "Get the current weather.",+ inputSchema: {+ properties: { city: { type: "string" } },+ required: ["city"],+ type: "object",+ },+ name: "get_weather",+ },+ ]),+ ).toBe(+ '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',+ );+ });++ it("drops trailing definitions to keep oversized content valid JSON", () => {+ const attribute = genAiToolDefinitionsAttribute([+ { inputSchema: { type: "object" }, name: "first" },+ {+ description: "x".repeat(CONTENT_ATTRIBUTE_LIMIT),+ inputSchema: { type: "object" },+ name: "oversized",+ },+ { inputSchema: { type: "object" }, name: "last" },+ ]);++ expect(attribute).toBeDefined();+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);+ expect(JSON.parse(attribute!)).toEqual([{ name: "first", parameters: { type: "object" } }]);+ });++ it("returns undefined when no tool definitions are available", () => {+ expect(genAiToolDefinitionsAttribute([])).toBeUndefined();+ expect(genAiToolDefinitionsAttribute(undefined)).toBeUndefined();+ });+});+describe("toolResultsContentAttribute", () => {it("returns the full payload when it fits", () => {const json = toolResultsContentAttribute([diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.tsindex 341bab5b7..4810380c1 100644--- a/packages/eve/src/tracing/agent-otel-content.ts+++ b/packages/eve/src/tracing/agent-otel-content.ts@@ -53,6 +53,31 @@ export function genAiSystemInstructionsAttribute(instructions: unknown): stringreturn fitSemanticText(text, (content) => [{ content, type: "text" }]);}+/** Serializes tool definitions, dropping trailing entries to keep the JSON whole. */+export function genAiToolDefinitionsAttribute(tools: unknown): string | undefined {+ if (!Array.isArray(tools)) return undefined;+ const definitions = tools.flatMap((tool): Record<string, unknown>[] => {+ if (!isRecord(tool) || typeof tool.name !== "string") return [];+ const definition: Record<string, unknown> = { name: tool.name };+ if (typeof tool.description === "string") definition.description = tool.description;+ if (tool.inputSchema !== undefined) definition.parameters = tool.inputSchema;+ return [definition];+ });+ if (definitions.length === 0) return undefined;++ const serialized: string[] = [];+ let length = 2;+ for (const definition of definitions) {+ const json = stringifyContent(definition);+ if (json === undefined) break;+ const nextLength = length + (serialized.length === 0 ? 0 : 1) + json.length;+ if (nextLength > CONTENT_ATTRIBUTE_LIMIT) break;+ serialized.push(json);+ length = nextLength;+ }+ return `[${serialized.join(",")}]`;+}+/** Serializes one model response using the OpenTelemetry GenAI message schema. */export function genAiOutputMessagesAttribute(content: readonly unknown[],diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 09792f1d4..0e8ec64db 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -107,6 +107,8 @@ function createRuntime(const agentOtelInput: Omit<AgentOtelInstrumentationInput, "tracePolicy"> & {tracePolicy?: TraceCapturePolicy;} = {+ // Pin the environment so an inherited EVE_DEV/VERCEL_ENV cannot relax audience ceilings.+ environment: "production",frameworkVersion: "test",idGenerator,recordInputs: true,@@ -136,6 +138,7 @@ async function emitAttempt(input: {readonly attemptError?: Error;readonly channelAudience?: ChannelAudience;readonly hooks: InstrumentationHooks;+ readonly modelTools?: readonly Record<string, unknown>[];readonly parentLineage?: InstrumentationParentLineage;readonly parentTraceContext?: InstrumentationTraceContext;readonly runInContext: InstrumentationContextRunner;@@ -190,6 +193,7 @@ async function emitAttempt(input: {],modelId: "claude-test",provider: "anthropic",+ tools: input.modelTools,},]);await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined });@@ -2726,6 +2730,17 @@ describe("createAgentOtelInstrumentation", () => {const runtime = createRuntime();await emitAttempt({hooks: runtime.hooks,+ modelTools: [+ {+ description: "Get the current weather.",+ inputSchema: {+ properties: { city: { type: "string" } },+ required: ["city"],+ type: "object",+ },+ name: "get_weather",+ },+ ],runInContext: runtime.runInContext,sessionId: "session-1",turnId: "turn-1",@@ -2748,6 +2763,8 @@ describe("createAgentOtelInstrumentation", () => {"gen_ai.response.finish_reasons": ["tool-calls"],"gen_ai.system_instructions":'[{"content":"You are a weather assistant (system prompt).","type":"text"}]',+ "gen_ai.tool.definitions":+ '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',});expect(model.attributes).not.toHaveProperty("ai.prompt.system");expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();@@ -2772,6 +2789,12 @@ describe("createAgentOtelInstrumentation", () => {await emitAttempt({channelAudience: "public",hooks: runtime.hooks,+ modelTools: [+ {+ inputSchema: { type: "object" },+ name: "get_weather",+ },+ ],runInContext: runtime.runInContext,sessionId: "session-redacted",turnId: "turn-redacted",@@ -2786,6 +2809,9 @@ describe("createAgentOtelInstrumentation", () => {expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty("gen_ai.system_instructions",);+ expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(+ "gen_ai.tool.definitions",+ );expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty("gen_ai.output.messages",);diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 3b9a24369..2d69f94e4 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -20,6 +20,7 @@ import {genAiInputMessagesAttribute,genAiOutputMessagesAttribute,genAiSystemInstructionsAttribute,+ genAiToolDefinitionsAttribute,textContentAttribute,toolResultsContentAttribute,} from "#tracing/agent-otel-content.js";@@ -388,6 +389,10 @@ export function createAgentOtelInstrumentation(if (genAiSystem !== undefined) {span.setAttribute("gen_ai.system_instructions", genAiSystem);}+ const toolDefinitions = genAiToolDefinitionsAttribute(event.input.tools);+ if (toolDefinitions !== undefined) {+ span.setAttribute("gen_ai.tool.definitions", toolDefinitions);+ }}const state = { context: trace.setSpan(attempt.context, span), span };getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);