Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Record tool definitions on GenAI chat spans (#3695)

roboeve[bot]880dd8b95445
diff --git a/.changeset/quiet-tools-explain.md b/.changeset/quiet-tools-explain.md
new file mode 100644
index 000000000..7c5c56953
--- /dev/null
+++ b/.changeset/quiet-tools-explain.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Record offered tool definitions on GenAI chat spans and expose them to lifecycle handlers when input capture is enabled.
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
index 91fbe118f..9d4e1b8f4 100644
--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
@@ -62,12 +62,15 @@ describe("createAiSdkHookBridge", () => {
it("publishes normalized model lifecycle to every provider", async () => {
const calls: string[] = [];
+ const startedEvents: InstrumentationModelCallStartedEvent[] = [];
+ const tools = [{ inputSchema: { type: "object" }, name: "get_weather" }];
const provider = (name: string): InstrumentationProviderDefinition => {
const states = new Map<string, string>();
return {
events: {
"model.call.started"(event) {
calls.push(`${name}:started:${event.idempotencyKey}`);
+ startedEvents.push(event);
states.set(event.idempotencyKey, `${name}-state`);
},
"model.call.completed"(event) {
@@ -77,13 +80,14 @@ describe("createAiSdkHookBridge", () => {
},
},
name,
+ tracePolicy: contentTracePolicy,
};
};
const hooks = createInstrumentationHooks([provider("a"), provider("b")]);
const bridge = createAiSdkHookBridge(scope, hooks);
await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
- { callId: "call-1", messages: [], modelId: "model", provider: "test", tools: undefined },
+ { callId: "call-1", messages: [], modelId: "model", provider: "test", tools },
]);
await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [
{
@@ -103,6 +107,10 @@ describe("createAiSdkHookBridge", () => {
`a:completed:${id}:a-state`,
`b:completed:${id}:b-state`,
]);
+ expect(startedEvents).toHaveLength(2);
+ expect(startedEvents[0]?.input?.tools).toEqual(tools);
+ expect(startedEvents[0]?.input?.tools).not.toBe(tools);
+ expect(Object.isFrozen(startedEvents[0]?.input?.tools)).toBe(true);
});
it("gives provider retries distinct identities and terminalizes failed calls", async () => {
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
index 9b295e5ba..80c8006f9 100644
--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
@@ -208,6 +208,7 @@ function toModelCallStarted(
? Object.freeze({
instructions: source.instructions,
messages: Object.freeze([...source.messages]),
+ tools: source.tools === undefined ? undefined : Object.freeze([...source.tools]),
})
: undefined,
model: Object.freeze({ modelId: source.modelId, provider: source.provider }),
diff --git a/packages/eve/src/instrumentation/lifecycle.ts b/packages/eve/src/instrumentation/lifecycle.ts
index 4292104dd..4e997975d 100644
--- a/packages/eve/src/instrumentation/lifecycle.ts
+++ b/packages/eve/src/instrumentation/lifecycle.ts
@@ -51,6 +51,7 @@ export interface InstrumentationUsage {
export interface InstrumentationModelInput {
readonly instructions?: unknown;
readonly messages: readonly unknown[];
+ readonly tools?: readonly unknown[];
}
/**
diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.ts
index 93f2ab53a..74dd467a0 100644
--- a/packages/eve/src/tracing/agent-otel-content.test.ts
+++ b/packages/eve/src/tracing/agent-otel-content.test.ts
@@ -6,6 +6,7 @@ import {
genAiInputMessagesAttribute,
genAiOutputMessagesAttribute,
genAiSystemInstructionsAttribute,
+ genAiToolDefinitionsAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
@@ -195,6 +196,47 @@ describe("GenAI message attributes", () => {
});
});
+describe("genAiToolDefinitionsAttribute", () => {
+ it("maps AI SDK input schemas to GenAI parameters", () => {
+ expect(
+ genAiToolDefinitionsAttribute([
+ {
+ description: "Get the current weather.",
+ inputSchema: {
+ properties: { city: { type: "string" } },
+ required: ["city"],
+ type: "object",
+ },
+ name: "get_weather",
+ },
+ ]),
+ ).toBe(
+ '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',
+ );
+ });
+
+ it("drops trailing definitions to keep oversized content valid JSON", () => {
+ const attribute = genAiToolDefinitionsAttribute([
+ { inputSchema: { type: "object" }, name: "first" },
+ {
+ description: "x".repeat(CONTENT_ATTRIBUTE_LIMIT),
+ inputSchema: { type: "object" },
+ name: "oversized",
+ },
+ { inputSchema: { type: "object" }, name: "last" },
+ ]);
+
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ expect(JSON.parse(attribute!)).toEqual([{ name: "first", parameters: { type: "object" } }]);
+ });
+
+ it("returns undefined when no tool definitions are available", () => {
+ expect(genAiToolDefinitionsAttribute([])).toBeUndefined();
+ expect(genAiToolDefinitionsAttribute(undefined)).toBeUndefined();
+ });
+});
+
describe("toolResultsContentAttribute", () => {
it("returns the full payload when it fits", () => {
const json = toolResultsContentAttribute([
diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.ts
index 341bab5b7..4810380c1 100644
--- a/packages/eve/src/tracing/agent-otel-content.ts
+++ b/packages/eve/src/tracing/agent-otel-content.ts
@@ -53,6 +53,31 @@ export function genAiSystemInstructionsAttribute(instructions: unknown): string
return fitSemanticText(text, (content) => [{ content, type: "text" }]);
}
+/** Serializes tool definitions, dropping trailing entries to keep the JSON whole. */
+export function genAiToolDefinitionsAttribute(tools: unknown): string | undefined {
+ if (!Array.isArray(tools)) return undefined;
+ const definitions = tools.flatMap((tool): Record<string, unknown>[] => {
+ if (!isRecord(tool) || typeof tool.name !== "string") return [];
+ const definition: Record<string, unknown> = { name: tool.name };
+ if (typeof tool.description === "string") definition.description = tool.description;
+ if (tool.inputSchema !== undefined) definition.parameters = tool.inputSchema;
+ return [definition];
+ });
+ if (definitions.length === 0) return undefined;
+
+ const serialized: string[] = [];
+ let length = 2;
+ for (const definition of definitions) {
+ const json = stringifyContent(definition);
+ if (json === undefined) break;
+ const nextLength = length + (serialized.length === 0 ? 0 : 1) + json.length;
+ if (nextLength > CONTENT_ATTRIBUTE_LIMIT) break;
+ serialized.push(json);
+ length = nextLength;
+ }
+ return `[${serialized.join(",")}]`;
+}
+
/** Serializes one model response using the OpenTelemetry GenAI message schema. */
export function genAiOutputMessagesAttribute(
content: readonly unknown[],
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 09792f1d4..0e8ec64db 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -107,6 +107,8 @@ function createRuntime(
const agentOtelInput: Omit<AgentOtelInstrumentationInput, "tracePolicy"> & {
tracePolicy?: TraceCapturePolicy;
} = {
+ // Pin the environment so an inherited EVE_DEV/VERCEL_ENV cannot relax audience ceilings.
+ environment: "production",
frameworkVersion: "test",
idGenerator,
recordInputs: true,
@@ -136,6 +138,7 @@ async function emitAttempt(input: {
readonly attemptError?: Error;
readonly channelAudience?: ChannelAudience;
readonly hooks: InstrumentationHooks;
+ readonly modelTools?: readonly Record<string, unknown>[];
readonly parentLineage?: InstrumentationParentLineage;
readonly parentTraceContext?: InstrumentationTraceContext;
readonly runInContext: InstrumentationContextRunner;
@@ -190,6 +193,7 @@ async function emitAttempt(input: {
],
modelId: "claude-test",
provider: "anthropic",
+ tools: input.modelTools,
},
]);
await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined });
@@ -2726,6 +2730,17 @@ describe("createAgentOtelInstrumentation", () => {
const runtime = createRuntime();
await emitAttempt({
hooks: runtime.hooks,
+ modelTools: [
+ {
+ description: "Get the current weather.",
+ inputSchema: {
+ properties: { city: { type: "string" } },
+ required: ["city"],
+ type: "object",
+ },
+ name: "get_weather",
+ },
+ ],
runInContext: runtime.runInContext,
sessionId: "session-1",
turnId: "turn-1",
@@ -2748,6 +2763,8 @@ describe("createAgentOtelInstrumentation", () => {
"gen_ai.response.finish_reasons": ["tool-calls"],
"gen_ai.system_instructions":
'[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
+ "gen_ai.tool.definitions":
+ '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',
});
expect(model.attributes).not.toHaveProperty("ai.prompt.system");
expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();
@@ -2772,6 +2789,12 @@ describe("createAgentOtelInstrumentation", () => {
await emitAttempt({
channelAudience: "public",
hooks: runtime.hooks,
+ modelTools: [
+ {
+ inputSchema: { type: "object" },
+ name: "get_weather",
+ },
+ ],
runInContext: runtime.runInContext,
sessionId: "session-redacted",
turnId: "turn-redacted",
@@ -2786,6 +2809,9 @@ describe("createAgentOtelInstrumentation", () => {
expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
"gen_ai.system_instructions",
);
+ expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
+ "gen_ai.tool.definitions",
+ );
expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
"gen_ai.output.messages",
);
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index 3b9a24369..2d69f94e4 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -20,6 +20,7 @@ import {
genAiInputMessagesAttribute,
genAiOutputMessagesAttribute,
genAiSystemInstructionsAttribute,
+ genAiToolDefinitionsAttribute,
textContentAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
@@ -388,6 +389,10 @@ export function createAgentOtelInstrumentation(
if (genAiSystem !== undefined) {
span.setAttribute("gen_ai.system_instructions", genAiSystem);
}
+ const toolDefinitions = genAiToolDefinitionsAttribute(event.input.tools);
+ if (toolDefinitions !== undefined) {
+ span.setAttribute("gen_ai.tool.definitions", toolDefinitions);
+ }
}
const state = { context: trace.setSpan(attempt.context, span), span };
getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);