Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): use GenAI system instructions in trace viewer (#4093)

Chad Hietalaaa73bcbeed6a
diff --git a/.changeset/clean-traces-use-genai-system-instructions.md b/.changeset/clean-traces-use-genai-system-instructions.md
new file mode 100644
index 000000000..e34ae9eb8
--- /dev/null
+++ b/.changeset/clean-traces-use-genai-system-instructions.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Trace viewers now use the GenAI system-instructions attribute instead of recording a duplicate system prompt attribute.
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
index b55c39fab..634e5f336 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.test.ts
@@ -443,7 +443,8 @@ describe("renderConversationItem", () => {
const turn = span("a".repeat(16), "agent.turn", 0, 0, undefined, {});
const step = span("b".repeat(16), "agent.step", 10, 100, turn.spanId, {});
const model = span("c".repeat(16), "ai.streamText.doStream", 20, 50, step.spanId, {
- "ai.prompt.system": "You are a test assistant. Be brief.",
+ "gen_ai.system_instructions":
+ '[{"content":"You are a test assistant. Be brief.","type":"text"}]',
"gen_ai.input.messages": JSON.stringify([
{ parts: [{ content: "hi", type: "text" }], role: "user" },
]),
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index c97a61246..1a5eadd0c 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -72,8 +72,9 @@ export function buildConversationItems(trace: LocalTrace): ConversationItem[] {
const entries: { readonly item: ConversationItem; readonly order: bigint }[] = [];
const firstSystemSpan = [...trace.spans]
.sort(compareLocalTraceSpans)
- .find((span) => isModelSpan(span) && typeof span.attributes["ai.prompt.system"] === "string");
- const systemText = firstSystemSpan?.attributes["ai.prompt.system"];
+ .find((span) => isModelSpan(span) && systemInstructionsText(span.attributes) !== undefined);
+ const systemText =
+ firstSystemSpan === undefined ? undefined : systemInstructionsText(firstSystemSpan.attributes);
if (firstSystemSpan !== undefined && typeof systemText === "string" && systemText.length > 0) {
entries.push({
item: {
@@ -182,6 +183,32 @@ export function buildConversationItems(trace: LocalTrace): ConversationItem[] {
.map((entry) => entry.item);
}
+function systemInstructionsText(attributes: Readonly<Record<string, unknown>>): string | undefined {
+ const value = attributes["gen_ai.system_instructions"];
+ if (typeof value !== "string") return undefined;
+ try {
+ const instructions: unknown = JSON.parse(value);
+ if (!Array.isArray(instructions)) return undefined;
+ const text = instructions
+ .flatMap((instruction: unknown) => {
+ if (
+ typeof instruction !== "object" ||
+ instruction === null ||
+ !("type" in instruction) ||
+ instruction.type !== "text" ||
+ !("content" in instruction) ||
+ typeof instruction.content !== "string"
+ )
+ return [];
+ return [instruction.content];
+ })
+ .join("\n\n");
+ return text.length > 0 ? text : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
function subagentFor(
span: LocalTraceSpan,
subagents: ReadonlyMap<string, ConversationSubagent>,
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-view.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-view.test.ts
index 3adda4058..22a041832 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-view.test.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-view.test.ts
@@ -54,7 +54,7 @@ function conversationSpans(): LocalTraceSpan[] {
"gen_ai.input.messages": JSON.stringify([
{ parts: [{ content: "hi", type: "text" }], role: "user" },
]),
- "ai.prompt.system": "You are a test assistant.",
+ "gen_ai.system_instructions": '[{"content":"You are a test assistant.","type":"text"}]',
"ai.response.text": "reply",
});
const action = span("d".repeat(16), "agent.action", 2100, 2400, turn.spanId, {
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-view.ts b/packages/eve/src/cli/dev/tui/traces/trace-view.ts
index 8308e0bc7..6954ab9c0 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-view.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-view.ts
@@ -192,7 +192,7 @@ function overlayRight(row: string, segment: string, width: number, margin: numbe
*/
const CONVERSATION_CONTENT_KEYS: ReadonlySet<string> = new Set([
"gen_ai.input.messages",
- "ai.prompt.system",
+ "gen_ai.system_instructions",
"ai.response.reasoning",
"ai.response.text",
"ai.response.tool_calls",
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.test.ts b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.test.ts
index 21ed62e2d..f62539fa9 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.test.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-viewer-state.test.ts
@@ -548,7 +548,7 @@ describe("applyLoadedTrace", () => {
"gen_ai.input.messages": JSON.stringify([
{ parts: [{ content: "hi", type: "text" }], role: "user" },
]),
- "ai.prompt.system": "system prompt",
+ "gen_ai.system_instructions": '[{"content":"system prompt","type":"text"}]',
"ai.response.text": "reply",
},
}),
diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.ts
index fd1e90246..505ee15cb 100644
--- a/packages/eve/src/tracing/agent-otel-content.ts
+++ b/packages/eve/src/tracing/agent-otel-content.ts
@@ -102,7 +102,7 @@ function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<s
return out;
}
-/** Normalizes the AI SDK's `instructions` prompt to plain text for `ai.prompt.system`. */
+/** Normalizes the AI SDK's `instructions` prompt to plain text. */
export function systemPromptAttribute(instructions: unknown): string | undefined {
if (typeof instructions === "string") return textContentAttribute(instructions);
if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined;
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 4d4cea292..fe9ed6fc8 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -2730,9 +2730,6 @@ describe("createAgentOtelInstrumentation", () => {
const spans = runtime.exporter.getFinishedSpans();
const model = byName(spans, "chat claude-test")[0]!;
const tool = byName(spans, "execute_tool weather")[0]!;
- expect(model.attributes["ai.prompt.system"]).toBe(
- "You are a weather assistant (system prompt).",
- );
expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls");
expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather");
expect(model.attributes["ai.response.text"]).toBe("Checking the weather.");
@@ -2746,6 +2743,7 @@ describe("createAgentOtelInstrumentation", () => {
"gen_ai.system_instructions":
'[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
});
+ expect(model.attributes).not.toHaveProperty("ai.prompt.system");
expect(model.attributes["agent.input.messages.delta"]).toBeUndefined();
// Provider-executed tools never reach the tool loop; their calls and
// results are captured off the model response content.
@@ -2783,6 +2781,9 @@ describe("createAgentOtelInstrumentation", () => {
expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
"gen_ai.input.messages",
);
+ expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
+ "gen_ai.system_instructions",
+ );
expect(byName(spans, "chat claude-test")[0]?.attributes).not.toHaveProperty(
"gen_ai.output.messages",
);
@@ -3034,11 +3035,8 @@ describe("createAgentOtelInstrumentation", () => {
}
const firstModel = byName(firstRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;
const secondModel = byName(secondRuntime.exporter.getFinishedSpans(), "chat claude-test")[0]!;
- expect(firstModel.attributes["ai.prompt.system"]).toBe(
- "You are a weather assistant (system prompt).",
- );
- expect(secondModel.attributes["ai.prompt.system"]).toBe(
- "You are a weather assistant (system prompt).",
+ expect(firstModel.attributes["gen_ai.system_instructions"]).toBe(
+ '[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
);
expect(secondModel.attributes["gen_ai.system_instructions"]).toBe(
'[{"content":"You are a weather assistant (system prompt).","type":"text"}]',
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index c837ea34a..a6ab0095b 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -20,7 +20,6 @@ import {
genAiInputMessagesAttribute,
genAiOutputMessagesAttribute,
genAiSystemInstructionsAttribute,
- systemPromptAttribute,
textContentAttribute,
toolResultsContentAttribute,
} from "#tracing/agent-otel-content.js";
@@ -387,8 +386,6 @@ export function createAgentOtelInstrumentation(
if (recordInputs && event.input !== undefined) {
const genAiMessages = genAiInputMessagesAttribute(event.input.messages);
if (genAiMessages !== undefined) span.setAttribute("gen_ai.input.messages", genAiMessages);
- const system = systemPromptAttribute(event.input.instructions);
- if (system !== undefined) span.setAttribute("ai.prompt.system", system);
const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);
if (genAiSystem !== undefined) {
span.setAttribute("gen_ai.system_instructions", genAiSystem);
diff --git a/packages/eve/src/tracing/content-attributes.test.ts b/packages/eve/src/tracing/content-attributes.test.ts
index 21d80e6a6..f60c580ea 100644
--- a/packages/eve/src/tracing/content-attributes.test.ts
+++ b/packages/eve/src/tracing/content-attributes.test.ts
@@ -8,6 +8,7 @@ const ATTRIBUTES = {
"agent.trace.content.input": true,
"agent.trace.content.output": true,
"gen_ai.input.messages": "what the user said",
+ "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
"ai.response.finish_reason": "stop",
"ai.response.text": "what the model said",
"gen_ai.request.model": "test-model",
@@ -55,6 +56,7 @@ describe("withoutDeclinedContent", () => {
"agent.trace.content.input": true,
"agent.trace.content.output": false,
"gen_ai.input.messages": "what the user said",
+ "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
"ai.response.finish_reason": "stop",
"gen_ai.request.model": "test-model",
"gen_ai.tool.call.arguments": "{}",
@@ -62,6 +64,15 @@ describe("withoutDeclinedContent", () => {
});
});
+ it("drops system instructions when the destination declined inputs", () => {
+ expect(
+ withoutDeclinedContent(
+ { "gen_ai.system_instructions": '[{"content":"private","type":"text"}]' },
+ { recordInputs: false, recordOutputs: true },
+ ),
+ ).toEqual({});
+ });
+
// The prefixes are shared: `ai.response.finish_reason` and `gen_ai.tool.name`
// say what happened rather than what was said, so declining content cannot
// cost a destination the ability to read its own traces.
diff --git a/packages/eve/src/tracing/content-attributes.ts b/packages/eve/src/tracing/content-attributes.ts
index 0f0417244..2409ab0cf 100644
--- a/packages/eve/src/tracing/content-attributes.ts
+++ b/packages/eve/src/tracing/content-attributes.ts
@@ -20,7 +20,6 @@ const INPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([
"agent.approval.request",
"ai.documents",
"ai.prompt",
- "ai.prompt.system",
"ai.prompt.toolChoice",
"ai.prompt.tools",
"ai.value",
diff --git a/packages/eve/test/tui-client/tui-traces.ts b/packages/eve/test/tui-client/tui-traces.ts
index 9952de5d1..b1253fe5b 100644
--- a/packages/eve/test/tui-client/tui-traces.ts
+++ b/packages/eve/test/tui-client/tui-traces.ts
@@ -115,7 +115,7 @@ void (async () => {
parentSpanId: step,
attributes: {
"gen_ai.request.model": "smoke-model-v1",
- "ai.prompt.system": SYSTEM_PROMPT,
+ "gen_ai.system_instructions": JSON.stringify([{ content: SYSTEM_PROMPT, type: "text" }]),
"ai.response.text": "smoke reply",
},
});