Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat(client): add tool-misuse trace-analysis dimension (#1394)

Morgan Carr256de33cc459
diff --git a/src/client/src/__tests__/fixtures/trace-analysis.example.json b/src/client/src/__tests__/fixtures/trace-analysis.example.json
index 8ddc9f39..b20ffa0c 100644
--- a/src/client/src/__tests__/fixtures/trace-analysis.example.json
+++ b/src/client/src/__tests__/fixtures/trace-analysis.example.json
@@ -34,6 +34,7 @@
"token_efficiency": [],
"path_analysis": [],
"prompt_injection": [],
+ "tool_misuse": [],
"totals": {
"span_count": 4,
"total_tokens": 12000,
diff --git a/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap b/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap
index 4df88fc0..85f81acc 100644
--- a/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap
+++ b/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap
@@ -1,6 +1,6 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
-exports[`trace analysis schema fixture has all seven dimensions and span refs on findings 1`] = `
+exports[`trace analysis schema fixture has all eight dimensions and span refs on findings 1`] = `
[
"strengths",
"improvements",
@@ -9,5 +9,6 @@ exports[`trace analysis schema fixture has all seven dimensions and span refs on
"token_efficiency",
"path_analysis",
"prompt_injection",
+ "tool_misuse",
]
`;
diff --git a/src/client/src/__tests__/lib/platform/chat/improvement.test.ts b/src/client/src/__tests__/lib/platform/chat/improvement.test.ts
index b4574e55..d9df182a 100644
--- a/src/client/src/__tests__/lib/platform/chat/improvement.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/improvement.test.ts
@@ -468,7 +468,7 @@ describe("streamTraceImprovementAnalysis", () => {
expect(streamText).not.toHaveBeenCalled();
});
- it("streams seven dimension analyses with grader passes, saves the run, and emits done data", async () => {
+ it("streams eight dimension analyses with grader passes, saves the run, and emits done data", async () => {
(getHeirarchyViaSpanId as jest.Mock).mockResolvedValue({
record: hierarchy,
err: null,
@@ -486,7 +486,7 @@ describe("streamTraceImprovementAnalysis", () => {
const doneEvent = events.find((event) => event.type === "done");
const insertValues = (dataCollector as jest.Mock).mock.calls[1][0].values[0];
- expect(streamText).toHaveBeenCalledTimes(14);
+ expect(streamText).toHaveBeenCalledTimes(16);
expect(getModelInstance).toHaveBeenCalledWith("openai", "sk-test", "gpt-4o-mini");
expect(dimensionEvents.map((event) => event.dimension)).toEqual([
"strengths",
@@ -496,6 +496,7 @@ describe("streamTraceImprovementAnalysis", () => {
"token_efficiency",
"path_analysis",
"prompt_injection",
+ "tool_misuse",
]);
expect(dimensionEvents[0].findings[0]).toEqual(
expect.objectContaining({
@@ -507,8 +508,8 @@ describe("streamTraceImprovementAnalysis", () => {
expect(insertValues.root_span_id).toBe("root-span");
expect(insertValues.selected_span_id).toBe("llm-span");
expect(insertValues.analysis_type).toBe("trace_analysis");
- expect(insertValues.prompt_tokens).toBe(140);
- expect(insertValues.completion_tokens).toBe(28);
+ expect(insertValues.prompt_tokens).toBe(160);
+ expect(insertValues.completion_tokens).toBe(32);
expect(JSON.parse(insertValues.analysis_json).totals).toEqual({
span_count: 3,
total_tokens: 170,
diff --git a/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts b/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts
index 85192d62..b46ec4fe 100644
--- a/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts
@@ -19,6 +19,7 @@ const expectedDimensions = [
"token_efficiency",
"path_analysis",
"prompt_injection",
+ "tool_misuse",
];
const expectedUiLabels = {
@@ -29,6 +30,7 @@ const expectedUiLabels = {
token_efficiency: "Token efficiency",
path_analysis: "Path",
prompt_injection: messages.TRACE_AI_PROMPT_INJECTION_UI_LABEL,
+ tool_misuse: messages.TRACE_AI_TOOL_MISUSE_UI_LABEL,
};
const expectedStreamLabels = {
@@ -39,11 +41,12 @@ const expectedStreamLabels = {
token_efficiency: "Token efficiency",
path_analysis: "Path analysis",
prompt_injection: messages.TRACE_AI_PROMPT_INJECTION_STREAM_LABEL,
+ tool_misuse: messages.TRACE_AI_TOOL_MISUSE_STREAM_LABEL,
};
describe("trace analysis dimension registry", () => {
- it("keeps the six existing dimensions in order and appends prompt injection", () => {
- expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS).toHaveLength(7);
+ it("keeps the existing dimensions in order and appends tool misuse", () => {
+ expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS).toHaveLength(8);
expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS.map(({ key }) => key)).toEqual(
expectedDimensions
);
@@ -120,6 +123,47 @@ describe("trace analysis dimension registry", () => {
}
});
+ it("defines a complete tool-misuse dimension through the registry contract", () => {
+ const definition = TRACE_ANALYSIS_DIMENSION_REGISTRY.tool_misuse;
+
+ expect(definition).toMatchObject({
+ key: "tool_misuse",
+ uiLabel: messages.TRACE_AI_TOOL_MISUSE_UI_LABEL,
+ streamLabel: messages.TRACE_AI_TOOL_MISUSE_STREAM_LABEL,
+ guidance: messages.TRACE_AI_TOOL_MISUSE_GUIDANCE,
+ emptyStateCopy: {
+ summary: messages.TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,
+ detail: messages.TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,
+ },
+ spanFields: [
+ "toolName",
+ "toolCallId",
+ "toolArgs",
+ "toolResult",
+ "systemPrompt",
+ "prompt",
+ "response",
+ ],
+ metricFields: [
+ "toolCallCount",
+ "toolsUsed",
+ "duplicateToolInputs",
+ "repeatedSpanNames",
+ "potentialRetrySequences",
+ "errorCount",
+ ],
+ });
+ for (const value of [
+ definition.uiLabel,
+ definition.streamLabel,
+ definition.guidance,
+ definition.emptyStateCopy.summary,
+ definition.emptyStateCopy.detail,
+ ]) {
+ expect(value.trim()).not.toBe("");
+ }
+ });
+
it("projects injection-bearing and clean span evidence through the existing selectors", () => {
const injectionBearingSpan = {
spanId: "injection-span",
@@ -198,6 +242,76 @@ describe("trace analysis dimension registry", () => {
);
});
+ it("projects tool-call context and sequence metrics through the existing selectors", () => {
+ const toolBearingSpan = {
+ spanId: "tool-call-span",
+ spanName: "agent.tool-call",
+ role: "assistant",
+ statusCode: "STATUS_CODE_ERROR",
+ statusMessage: "unexpected tool sequence",
+ durationMs: 12,
+ systemPrompt: "Use tools only to answer the current request.",
+ prompt: "Check the account balance without making changes.",
+ response: "The account balance is available.",
+ toolName: "payments.refund",
+ toolCallId: "call-tool-misuse",
+ toolArgs: '{"chargeId":"charge-123"}',
+ toolResult: '{"status":"refunded"}',
+ children: [],
+ };
+
+ expect(selectTraceAnalysisSpan(toolBearingSpan, "tool_misuse")).toEqual({
+ spanId: "tool-call-span",
+ spanName: "agent.tool-call",
+ role: "assistant",
+ statusCode: "STATUS_CODE_ERROR",
+ statusMessage: "unexpected tool sequence",
+ durationMs: 12,
+ error: undefined,
+ children: [],
+ toolName: "payments.refund",
+ toolCallId: "call-tool-misuse",
+ toolArgs: '{"chargeId":"charge-123"}',
+ toolResult: '{"status":"refunded"}',
+ systemPrompt: "Use tools only to answer the current request.",
+ prompt: "Check the account balance without making changes.",
+ response: "The account balance is available.",
+ });
+
+ const metrics = {
+ spanCount: 3,
+ maxDepth: 2,
+ errorCount: 1,
+ llmCallCount: 1,
+ toolCallCount: 2,
+ retrievalCallCount: 0,
+ modelsUsed: ["gpt-4o-mini"],
+ toolsUsed: ["accounts.balance", "payments.refund"],
+ duplicateToolInputs: [
+ {
+ key: 'payments.refund:{"chargeId":"charge-123"}',
+ count: 2,
+ spanIds: ["tool-call-span", "retry-span"],
+ },
+ ],
+ repeatedSpanNames: [
+ {
+ name: "tool.payments.refund",
+ count: 2,
+ spanIds: ["tool-call-span", "retry-span"],
+ },
+ ],
+ potentialRetrySequences: [
+ {
+ reason: "same tool followed the balance lookup",
+ spanIds: ["tool-call-span", "retry-span"],
+ },
+ ],
+ };
+
+ expect(selectTraceAnalysisMetrics(metrics, "tool_misuse")).toEqual(metrics);
+ });
+
it("leaves missing prompt-injection span fields undefined like other dimensions", () => {
const selected = selectTraceAnalysisSpan(
{
@@ -222,6 +336,48 @@ describe("trace analysis dimension registry", () => {
});
});
+ it("leaves missing tool-misuse evidence undefined like other dimensions", () => {
+ const selectedSpan = selectTraceAnalysisSpan(
+ {
+ spanId: "missing-tool-content",
+ spanName: "agent.empty",
+ durationMs: 1,
+ children: [],
+ },
+ "tool_misuse"
+ );
+ const selectedMetrics = selectTraceAnalysisMetrics(
+ {
+ spanCount: 1,
+ maxDepth: 1,
+ errorCount: 0,
+ llmCallCount: 1,
+ toolCallCount: 0,
+ retrievalCallCount: 0,
+ modelsUsed: ["gpt-4o-mini"],
+ toolsUsed: [],
+ },
+ "tool_misuse"
+ );
+
+ expect(selectedSpan).toMatchObject({
+ spanId: "missing-tool-content",
+ spanName: "agent.empty",
+ toolName: undefined,
+ toolCallId: undefined,
+ toolArgs: undefined,
+ toolResult: undefined,
+ systemPrompt: undefined,
+ prompt: undefined,
+ response: undefined,
+ });
+ expect(selectedMetrics).toMatchObject({
+ duplicateToolInputs: undefined,
+ repeatedSpanNames: undefined,
+ potentialRetrySequences: undefined,
+ });
+ });
+
it("exports every prompt-injection copy key from en.ts", () => {
const messageTable = messages as Record<string, unknown>;
for (const key of [
@@ -236,6 +392,20 @@ describe("trace analysis dimension registry", () => {
}
});
+ it("exports every tool-misuse copy key from en.ts", () => {
+ const messageTable = messages as Record<string, unknown>;
+ for (const key of [
+ "TRACE_AI_TOOL_MISUSE_UI_LABEL",
+ "TRACE_AI_TOOL_MISUSE_STREAM_LABEL",
+ "TRACE_AI_TOOL_MISUSE_GUIDANCE",
+ "TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY",
+ "TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL",
+ ]) {
+ expect(messageTable[key]).toEqual(expect.any(String));
+ expect((messageTable[key] as string).trim()).not.toBe("");
+ }
+ });
+
it("selects the same focused span and metric fields as the legacy branches", () => {
const child = {
spanId: "child-span",
diff --git a/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts b/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts
index a4e2b57b..57f680e4 100644
--- a/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts
+++ b/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts
@@ -6,7 +6,7 @@ import {
} from "@/types/trace-analysis";
describe("trace analysis schema fixture", () => {
- it("has all seven dimensions and span refs on findings", () => {
+ it("has all eight dimensions and span refs on findings", () => {
expect(Object.keys(fixture).filter((key) =>
(TRACE_ANALYSIS_DIMENSIONS as readonly string[]).includes(key)
)).toMatchSnapshot();
@@ -22,7 +22,7 @@ describe("trace analysis schema fixture", () => {
});
describe("emptyTraceAnalysis", () => {
- it("returns all seven dimension arrays as empty", () => {
+ it("returns all eight dimension arrays as empty", () => {
const analysis = emptyTraceAnalysis("trace-123");
expect(analysis.trace_id).toBe("trace-123");
expect(analysis.summary).toBe("");
@@ -48,7 +48,7 @@ describe("emptyTraceAnalysis", () => {
});
describe("ensureTraceAnalysisDimensions", () => {
- it("fills prompt_injection as empty when reading older stored analyses", () => {
+ it("fills appended security dimensions when reading older stored analyses", () => {
const legacyStored = {
trace_id: "legacy-trace",
summary: "pre-security-dimension run",
@@ -64,6 +64,7 @@ describe("ensureTraceAnalysisDimensions", () => {
const dimensions = ensureTraceAnalysisDimensions(legacyStored);
expect(dimensions.prompt_injection).toEqual([]);
+ expect(dimensions.tool_misuse).toEqual([]);
expect(dimensions.strengths).toEqual(fixture.strengths);
expect(Object.keys(dimensions).sort()).toEqual(
[...TRACE_ANALYSIS_DIMENSIONS].sort()
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index a8e2bd5a..66871616 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -1879,11 +1879,11 @@ export const TRACE_AI_STRENGTHS_EMPTY_DETAIL =
export const TRACE_AI_IMPROVEMENTS_UI_LABEL = "Improvements";
export const TRACE_AI_IMPROVEMENTS_STREAM_LABEL = "Improvements";
export const TRACE_AI_IMPROVEMENTS_GUIDANCE =
- "Find concrete general improvements that do not belong in cost, token_efficiency, wrong_turns, path_analysis, or prompt_injection. Avoid generic advice; cite spans.";
+ "Find concrete general improvements that do not belong in cost, token_efficiency, wrong_turns, path_analysis, prompt_injection, or tool_misuse. Avoid generic advice; cite spans.";
export const TRACE_AI_IMPROVEMENTS_EMPTY_SUMMARY =
"No general improvements are required right now.";
export const TRACE_AI_IMPROVEMENTS_EMPTY_DETAIL =
- "The trace did not show a broad improvement opportunity outside the more specific cost, token, path, wrong-turn, or prompt-injection categories.";
+ "The trace did not show a broad improvement opportunity outside the more specific cost, token, path, wrong-turn, prompt-injection, or tool-misuse categories.";
export const TRACE_AI_WRONG_TURNS_UI_LABEL = "Wrong turns";
export const TRACE_AI_WRONG_TURNS_STREAM_LABEL = "Wrong turns";
export const TRACE_AI_WRONG_TURNS_GUIDANCE =
@@ -1924,6 +1924,14 @@ export const TRACE_AI_PROMPT_INJECTION_EMPTY_SUMMARY =
"No prompt-injection or jailbreak evidence was surfaced.";
export const TRACE_AI_PROMPT_INJECTION_EMPTY_DETAIL =
"The analysis did not surface explicit instruction-override, role-confusion, or jailbreak-framing evidence in the selected span content. This does not establish that the trace is safe.";
+export const TRACE_AI_TOOL_MISUSE_UI_LABEL = "Tool misuse";
+export const TRACE_AI_TOOL_MISUSE_STREAM_LABEL = "Tool misuse";
+export const TRACE_AI_TOOL_MISUSE_GUIDANCE =
+ "Flag tool calls whose arguments or sequence diverge from the intent of the surrounding chat context. Cite the relevant spans and describe only the observed evidence; this is trace-review guidance, not a runtime guardrail.";
+export const TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY =
+ "No tool-misuse evidence was surfaced.";
+export const TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL =
+ "The analysis did not surface tool-call arguments or sequences that diverged from the intent of the surrounding chat context. This does not establish that the trace is safe.";
export const TRACE_AI_TOKENS_SAVED = (tokens: number | string) =>
`${tokens} tokens saved`;
export const TRACE_AI_USD_SAVED = (usd: number | string) => `$${usd} saved`;
diff --git a/src/client/src/lib/platform/chat/improvement.ts b/src/client/src/lib/platform/chat/improvement.ts
index e13da708..458f3509 100644
--- a/src/client/src/lib/platform/chat/improvement.ts
+++ b/src/client/src/lib/platform/chat/improvement.ts
@@ -132,6 +132,7 @@ type TraceAnalysis = {
token_efficiency: Finding[];
path_analysis: Finding[];
prompt_injection: Finding[];
+ tool_misuse: Finding[];
totals: {
span_count: number;
total_tokens: number;
diff --git a/src/client/src/lib/platform/chat/trace-analysis-registry.ts b/src/client/src/lib/platform/chat/trace-analysis-registry.ts
index d0bfacb1..b0c7b71d 100644
--- a/src/client/src/lib/platform/chat/trace-analysis-registry.ts
+++ b/src/client/src/lib/platform/chat/trace-analysis-registry.ts
@@ -29,6 +29,11 @@ import {
TRACE_AI_TOKEN_EFFICIENCY_GUIDANCE,
TRACE_AI_TOKEN_EFFICIENCY_STREAM_LABEL,
TRACE_AI_TOKEN_EFFICIENCY_UI_LABEL,
+ TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,
+ TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,
+ TRACE_AI_TOOL_MISUSE_GUIDANCE,
+ TRACE_AI_TOOL_MISUSE_STREAM_LABEL,
+ TRACE_AI_TOOL_MISUSE_UI_LABEL,
TRACE_AI_WRONG_TURNS_EMPTY_DETAIL,
TRACE_AI_WRONG_TURNS_EMPTY_SUMMARY,
TRACE_AI_WRONG_TURNS_GUIDANCE,
@@ -324,6 +329,33 @@ export const TRACE_ANALYSIS_DIMENSION_DEFINITIONS = Object.freeze([
],
metricFields: [],
}),
+ defineDimension({
+ key: "tool_misuse",
+ uiLabel: TRACE_AI_TOOL_MISUSE_UI_LABEL,
+ streamLabel: TRACE_AI_TOOL_MISUSE_STREAM_LABEL,
+ guidance: TRACE_AI_TOOL_MISUSE_GUIDANCE,
+ emptyStateCopy: {
+ summary: TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,
+ detail: TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,
+ },
+ spanFields: [
+ "toolName",
+ "toolCallId",
+ "toolArgs",
+ "toolResult",
+ "systemPrompt",
+ "prompt",
+ "response",
+ ],
+ metricFields: [
+ "toolCallCount",
+ "toolsUsed",
+ "duplicateToolInputs",
+ "repeatedSpanNames",
+ "potentialRetrySequences",
+ "errorCount",
+ ],
+ }),
] as const);
export type TraceAnalysisDimensionKey =