Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat(client): add tool-misuse trace-analysis dimension (#1394)
Morgan Carr256de33cc459
diff --git a/src/client/src/__tests__/fixtures/trace-analysis.example.json b/src/client/src/__tests__/fixtures/trace-analysis.example.jsonindex 8ddc9f39..b20ffa0c 100644--- a/src/client/src/__tests__/fixtures/trace-analysis.example.json+++ b/src/client/src/__tests__/fixtures/trace-analysis.example.json@@ -34,6 +34,7 @@"token_efficiency": [],"path_analysis": [],"prompt_injection": [],+ "tool_misuse": [],"totals": {"span_count": 4,"total_tokens": 12000,diff --git a/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap b/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snapindex 4df88fc0..85f81acc 100644--- a/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap+++ b/src/client/src/__tests__/lib/platform/chat/__snapshots__/trace-analysis.test.ts.snap@@ -1,6 +1,6 @@// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing-exports[`trace analysis schema fixture has all seven dimensions and span refs on findings 1`] = `+exports[`trace analysis schema fixture has all eight dimensions and span refs on findings 1`] = `["strengths","improvements",@@ -9,5 +9,6 @@ exports[`trace analysis schema fixture has all seven dimensions and span refs on"token_efficiency","path_analysis","prompt_injection",+ "tool_misuse",]`;diff --git a/src/client/src/__tests__/lib/platform/chat/improvement.test.ts b/src/client/src/__tests__/lib/platform/chat/improvement.test.tsindex b4574e55..d9df182a 100644--- a/src/client/src/__tests__/lib/platform/chat/improvement.test.ts+++ b/src/client/src/__tests__/lib/platform/chat/improvement.test.ts@@ -468,7 +468,7 @@ describe("streamTraceImprovementAnalysis", () => {expect(streamText).not.toHaveBeenCalled();});- it("streams seven dimension analyses with grader passes, saves the run, and emits done data", async () => {+ it("streams eight dimension analyses with grader passes, saves the run, and emits done data", async () => {(getHeirarchyViaSpanId as jest.Mock).mockResolvedValue({record: hierarchy,err: null,@@ -486,7 +486,7 @@ describe("streamTraceImprovementAnalysis", () => {const doneEvent = events.find((event) => event.type === "done");const insertValues = (dataCollector as jest.Mock).mock.calls[1][0].values[0];- expect(streamText).toHaveBeenCalledTimes(14);+ expect(streamText).toHaveBeenCalledTimes(16);expect(getModelInstance).toHaveBeenCalledWith("openai", "sk-test", "gpt-4o-mini");expect(dimensionEvents.map((event) => event.dimension)).toEqual(["strengths",@@ -496,6 +496,7 @@ describe("streamTraceImprovementAnalysis", () => {"token_efficiency","path_analysis","prompt_injection",+ "tool_misuse",]);expect(dimensionEvents[0].findings[0]).toEqual(expect.objectContaining({@@ -507,8 +508,8 @@ describe("streamTraceImprovementAnalysis", () => {expect(insertValues.root_span_id).toBe("root-span");expect(insertValues.selected_span_id).toBe("llm-span");expect(insertValues.analysis_type).toBe("trace_analysis");- expect(insertValues.prompt_tokens).toBe(140);- expect(insertValues.completion_tokens).toBe(28);+ expect(insertValues.prompt_tokens).toBe(160);+ expect(insertValues.completion_tokens).toBe(32);expect(JSON.parse(insertValues.analysis_json).totals).toEqual({span_count: 3,total_tokens: 170,diff --git a/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts b/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.tsindex 85192d62..b46ec4fe 100644--- a/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts+++ b/src/client/src/__tests__/lib/platform/chat/trace-analysis-registry.test.ts@@ -19,6 +19,7 @@ const expectedDimensions = ["token_efficiency","path_analysis","prompt_injection",+ "tool_misuse",];const expectedUiLabels = {@@ -29,6 +30,7 @@ const expectedUiLabels = {token_efficiency: "Token efficiency",path_analysis: "Path",prompt_injection: messages.TRACE_AI_PROMPT_INJECTION_UI_LABEL,+ tool_misuse: messages.TRACE_AI_TOOL_MISUSE_UI_LABEL,};const expectedStreamLabels = {@@ -39,11 +41,12 @@ const expectedStreamLabels = {token_efficiency: "Token efficiency",path_analysis: "Path analysis",prompt_injection: messages.TRACE_AI_PROMPT_INJECTION_STREAM_LABEL,+ tool_misuse: messages.TRACE_AI_TOOL_MISUSE_STREAM_LABEL,};describe("trace analysis dimension registry", () => {- it("keeps the six existing dimensions in order and appends prompt injection", () => {- expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS).toHaveLength(7);+ it("keeps the existing dimensions in order and appends tool misuse", () => {+ expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS).toHaveLength(8);expect(TRACE_ANALYSIS_DIMENSION_DEFINITIONS.map(({ key }) => key)).toEqual(expectedDimensions);@@ -120,6 +123,47 @@ describe("trace analysis dimension registry", () => {}});+ it("defines a complete tool-misuse dimension through the registry contract", () => {+ const definition = TRACE_ANALYSIS_DIMENSION_REGISTRY.tool_misuse;++ expect(definition).toMatchObject({+ key: "tool_misuse",+ uiLabel: messages.TRACE_AI_TOOL_MISUSE_UI_LABEL,+ streamLabel: messages.TRACE_AI_TOOL_MISUSE_STREAM_LABEL,+ guidance: messages.TRACE_AI_TOOL_MISUSE_GUIDANCE,+ emptyStateCopy: {+ summary: messages.TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,+ detail: messages.TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,+ },+ spanFields: [+ "toolName",+ "toolCallId",+ "toolArgs",+ "toolResult",+ "systemPrompt",+ "prompt",+ "response",+ ],+ metricFields: [+ "toolCallCount",+ "toolsUsed",+ "duplicateToolInputs",+ "repeatedSpanNames",+ "potentialRetrySequences",+ "errorCount",+ ],+ });+ for (const value of [+ definition.uiLabel,+ definition.streamLabel,+ definition.guidance,+ definition.emptyStateCopy.summary,+ definition.emptyStateCopy.detail,+ ]) {+ expect(value.trim()).not.toBe("");+ }+ });+it("projects injection-bearing and clean span evidence through the existing selectors", () => {const injectionBearingSpan = {spanId: "injection-span",@@ -198,6 +242,76 @@ describe("trace analysis dimension registry", () => {);});+ it("projects tool-call context and sequence metrics through the existing selectors", () => {+ const toolBearingSpan = {+ spanId: "tool-call-span",+ spanName: "agent.tool-call",+ role: "assistant",+ statusCode: "STATUS_CODE_ERROR",+ statusMessage: "unexpected tool sequence",+ durationMs: 12,+ systemPrompt: "Use tools only to answer the current request.",+ prompt: "Check the account balance without making changes.",+ response: "The account balance is available.",+ toolName: "payments.refund",+ toolCallId: "call-tool-misuse",+ toolArgs: '{"chargeId":"charge-123"}',+ toolResult: '{"status":"refunded"}',+ children: [],+ };++ expect(selectTraceAnalysisSpan(toolBearingSpan, "tool_misuse")).toEqual({+ spanId: "tool-call-span",+ spanName: "agent.tool-call",+ role: "assistant",+ statusCode: "STATUS_CODE_ERROR",+ statusMessage: "unexpected tool sequence",+ durationMs: 12,+ error: undefined,+ children: [],+ toolName: "payments.refund",+ toolCallId: "call-tool-misuse",+ toolArgs: '{"chargeId":"charge-123"}',+ toolResult: '{"status":"refunded"}',+ systemPrompt: "Use tools only to answer the current request.",+ prompt: "Check the account balance without making changes.",+ response: "The account balance is available.",+ });++ const metrics = {+ spanCount: 3,+ maxDepth: 2,+ errorCount: 1,+ llmCallCount: 1,+ toolCallCount: 2,+ retrievalCallCount: 0,+ modelsUsed: ["gpt-4o-mini"],+ toolsUsed: ["accounts.balance", "payments.refund"],+ duplicateToolInputs: [+ {+ key: 'payments.refund:{"chargeId":"charge-123"}',+ count: 2,+ spanIds: ["tool-call-span", "retry-span"],+ },+ ],+ repeatedSpanNames: [+ {+ name: "tool.payments.refund",+ count: 2,+ spanIds: ["tool-call-span", "retry-span"],+ },+ ],+ potentialRetrySequences: [+ {+ reason: "same tool followed the balance lookup",+ spanIds: ["tool-call-span", "retry-span"],+ },+ ],+ };++ expect(selectTraceAnalysisMetrics(metrics, "tool_misuse")).toEqual(metrics);+ });+it("leaves missing prompt-injection span fields undefined like other dimensions", () => {const selected = selectTraceAnalysisSpan({@@ -222,6 +336,48 @@ describe("trace analysis dimension registry", () => {});});+ it("leaves missing tool-misuse evidence undefined like other dimensions", () => {+ const selectedSpan = selectTraceAnalysisSpan(+ {+ spanId: "missing-tool-content",+ spanName: "agent.empty",+ durationMs: 1,+ children: [],+ },+ "tool_misuse"+ );+ const selectedMetrics = selectTraceAnalysisMetrics(+ {+ spanCount: 1,+ maxDepth: 1,+ errorCount: 0,+ llmCallCount: 1,+ toolCallCount: 0,+ retrievalCallCount: 0,+ modelsUsed: ["gpt-4o-mini"],+ toolsUsed: [],+ },+ "tool_misuse"+ );++ expect(selectedSpan).toMatchObject({+ spanId: "missing-tool-content",+ spanName: "agent.empty",+ toolName: undefined,+ toolCallId: undefined,+ toolArgs: undefined,+ toolResult: undefined,+ systemPrompt: undefined,+ prompt: undefined,+ response: undefined,+ });+ expect(selectedMetrics).toMatchObject({+ duplicateToolInputs: undefined,+ repeatedSpanNames: undefined,+ potentialRetrySequences: undefined,+ });+ });+it("exports every prompt-injection copy key from en.ts", () => {const messageTable = messages as Record<string, unknown>;for (const key of [@@ -236,6 +392,20 @@ describe("trace analysis dimension registry", () => {}});+ it("exports every tool-misuse copy key from en.ts", () => {+ const messageTable = messages as Record<string, unknown>;+ for (const key of [+ "TRACE_AI_TOOL_MISUSE_UI_LABEL",+ "TRACE_AI_TOOL_MISUSE_STREAM_LABEL",+ "TRACE_AI_TOOL_MISUSE_GUIDANCE",+ "TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY",+ "TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL",+ ]) {+ expect(messageTable[key]).toEqual(expect.any(String));+ expect((messageTable[key] as string).trim()).not.toBe("");+ }+ });+it("selects the same focused span and metric fields as the legacy branches", () => {const child = {spanId: "child-span",diff --git a/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts b/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.tsindex a4e2b57b..57f680e4 100644--- a/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts+++ b/src/client/src/__tests__/lib/platform/chat/trace-analysis.test.ts@@ -6,7 +6,7 @@ import {} from "@/types/trace-analysis";describe("trace analysis schema fixture", () => {- it("has all seven dimensions and span refs on findings", () => {+ it("has all eight dimensions and span refs on findings", () => {expect(Object.keys(fixture).filter((key) =>(TRACE_ANALYSIS_DIMENSIONS as readonly string[]).includes(key))).toMatchSnapshot();@@ -22,7 +22,7 @@ describe("trace analysis schema fixture", () => {});describe("emptyTraceAnalysis", () => {- it("returns all seven dimension arrays as empty", () => {+ it("returns all eight dimension arrays as empty", () => {const analysis = emptyTraceAnalysis("trace-123");expect(analysis.trace_id).toBe("trace-123");expect(analysis.summary).toBe("");@@ -48,7 +48,7 @@ describe("emptyTraceAnalysis", () => {});describe("ensureTraceAnalysisDimensions", () => {- it("fills prompt_injection as empty when reading older stored analyses", () => {+ it("fills appended security dimensions when reading older stored analyses", () => {const legacyStored = {trace_id: "legacy-trace",summary: "pre-security-dimension run",@@ -64,6 +64,7 @@ describe("ensureTraceAnalysisDimensions", () => {const dimensions = ensureTraceAnalysisDimensions(legacyStored);expect(dimensions.prompt_injection).toEqual([]);+ expect(dimensions.tool_misuse).toEqual([]);expect(dimensions.strengths).toEqual(fixture.strengths);expect(Object.keys(dimensions).sort()).toEqual([...TRACE_ANALYSIS_DIMENSIONS].sort()diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex a8e2bd5a..66871616 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -1879,11 +1879,11 @@ export const TRACE_AI_STRENGTHS_EMPTY_DETAIL =export const TRACE_AI_IMPROVEMENTS_UI_LABEL = "Improvements";export const TRACE_AI_IMPROVEMENTS_STREAM_LABEL = "Improvements";export const TRACE_AI_IMPROVEMENTS_GUIDANCE =- "Find concrete general improvements that do not belong in cost, token_efficiency, wrong_turns, path_analysis, or prompt_injection. Avoid generic advice; cite spans.";+ "Find concrete general improvements that do not belong in cost, token_efficiency, wrong_turns, path_analysis, prompt_injection, or tool_misuse. Avoid generic advice; cite spans.";export const TRACE_AI_IMPROVEMENTS_EMPTY_SUMMARY ="No general improvements are required right now.";export const TRACE_AI_IMPROVEMENTS_EMPTY_DETAIL =- "The trace did not show a broad improvement opportunity outside the more specific cost, token, path, wrong-turn, or prompt-injection categories.";+ "The trace did not show a broad improvement opportunity outside the more specific cost, token, path, wrong-turn, prompt-injection, or tool-misuse categories.";export const TRACE_AI_WRONG_TURNS_UI_LABEL = "Wrong turns";export const TRACE_AI_WRONG_TURNS_STREAM_LABEL = "Wrong turns";export const TRACE_AI_WRONG_TURNS_GUIDANCE =@@ -1924,6 +1924,14 @@ export const TRACE_AI_PROMPT_INJECTION_EMPTY_SUMMARY ="No prompt-injection or jailbreak evidence was surfaced.";export const TRACE_AI_PROMPT_INJECTION_EMPTY_DETAIL ="The analysis did not surface explicit instruction-override, role-confusion, or jailbreak-framing evidence in the selected span content. This does not establish that the trace is safe.";+export const TRACE_AI_TOOL_MISUSE_UI_LABEL = "Tool misuse";+export const TRACE_AI_TOOL_MISUSE_STREAM_LABEL = "Tool misuse";+export const TRACE_AI_TOOL_MISUSE_GUIDANCE =+ "Flag tool calls whose arguments or sequence diverge from the intent of the surrounding chat context. Cite the relevant spans and describe only the observed evidence; this is trace-review guidance, not a runtime guardrail.";+export const TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY =+ "No tool-misuse evidence was surfaced.";+export const TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL =+ "The analysis did not surface tool-call arguments or sequences that diverged from the intent of the surrounding chat context. This does not establish that the trace is safe.";export const TRACE_AI_TOKENS_SAVED = (tokens: number | string) =>`${tokens} tokens saved`;export const TRACE_AI_USD_SAVED = (usd: number | string) => `$${usd} saved`;diff --git a/src/client/src/lib/platform/chat/improvement.ts b/src/client/src/lib/platform/chat/improvement.tsindex e13da708..458f3509 100644--- a/src/client/src/lib/platform/chat/improvement.ts+++ b/src/client/src/lib/platform/chat/improvement.ts@@ -132,6 +132,7 @@ type TraceAnalysis = {token_efficiency: Finding[];path_analysis: Finding[];prompt_injection: Finding[];+ tool_misuse: Finding[];totals: {span_count: number;total_tokens: number;diff --git a/src/client/src/lib/platform/chat/trace-analysis-registry.ts b/src/client/src/lib/platform/chat/trace-analysis-registry.tsindex d0bfacb1..b0c7b71d 100644--- a/src/client/src/lib/platform/chat/trace-analysis-registry.ts+++ b/src/client/src/lib/platform/chat/trace-analysis-registry.ts@@ -29,6 +29,11 @@ import {TRACE_AI_TOKEN_EFFICIENCY_GUIDANCE,TRACE_AI_TOKEN_EFFICIENCY_STREAM_LABEL,TRACE_AI_TOKEN_EFFICIENCY_UI_LABEL,+ TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,+ TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,+ TRACE_AI_TOOL_MISUSE_GUIDANCE,+ TRACE_AI_TOOL_MISUSE_STREAM_LABEL,+ TRACE_AI_TOOL_MISUSE_UI_LABEL,TRACE_AI_WRONG_TURNS_EMPTY_DETAIL,TRACE_AI_WRONG_TURNS_EMPTY_SUMMARY,TRACE_AI_WRONG_TURNS_GUIDANCE,@@ -324,6 +329,33 @@ export const TRACE_ANALYSIS_DIMENSION_DEFINITIONS = Object.freeze([],metricFields: [],}),+ defineDimension({+ key: "tool_misuse",+ uiLabel: TRACE_AI_TOOL_MISUSE_UI_LABEL,+ streamLabel: TRACE_AI_TOOL_MISUSE_STREAM_LABEL,+ guidance: TRACE_AI_TOOL_MISUSE_GUIDANCE,+ emptyStateCopy: {+ summary: TRACE_AI_TOOL_MISUSE_EMPTY_SUMMARY,+ detail: TRACE_AI_TOOL_MISUSE_EMPTY_DETAIL,+ },+ spanFields: [+ "toolName",+ "toolCallId",+ "toolArgs",+ "toolResult",+ "systemPrompt",+ "prompt",+ "response",+ ],+ metricFields: [+ "toolCallCount",+ "toolsUsed",+ "duplicateToolInputs",+ "repeatedSpanNames",+ "potentialRetrySequences",+ "errorCount",+ ],+ }),] as const);export type TraceAnalysisDimensionKey =