The world's first agentic reverse engineer.
feat: add analyze_function_batch to LLM client
Amruthd88146e5cc79
diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.pyindex 96121f3..e5bd5d8 100644--- a/kong/agent/analyzer.py+++ b/kong/agent/analyzer.py@@ -39,6 +39,8 @@ class LLMClient(Protocol):max_rounds: int = 10,) -> LLMResponse: ...+ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]: ...+@dataclassclass LLMResponse:diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.pyindex 146ea28..c45df49 100644--- a/kong/agent/supervisor.py+++ b/kong/agent/supervisor.py@@ -23,7 +23,7 @@ from kong.export.source import ExportData, export_sourcefrom kong.export.structured import export_jsonfrom kong.ghidra.client import GhidraClientfrom kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntry-from kong.llm.client import TokenUsage+from kong.llm.usage import TokenUsagefrom kong.normalizer.syntactic import normalizefrom kong.synthesis.semantic import SemanticSynthesizer, SynthesisResultdiff --git a/kong/export/source.py b/kong/export/source.pyindex d49f4ad..6307271 100644--- a/kong/export/source.py+++ b/kong/export/source.py@@ -2,11 +2,10 @@ from __future__ import annotationsfrom dataclasses import dataclassfrom pathlib import Path-from kong.agent.models import FunctionResultfrom kong.agent.models import AnalysisStatsfrom kong.ghidra.types import BinaryInfo-from kong.llm.client import TokenUsage+from kong.llm.usage import TokenUsageSECTION_ORDER: list[tuple[str, str]] = [("crypto", "Crypto"),diff --git a/kong/llm/client.py b/kong/llm/client.pyindex a51de43..93c9c96 100644--- a/kong/llm/client.py+++ b/kong/llm/client.py@@ -8,77 +8,20 @@ Tracks token usage and cost per call. Supports both simplefrom __future__ import annotationsimport logging-from dataclasses import dataclass, fieldfrom typing import Anyimport anthropicfrom kong.agent.analyzer import Analyzer, LLMResponse-from kong.agent.prompts import OUTPUT_SCHEMA, SYSTEM_PROMPT+from kong.agent.prompts import BATCH_OUTPUT_SCHEMA, BATCH_SYSTEM_PROMPT, OUTPUT_SCHEMA, SYSTEM_PROMPTfrom kong.llm.tools import ToolExecutor+from kong.llm.usage import ModelTokenUsage, TokenUsage, _PRICINGlogger = logging.getLogger(__name__)-# Pricing per million tokens (input/output) by model.-# Source: https://docs.anthropic.com/en/docs/about-claude/models-_PRICING: dict[str, tuple[float, float]] = {- "claude-sonnet-4-20250514": (3.0, 15.0),- "claude-haiku-4-5-20251001": (1.0, 5.0),-}-DEFAULT_MODEL = "claude-sonnet-4-20250514"-@dataclass-class ModelTokenUsage:- input_tokens: int = 0- output_tokens: int = 0- cache_creation_tokens: int = 0- cache_read_tokens: int = 0- calls: int = 0-- def cost_usd(self, model: str) -> float:- input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))- cache_write_rate = input_rate * 1.25- cache_read_rate = input_rate * 0.10- return (- (self.input_tokens / 1_000_000) * input_rate- + (self.output_tokens / 1_000_000) * output_rate- + (self.cache_creation_tokens / 1_000_000) * cache_write_rate- + (self.cache_read_tokens / 1_000_000) * cache_read_rate- )---@dataclass-class TokenUsage:- by_model: dict[str, ModelTokenUsage] = field(default_factory=dict)-- def _get(self, model: str) -> ModelTokenUsage:- if model not in self.by_model:- self.by_model[model] = ModelTokenUsage()- return self.by_model[model]-- @property- def input_tokens(self) -> int:- return sum(m.input_tokens for m in self.by_model.values())-- @property- def output_tokens(self) -> int:- return sum(m.output_tokens for m in self.by_model.values())-- @property- def total_tokens(self) -> int:- return self.input_tokens + self.output_tokens-- @property- def calls(self) -> int:- return sum(m.calls for m in self.by_model.values())-- @property- def total_cost_usd(self) -> float:- return sum(m.cost_usd(model) for model, m in self.by_model.items())--class AnthropicClient:"""Concrete LLM client using the Anthropic SDK.@@ -130,6 +73,31 @@ class AnthropicClient:response.raw = raw_textreturn response+ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]:+ """Send a batch analysis prompt and return parsed list of responses."""+ effective_model = model or self.model+ message = self._client.messages.create(+ model=effective_model,+ max_tokens=self.max_tokens * 2,+ system=[{+ "type": "text",+ "text": f"{BATCH_SYSTEM_PROMPT}\n\n{BATCH_OUTPUT_SCHEMA}",+ "cache_control": {"type": "ephemeral"},+ }],+ messages=[+ {"role": "user", "content": prompt},+ ],+ )++ raw_text = self._extract_text(message)+ self._record_usage(message, effective_model)++ responses = Analyzer.parse_llm_json_batch(raw_text)+ for resp in responses:+ resp.input_tokens = message.usage.input_tokens+ resp.output_tokens = message.usage.output_tokens+ return responses+def analyze_with_tools(self,prompt: str,diff --git a/kong/llm/usage.py b/kong/llm/usage.pynew file mode 100644index 0000000..1e862ca--- /dev/null+++ b/kong/llm/usage.py@@ -0,0 +1,60 @@+"""Token usage tracking for LLM API calls."""++from __future__ import annotations++from dataclasses import dataclass, field++_PRICING: dict[str, tuple[float, float]] = {+ "claude-sonnet-4-20250514": (3.0, 15.0),+ "claude-haiku-4-5-20251001": (1.0, 5.0),+}+++@dataclass+class ModelTokenUsage:+ input_tokens: int = 0+ output_tokens: int = 0+ cache_creation_tokens: int = 0+ cache_read_tokens: int = 0+ calls: int = 0++ def cost_usd(self, model: str) -> float:+ input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))+ cache_write_rate = input_rate * 1.25+ cache_read_rate = input_rate * 0.10+ return (+ (self.input_tokens / 1_000_000) * input_rate+ + (self.output_tokens / 1_000_000) * output_rate+ + (self.cache_creation_tokens / 1_000_000) * cache_write_rate+ + (self.cache_read_tokens / 1_000_000) * cache_read_rate+ )+++@dataclass+class TokenUsage:+ by_model: dict[str, ModelTokenUsage] = field(default_factory=dict)++ def _get(self, model: str) -> ModelTokenUsage:+ if model not in self.by_model:+ self.by_model[model] = ModelTokenUsage()+ return self.by_model[model]++ @property+ def input_tokens(self) -> int:+ return sum(m.input_tokens for m in self.by_model.values())++ @property+ def output_tokens(self) -> int:+ return sum(m.output_tokens for m in self.by_model.values())++ @property+ def total_tokens(self) -> int:+ return self.input_tokens + self.output_tokens++ @property+ def calls(self) -> int:+ return sum(m.calls for m in self.by_model.values())++ @property+ def total_cost_usd(self) -> float:+ return sum(m.cost_usd(model) for model, m in self.by_model.items())diff --git a/tests/test_llm_client.py b/tests/test_llm_client.pyindex b056119..f9a8934 100644--- a/tests/test_llm_client.py+++ b/tests/test_llm_client.py@@ -2,6 +2,7 @@from __future__ import annotations+import jsonfrom unittest.mock import MagicMock, patchfrom kong.llm.client import (@@ -206,3 +207,62 @@ class TestPricingValues:sonnet_in, sonnet_out = _PRICING["claude-sonnet-4-20250514"]assert haiku_in < sonnet_inassert haiku_out < sonnet_out+++class TestAnalyzeFunctionBatch:+ def test_batch_returns_list_of_responses(self) -> None:+ mock_anthropic = MagicMock()+ mock_message = MagicMock()+ mock_message.content = [MagicMock(type="text", text=json.dumps([+ {"address": "0x1000", "name": "foo", "confidence": 80},+ {"address": "0x2000", "name": "bar", "confidence": 70},+ ]))]+ mock_message.usage = MagicMock(+ input_tokens=100, output_tokens=50,+ cache_creation_input_tokens=0, cache_read_input_tokens=0,+ )+ mock_anthropic.messages.create.return_value = mock_message++ client = AnthropicClient(api_key="test")+ client._client = mock_anthropic+ results = client.analyze_function_batch("batch prompt", model="claude-haiku-4-5-20251001")++ assert len(results) == 2+ assert results[0].name == "foo"+ assert results[1].name == "bar"++ def test_batch_uses_batch_system_prompt(self) -> None:+ mock_anthropic = MagicMock()+ mock_message = MagicMock()+ mock_message.content = [MagicMock(type="text", text='[{"name": "f", "confidence": 50}]')]+ mock_message.usage = MagicMock(+ input_tokens=100, output_tokens=50,+ cache_creation_input_tokens=0, cache_read_input_tokens=0,+ )+ mock_anthropic.messages.create.return_value = mock_message++ client = AnthropicClient(api_key="test")+ client._client = mock_anthropic+ client.analyze_function_batch("prompt")++ call_kwargs = mock_anthropic.messages.create.call_args+ system_text = call_kwargs.kwargs["system"][0]["text"]+ assert "multiple" in system_text.lower() or "batch" in system_text.lower()++ def test_batch_records_usage(self) -> None:+ mock_anthropic = MagicMock()+ mock_message = MagicMock()+ mock_message.content = [MagicMock(type="text", text='[{"name": "f", "confidence": 50}]')]+ mock_message.usage = MagicMock(+ input_tokens=500, output_tokens=200,+ cache_creation_input_tokens=100, cache_read_input_tokens=50,+ )+ mock_anthropic.messages.create.return_value = mock_message++ client = AnthropicClient(api_key="test")+ client._client = mock_anthropic+ client.analyze_function_batch("prompt")++ assert client.usage.input_tokens == 500+ assert client.usage.output_tokens == 200+ assert client.usage.calls == 1