Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

feat: add analyze_function_batch to LLM client

Amruthd88146e5cc79
diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.py
index 96121f3..e5bd5d8 100644
--- a/kong/agent/analyzer.py
+++ b/kong/agent/analyzer.py
@@ -39,6 +39,8 @@ class LLMClient(Protocol):
max_rounds: int = 10,
) -> LLMResponse: ...
+ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]: ...
+
@dataclass
class LLMResponse:
diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.py
index 146ea28..c45df49 100644
--- a/kong/agent/supervisor.py
+++ b/kong/agent/supervisor.py
@@ -23,7 +23,7 @@ from kong.export.source import ExportData, export_source
from kong.export.structured import export_json
from kong.ghidra.client import GhidraClient
from kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntry
-from kong.llm.client import TokenUsage
+from kong.llm.usage import TokenUsage
from kong.normalizer.syntactic import normalize
from kong.synthesis.semantic import SemanticSynthesizer, SynthesisResult
diff --git a/kong/export/source.py b/kong/export/source.py
index d49f4ad..6307271 100644
--- a/kong/export/source.py
+++ b/kong/export/source.py
@@ -2,11 +2,10 @@ from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
-
from kong.agent.models import FunctionResult
from kong.agent.models import AnalysisStats
from kong.ghidra.types import BinaryInfo
-from kong.llm.client import TokenUsage
+from kong.llm.usage import TokenUsage
SECTION_ORDER: list[tuple[str, str]] = [
("crypto", "Crypto"),
diff --git a/kong/llm/client.py b/kong/llm/client.py
index a51de43..93c9c96 100644
--- a/kong/llm/client.py
+++ b/kong/llm/client.py
@@ -8,77 +8,20 @@ Tracks token usage and cost per call. Supports both simple
from __future__ import annotations
import logging
-from dataclasses import dataclass, field
from typing import Any
import anthropic
from kong.agent.analyzer import Analyzer, LLMResponse
-from kong.agent.prompts import OUTPUT_SCHEMA, SYSTEM_PROMPT
+from kong.agent.prompts import BATCH_OUTPUT_SCHEMA, BATCH_SYSTEM_PROMPT, OUTPUT_SCHEMA, SYSTEM_PROMPT
from kong.llm.tools import ToolExecutor
+from kong.llm.usage import ModelTokenUsage, TokenUsage, _PRICING
logger = logging.getLogger(__name__)
-# Pricing per million tokens (input/output) by model.
-# Source: https://docs.anthropic.com/en/docs/about-claude/models
-_PRICING: dict[str, tuple[float, float]] = {
- "claude-sonnet-4-20250514": (3.0, 15.0),
- "claude-haiku-4-5-20251001": (1.0, 5.0),
-}
-
DEFAULT_MODEL = "claude-sonnet-4-20250514"
-@dataclass
-class ModelTokenUsage:
- input_tokens: int = 0
- output_tokens: int = 0
- cache_creation_tokens: int = 0
- cache_read_tokens: int = 0
- calls: int = 0
-
- def cost_usd(self, model: str) -> float:
- input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))
- cache_write_rate = input_rate * 1.25
- cache_read_rate = input_rate * 0.10
- return (
- (self.input_tokens / 1_000_000) * input_rate
- + (self.output_tokens / 1_000_000) * output_rate
- + (self.cache_creation_tokens / 1_000_000) * cache_write_rate
- + (self.cache_read_tokens / 1_000_000) * cache_read_rate
- )
-
-
-@dataclass
-class TokenUsage:
- by_model: dict[str, ModelTokenUsage] = field(default_factory=dict)
-
- def _get(self, model: str) -> ModelTokenUsage:
- if model not in self.by_model:
- self.by_model[model] = ModelTokenUsage()
- return self.by_model[model]
-
- @property
- def input_tokens(self) -> int:
- return sum(m.input_tokens for m in self.by_model.values())
-
- @property
- def output_tokens(self) -> int:
- return sum(m.output_tokens for m in self.by_model.values())
-
- @property
- def total_tokens(self) -> int:
- return self.input_tokens + self.output_tokens
-
- @property
- def calls(self) -> int:
- return sum(m.calls for m in self.by_model.values())
-
- @property
- def total_cost_usd(self) -> float:
- return sum(m.cost_usd(model) for model, m in self.by_model.items())
-
-
class AnthropicClient:
"""Concrete LLM client using the Anthropic SDK.
@@ -130,6 +73,31 @@ class AnthropicClient:
response.raw = raw_text
return response
+ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]:
+ """Send a batch analysis prompt and return parsed list of responses."""
+ effective_model = model or self.model
+ message = self._client.messages.create(
+ model=effective_model,
+ max_tokens=self.max_tokens * 2,
+ system=[{
+ "type": "text",
+ "text": f"{BATCH_SYSTEM_PROMPT}\n\n{BATCH_OUTPUT_SCHEMA}",
+ "cache_control": {"type": "ephemeral"},
+ }],
+ messages=[
+ {"role": "user", "content": prompt},
+ ],
+ )
+
+ raw_text = self._extract_text(message)
+ self._record_usage(message, effective_model)
+
+ responses = Analyzer.parse_llm_json_batch(raw_text)
+ for resp in responses:
+ resp.input_tokens = message.usage.input_tokens
+ resp.output_tokens = message.usage.output_tokens
+ return responses
+
def analyze_with_tools(
self,
prompt: str,
diff --git a/kong/llm/usage.py b/kong/llm/usage.py
new file mode 100644
index 0000000..1e862ca
--- /dev/null
+++ b/kong/llm/usage.py
@@ -0,0 +1,60 @@
+"""Token usage tracking for LLM API calls."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+_PRICING: dict[str, tuple[float, float]] = {
+ "claude-sonnet-4-20250514": (3.0, 15.0),
+ "claude-haiku-4-5-20251001": (1.0, 5.0),
+}
+
+
+@dataclass
+class ModelTokenUsage:
+ input_tokens: int = 0
+ output_tokens: int = 0
+ cache_creation_tokens: int = 0
+ cache_read_tokens: int = 0
+ calls: int = 0
+
+ def cost_usd(self, model: str) -> float:
+ input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))
+ cache_write_rate = input_rate * 1.25
+ cache_read_rate = input_rate * 0.10
+ return (
+ (self.input_tokens / 1_000_000) * input_rate
+ + (self.output_tokens / 1_000_000) * output_rate
+ + (self.cache_creation_tokens / 1_000_000) * cache_write_rate
+ + (self.cache_read_tokens / 1_000_000) * cache_read_rate
+ )
+
+
+@dataclass
+class TokenUsage:
+ by_model: dict[str, ModelTokenUsage] = field(default_factory=dict)
+
+ def _get(self, model: str) -> ModelTokenUsage:
+ if model not in self.by_model:
+ self.by_model[model] = ModelTokenUsage()
+ return self.by_model[model]
+
+ @property
+ def input_tokens(self) -> int:
+ return sum(m.input_tokens for m in self.by_model.values())
+
+ @property
+ def output_tokens(self) -> int:
+ return sum(m.output_tokens for m in self.by_model.values())
+
+ @property
+ def total_tokens(self) -> int:
+ return self.input_tokens + self.output_tokens
+
+ @property
+ def calls(self) -> int:
+ return sum(m.calls for m in self.by_model.values())
+
+ @property
+ def total_cost_usd(self) -> float:
+ return sum(m.cost_usd(model) for model, m in self.by_model.items())
diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py
index b056119..f9a8934 100644
--- a/tests/test_llm_client.py
+++ b/tests/test_llm_client.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
from unittest.mock import MagicMock, patch
from kong.llm.client import (
@@ -206,3 +207,62 @@ class TestPricingValues:
sonnet_in, sonnet_out = _PRICING["claude-sonnet-4-20250514"]
assert haiku_in < sonnet_in
assert haiku_out < sonnet_out
+
+
+class TestAnalyzeFunctionBatch:
+ def test_batch_returns_list_of_responses(self) -> None:
+ mock_anthropic = MagicMock()
+ mock_message = MagicMock()
+ mock_message.content = [MagicMock(type="text", text=json.dumps([
+ {"address": "0x1000", "name": "foo", "confidence": 80},
+ {"address": "0x2000", "name": "bar", "confidence": 70},
+ ]))]
+ mock_message.usage = MagicMock(
+ input_tokens=100, output_tokens=50,
+ cache_creation_input_tokens=0, cache_read_input_tokens=0,
+ )
+ mock_anthropic.messages.create.return_value = mock_message
+
+ client = AnthropicClient(api_key="test")
+ client._client = mock_anthropic
+ results = client.analyze_function_batch("batch prompt", model="claude-haiku-4-5-20251001")
+
+ assert len(results) == 2
+ assert results[0].name == "foo"
+ assert results[1].name == "bar"
+
+ def test_batch_uses_batch_system_prompt(self) -> None:
+ mock_anthropic = MagicMock()
+ mock_message = MagicMock()
+ mock_message.content = [MagicMock(type="text", text='[{"name": "f", "confidence": 50}]')]
+ mock_message.usage = MagicMock(
+ input_tokens=100, output_tokens=50,
+ cache_creation_input_tokens=0, cache_read_input_tokens=0,
+ )
+ mock_anthropic.messages.create.return_value = mock_message
+
+ client = AnthropicClient(api_key="test")
+ client._client = mock_anthropic
+ client.analyze_function_batch("prompt")
+
+ call_kwargs = mock_anthropic.messages.create.call_args
+ system_text = call_kwargs.kwargs["system"][0]["text"]
+ assert "multiple" in system_text.lower() or "batch" in system_text.lower()
+
+ def test_batch_records_usage(self) -> None:
+ mock_anthropic = MagicMock()
+ mock_message = MagicMock()
+ mock_message.content = [MagicMock(type="text", text='[{"name": "f", "confidence": 50}]')]
+ mock_message.usage = MagicMock(
+ input_tokens=500, output_tokens=200,
+ cache_creation_input_tokens=100, cache_read_input_tokens=50,
+ )
+ mock_anthropic.messages.create.return_value = mock_message
+
+ client = AnthropicClient(api_key="test")
+ client._client = mock_anthropic
+ client.analyze_function_batch("prompt")
+
+ assert client.usage.input_tokens == 500
+ assert client.usage.output_tokens == 200
+ assert client.usage.calls == 1