The world's first agentic reverse engineer.
WIP: v1 for openai sdk. needs testing but more importantly needs a setup wizard
Amruthec8ebb009aa5
diff --git a/kong/__main__.py b/kong/__main__.pyindex 46ea46d..db12a48 100644--- a/kong/__main__.py+++ b/kong/__main__.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport osfrom collections import Counterfrom pathlib import Path+from typing import TYPE_CHECKINGimport clickfrom rich.console import Console@@ -14,14 +15,34 @@ from kong import __version__from kong.agent.events import Event, EventTypefrom kong.agent.supervisor import Supervisorfrom kong.banner import check_api_key, print_analyze_header, print_banner, print_setup_needed-from kong.config import GhidraConfig, KongConfig, OutputConfig+from kong.config import GhidraConfig, KongConfig, LLMConfig, LLMProvider, OutputConfigfrom kong.evals.harness import score as eval_scorefrom kong.ghidra.client import GhidraClient, GhidraClientError-from kong.llm.client import AnthropicClient+from kong.llm.usage import TokenUsage++if TYPE_CHECKING:+ from kong.agent.analyzer import LLMClientconsole = Console()+_DEFAULT_MODELS: dict[LLMProvider, str] = {+ LLMProvider.ANTHROPIC: "claude-opus-4-6",+ LLMProvider.OPENAI: "gpt-4o",+}+++def create_llm_client(config: LLMConfig) -> LLMClient:+ """Instantiate the appropriate LLM client based on provider config."""+ # will generalize further in the future+ model = config.model or _DEFAULT_MODELS[config.provider]+ if config.provider is LLMProvider.OPENAI:+ from kong.llm.openai_client import OpenAIClient+ return OpenAIClient(model=model, api_key=config.api_key)+ from kong.llm.client import AnthropicClient+ return AnthropicClient(model=model, api_key=config.api_key)++@click.group(invoke_without_command=True)@click.version_option(version=__version__, prog_name="kong")@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")@@ -45,7 +66,7 @@ def cli(ctx: click.Context, verbose: bool) -> None:console.print("Run [bold]kong --help[/bold] for all options.")-def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> None:+def _print_final_stats(supervisor: Supervisor, llm_client: LLMClient) -> None:stats = supervisor.statsconsole.print()console.print(@@ -57,13 +78,16 @@ def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> Nf"{stats.medium_confidence} med, {stats.low_confidence} low")console.print(f"[bold]LLM calls:[/bold] {stats.llm_calls}")- for model_name, mu in llm_client.usage.by_model.items():- short_name = model_name.split("-")[1] if "-" in model_name else model_name- console.print(- f" {short_name}: {mu.calls} calls, "- f"${mu.cost_usd(model_name):.4f}"- )- console.print(f"[bold]Cost:[/bold] ${llm_client.total_cost_usd:.4f}")+ usage = getattr(llm_client, "usage", None)+ if isinstance(usage, TokenUsage):+ for model_name, mu in usage.by_model.items():+ short_name = model_name.split("-")[1] if "-" in model_name else model_name+ console.print(+ f" {short_name}: {mu.calls} calls, "+ f"${mu.cost_usd(model_name):.4f}"+ )+ total_cost = getattr(llm_client, "total_cost_usd", 0.0)+ console.print(f"[bold]Cost:[/bold] ${total_cost:.4f}")console.print(f"[bold]Duration:[/bold] {stats.duration_seconds:.1f}s")@@ -85,6 +109,13 @@ def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> Nhelp="Output formats.",)@click.option("--ghidra-dir", default=None, help="Ghidra installation directory.")+ "--provider", "-p",+ type=click.Choice([p.value for p in LLMProvider], case_sensitive=False),+ default=LLMProvider.ANTHROPIC.value,+ help="LLM provider (anthropic or openai).",+)[email protected]("--model", "-m", default=None, help="Override the LLM model name.")@click.pass_contextdef analyze(ctx: click.Context,@@ -93,10 +124,14 @@ def analyze(output: str,formats: tuple[str, ...],ghidra_dir: str | None,+ provider: str,+ model: str | None,) -> None:"""Analyze a binary with Kong's autonomous agent."""+ llm_provider = LLMProvider(provider)config = KongConfig(ghidra=GhidraConfig(install_dir=ghidra_dir),+ llm=LLMConfig(provider=llm_provider, model=model),output=OutputConfig(directory=Path(output), formats=list(formats)),headless=headless,verbose=ctx.obj["verbose"],@@ -111,8 +146,8 @@ def analyze(formats=config.output.formats,)- if not check_api_key():- print_setup_needed(console)+ if not check_api_key(llm_provider):+ print_setup_needed(console, llm_provider)raise SystemExit(1)if not config.ghidra.install_dir:@@ -140,7 +175,7 @@ def analyze(info = client.get_binary_info()console.print(f"[green]Loaded.[/green] {info.arch} {info.format} ({info.compiler})")- llm_client = AnthropicClient()+ llm_client = create_llm_client(config.llm)def print_event(event: Event) -> None:style = {@@ -169,7 +204,6 @@ def analyze(_print_final_stats(supervisor, llm_client)client.close()else:- # Deferred to avoid loading Textual when running --headlessfrom kong.tui.app import KongAppapp = KongApp(supervisor)try:@@ -228,29 +262,28 @@ def setup() -> None:print_banner(console)console.print()- console.print("[bold]Step 1: Anthropic API Key[/bold]")- console.print()+ from kong.banner import _ENV_VARS, _KEY_EXAMPLES, _KEY_URLS- current_key = os.environ.get("ANTHROPIC_API_KEY", "")- if current_key:- masked = current_key[:7] + "..." + current_key[-4:]- console.print(f" [green]Found:[/green] {masked}")- else:- console.print(" [yellow]Not set.[/yellow]")- console.print()- console.print(" Get your key at: [bold]https://console.anthropic.com/settings/keys[/bold]")+ step = 1+ for provider in LLMProvider:+ env_var = _ENV_VARS[provider]+ console.print(f"[bold]Step {step}: {provider.value.title()} API Key[/bold]")console.print()- key = click.prompt(" Enter your API key", hide_input=True)- if not key.startswith("sk-ant-"):- console.print(" [red]Invalid key format. Keys start with sk-ant-[/red]")- raise SystemExit(1)++ current_key = os.environ.get(env_var, "")+ if current_key:+ masked = current_key[:7] + "..." + current_key[-4:]+ console.print(f" [green]Found:[/green] {masked}")+ else:+ console.print(" [yellow]Not set.[/yellow]")+ console.print()+ console.print(f" Get your key at: [bold]{_KEY_URLS[provider]}[/bold]")+ console.print(f" [bold]export {env_var}={_KEY_EXAMPLES[provider]}[/bold]")+console.print()- console.print(" Add to your shell profile:")- console.print(f' [bold]export ANTHROPIC_API_KEY="{key}"[/bold]')- os.environ["ANTHROPIC_API_KEY"] = key+ step += 1- console.print()- console.print("[bold]Step 2: Ghidra[/bold]")+ console.print(f"[bold]Step {step}: Ghidra[/bold]")console.print()ghidra_config = GhidraConfig()@@ -266,7 +299,8 @@ def setup() -> None:console.print(" Then set [bold]GHIDRA_INSTALL_DIR[/bold] to the install path.")console.print()- all_good = check_api_key() and ghidra_config.install_dir+ has_any_key = any(check_api_key(p) for p in LLMProvider)+ all_good = has_any_key and ghidra_config.install_dirif all_good:console.print("[bold green]All set! Run [bold]kong analyze <binary>[/bold] to get started.[/bold green]")else:diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.pyindex d246b30..977d70c 100644--- a/kong/agent/analyzer.py+++ b/kong/agent/analyzer.py@@ -23,6 +23,7 @@ from kong.normalizer.syntactic import normalizeif TYPE_CHECKING:from kong.agent.deobfuscator import Deobfuscatorfrom kong.llm.tools import ToolExecutor, ToolSchema+ from kong.llm.usage import TokenUsagelogger = logging.getLogger(__name__)@@ -59,6 +60,12 @@ def strip_markdown_fences(text: str) -> str:class LLMClient(Protocol):"""Protocol for LLM interaction."""+ model: str+ usage: TokenUsage++ @property+ def total_cost_usd(self) -> float: ...+def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse: ...def analyze_with_tools(diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.pyindex a3f4ec4..25d3c50 100644--- a/kong/agent/supervisor.py+++ b/kong/agent/supervisor.py@@ -23,16 +23,13 @@ from kong.export.source import ExportData, export_sourcefrom kong.export.structured import export_jsonfrom kong.ghidra.client import GhidraClientfrom kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntry+from kong.llm.limits import get_model_limitsfrom kong.llm.usage import TokenUsagefrom kong.normalizer.syntactic import normalizefrom kong.synthesis.semantic import SemanticSynthesizer, SynthesisResultlogger = logging.getLogger(__name__)-OPUS_MODEL = "claude-opus-4-6"-MAX_PROMPT_CHARS = 400_000-MAX_CHUNK_FUNCTIONS = 120-class Supervisor:"""Main agent loop that orchestrates the full analysis pipeline.@@ -305,9 +302,12 @@ class Supervisor:"""Split items into chunks by building the actual prompt and measuring size."""assert self.binary_info is not None+ model_name = getattr(self.llm_client, "model", "") or ""+ limits = get_model_limits(model_name)+overhead = self._build_chunk_prompt([])overhead_len = len(overhead)- available = MAX_PROMPT_CHARS - overhead_len+ available = limits.max_prompt_chars - overhead_lenchunks: list[list[tuple[WorkItem, str]]] = []current_chunk: list[tuple[WorkItem, str]] = []@@ -323,7 +323,7 @@ class Supervisor:chunk_full = (current_chars + entry_len > available- or len(current_chunk) >= MAX_CHUNK_FUNCTIONS+ or len(current_chunk) >= limits.max_chunk_functions)if current_chunk and chunk_full:chunks.append(current_chunk)@@ -385,7 +385,7 @@ class Supervisor:try:responses = self.llm_client.analyze_function_batch(- prompt, model=OPUS_MODEL,+ prompt, model=self.llm_client.model,)except Exception as e:logger.warning("Chunk %d/%d failed: %s", chunk_num, total_chunks, e)@@ -502,7 +502,7 @@ class Supervisor:binary_info=self.binary_info,known_results=self.results,strings=self.strings,- model=OPUS_MODEL,+ model=self.llm_client.model,)if result.obfuscation_techniques:@@ -661,7 +661,7 @@ class Supervisor:synthesizer = SemanticSynthesizer(self.llm_client)try:synthesis_result = synthesizer.synthesize(- list(self.results.values()), decompilations, model=OPUS_MODEL,+ list(self.results.values()), decompilations, model=self.llm_client.model,)except Exception as e:logger.warning("Synthesis failed: %s", e)diff --git a/kong/banner.py b/kong/banner.pyindex 27d823c..529925c 100644--- a/kong/banner.py+++ b/kong/banner.py@@ -10,6 +10,7 @@ from rich.panel import Panelfrom rich.text import Textfrom kong import __version__+from kong.config import LLMProviderKONG_ASCII = r"""██╗ ██╗ ██████╗ ███╗ ██╗ ██████╗@@ -39,12 +40,33 @@ def print_banner(console: Console) -> None:))-def _load_dotenv() -> None:- """Load ANTHROPIC_API_KEY from .env if not already in the environment.+_ENV_VARS: dict[LLMProvider, str] = {+ LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY",+ LLMProvider.OPENAI: "OPENAI_API_KEY",+}++_KEY_EXAMPLES: dict[LLMProvider, str] = {+ LLMProvider.ANTHROPIC: "sk-ant-...",+ LLMProvider.OPENAI: "sk-...",+}++_KEY_URLS: dict[LLMProvider, str] = {+ LLMProvider.ANTHROPIC: "https://console.anthropic.com/settings/keys",+ LLMProvider.OPENAI: "https://platform.openai.com/api-keys",+}+++def _env_var_for(provider: LLMProvider) -> str:+ return _ENV_VARS[provider]+++def _load_dotenv(provider: LLMProvider = LLMProvider.ANTHROPIC) -> None:+ """Load the API key for *provider* from .env if not already set.Checks cwd first, then the package root (two levels up from this file)."""- if os.environ.get("ANTHROPIC_API_KEY"):+ env_var = _env_var_for(provider)+ if os.environ.get(env_var):returnpackage_root = Path(__file__).resolve().parent.parentfor candidate in (Path.cwd() / ".env", package_root / ".env"):@@ -54,27 +76,28 @@ def _load_dotenv() -> None:line = line.strip()if not line or line.startswith("#"):continue- if line.startswith("ANTHROPIC_API_KEY="):+ if line.startswith(f"{env_var}="):value = line.split("=", 1)[1].strip().strip("\"'")if value:- os.environ["ANTHROPIC_API_KEY"] = value+ os.environ[env_var] = valuereturn-def check_api_key() -> bool:- _load_dotenv()- return bool(os.environ.get("ANTHROPIC_API_KEY"))+def check_api_key(provider: LLMProvider = LLMProvider.ANTHROPIC) -> bool:+ _load_dotenv(provider)+ return bool(os.environ.get(_env_var_for(provider)))-def print_setup_needed(console: Console) -> None:+def print_setup_needed(console: Console, provider: LLMProvider = LLMProvider.ANTHROPIC) -> None:+ env_var = _env_var_for(provider)console.print()- console.print("[bold red]ANTHROPIC_API_KEY is not set.[/bold red]")+ console.print(f"[bold red]{env_var} is not set.[/bold red]")console.print()- console.print("Kong requires an Anthropic API key to analyze binaries.")+ console.print(f"Kong requires a {provider.value.title()} API key to analyze binaries.")console.print("Run [bold cyan]kong setup[/bold cyan] for guided configuration.")console.print()console.print("Or set it directly:")- console.print(" [bold]export ANTHROPIC_API_KEY=sk-ant-...[/bold]")+ console.print(f" [bold]export {env_var}={_KEY_EXAMPLES[provider]}[/bold]")def print_analyze_header(diff --git a/kong/config.py b/kong/config.pyindex 020a9b3..3578821 100644--- a/kong/config.py+++ b/kong/config.py@@ -3,10 +3,17 @@from __future__ import annotationsfrom dataclasses import dataclass, field+from enum import Enumfrom pathlib import Path+from kong.ghidra.environment import find_ghidra_install+class LLMProvider(Enum):+ ANTHROPIC = "anthropic"+ OPENAI = "openai"++@dataclassclass GhidraConfig:install_dir: str | None = None@@ -24,9 +31,17 @@ class OutputConfig:formats: list[str] = field(default_factory=lambda: ["source", "json", "ghidra"])+@dataclass+class LLMConfig:+ provider: LLMProvider = LLMProvider.ANTHROPIC+ model: str | None = None+ api_key: str | None = None++@dataclassclass KongConfig:ghidra: GhidraConfig = field(default_factory=GhidraConfig)+ llm: LLMConfig = field(default_factory=LLMConfig)output: OutputConfig = field(default_factory=OutputConfig)headless: bool = Falseverbose: bool = Falsediff --git a/kong/llm/__init__.py b/kong/llm/__init__.pyindex 6af7e8f..98a3e35 100644--- a/kong/llm/__init__.py+++ b/kong/llm/__init__.py@@ -1,7 +1,14 @@"""LLM integration layer."""from kong.llm.client import AnthropicClient+from kong.llm.openai_client import OpenAIClientfrom kong.llm.usage import TokenUsagefrom kong.llm.tools import DEOBFUSCATION_TOOLS, ToolExecutor-__all__ = ["AnthropicClient", "DEOBFUSCATION_TOOLS", "TokenUsage", "ToolExecutor"]+__all__ = [+ "AnthropicClient",+ "DEOBFUSCATION_TOOLS",+ "OpenAIClient",+ "TokenUsage",+ "ToolExecutor",+]diff --git a/kong/llm/limits.py b/kong/llm/limits.pynew file mode 100644index 0000000..833780b--- /dev/null+++ b/kong/llm/limits.py@@ -0,0 +1,105 @@+"""Model-specific limits and rate limiting for LLM API calls."""++from __future__ import annotations++import threading+import time+from dataclasses import dataclass+++@dataclass(frozen=True)+class ModelLimits:+ """Context window and chunking parameters for a specific model.++ max_prompt_chars: safe upper bound on prompt length in characters,+ leaving room for system prompt and output within the context window.+ max_chunk_functions: cap on how many functions to pack into a single+ batch call (prevents the LLM from losing track of results).+ max_output_tokens: maximum output tokens for batch calls.+ """++ max_prompt_chars: int+ max_chunk_functions: int+ max_output_tokens: int+++_DEFAULT_LIMITS = ModelLimits(+ max_prompt_chars=400_000,+ max_chunk_functions=120,+ max_output_tokens=16384,+)++MODEL_LIMITS: dict[str, ModelLimits] = {+ # Anthropic — 200k token context (~800k chars) so 400k chars is a safe prompt cap.+ "claude-opus-4-6": _DEFAULT_LIMITS,+ "claude-sonnet-4-6": _DEFAULT_LIMITS,+ "claude-sonnet-4-20250514": _DEFAULT_LIMITS,+ "claude-haiku-4-5-20251001": _DEFAULT_LIMITS,+ # OpenAI — 128k token context (~512k chars) so 350k chars leaves room for overhead.+ "gpt-4o": ModelLimits(350_000, 80, 16384),+ "gpt-4o-2024-11-20": ModelLimits(350_000, 80, 16384),+ "gpt-4o-mini": ModelLimits(350_000, 80, 16384),+ "gpt-4o-mini-2024-07-18": ModelLimits(350_000, 80, 16384),+ # OpenAI reasoning models — 200k context so more expensive, smaller batches.+ "o1": ModelLimits(400_000, 40, 32768),+ "o3-mini": ModelLimits(400_000, 60, 32768),+}+++def get_model_limits(model: str) -> ModelLimits:+ """Look up chunking limits for a model, falling back to defaults."""+ return MODEL_LIMITS.get(model, _DEFAULT_LIMITS)+++class RateLimiter:+ """Token-bucket style rate limiter for API calls.++ Tracks request timestamps and sleeps proactively to stay below the+ configured requests-per-minute (RPM) and tokens-per-minute (TPM) limits.+ """++ def __init__(+ self,+ requests_per_minute: int | None = None,+ tokens_per_minute: int | None = None,+ ) -> None:+ self._rpm = requests_per_minute+ self._tpm = tokens_per_minute+ self._lock = threading.Lock()+ self._request_times: list[float] = []+ self._token_log: list[tuple[float, int]] = []++ def wait_if_needed(self, estimated_tokens: int = 0) -> None:+ """Block until it's safe to make the next request."""+ with self._lock:+ now = time.monotonic()+ window_start = now - 60.0++ if self._rpm is not None:+ self._request_times = [+ t for t in self._request_times if t > window_start+ ]+ if len(self._request_times) >= self._rpm:+ sleep_until = self._request_times[0] + 60.0+ wait = sleep_until - now+ if wait > 0:+ time.sleep(wait)++ if self._tpm is not None and estimated_tokens > 0:+ self._token_log = [+ (t, n) for t, n in self._token_log if t > window_start+ ]+ total_tokens = sum(n for _, n in self._token_log)+ if total_tokens + estimated_tokens > self._tpm:+ sleep_until = self._token_log[0][0] + 60.0 if self._token_log else now+ wait = sleep_until - time.monotonic()+ if wait > 0:+ time.sleep(wait)++ def record_request(self, tokens_used: int = 0) -> None:+ """Record that a request was made."""+ with self._lock:+ now = time.monotonic()+ self._request_times.append(now)+ if tokens_used > 0:+ self._token_log.append((now, tokens_used))diff --git a/kong/llm/openai_client.py b/kong/llm/openai_client.pynew file mode 100644index 0000000..f8b9d6b--- /dev/null+++ b/kong/llm/openai_client.py@@ -0,0 +1,214 @@+"""OpenAI SDK wrapper for function analysis.++Implements the LLMClient protocol expected by the Analyzer.+Tracks token usage and cost per call. Supports both simple+(single-shot JSON) and tool-use (agentic loop) interactions.+"""++from __future__ import annotations++import json+import logging+from typing import Any++import openai++from kong.agent.analyzer import Analyzer, LLMResponse+from kong.agent.prompts import BATCH_OUTPUT_SCHEMA, BATCH_SYSTEM_PROMPT, OUTPUT_SCHEMA, SYSTEM_PROMPT+from kong.llm.tools import ToolExecutor+from kong.llm.usage import TokenUsage++logger = logging.getLogger(__name__)++DEFAULT_MODEL = "gpt-4o"+++def _convert_tools_to_openai(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Convert Anthropic-style tool schemas to OpenAI function-calling format."""+ return [+ {+ "type": "function",+ "function": {+ "name": tool["name"],+ "description": tool["description"],+ "parameters": tool["input_schema"],+ },+ }+ for tool in tools+ ]+++class OpenAIClient:+ """Concrete LLM client using the OpenAI SDK.++ Satisfies the LLMClient protocol from kong.agent.analyzer.++ Usage::++ client = OpenAIClient()+ response = client.analyze_function(prompt)+ response = client.analyze_with_tools(prompt, system, tools, executor)+ """++ def __init__(+ self,+ model: str = DEFAULT_MODEL,+ max_tokens: int = 2048,+ api_key: str | None = None,+ ) -> None:+ self.model = model+ self.max_tokens = max_tokens+ self._client = openai.OpenAI(api_key=api_key, max_retries=5)+ self.usage = TokenUsage()++ def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse:+ """Send an analysis prompt and return parsed response (no tools)."""+ effective_model = model or self.model+ response = self._client.chat.completions.create(+ model=effective_model,+ max_tokens=self.max_tokens,+ response_format={"type": "json_object"},+ messages=[+ {"role": "system", "content": f"{SYSTEM_PROMPT}\n\n{OUTPUT_SCHEMA}"},+ {"role": "user", "content": prompt},+ ],+ )++ raw_text = response.choices[0].message.content or ""+ self._record_usage(response, effective_model)++ result = Analyzer.parse_llm_json(raw_text)+ result.input_tokens = response.usage.prompt_tokens+ result.output_tokens = response.usage.completion_tokens+ result.raw = raw_text+ return result++ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]:+ """Send a batch analysis prompt and return parsed list of responses."""+ effective_model = model or self.model+ response = self._client.chat.completions.create(+ model=effective_model,+ max_tokens=16384,+ messages=[+ {"role": "system", "content": f"{BATCH_SYSTEM_PROMPT}\n\n{BATCH_OUTPUT_SCHEMA}"},+ {"role": "user", "content": prompt},+ ],+ )++ raw_text = response.choices[0].message.content or ""+ self._record_usage(response, effective_model)++ responses = Analyzer.parse_llm_json_batch(raw_text)+ for resp in responses:+ resp.input_tokens = response.usage.prompt_tokens+ resp.output_tokens = response.usage.completion_tokens+ return responses++ def analyze_with_tools(+ self,+ prompt: str,+ system: str,+ tools: list[dict[str, Any]],+ tool_executor: ToolExecutor,+ max_rounds: int = 10,+ ) -> LLMResponse:+ """Run an agentic tool-use loop.++ Sends the prompt with tool definitions. When the model returns+ tool_calls, executes each tool via *tool_executor* and feeds results+ back. Repeats until the model returns a final text response or+ *max_rounds* is exhausted.+ """+ openai_tools = _convert_tools_to_openai(tools)++ messages: list[dict[str, Any]] = [+ {"role": "system", "content": f"{system}\n\n{OUTPUT_SCHEMA}"},+ {"role": "user", "content": prompt},+ ]++ total_input = 0+ total_output = 0++ last_text = ""+ for _ in range(max_rounds):+ response = self._client.chat.completions.create(+ model=self.model,+ max_tokens=self.max_tokens,+ tools=openai_tools,+ messages=messages,+ )++ usage = response.usage+ total_input += usage.prompt_tokens+ total_output += usage.completion_tokens+ self._record_usage(response, self.model)++ choice = response.choices[0]++ if choice.finish_reason != "tool_calls":+ last_text = choice.message.content or ""+ result = Analyzer.parse_llm_json(last_text)+ result.input_tokens = total_input+ result.output_tokens = total_output+ result.raw = last_text+ return result++ assistant_msg: dict[str, Any] = {+ "role": "assistant",+ "content": choice.message.content,+ "tool_calls": [+ {+ "id": tc.id,+ "type": "function",+ "function": {+ "name": tc.function.name,+ "arguments": tc.function.arguments,+ },+ }+ for tc in choice.message.tool_calls+ ],+ }+ messages.append(assistant_msg)++ for tool_call in choice.message.tool_calls:+ arguments = json.loads(tool_call.function.arguments)+ result_str = tool_executor.execute(tool_call.function.name, arguments)+ messages.append({+ "role": "tool",+ "tool_call_id": tool_call.id,+ "content": result_str,+ })++ result = Analyzer.parse_llm_json(last_text)+ result.input_tokens = total_input+ result.output_tokens = total_output+ result.raw = last_text+ return result++ @property+ def total_cost_usd(self) -> float:+ return self.usage.total_cost_usd++ def _record_usage(self, response: Any, model: str | None = None) -> None:+ effective_model = model or self.model+ usage = response.usage+ mu = self.usage._get(effective_model)+ mu.input_tokens += usage.prompt_tokens+ mu.output_tokens += usage.completion_tokens+ cached = getattr(+ getattr(usage, "prompt_tokens_details", None),+ "cached_tokens",+ 0,+ ) or 0+ mu.cache_read_tokens += cached+ mu.calls += 1+ logger.debug(+ "LLM [%s]: %d in / %d out / %d cached tokens "+ "(total: %d calls, $%.4f)",+ effective_model,+ usage.prompt_tokens,+ usage.completion_tokens,+ cached,+ self.usage.calls,+ self.usage.total_cost_usd,+ )diff --git a/kong/llm/usage.py b/kong/llm/usage.pyindex 12c32e0..ba4d25a 100644--- a/kong/llm/usage.py+++ b/kong/llm/usage.py@@ -4,14 +4,43 @@ from __future__ import annotationsfrom dataclasses import dataclass, field-_PRICING: dict[str, tuple[float, float]] = {- "claude-opus-4-6": (5.0, 25.0),- "claude-sonnet-4-6": (3.0, 15.0),- "claude-sonnet-4-20250514": (3.0, 15.0),- "claude-haiku-4-5-20251001": (1.0, 5.0),++@dataclass(frozen=True)+class PricingTier:+ """Per-model pricing in USD per 1M tokens.++ Rates sourced from published Anthropic/OpenAI API pricing pages.+ """++ input_rate: float+ output_rate: float+ cache_write_rate: float = 0.0+ cache_read_rate: float = 0.0+++_DEFAULT_PRICING = PricingTier(input_rate=3.0, output_rate=15.0)++PRICING_REGISTRY: dict[str, PricingTier] = {+ # Anthropic: cache_write = input * 1.25, cache_read = input * 0.10+ "claude-opus-4-6": PricingTier(5.0, 25.0, cache_write_rate=6.25, cache_read_rate=0.50),+ "claude-sonnet-4-6": PricingTier(3.0, 15.0, cache_write_rate=3.75, cache_read_rate=0.30),+ "claude-sonnet-4-20250514": PricingTier(3.0, 15.0, cache_write_rate=3.75, cache_read_rate=0.30),+ "claude-haiku-4-5-20251001": PricingTier(1.0, 5.0, cache_write_rate=1.25, cache_read_rate=0.10),+ # OpenAI: cached input is 50% of input rate, no cache write billing+ "gpt-4o": PricingTier(2.50, 10.00, cache_read_rate=1.25),+ "gpt-4o-2024-11-20": PricingTier(2.50, 10.00, cache_read_rate=1.25),+ "gpt-4o-mini": PricingTier(0.15, 0.60, cache_read_rate=0.075),+ "gpt-4o-mini-2024-07-18": PricingTier(0.15, 0.60, cache_read_rate=0.075),+ "o1": PricingTier(15.00, 60.00, cache_read_rate=7.50),+ "o3-mini": PricingTier(1.10, 4.40, cache_read_rate=0.55),}+def get_pricing(model: str) -> PricingTier:+ """Look up pricing for a model, falling back to a sensible default."""+ return PRICING_REGISTRY.get(model, _DEFAULT_PRICING)++@dataclassclass ModelTokenUsage:input_tokens: int = 0@@ -21,14 +50,12 @@ class ModelTokenUsage:calls: int = 0def cost_usd(self, model: str) -> float:- input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))- cache_write_rate = input_rate * 1.25- cache_read_rate = input_rate * 0.10+ tier = get_pricing(model)return (- (self.input_tokens / 1_000_000) * input_rate- + (self.output_tokens / 1_000_000) * output_rate- + (self.cache_creation_tokens / 1_000_000) * cache_write_rate- + (self.cache_read_tokens / 1_000_000) * cache_read_rate+ (self.input_tokens / 1_000_000) * tier.input_rate+ + (self.output_tokens / 1_000_000) * tier.output_rate+ + (self.cache_creation_tokens / 1_000_000) * tier.cache_write_rate+ + (self.cache_read_tokens / 1_000_000) * tier.cache_read_rate)diff --git a/pyproject.toml b/pyproject.tomlindex d375980..a4a645c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -41,6 +41,9 @@ Issues = "https://github.com/amruth-sn/kong/issues"kong = "kong.__main__:cli"[project.optional-dependencies]+openai = [+ "openai>=1.0",+]dev = ["pytest>=8.0","pytest-asyncio>=0.24",diff --git a/tests/test_openai_client.py b/tests/test_openai_client.pynew file mode 100644index 0000000..36fcaa5--- /dev/null+++ b/tests/test_openai_client.py@@ -0,0 +1,296 @@+"""Tests for the OpenAI LLM client wrapper."""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++from kong.llm.openai_client import OpenAIClient, _convert_tools_to_openai+from kong.llm.usage import ModelTokenUsage, TokenUsage+++def _mock_response(text: str, prompt_tokens: int = 100, completion_tokens: int = 50):+ """Create a mock OpenAI ChatCompletion response."""+ message = MagicMock()+ message.content = text+ message.tool_calls = None++ choice = MagicMock()+ choice.message = message+ choice.finish_reason = "stop"++ usage = MagicMock()+ usage.prompt_tokens = prompt_tokens+ usage.completion_tokens = completion_tokens+ usage.prompt_tokens_details = None++ resp = MagicMock()+ resp.choices = [choice]+ resp.usage = usage+ return resp+++class TestOpenAIClient:+ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_analyze_function_parses_json(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ '{"name": "parse_config", "confidence": 85, "classification": "parser"}'+ )++ client = OpenAIClient(api_key="test-key")+ response = client.analyze_function("analyze this function")++ assert response.name == "parse_config"+ assert response.confidence == 85+ assert response.classification == "parser"++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_tracks_token_usage(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ '{"name": "f"}', prompt_tokens=200, completion_tokens=80+ )++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("prompt1")++ assert client.usage.input_tokens == 200+ assert client.usage.output_tokens == 80+ assert client.usage.calls == 1++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_accumulates_across_calls(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ '{"name": "f"}', prompt_tokens=100, completion_tokens=50+ )++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("p1")+ client.analyze_function("p2")++ assert client.usage.input_tokens == 200+ assert client.usage.output_tokens == 100+ assert client.usage.calls == 2++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_sends_system_message(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("test prompt")++ call_kwargs = mock_client.chat.completions.create.call_args+ messages = call_kwargs.kwargs["messages"]+ assert messages[0]["role"] == "system"+ assert "reverse engineer" in messages[0]["content"].lower()+ assert messages[1]["role"] == "user"+ assert messages[1]["content"] == "test prompt"++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_uses_json_mode_for_single_analysis(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("prompt")++ call_kwargs = mock_client.chat.completions.create.call_args+ assert call_kwargs.kwargs["response_format"] == {"type": "json_object"}++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_model_override(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("prompt", model="gpt-4o-mini")++ call_kwargs = mock_client.chat.completions.create.call_args+ assert call_kwargs.kwargs["model"] == "gpt-4o-mini"++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_per_model_cost_tracking(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ '{"name": "f"}', prompt_tokens=100, completion_tokens=50+ )++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("p1")+ client.analyze_function("p2", model="gpt-4o-mini")++ assert len(client.usage.by_model) == 2+ assert "gpt-4o" in client.usage.by_model+ assert "gpt-4o-mini" in client.usage.by_model+ assert client.usage.calls == 2++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_handles_malformed_response(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ "I cannot analyze this function because reasons."+ )++ client = OpenAIClient(api_key="test-key")+ response = client.analyze_function("prompt")++ assert response.name == ""+ assert "Failed to parse" in response.reasoning++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_handles_empty_content(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ resp = _mock_response("")+ resp.choices[0].message.content = None+ mock_client.chat.completions.create.return_value = resp++ client = OpenAIClient(api_key="test-key")+ response = client.analyze_function("prompt")++ assert response.name == ""++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_records_cached_tokens(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ resp = _mock_response('{"name": "f"}', prompt_tokens=200, completion_tokens=50)+ details = MagicMock()+ details.cached_tokens = 100+ resp.usage.prompt_tokens_details = details+ mock_client.chat.completions.create.return_value = resp++ client = OpenAIClient(api_key="test-key")+ client.analyze_function("prompt")++ mu = client.usage.by_model["gpt-4o"]+ assert mu.cache_read_tokens == 100+++class TestOpenAIBatch:+ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_batch_returns_list_of_responses(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ json.dumps([+ {"address": "0x1000", "name": "foo", "confidence": 80},+ {"address": "0x2000", "name": "bar", "confidence": 70},+ ])+ )++ client = OpenAIClient(api_key="test-key")+ results = client.analyze_function_batch("batch prompt")++ assert len(results) == 2+ assert results[0].name == "foo"+ assert results[1].name == "bar"++ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_batch_uses_batch_system_prompt(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.chat.completions.create.return_value = _mock_response(+ '[{"name": "f", "confidence": 50}]'+ )++ client = OpenAIClient(api_key="test-key")+ client.analyze_function_batch("prompt")++ call_kwargs = mock_client.chat.completions.create.call_args+ system_text = call_kwargs.kwargs["messages"][0]["content"]+ assert "multiple" in system_text.lower() or "batch" in system_text.lower()+++class TestOpenAIToolUse:+ @patch("kong.llm.openai_client.openai.OpenAI")+ def test_tool_loop_executes_tools(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client++ tool_call = MagicMock()+ tool_call.id = "call_123"+ tool_call.function.name = "simplify_expression"+ tool_call.function.arguments = '{"expression": "x + 0"}'++ tool_response = _mock_response("", prompt_tokens=100, completion_tokens=20)+ tool_response.choices[0].finish_reason = "tool_calls"+ tool_response.choices[0].message.content = None+ tool_response.choices[0].message.tool_calls = [tool_call]++ final_response = _mock_response(+ '{"name": "clean_func", "confidence": 75}',+ prompt_tokens=150, completion_tokens=40,+ )++ mock_client.chat.completions.create.side_effect = [tool_response, final_response]++ executor = MagicMock()+ executor.execute.return_value = '{"simplified": "x"}'++ tools = [{+ "name": "simplify_expression",+ "description": "Simplify an expression.",+ "input_schema": {"type": "object", "properties": {"expression": {"type": "string"}}},+ }]++ client = OpenAIClient(api_key="test-key")+ result = client.analyze_with_tools("prompt", "system", tools, executor)++ assert result.name == "clean_func"+ assert result.input_tokens == 250+ assert result.output_tokens == 60+ executor.execute.assert_called_once_with("simplify_expression", {"expression": "x + 0"})+++class TestConvertTools:+ def test_converts_anthropic_to_openai_format(self):+ anthropic_tools = [{+ "name": "test_tool",+ "description": "A test tool.",+ "input_schema": {+ "type": "object",+ "properties": {"x": {"type": "string"}},+ "required": ["x"],+ },+ }]++ openai_tools = _convert_tools_to_openai(anthropic_tools)++ assert len(openai_tools) == 1+ assert openai_tools[0]["type"] == "function"+ assert openai_tools[0]["function"]["name"] == "test_tool"+ assert openai_tools[0]["function"]["description"] == "A test tool."+ assert openai_tools[0]["function"]["parameters"]["type"] == "object"+ assert "x" in openai_tools[0]["function"]["parameters"]["properties"]+++class TestOpenAICostCalculation:+ def test_gpt4o_cost(self):+ mu = ModelTokenUsage(input_tokens=1_000_000, output_tokens=1_000_000)+ cost = mu.cost_usd("gpt-4o")+ assert cost == 2.50 + 10.00++ def test_gpt4o_mini_cost(self):+ mu = ModelTokenUsage(input_tokens=1_000_000, output_tokens=1_000_000)+ cost = mu.cost_usd("gpt-4o-mini")+ assert cost == 0.15 + 0.60++ def test_gpt4o_cached_tokens_cost(self):+ mu = ModelTokenUsage(+ input_tokens=0, output_tokens=0,+ cache_read_tokens=1_000_000,+ )+ cost = mu.cost_usd("gpt-4o")+ assert cost == 1.25diff --git a/uv.lock b/uv.lockindex 87c2b28..9c0f926 100644--- a/uv.lock+++ b/uv.lock@@ -271,7 +271,7 @@ wheels = [[[package]]name = "kong-re"-version = "0.1.0"+version = "0.1.2"source = { editable = "." }dependencies = [{ name = "anthropic" },@@ -290,6 +290,9 @@ dev = [{ name = "pytest" },{ name = "pytest-asyncio" },]+openai = [+ { name = "openai" },+][package.metadata]requires-dist = [@@ -298,6 +301,7 @@ requires-dist = [{ name = "jpype1", specifier = ">=1.6.0" },{ name = "json-repair", specifier = ">=0.58.5" },{ name = "lief", specifier = ">=0.15" },+ { name = "openai", marker = "extra == 'openai'", specifier = ">=1.0" },{ name = "pyghidra", specifier = ">=2.2.1" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24" },@@ -305,7 +309,7 @@ requires-dist = [{ name = "textual", specifier = ">=1.0" },{ name = "z3-solver", specifier = ">=4.12" },]-provides-extras = ["dev"]+provides-extras = ["openai", "dev"][[package]]name = "lief"@@ -409,6 +413,25 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },]+[[package]]+name = "openai"+version = "2.28.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "anyio" },+ { name = "distro" },+ { name = "httpx" },+ { name = "jiter" },+ { name = "pydantic" },+ { name = "sniffio" },+ { name = "tqdm" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/56/87/eb0abb4ef88ddb95b3c13149384c4c288f584f3be17d6a4f63f8c3e3c226/openai-2.28.0.tar.gz", hash = "sha256:bb7fdff384d2a787fa82e8822d1dd3c02e8cf901d60f1df523b7da03cbb6d48d", size = 670334, upload-time = "2026-03-13T19:56:27.306Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c0/5a/df122348638885526e53140e9c6b0d844af7312682b3bde9587eebc28b47/openai-2.28.0-py3-none-any.whl", hash = "sha256:79aa5c45dba7fef84085701c235cf13ba88485e1ef4f8dfcedc44fc2a698fc1d", size = 1141218, upload-time = "2026-03-13T19:56:25.46Z" },+]+[[package]]name = "packaging"version = "26.0"@@ -638,6 +661,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/d3/be/e191c2a15da20530fde03564564e3e4b4220eb9d687d4014957e5c6a5e85/textual-8.0.0-py3-none-any.whl", hash = "sha256:8908f4ebe93a6b4f77ca7262197784a52162bc88b05f4ecf50ac93a92d49bb8f", size = 718904, upload-time = "2026-02-16T17:12:11.962Z" },]+[[package]]+name = "tqdm"+version = "4.67.3"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "colorama", marker = "sys_platform == 'win32'" },+]+sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" },+]+[[package]]name = "typing-extensions"version = "4.15.0"