Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

WIP: v1 for openai sdk. needs testing but more importantly needs a setup wizard

Amruthec8ebb009aa5
diff --git a/kong/__main__.py b/kong/__main__.py
index 46ea46d..db12a48 100644
--- a/kong/__main__.py
+++ b/kong/__main__.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import os
from collections import Counter
from pathlib import Path
+from typing import TYPE_CHECKING
import click
from rich.console import Console
@@ -14,14 +15,34 @@ from kong import __version__
from kong.agent.events import Event, EventType
from kong.agent.supervisor import Supervisor
from kong.banner import check_api_key, print_analyze_header, print_banner, print_setup_needed
-from kong.config import GhidraConfig, KongConfig, OutputConfig
+from kong.config import GhidraConfig, KongConfig, LLMConfig, LLMProvider, OutputConfig
from kong.evals.harness import score as eval_score
from kong.ghidra.client import GhidraClient, GhidraClientError
-from kong.llm.client import AnthropicClient
+from kong.llm.usage import TokenUsage
+
+if TYPE_CHECKING:
+ from kong.agent.analyzer import LLMClient
console = Console()
+_DEFAULT_MODELS: dict[LLMProvider, str] = {
+ LLMProvider.ANTHROPIC: "claude-opus-4-6",
+ LLMProvider.OPENAI: "gpt-4o",
+}
+
+
+def create_llm_client(config: LLMConfig) -> LLMClient:
+ """Instantiate the appropriate LLM client based on provider config."""
+ # will generalize further in the future
+ model = config.model or _DEFAULT_MODELS[config.provider]
+ if config.provider is LLMProvider.OPENAI:
+ from kong.llm.openai_client import OpenAIClient
+ return OpenAIClient(model=model, api_key=config.api_key)
+ from kong.llm.client import AnthropicClient
+ return AnthropicClient(model=model, api_key=config.api_key)
+
+
@click.group(invoke_without_command=True)
@click.version_option(version=__version__, prog_name="kong")
@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
@@ -45,7 +66,7 @@ def cli(ctx: click.Context, verbose: bool) -> None:
console.print("Run [bold]kong --help[/bold] for all options.")
-def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> None:
+def _print_final_stats(supervisor: Supervisor, llm_client: LLMClient) -> None:
stats = supervisor.stats
console.print()
console.print(
@@ -57,13 +78,16 @@ def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> N
f"{stats.medium_confidence} med, {stats.low_confidence} low"
)
console.print(f"[bold]LLM calls:[/bold] {stats.llm_calls}")
- for model_name, mu in llm_client.usage.by_model.items():
- short_name = model_name.split("-")[1] if "-" in model_name else model_name
- console.print(
- f" {short_name}: {mu.calls} calls, "
- f"${mu.cost_usd(model_name):.4f}"
- )
- console.print(f"[bold]Cost:[/bold] ${llm_client.total_cost_usd:.4f}")
+ usage = getattr(llm_client, "usage", None)
+ if isinstance(usage, TokenUsage):
+ for model_name, mu in usage.by_model.items():
+ short_name = model_name.split("-")[1] if "-" in model_name else model_name
+ console.print(
+ f" {short_name}: {mu.calls} calls, "
+ f"${mu.cost_usd(model_name):.4f}"
+ )
+ total_cost = getattr(llm_client, "total_cost_usd", 0.0)
+ console.print(f"[bold]Cost:[/bold] ${total_cost:.4f}")
console.print(f"[bold]Duration:[/bold] {stats.duration_seconds:.1f}s")
@@ -85,6 +109,13 @@ def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> N
help="Output formats.",
)
@click.option("--ghidra-dir", default=None, help="Ghidra installation directory.")
+ "--provider", "-p",
+ type=click.Choice([p.value for p in LLMProvider], case_sensitive=False),
+ default=LLMProvider.ANTHROPIC.value,
+ help="LLM provider (anthropic or openai).",
+)
[email protected]("--model", "-m", default=None, help="Override the LLM model name.")
@click.pass_context
def analyze(
ctx: click.Context,
@@ -93,10 +124,14 @@ def analyze(
output: str,
formats: tuple[str, ...],
ghidra_dir: str | None,
+ provider: str,
+ model: str | None,
) -> None:
"""Analyze a binary with Kong's autonomous agent."""
+ llm_provider = LLMProvider(provider)
config = KongConfig(
ghidra=GhidraConfig(install_dir=ghidra_dir),
+ llm=LLMConfig(provider=llm_provider, model=model),
output=OutputConfig(directory=Path(output), formats=list(formats)),
headless=headless,
verbose=ctx.obj["verbose"],
@@ -111,8 +146,8 @@ def analyze(
formats=config.output.formats,
)
- if not check_api_key():
- print_setup_needed(console)
+ if not check_api_key(llm_provider):
+ print_setup_needed(console, llm_provider)
raise SystemExit(1)
if not config.ghidra.install_dir:
@@ -140,7 +175,7 @@ def analyze(
info = client.get_binary_info()
console.print(f"[green]Loaded.[/green] {info.arch} {info.format} ({info.compiler})")
- llm_client = AnthropicClient()
+ llm_client = create_llm_client(config.llm)
def print_event(event: Event) -> None:
style = {
@@ -169,7 +204,6 @@ def analyze(
_print_final_stats(supervisor, llm_client)
client.close()
else:
- # Deferred to avoid loading Textual when running --headless
from kong.tui.app import KongApp
app = KongApp(supervisor)
try:
@@ -228,29 +262,28 @@ def setup() -> None:
print_banner(console)
console.print()
- console.print("[bold]Step 1: Anthropic API Key[/bold]")
- console.print()
+ from kong.banner import _ENV_VARS, _KEY_EXAMPLES, _KEY_URLS
- current_key = os.environ.get("ANTHROPIC_API_KEY", "")
- if current_key:
- masked = current_key[:7] + "..." + current_key[-4:]
- console.print(f" [green]Found:[/green] {masked}")
- else:
- console.print(" [yellow]Not set.[/yellow]")
- console.print()
- console.print(" Get your key at: [bold]https://console.anthropic.com/settings/keys[/bold]")
+ step = 1
+ for provider in LLMProvider:
+ env_var = _ENV_VARS[provider]
+ console.print(f"[bold]Step {step}: {provider.value.title()} API Key[/bold]")
console.print()
- key = click.prompt(" Enter your API key", hide_input=True)
- if not key.startswith("sk-ant-"):
- console.print(" [red]Invalid key format. Keys start with sk-ant-[/red]")
- raise SystemExit(1)
+
+ current_key = os.environ.get(env_var, "")
+ if current_key:
+ masked = current_key[:7] + "..." + current_key[-4:]
+ console.print(f" [green]Found:[/green] {masked}")
+ else:
+ console.print(" [yellow]Not set.[/yellow]")
+ console.print()
+ console.print(f" Get your key at: [bold]{_KEY_URLS[provider]}[/bold]")
+ console.print(f" [bold]export {env_var}={_KEY_EXAMPLES[provider]}[/bold]")
+
console.print()
- console.print(" Add to your shell profile:")
- console.print(f' [bold]export ANTHROPIC_API_KEY="{key}"[/bold]')
- os.environ["ANTHROPIC_API_KEY"] = key
+ step += 1
- console.print()
- console.print("[bold]Step 2: Ghidra[/bold]")
+ console.print(f"[bold]Step {step}: Ghidra[/bold]")
console.print()
ghidra_config = GhidraConfig()
@@ -266,7 +299,8 @@ def setup() -> None:
console.print(" Then set [bold]GHIDRA_INSTALL_DIR[/bold] to the install path.")
console.print()
- all_good = check_api_key() and ghidra_config.install_dir
+ has_any_key = any(check_api_key(p) for p in LLMProvider)
+ all_good = has_any_key and ghidra_config.install_dir
if all_good:
console.print("[bold green]All set! Run [bold]kong analyze <binary>[/bold] to get started.[/bold green]")
else:
diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.py
index d246b30..977d70c 100644
--- a/kong/agent/analyzer.py
+++ b/kong/agent/analyzer.py
@@ -23,6 +23,7 @@ from kong.normalizer.syntactic import normalize
if TYPE_CHECKING:
from kong.agent.deobfuscator import Deobfuscator
from kong.llm.tools import ToolExecutor, ToolSchema
+ from kong.llm.usage import TokenUsage
logger = logging.getLogger(__name__)
@@ -59,6 +60,12 @@ def strip_markdown_fences(text: str) -> str:
class LLMClient(Protocol):
"""Protocol for LLM interaction."""
+ model: str
+ usage: TokenUsage
+
+ @property
+ def total_cost_usd(self) -> float: ...
+
def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse: ...
def analyze_with_tools(
diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.py
index a3f4ec4..25d3c50 100644
--- a/kong/agent/supervisor.py
+++ b/kong/agent/supervisor.py
@@ -23,16 +23,13 @@ from kong.export.source import ExportData, export_source
from kong.export.structured import export_json
from kong.ghidra.client import GhidraClient
from kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntry
+from kong.llm.limits import get_model_limits
from kong.llm.usage import TokenUsage
from kong.normalizer.syntactic import normalize
from kong.synthesis.semantic import SemanticSynthesizer, SynthesisResult
logger = logging.getLogger(__name__)
-OPUS_MODEL = "claude-opus-4-6"
-MAX_PROMPT_CHARS = 400_000
-MAX_CHUNK_FUNCTIONS = 120
-
class Supervisor:
"""Main agent loop that orchestrates the full analysis pipeline.
@@ -305,9 +302,12 @@ class Supervisor:
"""Split items into chunks by building the actual prompt and measuring size."""
assert self.binary_info is not None
+ model_name = getattr(self.llm_client, "model", "") or ""
+ limits = get_model_limits(model_name)
+
overhead = self._build_chunk_prompt([])
overhead_len = len(overhead)
- available = MAX_PROMPT_CHARS - overhead_len
+ available = limits.max_prompt_chars - overhead_len
chunks: list[list[tuple[WorkItem, str]]] = []
current_chunk: list[tuple[WorkItem, str]] = []
@@ -323,7 +323,7 @@ class Supervisor:
chunk_full = (
current_chars + entry_len > available
- or len(current_chunk) >= MAX_CHUNK_FUNCTIONS
+ or len(current_chunk) >= limits.max_chunk_functions
)
if current_chunk and chunk_full:
chunks.append(current_chunk)
@@ -385,7 +385,7 @@ class Supervisor:
try:
responses = self.llm_client.analyze_function_batch(
- prompt, model=OPUS_MODEL,
+ prompt, model=self.llm_client.model,
)
except Exception as e:
logger.warning("Chunk %d/%d failed: %s", chunk_num, total_chunks, e)
@@ -502,7 +502,7 @@ class Supervisor:
binary_info=self.binary_info,
known_results=self.results,
strings=self.strings,
- model=OPUS_MODEL,
+ model=self.llm_client.model,
)
if result.obfuscation_techniques:
@@ -661,7 +661,7 @@ class Supervisor:
synthesizer = SemanticSynthesizer(self.llm_client)
try:
synthesis_result = synthesizer.synthesize(
- list(self.results.values()), decompilations, model=OPUS_MODEL,
+ list(self.results.values()), decompilations, model=self.llm_client.model,
)
except Exception as e:
logger.warning("Synthesis failed: %s", e)
diff --git a/kong/banner.py b/kong/banner.py
index 27d823c..529925c 100644
--- a/kong/banner.py
+++ b/kong/banner.py
@@ -10,6 +10,7 @@ from rich.panel import Panel
from rich.text import Text
from kong import __version__
+from kong.config import LLMProvider
KONG_ASCII = r"""
██╗ ██╗ ██████╗ ███╗ ██╗ ██████╗
@@ -39,12 +40,33 @@ def print_banner(console: Console) -> None:
))
-def _load_dotenv() -> None:
- """Load ANTHROPIC_API_KEY from .env if not already in the environment.
+_ENV_VARS: dict[LLMProvider, str] = {
+ LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY",
+ LLMProvider.OPENAI: "OPENAI_API_KEY",
+}
+
+_KEY_EXAMPLES: dict[LLMProvider, str] = {
+ LLMProvider.ANTHROPIC: "sk-ant-...",
+ LLMProvider.OPENAI: "sk-...",
+}
+
+_KEY_URLS: dict[LLMProvider, str] = {
+ LLMProvider.ANTHROPIC: "https://console.anthropic.com/settings/keys",
+ LLMProvider.OPENAI: "https://platform.openai.com/api-keys",
+}
+
+
+def _env_var_for(provider: LLMProvider) -> str:
+ return _ENV_VARS[provider]
+
+
+def _load_dotenv(provider: LLMProvider = LLMProvider.ANTHROPIC) -> None:
+ """Load the API key for *provider* from .env if not already set.
Checks cwd first, then the package root (two levels up from this file).
"""
- if os.environ.get("ANTHROPIC_API_KEY"):
+ env_var = _env_var_for(provider)
+ if os.environ.get(env_var):
return
package_root = Path(__file__).resolve().parent.parent
for candidate in (Path.cwd() / ".env", package_root / ".env"):
@@ -54,27 +76,28 @@ def _load_dotenv() -> None:
line = line.strip()
if not line or line.startswith("#"):
continue
- if line.startswith("ANTHROPIC_API_KEY="):
+ if line.startswith(f"{env_var}="):
value = line.split("=", 1)[1].strip().strip("\"'")
if value:
- os.environ["ANTHROPIC_API_KEY"] = value
+ os.environ[env_var] = value
return
-def check_api_key() -> bool:
- _load_dotenv()
- return bool(os.environ.get("ANTHROPIC_API_KEY"))
+def check_api_key(provider: LLMProvider = LLMProvider.ANTHROPIC) -> bool:
+ _load_dotenv(provider)
+ return bool(os.environ.get(_env_var_for(provider)))
-def print_setup_needed(console: Console) -> None:
+def print_setup_needed(console: Console, provider: LLMProvider = LLMProvider.ANTHROPIC) -> None:
+ env_var = _env_var_for(provider)
console.print()
- console.print("[bold red]ANTHROPIC_API_KEY is not set.[/bold red]")
+ console.print(f"[bold red]{env_var} is not set.[/bold red]")
console.print()
- console.print("Kong requires an Anthropic API key to analyze binaries.")
+ console.print(f"Kong requires a {provider.value.title()} API key to analyze binaries.")
console.print("Run [bold cyan]kong setup[/bold cyan] for guided configuration.")
console.print()
console.print("Or set it directly:")
- console.print(" [bold]export ANTHROPIC_API_KEY=sk-ant-...[/bold]")
+ console.print(f" [bold]export {env_var}={_KEY_EXAMPLES[provider]}[/bold]")
def print_analyze_header(
diff --git a/kong/config.py b/kong/config.py
index 020a9b3..3578821 100644
--- a/kong/config.py
+++ b/kong/config.py
@@ -3,10 +3,17 @@
from __future__ import annotations
from dataclasses import dataclass, field
+from enum import Enum
from pathlib import Path
+
from kong.ghidra.environment import find_ghidra_install
+class LLMProvider(Enum):
+ ANTHROPIC = "anthropic"
+ OPENAI = "openai"
+
+
@dataclass
class GhidraConfig:
install_dir: str | None = None
@@ -24,9 +31,17 @@ class OutputConfig:
formats: list[str] = field(default_factory=lambda: ["source", "json", "ghidra"])
+@dataclass
+class LLMConfig:
+ provider: LLMProvider = LLMProvider.ANTHROPIC
+ model: str | None = None
+ api_key: str | None = None
+
+
@dataclass
class KongConfig:
ghidra: GhidraConfig = field(default_factory=GhidraConfig)
+ llm: LLMConfig = field(default_factory=LLMConfig)
output: OutputConfig = field(default_factory=OutputConfig)
headless: bool = False
verbose: bool = False
diff --git a/kong/llm/__init__.py b/kong/llm/__init__.py
index 6af7e8f..98a3e35 100644
--- a/kong/llm/__init__.py
+++ b/kong/llm/__init__.py
@@ -1,7 +1,14 @@
"""LLM integration layer."""
from kong.llm.client import AnthropicClient
+from kong.llm.openai_client import OpenAIClient
from kong.llm.usage import TokenUsage
from kong.llm.tools import DEOBFUSCATION_TOOLS, ToolExecutor
-__all__ = ["AnthropicClient", "DEOBFUSCATION_TOOLS", "TokenUsage", "ToolExecutor"]
+__all__ = [
+ "AnthropicClient",
+ "DEOBFUSCATION_TOOLS",
+ "OpenAIClient",
+ "TokenUsage",
+ "ToolExecutor",
+]
diff --git a/kong/llm/limits.py b/kong/llm/limits.py
new file mode 100644
index 0000000..833780b
--- /dev/null
+++ b/kong/llm/limits.py
@@ -0,0 +1,105 @@
+"""Model-specific limits and rate limiting for LLM API calls."""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelLimits:
+ """Context window and chunking parameters for a specific model.
+
+ max_prompt_chars: safe upper bound on prompt length in characters,
+ leaving room for system prompt and output within the context window.
+ max_chunk_functions: cap on how many functions to pack into a single
+ batch call (prevents the LLM from losing track of results).
+ max_output_tokens: maximum output tokens for batch calls.
+ """
+
+ max_prompt_chars: int
+ max_chunk_functions: int
+ max_output_tokens: int
+
+
+_DEFAULT_LIMITS = ModelLimits(
+ max_prompt_chars=400_000,
+ max_chunk_functions=120,
+ max_output_tokens=16384,
+)
+
+MODEL_LIMITS: dict[str, ModelLimits] = {
+ # Anthropic — 200k token context (~800k chars) so 400k chars is a safe prompt cap.
+ "claude-opus-4-6": _DEFAULT_LIMITS,
+ "claude-sonnet-4-6": _DEFAULT_LIMITS,
+ "claude-sonnet-4-20250514": _DEFAULT_LIMITS,
+ "claude-haiku-4-5-20251001": _DEFAULT_LIMITS,
+ # OpenAI — 128k token context (~512k chars) so 350k chars leaves room for overhead.
+ "gpt-4o": ModelLimits(350_000, 80, 16384),
+ "gpt-4o-2024-11-20": ModelLimits(350_000, 80, 16384),
+ "gpt-4o-mini": ModelLimits(350_000, 80, 16384),
+ "gpt-4o-mini-2024-07-18": ModelLimits(350_000, 80, 16384),
+ # OpenAI reasoning models — 200k context so more expensive, smaller batches.
+ "o1": ModelLimits(400_000, 40, 32768),
+ "o3-mini": ModelLimits(400_000, 60, 32768),
+}
+
+
+def get_model_limits(model: str) -> ModelLimits:
+ """Look up chunking limits for a model, falling back to defaults."""
+ return MODEL_LIMITS.get(model, _DEFAULT_LIMITS)
+
+
+class RateLimiter:
+ """Token-bucket style rate limiter for API calls.
+
+ Tracks request timestamps and sleeps proactively to stay below the
+ configured requests-per-minute (RPM) and tokens-per-minute (TPM) limits.
+ """
+
+ def __init__(
+ self,
+ requests_per_minute: int | None = None,
+ tokens_per_minute: int | None = None,
+ ) -> None:
+ self._rpm = requests_per_minute
+ self._tpm = tokens_per_minute
+ self._lock = threading.Lock()
+ self._request_times: list[float] = []
+ self._token_log: list[tuple[float, int]] = []
+
+ def wait_if_needed(self, estimated_tokens: int = 0) -> None:
+ """Block until it's safe to make the next request."""
+ with self._lock:
+ now = time.monotonic()
+ window_start = now - 60.0
+
+ if self._rpm is not None:
+ self._request_times = [
+ t for t in self._request_times if t > window_start
+ ]
+ if len(self._request_times) >= self._rpm:
+ sleep_until = self._request_times[0] + 60.0
+ wait = sleep_until - now
+ if wait > 0:
+ time.sleep(wait)
+
+ if self._tpm is not None and estimated_tokens > 0:
+ self._token_log = [
+ (t, n) for t, n in self._token_log if t > window_start
+ ]
+ total_tokens = sum(n for _, n in self._token_log)
+ if total_tokens + estimated_tokens > self._tpm:
+ sleep_until = self._token_log[0][0] + 60.0 if self._token_log else now
+ wait = sleep_until - time.monotonic()
+ if wait > 0:
+ time.sleep(wait)
+
+ def record_request(self, tokens_used: int = 0) -> None:
+ """Record that a request was made."""
+ with self._lock:
+ now = time.monotonic()
+ self._request_times.append(now)
+ if tokens_used > 0:
+ self._token_log.append((now, tokens_used))
diff --git a/kong/llm/openai_client.py b/kong/llm/openai_client.py
new file mode 100644
index 0000000..f8b9d6b
--- /dev/null
+++ b/kong/llm/openai_client.py
@@ -0,0 +1,214 @@
+"""OpenAI SDK wrapper for function analysis.
+
+Implements the LLMClient protocol expected by the Analyzer.
+Tracks token usage and cost per call. Supports both simple
+(single-shot JSON) and tool-use (agentic loop) interactions.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+import openai
+
+from kong.agent.analyzer import Analyzer, LLMResponse
+from kong.agent.prompts import BATCH_OUTPUT_SCHEMA, BATCH_SYSTEM_PROMPT, OUTPUT_SCHEMA, SYSTEM_PROMPT
+from kong.llm.tools import ToolExecutor
+from kong.llm.usage import TokenUsage
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "gpt-4o"
+
+
+def _convert_tools_to_openai(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Convert Anthropic-style tool schemas to OpenAI function-calling format."""
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": tool["name"],
+ "description": tool["description"],
+ "parameters": tool["input_schema"],
+ },
+ }
+ for tool in tools
+ ]
+
+
+class OpenAIClient:
+ """Concrete LLM client using the OpenAI SDK.
+
+ Satisfies the LLMClient protocol from kong.agent.analyzer.
+
+ Usage::
+
+ client = OpenAIClient()
+ response = client.analyze_function(prompt)
+ response = client.analyze_with_tools(prompt, system, tools, executor)
+ """
+
+ def __init__(
+ self,
+ model: str = DEFAULT_MODEL,
+ max_tokens: int = 2048,
+ api_key: str | None = None,
+ ) -> None:
+ self.model = model
+ self.max_tokens = max_tokens
+ self._client = openai.OpenAI(api_key=api_key, max_retries=5)
+ self.usage = TokenUsage()
+
+ def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse:
+ """Send an analysis prompt and return parsed response (no tools)."""
+ effective_model = model or self.model
+ response = self._client.chat.completions.create(
+ model=effective_model,
+ max_tokens=self.max_tokens,
+ response_format={"type": "json_object"},
+ messages=[
+ {"role": "system", "content": f"{SYSTEM_PROMPT}\n\n{OUTPUT_SCHEMA}"},
+ {"role": "user", "content": prompt},
+ ],
+ )
+
+ raw_text = response.choices[0].message.content or ""
+ self._record_usage(response, effective_model)
+
+ result = Analyzer.parse_llm_json(raw_text)
+ result.input_tokens = response.usage.prompt_tokens
+ result.output_tokens = response.usage.completion_tokens
+ result.raw = raw_text
+ return result
+
+ def analyze_function_batch(self, prompt: str, *, model: str | None = None) -> list[LLMResponse]:
+ """Send a batch analysis prompt and return parsed list of responses."""
+ effective_model = model or self.model
+ response = self._client.chat.completions.create(
+ model=effective_model,
+ max_tokens=16384,
+ messages=[
+ {"role": "system", "content": f"{BATCH_SYSTEM_PROMPT}\n\n{BATCH_OUTPUT_SCHEMA}"},
+ {"role": "user", "content": prompt},
+ ],
+ )
+
+ raw_text = response.choices[0].message.content or ""
+ self._record_usage(response, effective_model)
+
+ responses = Analyzer.parse_llm_json_batch(raw_text)
+ for resp in responses:
+ resp.input_tokens = response.usage.prompt_tokens
+ resp.output_tokens = response.usage.completion_tokens
+ return responses
+
+ def analyze_with_tools(
+ self,
+ prompt: str,
+ system: str,
+ tools: list[dict[str, Any]],
+ tool_executor: ToolExecutor,
+ max_rounds: int = 10,
+ ) -> LLMResponse:
+ """Run an agentic tool-use loop.
+
+ Sends the prompt with tool definitions. When the model returns
+ tool_calls, executes each tool via *tool_executor* and feeds results
+ back. Repeats until the model returns a final text response or
+ *max_rounds* is exhausted.
+ """
+ openai_tools = _convert_tools_to_openai(tools)
+
+ messages: list[dict[str, Any]] = [
+ {"role": "system", "content": f"{system}\n\n{OUTPUT_SCHEMA}"},
+ {"role": "user", "content": prompt},
+ ]
+
+ total_input = 0
+ total_output = 0
+
+ last_text = ""
+ for _ in range(max_rounds):
+ response = self._client.chat.completions.create(
+ model=self.model,
+ max_tokens=self.max_tokens,
+ tools=openai_tools,
+ messages=messages,
+ )
+
+ usage = response.usage
+ total_input += usage.prompt_tokens
+ total_output += usage.completion_tokens
+ self._record_usage(response, self.model)
+
+ choice = response.choices[0]
+
+ if choice.finish_reason != "tool_calls":
+ last_text = choice.message.content or ""
+ result = Analyzer.parse_llm_json(last_text)
+ result.input_tokens = total_input
+ result.output_tokens = total_output
+ result.raw = last_text
+ return result
+
+ assistant_msg: dict[str, Any] = {
+ "role": "assistant",
+ "content": choice.message.content,
+ "tool_calls": [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {
+ "name": tc.function.name,
+ "arguments": tc.function.arguments,
+ },
+ }
+ for tc in choice.message.tool_calls
+ ],
+ }
+ messages.append(assistant_msg)
+
+ for tool_call in choice.message.tool_calls:
+ arguments = json.loads(tool_call.function.arguments)
+ result_str = tool_executor.execute(tool_call.function.name, arguments)
+ messages.append({
+ "role": "tool",
+ "tool_call_id": tool_call.id,
+ "content": result_str,
+ })
+
+ result = Analyzer.parse_llm_json(last_text)
+ result.input_tokens = total_input
+ result.output_tokens = total_output
+ result.raw = last_text
+ return result
+
+ @property
+ def total_cost_usd(self) -> float:
+ return self.usage.total_cost_usd
+
+ def _record_usage(self, response: Any, model: str | None = None) -> None:
+ effective_model = model or self.model
+ usage = response.usage
+ mu = self.usage._get(effective_model)
+ mu.input_tokens += usage.prompt_tokens
+ mu.output_tokens += usage.completion_tokens
+ cached = getattr(
+ getattr(usage, "prompt_tokens_details", None),
+ "cached_tokens",
+ 0,
+ ) or 0
+ mu.cache_read_tokens += cached
+ mu.calls += 1
+ logger.debug(
+ "LLM [%s]: %d in / %d out / %d cached tokens "
+ "(total: %d calls, $%.4f)",
+ effective_model,
+ usage.prompt_tokens,
+ usage.completion_tokens,
+ cached,
+ self.usage.calls,
+ self.usage.total_cost_usd,
+ )
diff --git a/kong/llm/usage.py b/kong/llm/usage.py
index 12c32e0..ba4d25a 100644
--- a/kong/llm/usage.py
+++ b/kong/llm/usage.py
@@ -4,14 +4,43 @@ from __future__ import annotations
from dataclasses import dataclass, field
-_PRICING: dict[str, tuple[float, float]] = {
- "claude-opus-4-6": (5.0, 25.0),
- "claude-sonnet-4-6": (3.0, 15.0),
- "claude-sonnet-4-20250514": (3.0, 15.0),
- "claude-haiku-4-5-20251001": (1.0, 5.0),
+
+@dataclass(frozen=True)
+class PricingTier:
+ """Per-model pricing in USD per 1M tokens.
+
+ Rates sourced from published Anthropic/OpenAI API pricing pages.
+ """
+
+ input_rate: float
+ output_rate: float
+ cache_write_rate: float = 0.0
+ cache_read_rate: float = 0.0
+
+
+_DEFAULT_PRICING = PricingTier(input_rate=3.0, output_rate=15.0)
+
+PRICING_REGISTRY: dict[str, PricingTier] = {
+ # Anthropic: cache_write = input * 1.25, cache_read = input * 0.10
+ "claude-opus-4-6": PricingTier(5.0, 25.0, cache_write_rate=6.25, cache_read_rate=0.50),
+ "claude-sonnet-4-6": PricingTier(3.0, 15.0, cache_write_rate=3.75, cache_read_rate=0.30),
+ "claude-sonnet-4-20250514": PricingTier(3.0, 15.0, cache_write_rate=3.75, cache_read_rate=0.30),
+ "claude-haiku-4-5-20251001": PricingTier(1.0, 5.0, cache_write_rate=1.25, cache_read_rate=0.10),
+ # OpenAI: cached input is 50% of input rate, no cache write billing
+ "gpt-4o": PricingTier(2.50, 10.00, cache_read_rate=1.25),
+ "gpt-4o-2024-11-20": PricingTier(2.50, 10.00, cache_read_rate=1.25),
+ "gpt-4o-mini": PricingTier(0.15, 0.60, cache_read_rate=0.075),
+ "gpt-4o-mini-2024-07-18": PricingTier(0.15, 0.60, cache_read_rate=0.075),
+ "o1": PricingTier(15.00, 60.00, cache_read_rate=7.50),
+ "o3-mini": PricingTier(1.10, 4.40, cache_read_rate=0.55),
}
+def get_pricing(model: str) -> PricingTier:
+ """Look up pricing for a model, falling back to a sensible default."""
+ return PRICING_REGISTRY.get(model, _DEFAULT_PRICING)
+
+
@dataclass
class ModelTokenUsage:
input_tokens: int = 0
@@ -21,14 +50,12 @@ class ModelTokenUsage:
calls: int = 0
def cost_usd(self, model: str) -> float:
- input_rate, output_rate = _PRICING.get(model, (3.0, 15.0))
- cache_write_rate = input_rate * 1.25
- cache_read_rate = input_rate * 0.10
+ tier = get_pricing(model)
return (
- (self.input_tokens / 1_000_000) * input_rate
- + (self.output_tokens / 1_000_000) * output_rate
- + (self.cache_creation_tokens / 1_000_000) * cache_write_rate
- + (self.cache_read_tokens / 1_000_000) * cache_read_rate
+ (self.input_tokens / 1_000_000) * tier.input_rate
+ + (self.output_tokens / 1_000_000) * tier.output_rate
+ + (self.cache_creation_tokens / 1_000_000) * tier.cache_write_rate
+ + (self.cache_read_tokens / 1_000_000) * tier.cache_read_rate
)
diff --git a/pyproject.toml b/pyproject.toml
index d375980..a4a645c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,9 @@ Issues = "https://github.com/amruth-sn/kong/issues"
kong = "kong.__main__:cli"
[project.optional-dependencies]
+openai = [
+ "openai>=1.0",
+]
dev = [
"pytest>=8.0",
"pytest-asyncio>=0.24",
diff --git a/tests/test_openai_client.py b/tests/test_openai_client.py
new file mode 100644
index 0000000..36fcaa5
--- /dev/null
+++ b/tests/test_openai_client.py
@@ -0,0 +1,296 @@
+"""Tests for the OpenAI LLM client wrapper."""
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+from kong.llm.openai_client import OpenAIClient, _convert_tools_to_openai
+from kong.llm.usage import ModelTokenUsage, TokenUsage
+
+
+def _mock_response(text: str, prompt_tokens: int = 100, completion_tokens: int = 50):
+ """Create a mock OpenAI ChatCompletion response."""
+ message = MagicMock()
+ message.content = text
+ message.tool_calls = None
+
+ choice = MagicMock()
+ choice.message = message
+ choice.finish_reason = "stop"
+
+ usage = MagicMock()
+ usage.prompt_tokens = prompt_tokens
+ usage.completion_tokens = completion_tokens
+ usage.prompt_tokens_details = None
+
+ resp = MagicMock()
+ resp.choices = [choice]
+ resp.usage = usage
+ return resp
+
+
+class TestOpenAIClient:
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_analyze_function_parses_json(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ '{"name": "parse_config", "confidence": 85, "classification": "parser"}'
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ response = client.analyze_function("analyze this function")
+
+ assert response.name == "parse_config"
+ assert response.confidence == 85
+ assert response.classification == "parser"
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_tracks_token_usage(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ '{"name": "f"}', prompt_tokens=200, completion_tokens=80
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("prompt1")
+
+ assert client.usage.input_tokens == 200
+ assert client.usage.output_tokens == 80
+ assert client.usage.calls == 1
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_accumulates_across_calls(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ '{"name": "f"}', prompt_tokens=100, completion_tokens=50
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("p1")
+ client.analyze_function("p2")
+
+ assert client.usage.input_tokens == 200
+ assert client.usage.output_tokens == 100
+ assert client.usage.calls == 2
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_sends_system_message(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("test prompt")
+
+ call_kwargs = mock_client.chat.completions.create.call_args
+ messages = call_kwargs.kwargs["messages"]
+ assert messages[0]["role"] == "system"
+ assert "reverse engineer" in messages[0]["content"].lower()
+ assert messages[1]["role"] == "user"
+ assert messages[1]["content"] == "test prompt"
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_uses_json_mode_for_single_analysis(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("prompt")
+
+ call_kwargs = mock_client.chat.completions.create.call_args
+ assert call_kwargs.kwargs["response_format"] == {"type": "json_object"}
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_model_override(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response('{"name": "f"}')
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("prompt", model="gpt-4o-mini")
+
+ call_kwargs = mock_client.chat.completions.create.call_args
+ assert call_kwargs.kwargs["model"] == "gpt-4o-mini"
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_per_model_cost_tracking(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ '{"name": "f"}', prompt_tokens=100, completion_tokens=50
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("p1")
+ client.analyze_function("p2", model="gpt-4o-mini")
+
+ assert len(client.usage.by_model) == 2
+ assert "gpt-4o" in client.usage.by_model
+ assert "gpt-4o-mini" in client.usage.by_model
+ assert client.usage.calls == 2
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_handles_malformed_response(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ "I cannot analyze this function because reasons."
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ response = client.analyze_function("prompt")
+
+ assert response.name == ""
+ assert "Failed to parse" in response.reasoning
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_handles_empty_content(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ resp = _mock_response("")
+ resp.choices[0].message.content = None
+ mock_client.chat.completions.create.return_value = resp
+
+ client = OpenAIClient(api_key="test-key")
+ response = client.analyze_function("prompt")
+
+ assert response.name == ""
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_records_cached_tokens(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ resp = _mock_response('{"name": "f"}', prompt_tokens=200, completion_tokens=50)
+ details = MagicMock()
+ details.cached_tokens = 100
+ resp.usage.prompt_tokens_details = details
+ mock_client.chat.completions.create.return_value = resp
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function("prompt")
+
+ mu = client.usage.by_model["gpt-4o"]
+ assert mu.cache_read_tokens == 100
+
+
+class TestOpenAIBatch:
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_batch_returns_list_of_responses(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ json.dumps([
+ {"address": "0x1000", "name": "foo", "confidence": 80},
+ {"address": "0x2000", "name": "bar", "confidence": 70},
+ ])
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ results = client.analyze_function_batch("batch prompt")
+
+ assert len(results) == 2
+ assert results[0].name == "foo"
+ assert results[1].name == "bar"
+
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_batch_uses_batch_system_prompt(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.return_value = _mock_response(
+ '[{"name": "f", "confidence": 50}]'
+ )
+
+ client = OpenAIClient(api_key="test-key")
+ client.analyze_function_batch("prompt")
+
+ call_kwargs = mock_client.chat.completions.create.call_args
+ system_text = call_kwargs.kwargs["messages"][0]["content"]
+ assert "multiple" in system_text.lower() or "batch" in system_text.lower()
+
+
+class TestOpenAIToolUse:
+ @patch("kong.llm.openai_client.openai.OpenAI")
+ def test_tool_loop_executes_tools(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+
+ tool_call = MagicMock()
+ tool_call.id = "call_123"
+ tool_call.function.name = "simplify_expression"
+ tool_call.function.arguments = '{"expression": "x + 0"}'
+
+ tool_response = _mock_response("", prompt_tokens=100, completion_tokens=20)
+ tool_response.choices[0].finish_reason = "tool_calls"
+ tool_response.choices[0].message.content = None
+ tool_response.choices[0].message.tool_calls = [tool_call]
+
+ final_response = _mock_response(
+ '{"name": "clean_func", "confidence": 75}',
+ prompt_tokens=150, completion_tokens=40,
+ )
+
+ mock_client.chat.completions.create.side_effect = [tool_response, final_response]
+
+ executor = MagicMock()
+ executor.execute.return_value = '{"simplified": "x"}'
+
+ tools = [{
+ "name": "simplify_expression",
+ "description": "Simplify an expression.",
+ "input_schema": {"type": "object", "properties": {"expression": {"type": "string"}}},
+ }]
+
+ client = OpenAIClient(api_key="test-key")
+ result = client.analyze_with_tools("prompt", "system", tools, executor)
+
+ assert result.name == "clean_func"
+ assert result.input_tokens == 250
+ assert result.output_tokens == 60
+ executor.execute.assert_called_once_with("simplify_expression", {"expression": "x + 0"})
+
+
+class TestConvertTools:
+ def test_converts_anthropic_to_openai_format(self):
+ anthropic_tools = [{
+ "name": "test_tool",
+ "description": "A test tool.",
+ "input_schema": {
+ "type": "object",
+ "properties": {"x": {"type": "string"}},
+ "required": ["x"],
+ },
+ }]
+
+ openai_tools = _convert_tools_to_openai(anthropic_tools)
+
+ assert len(openai_tools) == 1
+ assert openai_tools[0]["type"] == "function"
+ assert openai_tools[0]["function"]["name"] == "test_tool"
+ assert openai_tools[0]["function"]["description"] == "A test tool."
+ assert openai_tools[0]["function"]["parameters"]["type"] == "object"
+ assert "x" in openai_tools[0]["function"]["parameters"]["properties"]
+
+
+class TestOpenAICostCalculation:
+ def test_gpt4o_cost(self):
+ mu = ModelTokenUsage(input_tokens=1_000_000, output_tokens=1_000_000)
+ cost = mu.cost_usd("gpt-4o")
+ assert cost == 2.50 + 10.00
+
+ def test_gpt4o_mini_cost(self):
+ mu = ModelTokenUsage(input_tokens=1_000_000, output_tokens=1_000_000)
+ cost = mu.cost_usd("gpt-4o-mini")
+ assert cost == 0.15 + 0.60
+
+ def test_gpt4o_cached_tokens_cost(self):
+ mu = ModelTokenUsage(
+ input_tokens=0, output_tokens=0,
+ cache_read_tokens=1_000_000,
+ )
+ cost = mu.cost_usd("gpt-4o")
+ assert cost == 1.25
diff --git a/uv.lock b/uv.lock
index 87c2b28..9c0f926 100644
--- a/uv.lock
+++ b/uv.lock
@@ -271,7 +271,7 @@ wheels = [
[[package]]
name = "kong-re"
-version = "0.1.0"
+version = "0.1.2"
source = { editable = "." }
dependencies = [
{ name = "anthropic" },
@@ -290,6 +290,9 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
]
+openai = [
+ { name = "openai" },
+]
[package.metadata]
requires-dist = [
@@ -298,6 +301,7 @@ requires-dist = [
{ name = "jpype1", specifier = ">=1.6.0" },
{ name = "json-repair", specifier = ">=0.58.5" },
{ name = "lief", specifier = ">=0.15" },
+ { name = "openai", marker = "extra == 'openai'", specifier = ">=1.0" },
{ name = "pyghidra", specifier = ">=2.2.1" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24" },
@@ -305,7 +309,7 @@ requires-dist = [
{ name = "textual", specifier = ">=1.0" },
{ name = "z3-solver", specifier = ">=4.12" },
]
-provides-extras = ["dev"]
+provides-extras = ["openai", "dev"]
[[package]]
name = "lief"
@@ -409,6 +413,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
]
+[[package]]
+name = "openai"
+version = "2.28.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "distro" },
+ { name = "httpx" },
+ { name = "jiter" },
+ { name = "pydantic" },
+ { name = "sniffio" },
+ { name = "tqdm" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/56/87/eb0abb4ef88ddb95b3c13149384c4c288f584f3be17d6a4f63f8c3e3c226/openai-2.28.0.tar.gz", hash = "sha256:bb7fdff384d2a787fa82e8822d1dd3c02e8cf901d60f1df523b7da03cbb6d48d", size = 670334, upload-time = "2026-03-13T19:56:27.306Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c0/5a/df122348638885526e53140e9c6b0d844af7312682b3bde9587eebc28b47/openai-2.28.0-py3-none-any.whl", hash = "sha256:79aa5c45dba7fef84085701c235cf13ba88485e1ef4f8dfcedc44fc2a698fc1d", size = 1141218, upload-time = "2026-03-13T19:56:25.46Z" },
+]
+
[[package]]
name = "packaging"
version = "26.0"
@@ -638,6 +661,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d3/be/e191c2a15da20530fde03564564e3e4b4220eb9d687d4014957e5c6a5e85/textual-8.0.0-py3-none-any.whl", hash = "sha256:8908f4ebe93a6b4f77ca7262197784a52162bc88b05f4ecf50ac93a92d49bb8f", size = 718904, upload-time = "2026-02-16T17:12:11.962Z" },
]
+[[package]]
+name = "tqdm"
+version = "4.67.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" },
+]
+
[[package]]
name = "typing-extensions"
version = "4.15.0"