Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

feat(cli): add kong eval command for scoring against ground truth

Amruthb09d005fd76b
diff --git a/kong/__main__.py b/kong/__main__.py
index 0dcb652..966b25e 100644
--- a/kong/__main__.py
+++ b/kong/__main__.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import os
+from collections import Counter
from pathlib import Path
import click
@@ -9,17 +11,18 @@ from rich.console import Console
from rich.markup import escape
from kong import __version__
-from collections import Counter
-
from kong.agent.events import Event, EventType
from kong.agent.supervisor import Supervisor
+from kong.banner import check_api_key, print_analyze_header, print_banner, print_setup_needed
from kong.config import GhidraConfig, KongConfig, OutputConfig
+from kong.evals.harness import score as eval_score
from kong.ghidra.client import GhidraClient, GhidraClientError
from kong.llm.client import AnthropicClient
+
console = Console()
[email protected](invoke_without_command=True)
@click.version_option(version=__version__, prog_name="kong")
@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
@click.pass_context
@@ -32,6 +35,15 @@ def cli(ctx: click.Context, verbose: bool) -> None:
ctx.ensure_object(dict)
ctx.obj["verbose"] = verbose
+ if ctx.invoked_subcommand is None:
+ print_banner(console)
+ console.print()
+ console.print("Usage: [bold]kong analyze <binary>[/bold]")
+ console.print(" [bold]kong info <binary>[/bold]")
+ console.print(" [bold]kong setup[/bold]")
+ console.print()
+ console.print("Run [bold]kong --help[/bold] for all options.")
+
def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> None:
stats = supervisor.stats
@@ -91,11 +103,17 @@ def analyze(
)
binary_path = Path(binary).resolve()
- console.print(f"[bold]Kong v{__version__}[/bold]")
- console.print(f"Binary: {binary_path}")
- console.print(f"Output: {config.output.directory}")
- console.print(f"Formats: {', '.join(config.output.formats)}")
- console.print()
+
+ print_analyze_header(
+ console,
+ binary_path=str(binary_path),
+ output_dir=str(config.output.directory),
+ formats=config.output.formats,
+ )
+
+ if not check_api_key():
+ print_setup_needed(console)
+ raise SystemExit(1)
if not config.ghidra.install_dir:
console.print("[red]Ghidra is not installed or not found.[/red]")
@@ -204,6 +222,88 @@ def info(binary: str, ghidra_dir: str | None) -> None:
client.close()
+def setup() -> None:
+ """Guided setup for Kong."""
+ print_banner(console)
+ console.print()
+
+ console.print("[bold]Step 1: Anthropic API Key[/bold]")
+ console.print()
+
+ current_key = os.environ.get("ANTHROPIC_API_KEY", "")
+ if current_key:
+ masked = current_key[:7] + "..." + current_key[-4:]
+ console.print(f" [green]Found:[/green] {masked}")
+ else:
+ console.print(" [yellow]Not set.[/yellow]")
+ console.print()
+ console.print(" Get your key at: [bold]https://console.anthropic.com/settings/keys[/bold]")
+ console.print()
+ key = click.prompt(" Enter your API key", hide_input=True)
+ if not key.startswith("sk-ant-"):
+ console.print(" [red]Invalid key format. Keys start with sk-ant-[/red]")
+ raise SystemExit(1)
+ console.print()
+ console.print(" Add to your shell profile:")
+ console.print(f' [bold]export ANTHROPIC_API_KEY="{key}"[/bold]')
+ os.environ["ANTHROPIC_API_KEY"] = key
+
+ console.print()
+ console.print("[bold]Step 2: Ghidra[/bold]")
+ console.print()
+
+ ghidra_config = GhidraConfig()
+ if ghidra_config.install_dir:
+ console.print(f" [green]Found:[/green] {ghidra_config.install_dir}")
+ else:
+ console.print(" [yellow]Not found.[/yellow]")
+ console.print()
+ console.print(" Install Ghidra:")
+ console.print(" [bold]brew install ghidra[/bold] (macOS)")
+ console.print(" Or download from [bold]https://ghidra-sre.org[/bold]")
+ console.print()
+ console.print(" Then set [bold]GHIDRA_INSTALL_DIR[/bold] to the install path.")
+
+ console.print()
+ all_good = check_api_key() and ghidra_config.install_dir
+ if all_good:
+ console.print("[bold green]All set! Run [bold]kong analyze <binary>[/bold] to get started.[/bold green]")
+ else:
+ console.print("[yellow]Some dependencies are missing. See above for instructions.[/yellow]")
+
+
[email protected](name="eval")
[email protected]("analysis_json", type=click.Path(exists=True, dir_okay=False))
[email protected]("source_file", type=click.Path(exists=True, dir_okay=False))
+def eval_cmd(analysis_json: str, source_file: str) -> None:
+ """Score a Kong analysis against ground truth source code."""
+ scorecard = eval_score(
+ analysis_path=Path(analysis_json),
+ source_path=Path(source_file),
+ )
+
+ console.print(f"[bold]Binary:[/bold] {scorecard.binary}")
+ console.print(f"[bold]Functions:[/bold] {scorecard.functions_analyzed} analyzed / {scorecard.total_functions} in source")
+ console.print(f"[bold]Symbol Accuracy:[/bold] {scorecard.symbol_accuracy:.1%}")
+ console.print(f"[bold]Type Accuracy:[/bold] {scorecard.type_accuracy:.1%}")
+ console.print()
+
+ console.print("[bold]Per-Function Scores:[/bold]")
+ for pf in scorecard.per_function:
+ pred = pf["predicted_name"]
+ truth = pf["truth_name"]
+ sym = pf["symbol_score"]
+ typ = pf["type_score"]
+ match_indicator = "[green]OK[/green]" if sym >= 0.8 else "[yellow]~~[/yellow]" if sym > 0 else "[red]NO[/red]"
+ console.print(f" {match_indicator} {pred:30s} -> {truth:20s} sym={sym:.2f} type={typ:.2f}")
+
+ console.print()
+ console.print(f"[bold]LLM Calls:[/bold] {scorecard.llm_calls}")
+ console.print(f"[bold]Duration:[/bold] {scorecard.duration_seconds:.1f}s")
+ console.print(f"[bold]Cost:[/bold] ${scorecard.cost_usd:.4f}")
+
+
def main() -> None:
cli()
diff --git a/kong/evals/harness.py b/kong/evals/harness.py
new file mode 100644
index 0000000..ada43c2
--- /dev/null
+++ b/kong/evals/harness.py
@@ -0,0 +1,73 @@
+"""Evaluation harness for scoring Kong analysis against ground truth."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class Scorecard:
+ binary: str = ""
+ total_functions: int = 0
+ functions_analyzed: int = 0
+ symbol_accuracy: float = 0.0
+ type_accuracy: float = 0.0
+ per_function: list[dict[str, str | float]] = field(default_factory=list)
+ llm_calls: int = 0
+ duration_seconds: float = 0.0
+ cost_usd: float = 0.0
+
+
+def score(analysis_path: Path, source_path: Path) -> Scorecard:
+ """Score a Kong analysis.json against ground truth C source code."""
+ import json
+ import re
+
+ analysis = json.loads(analysis_path.read_text())
+
+ source_text = source_path.read_text()
+ source_functions = re.findall(
+ r"(?:[\w\s\*]+)\s+(\w+)\s*\([^)]*\)",
+ source_text,
+ )
+ source_func_set = set(source_functions)
+
+ binary_name = analysis.get("binary", {}).get("name", "")
+ stats = analysis.get("stats", {})
+ analyzed_functions = analysis.get("functions", [])
+
+ per_function: list[dict[str, str | float]] = []
+ symbol_matches = 0
+ type_matches = 0
+
+ for func in analyzed_functions:
+ predicted_name = func.get("name", "")
+ best_match = predicted_name if predicted_name in source_func_set else ""
+ sym_score = 1.0 if best_match else 0.0
+ typ_score = 1.0 if best_match else 0.0
+
+ if sym_score > 0:
+ symbol_matches += 1
+ type_matches += 1
+
+ per_function.append({
+ "predicted_name": predicted_name,
+ "truth_name": best_match or "???",
+ "symbol_score": sym_score,
+ "type_score": typ_score,
+ })
+
+ total = max(len(source_func_set), 1)
+
+ return Scorecard(
+ binary=binary_name,
+ total_functions=len(source_func_set),
+ functions_analyzed=len(analyzed_functions),
+ symbol_accuracy=symbol_matches / total,
+ type_accuracy=type_matches / total,
+ per_function=per_function,
+ llm_calls=stats.get("llm_calls", 0),
+ duration_seconds=stats.get("duration_seconds", 0.0),
+ cost_usd=stats.get("cost_usd", 0.0),
+ )
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 265d399..f1d3041 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from unittest.mock import patch
+
from click.testing import CliRunner
from kong.__main__ import cli
@@ -37,13 +38,29 @@ def test_analyze_missing_binary():
assert result.exit_code != 0
+def test_analyze_no_api_key(tmp_path):
+ """When ANTHROPIC_API_KEY is not set, show setup instructions."""
+ binary = tmp_path / "test_binary"
+ binary.write_bytes(b"\x00" * 16)
+
+ runner = CliRunner()
+ with patch.dict("os.environ", {}, clear=True), \
+ patch("kong.__main__.check_api_key", return_value=False):
+ result = runner.invoke(cli, ["analyze", str(binary)])
+
+ assert result.exit_code != 0
+ assert "anthropic_api_key" in result.output.lower()
+ assert "kong setup" in result.output.lower()
+
+
def test_analyze_no_ghidra_installed(tmp_path):
"""When Ghidra is not installed, show install instructions."""
binary = tmp_path / "test_binary"
binary.write_bytes(b"\x00" * 16)
runner = CliRunner()
- with patch("kong.config.find_ghidra_install", return_value=None):
+ with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "sk-ant-test"}), \
+ patch("kong.config.find_ghidra_install", return_value=None):
result = runner.invoke(cli, ["analyze", str(binary)])
assert result.exit_code != 0
@@ -51,7 +68,61 @@ def test_analyze_no_ghidra_installed(tmp_path):
assert "brew install ghidra" in result.output
+def test_bare_kong_shows_banner():
+ """Running 'kong' with no subcommand shows banner and usage."""
+ runner = CliRunner()
+ result = runner.invoke(cli, [])
+
+ assert result.exit_code == 0
+ assert "KONG" in result.output or "kong" in result.output.lower()
+ assert "kong analyze" in result.output.lower()
+ assert "kong setup" in result.output.lower()
+
+
+def test_setup_with_key_set():
+ """Setup command detects existing API key."""
+ runner = CliRunner()
+ with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "sk-ant-test1234"}):
+ result = runner.invoke(cli, ["setup"])
+
+ assert result.exit_code == 0
+ assert "sk-ant-" in result.output
+ assert "..." in result.output
+
+
def test_info_requires_binary_arg():
runner = CliRunner()
result = runner.invoke(cli, ["info"])
assert result.exit_code != 0
+
+
+def test_eval_help():
+ runner = CliRunner()
+ result = runner.invoke(cli, ["eval", "--help"])
+ assert result.exit_code == 0
+ assert "score" in result.output.lower() or "analysis" in result.output.lower()
+
+
+def test_eval_with_test_data(tmp_path):
+ """Run eval against a simple test case."""
+ import json
+
+ analysis = {
+ "binary": {"name": "test"},
+ "stats": {"llm_calls": 5, "duration_seconds": 10.0, "cost_usd": 0.05},
+ "functions": [
+ {"name": "hash_string", "signature": "uint hash_string(byte *str)", "confidence": 95, "address": "0x1000"},
+ ],
+ }
+ analysis_path = tmp_path / "analysis.json"
+ analysis_path.write_text(json.dumps(analysis))
+
+ source = "unsigned int hash_string(const char *s) {\n return 0;\n}\n"
+ source_path = tmp_path / "test.c"
+ source_path.write_text(source)
+
+ runner = CliRunner()
+ result = runner.invoke(cli, ["eval", str(analysis_path), str(source_path)])
+ assert result.exit_code == 0
+ assert "hash_string" in result.output
+ assert "Symbol Accuracy" in result.output