The world's first agentic reverse engineer.
feat(cli): add kong eval command for scoring against ground truth
Amruthb09d005fd76b
diff --git a/kong/__main__.py b/kong/__main__.pyindex 0dcb652..966b25e 100644--- a/kong/__main__.py+++ b/kong/__main__.py@@ -2,6 +2,8 @@from __future__ import annotations+import os+from collections import Counterfrom pathlib import Pathimport click@@ -9,17 +11,18 @@ from rich.console import Consolefrom rich.markup import escapefrom kong import __version__-from collections import Counter-from kong.agent.events import Event, EventTypefrom kong.agent.supervisor import Supervisor+from kong.banner import check_api_key, print_analyze_header, print_banner, print_setup_neededfrom kong.config import GhidraConfig, KongConfig, OutputConfig+from kong.evals.harness import score as eval_scorefrom kong.ghidra.client import GhidraClient, GhidraClientErrorfrom kong.llm.client import AnthropicClient+console = Console()[email protected](invoke_without_command=True)@click.version_option(version=__version__, prog_name="kong")@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")@click.pass_context@@ -32,6 +35,15 @@ def cli(ctx: click.Context, verbose: bool) -> None:ctx.ensure_object(dict)ctx.obj["verbose"] = verbose+ if ctx.invoked_subcommand is None:+ print_banner(console)+ console.print()+ console.print("Usage: [bold]kong analyze <binary>[/bold]")+ console.print(" [bold]kong info <binary>[/bold]")+ console.print(" [bold]kong setup[/bold]")+ console.print()+ console.print("Run [bold]kong --help[/bold] for all options.")+def _print_final_stats(supervisor: Supervisor, llm_client: AnthropicClient) -> None:stats = supervisor.stats@@ -91,11 +103,17 @@ def analyze()binary_path = Path(binary).resolve()- console.print(f"[bold]Kong v{__version__}[/bold]")- console.print(f"Binary: {binary_path}")- console.print(f"Output: {config.output.directory}")- console.print(f"Formats: {', '.join(config.output.formats)}")- console.print()++ print_analyze_header(+ console,+ binary_path=str(binary_path),+ output_dir=str(config.output.directory),+ formats=config.output.formats,+ )++ if not check_api_key():+ print_setup_needed(console)+ raise SystemExit(1)if not config.ghidra.install_dir:console.print("[red]Ghidra is not installed or not found.[/red]")@@ -204,6 +222,88 @@ def info(binary: str, ghidra_dir: str | None) -> None:client.close()+def setup() -> None:+ """Guided setup for Kong."""+ print_banner(console)+ console.print()++ console.print("[bold]Step 1: Anthropic API Key[/bold]")+ console.print()++ current_key = os.environ.get("ANTHROPIC_API_KEY", "")+ if current_key:+ masked = current_key[:7] + "..." + current_key[-4:]+ console.print(f" [green]Found:[/green] {masked}")+ else:+ console.print(" [yellow]Not set.[/yellow]")+ console.print()+ console.print(" Get your key at: [bold]https://console.anthropic.com/settings/keys[/bold]")+ console.print()+ key = click.prompt(" Enter your API key", hide_input=True)+ if not key.startswith("sk-ant-"):+ console.print(" [red]Invalid key format. Keys start with sk-ant-[/red]")+ raise SystemExit(1)+ console.print()+ console.print(" Add to your shell profile:")+ console.print(f' [bold]export ANTHROPIC_API_KEY="{key}"[/bold]')+ os.environ["ANTHROPIC_API_KEY"] = key++ console.print()+ console.print("[bold]Step 2: Ghidra[/bold]")+ console.print()++ ghidra_config = GhidraConfig()+ if ghidra_config.install_dir:+ console.print(f" [green]Found:[/green] {ghidra_config.install_dir}")+ else:+ console.print(" [yellow]Not found.[/yellow]")+ console.print()+ console.print(" Install Ghidra:")+ console.print(" [bold]brew install ghidra[/bold] (macOS)")+ console.print(" Or download from [bold]https://ghidra-sre.org[/bold]")+ console.print()+ console.print(" Then set [bold]GHIDRA_INSTALL_DIR[/bold] to the install path.")++ console.print()+ all_good = check_api_key() and ghidra_config.install_dir+ if all_good:+ console.print("[bold green]All set! Run [bold]kong analyze <binary>[/bold] to get started.[/bold green]")+ else:+ console.print("[yellow]Some dependencies are missing. See above for instructions.[/yellow]")++[email protected](name="eval")[email protected]("analysis_json", type=click.Path(exists=True, dir_okay=False))[email protected]("source_file", type=click.Path(exists=True, dir_okay=False))+def eval_cmd(analysis_json: str, source_file: str) -> None:+ """Score a Kong analysis against ground truth source code."""+ scorecard = eval_score(+ analysis_path=Path(analysis_json),+ source_path=Path(source_file),+ )++ console.print(f"[bold]Binary:[/bold] {scorecard.binary}")+ console.print(f"[bold]Functions:[/bold] {scorecard.functions_analyzed} analyzed / {scorecard.total_functions} in source")+ console.print(f"[bold]Symbol Accuracy:[/bold] {scorecard.symbol_accuracy:.1%}")+ console.print(f"[bold]Type Accuracy:[/bold] {scorecard.type_accuracy:.1%}")+ console.print()++ console.print("[bold]Per-Function Scores:[/bold]")+ for pf in scorecard.per_function:+ pred = pf["predicted_name"]+ truth = pf["truth_name"]+ sym = pf["symbol_score"]+ typ = pf["type_score"]+ match_indicator = "[green]OK[/green]" if sym >= 0.8 else "[yellow]~~[/yellow]" if sym > 0 else "[red]NO[/red]"+ console.print(f" {match_indicator} {pred:30s} -> {truth:20s} sym={sym:.2f} type={typ:.2f}")++ console.print()+ console.print(f"[bold]LLM Calls:[/bold] {scorecard.llm_calls}")+ console.print(f"[bold]Duration:[/bold] {scorecard.duration_seconds:.1f}s")+ console.print(f"[bold]Cost:[/bold] ${scorecard.cost_usd:.4f}")++def main() -> None:cli()diff --git a/kong/evals/harness.py b/kong/evals/harness.pynew file mode 100644index 0000000..ada43c2--- /dev/null+++ b/kong/evals/harness.py@@ -0,0 +1,73 @@+"""Evaluation harness for scoring Kong analysis against ground truth."""++from __future__ import annotations++from dataclasses import dataclass, field+from pathlib import Path+++@dataclass+class Scorecard:+ binary: str = ""+ total_functions: int = 0+ functions_analyzed: int = 0+ symbol_accuracy: float = 0.0+ type_accuracy: float = 0.0+ per_function: list[dict[str, str | float]] = field(default_factory=list)+ llm_calls: int = 0+ duration_seconds: float = 0.0+ cost_usd: float = 0.0+++def score(analysis_path: Path, source_path: Path) -> Scorecard:+ """Score a Kong analysis.json against ground truth C source code."""+ import json+ import re++ analysis = json.loads(analysis_path.read_text())++ source_text = source_path.read_text()+ source_functions = re.findall(+ r"(?:[\w\s\*]+)\s+(\w+)\s*\([^)]*\)",+ source_text,+ )+ source_func_set = set(source_functions)++ binary_name = analysis.get("binary", {}).get("name", "")+ stats = analysis.get("stats", {})+ analyzed_functions = analysis.get("functions", [])++ per_function: list[dict[str, str | float]] = []+ symbol_matches = 0+ type_matches = 0++ for func in analyzed_functions:+ predicted_name = func.get("name", "")+ best_match = predicted_name if predicted_name in source_func_set else ""+ sym_score = 1.0 if best_match else 0.0+ typ_score = 1.0 if best_match else 0.0++ if sym_score > 0:+ symbol_matches += 1+ type_matches += 1++ per_function.append({+ "predicted_name": predicted_name,+ "truth_name": best_match or "???",+ "symbol_score": sym_score,+ "type_score": typ_score,+ })++ total = max(len(source_func_set), 1)++ return Scorecard(+ binary=binary_name,+ total_functions=len(source_func_set),+ functions_analyzed=len(analyzed_functions),+ symbol_accuracy=symbol_matches / total,+ type_accuracy=type_matches / total,+ per_function=per_function,+ llm_calls=stats.get("llm_calls", 0),+ duration_seconds=stats.get("duration_seconds", 0.0),+ cost_usd=stats.get("cost_usd", 0.0),+ )diff --git a/tests/test_cli.py b/tests/test_cli.pyindex 265d399..f1d3041 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -4,6 +4,7 @@ from __future__ import annotationsfrom unittest.mock import patch+from click.testing import CliRunnerfrom kong.__main__ import cli@@ -37,13 +38,29 @@ def test_analyze_missing_binary():assert result.exit_code != 0+def test_analyze_no_api_key(tmp_path):+ """When ANTHROPIC_API_KEY is not set, show setup instructions."""+ binary = tmp_path / "test_binary"+ binary.write_bytes(b"\x00" * 16)++ runner = CliRunner()+ with patch.dict("os.environ", {}, clear=True), \+ patch("kong.__main__.check_api_key", return_value=False):+ result = runner.invoke(cli, ["analyze", str(binary)])++ assert result.exit_code != 0+ assert "anthropic_api_key" in result.output.lower()+ assert "kong setup" in result.output.lower()++def test_analyze_no_ghidra_installed(tmp_path):"""When Ghidra is not installed, show install instructions."""binary = tmp_path / "test_binary"binary.write_bytes(b"\x00" * 16)runner = CliRunner()- with patch("kong.config.find_ghidra_install", return_value=None):+ with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "sk-ant-test"}), \+ patch("kong.config.find_ghidra_install", return_value=None):result = runner.invoke(cli, ["analyze", str(binary)])assert result.exit_code != 0@@ -51,7 +68,61 @@ def test_analyze_no_ghidra_installed(tmp_path):assert "brew install ghidra" in result.output+def test_bare_kong_shows_banner():+ """Running 'kong' with no subcommand shows banner and usage."""+ runner = CliRunner()+ result = runner.invoke(cli, [])++ assert result.exit_code == 0+ assert "KONG" in result.output or "kong" in result.output.lower()+ assert "kong analyze" in result.output.lower()+ assert "kong setup" in result.output.lower()+++def test_setup_with_key_set():+ """Setup command detects existing API key."""+ runner = CliRunner()+ with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "sk-ant-test1234"}):+ result = runner.invoke(cli, ["setup"])++ assert result.exit_code == 0+ assert "sk-ant-" in result.output+ assert "..." in result.output++def test_info_requires_binary_arg():runner = CliRunner()result = runner.invoke(cli, ["info"])assert result.exit_code != 0+++def test_eval_help():+ runner = CliRunner()+ result = runner.invoke(cli, ["eval", "--help"])+ assert result.exit_code == 0+ assert "score" in result.output.lower() or "analysis" in result.output.lower()+++def test_eval_with_test_data(tmp_path):+ """Run eval against a simple test case."""+ import json++ analysis = {+ "binary": {"name": "test"},+ "stats": {"llm_calls": 5, "duration_seconds": 10.0, "cost_usd": 0.05},+ "functions": [+ {"name": "hash_string", "signature": "uint hash_string(byte *str)", "confidence": 95, "address": "0x1000"},+ ],+ }+ analysis_path = tmp_path / "analysis.json"+ analysis_path.write_text(json.dumps(analysis))++ source = "unsigned int hash_string(const char *s) {\n return 0;\n}\n"+ source_path = tmp_path / "test.c"+ source_path.write_text(source)++ runner = CliRunner()+ result = runner.invoke(cli, ["eval", str(analysis_path), str(source_path)])+ assert result.exit_code == 0+ assert "hash_string" in result.output+ assert "Symbol Accuracy" in result.output