The world's first agentic reverse engineer.
feat: add eval harness with symbol and type accuracy metrics
Amruth66ac6457d321
diff --git a/kong/__main__.py b/kong/__main__.pyindex 966b25e..015a81d 100644--- a/kong/__main__.py+++ b/kong/__main__.py@@ -293,8 +293,8 @@ def eval_cmd(analysis_json: str, source_file: str) -> None:for pf in scorecard.per_function:pred = pf["predicted_name"]truth = pf["truth_name"]- sym = pf["symbol_score"]- typ = pf["type_score"]+ sym = pf["symbol_accuracy"]+ typ = pf["type_accuracy"]match_indicator = "[green]OK[/green]" if sym >= 0.8 else "[yellow]~~[/yellow]" if sym > 0 else "[red]NO[/red]"console.print(f" {match_indicator} {pred:30s} -> {truth:20s} sym={sym:.2f} type={typ:.2f}")diff --git a/kong/evals/__init__.py b/kong/evals/__init__.pyindex e69de29..197e460 100644--- a/kong/evals/__init__.py+++ b/kong/evals/__init__.py@@ -0,0 +1 @@+"""Evaluation framework for scoring Kong analysis output against ground truth."""diff --git a/kong/evals/harness.py b/kong/evals/harness.pyindex ada43c2..e63c5dd 100644--- a/kong/evals/harness.py+++ b/kong/evals/harness.py@@ -1,71 +1,137 @@-"""Evaluation harness for scoring Kong analysis against ground truth."""+"""Eval harness: compare Kong analysis output against ground truth source code."""from __future__ import annotations+import json+import refrom dataclasses import dataclass, fieldfrom pathlib import Path+from kong.evals.metrics import symbol_accuracy, type_accuracy+++@dataclass+class GroundTruth:+ functions: list[dict[str, str]]+@dataclassclass Scorecard:- binary: str = ""- total_functions: int = 0- functions_analyzed: int = 0- symbol_accuracy: float = 0.0- type_accuracy: float = 0.0- per_function: list[dict[str, str | float]] = field(default_factory=list)- llm_calls: int = 0- duration_seconds: float = 0.0- cost_usd: float = 0.0+ binary: str+ total_functions: int+ functions_analyzed: int+ symbol_accuracy: float+ type_accuracy: float+ per_function: list[dict[str, str | float]]+ llm_calls: int+ duration_seconds: float+ cost_usd: float-def score(analysis_path: Path, source_path: Path) -> Scorecard:- """Score a Kong analysis.json against ground truth C source code."""- import json- import re+_FUNC_PATTERN = re.compile(+ r"^(?:static\s+)?"+ r"([\w\s\*]+?)"+ r"\s+(\w+)\s*"+ r"\(([^)]*)\)\s*\{",+ re.MULTILINE,+)- analysis = json.loads(analysis_path.read_text())- source_text = source_path.read_text()- source_functions = re.findall(- r"(?:[\w\s\*]+)\s+(\w+)\s*\([^)]*\)",- source_text,- )- source_func_set = set(source_functions)+def extract_ground_truth(source_path: Path) -> GroundTruth:+ """Parse a C source file to extract function names and signatures."""+ text = source_path.read_text()+ functions: list[dict[str, str]] = []- binary_name = analysis.get("binary", {}).get("name", "")- stats = analysis.get("stats", {})- analyzed_functions = analysis.get("functions", [])+ for match in _FUNC_PATTERN.finditer(text):+ return_type = match.group(1).strip()+ name = match.group(2).strip()+ params = match.group(3).strip()+ signature = f"{return_type} {name}({params})"+ functions.append({"name": name, "signature": signature})- per_function: list[dict[str, str | float]] = []- symbol_matches = 0- type_matches = 0+ return GroundTruth(functions=functions)+++def load_analysis(analysis_path: Path) -> list[dict[str, str]]:+ """Load Kong's analysis.json and return the list of function entries."""+ data = json.loads(analysis_path.read_text())+ return data["functions"]+++def match_functions(+ predicted: list[dict[str, str]],+ truth: list[dict[str, str]],+) -> list[tuple[dict[str, str], dict[str, str] | None, float]]:+ """Match predicted functions to truth entries by best symbol_accuracy (greedy, no reuse)."""+ used_truth_indices: set[int] = set()+ results: list[tuple[dict[str, str], dict[str, str] | None, float]] = []- for func in analyzed_functions:- predicted_name = func.get("name", "")- best_match = predicted_name if predicted_name in source_func_set else ""- sym_score = 1.0 if best_match else 0.0- typ_score = 1.0 if best_match else 0.0+ scored_pairs: list[tuple[int, int, float]] = []+ for pi, pred in enumerate(predicted):+ for ti, tr in enumerate(truth):+ sc = symbol_accuracy(pred["name"], tr["name"])+ scored_pairs.append((pi, ti, sc))- if sym_score > 0:- symbol_matches += 1- type_matches += 1+ scored_pairs.sort(key=lambda x: x[2], reverse=True)+ matched_pred: dict[int, tuple[int, float]] = {}+ for pi, ti, sc in scored_pairs:+ if pi in matched_pred or ti in used_truth_indices:+ continue+ if sc > 0.0:+ matched_pred[pi] = (ti, sc)+ used_truth_indices.add(ti)++ for pi, pred in enumerate(predicted):+ if pi in matched_pred:+ ti, sc = matched_pred[pi]+ results.append((pred, truth[ti], sc))+ else:+ results.append((pred, None, 0.0))++ return results+++def score(analysis_path: Path, source_path: Path) -> Scorecard:+ """Load analysis and source, match functions, compute scores."""+ raw = json.loads(analysis_path.read_text())+ predicted = load_analysis(analysis_path)+ ground_truth = extract_ground_truth(source_path)++ matches = match_functions(predicted, ground_truth.functions)++ per_function: list[dict[str, str | float]] = []+ sym_scores: list[float] = []+ type_scores: list[float] = []++ for pred, tr, _ in matches:+ if tr is not None:+ sym_sc = symbol_accuracy(pred["name"], tr["name"])+ type_sc = type_accuracy(pred["signature"], tr["signature"])+ else:+ sym_sc = 0.0+ type_sc = 0.0++ sym_scores.append(sym_sc)+ type_scores.append(type_sc)per_function.append({- "predicted_name": predicted_name,- "truth_name": best_match or "???",- "symbol_score": sym_score,- "type_score": typ_score,+ "predicted_name": pred["name"],+ "truth_name": tr["name"] if tr else "",+ "symbol_accuracy": sym_sc,+ "type_accuracy": type_sc,})- total = max(len(source_func_set), 1)+ avg_sym = sum(sym_scores) / len(sym_scores) if sym_scores else 0.0+ avg_type = sum(type_scores) / len(type_scores) if type_scores else 0.0++ stats = raw.get("stats", {})return Scorecard(- binary=binary_name,- total_functions=len(source_func_set),- functions_analyzed=len(analyzed_functions),- symbol_accuracy=symbol_matches / total,- type_accuracy=type_matches / total,+ binary=raw.get("binary", {}).get("name", ""),+ total_functions=stats.get("total_functions", 0),+ functions_analyzed=stats.get("analyzed", 0),+ symbol_accuracy=avg_sym,+ type_accuracy=avg_type,per_function=per_function,llm_calls=stats.get("llm_calls", 0),duration_seconds=stats.get("duration_seconds", 0.0),diff --git a/kong/evals/metrics.py b/kong/evals/metrics.pynew file mode 100644index 0000000..85df10c--- /dev/null+++ b/kong/evals/metrics.py@@ -0,0 +1,144 @@+"""Scoring functions for evaluating Kong analysis output against ground truth."""++from __future__ import annotations++import re+++def _normalize_name(name: str) -> list[str]:+ """Split camelCase/snake_case/PascalCase into lowercase word tokens."""+ parts = name.replace("_", " ").strip().split()+ tokens: list[str] = []+ for part in parts:+ splits = re.sub(r"([a-z])([A-Z])", r"\1 \2", part).split()+ tokens.extend(s.lower() for s in splits if s)+ return tokens+++def symbol_accuracy(predicted: str, truth: str) -> float:+ """Score predicted symbol name against ground truth.++ Returns 1.0 for exact match, 0.8 for same words different order,+ Jaccard-based (capped at 0.6) for partial overlap, 0.0 for no overlap.+ """+ if predicted == truth:+ return 1.0++ pred_tokens = _normalize_name(predicted)+ truth_tokens = _normalize_name(truth)++ if not pred_tokens or not truth_tokens:+ return 0.0++ pred_set = set(pred_tokens)+ truth_set = set(truth_tokens)++ if pred_set == truth_set:+ return 0.8++ intersection = pred_set & truth_set+ if not intersection:+ return 0.0++ union = pred_set | truth_set+ jaccard = len(intersection) / len(union)+ return min(jaccard, 0.6)+++def _strip_qualifiers(type_str: str) -> str:+ """Remove const/unsigned/signed qualifiers from a type string."""+ stripped = type_str+ for qualifier in ("const", "unsigned", "signed"):+ stripped = stripped.replace(qualifier, "")+ return " ".join(stripped.split())+++def _parse_signature(sig: str) -> tuple[str, list[str]]:+ """Extract (return_type, [param_types]) from a C signature string."""+ paren_match = re.match(r"^(.*?)\s*\w+\s*\((.*)\)\s*$", sig)+ if not paren_match:+ return ("", [])++ return_type = paren_match.group(1).strip()+ params_str = paren_match.group(2).strip()++ if not params_str or params_str == "void":+ return (return_type, [])++ param_types: list[str] = []+ for param in params_str.split(","):+ param = param.strip()+ pointer_match = re.match(r"^(.*\*)\s*\w*$", param)+ if pointer_match:+ param_types.append(pointer_match.group(1).strip())+ else:+ parts = param.rsplit(None, 1)+ if len(parts) >= 2:+ param_types.append(parts[0].strip())+ else:+ param_types.append(param.strip())++ return (return_type, param_types)+++def type_accuracy(predicted_sig: str, truth_sig: str) -> float:+ """Score predicted type signature against ground truth.++ Exact match: 1.0. Otherwise scored by components:+ return type match (0.4), param count match (0.3),+ individual param type matches (up to 0.3).+ """+ if predicted_sig == truth_sig:+ return 1.0++ pred_ret, pred_params = _parse_signature(predicted_sig)+ truth_ret, truth_params = _parse_signature(truth_sig)++ score = 0.0++ if _strip_qualifiers(pred_ret) == _strip_qualifiers(truth_ret):+ score += 0.4++ if len(pred_params) == len(truth_params):+ score += 0.3++ if pred_params and truth_params:+ match_count = sum(+ 1 for p, t in zip(pred_params, truth_params)+ if _strip_qualifiers(p) == _strip_qualifiers(t)+ )+ max_params = max(len(pred_params), len(truth_params))+ score += 0.3 * (match_count / max_params)+ elif not pred_params and not truth_params:+ score += 0.3++ return score+++def overall_score(+ predicted: list[dict[str, str]],+ truth: list[dict[str, str]],+) -> dict[str, float]:+ """Compute aggregate symbol and type accuracy across matched functions."""+ from kong.evals.harness import match_functions++ matches = match_functions(predicted, truth)++ if not matches:+ return {"symbol_accuracy": 0.0, "type_accuracy": 0.0}++ sym_scores: list[float] = []+ type_scores: list[float] = []++ for pred, tr, _ in matches:+ if tr is not None:+ sym_scores.append(symbol_accuracy(pred["name"], tr["name"]))+ type_scores.append(type_accuracy(pred["signature"], tr["signature"]))+ else:+ sym_scores.append(0.0)+ type_scores.append(0.0)++ return {+ "symbol_accuracy": sum(sym_scores) / len(sym_scores),+ "type_accuracy": sum(type_scores) / len(type_scores),+ }diff --git a/tests/test_evals.py b/tests/test_evals.pynew file mode 100644index 0000000..e1e9bc9--- /dev/null+++ b/tests/test_evals.py@@ -0,0 +1,274 @@+"""Tests for the eval harness and scoring metrics."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from kong.evals.metrics import (+ _normalize_name,+ _parse_signature,+ overall_score,+ symbol_accuracy,+ type_accuracy,+)+from kong.evals.harness import (+ GroundTruth,+ Scorecard,+ extract_ground_truth,+ load_analysis,+ match_functions,+ score,+)+++class TestNormalizeName:+ def test_snake_case(self) -> None:+ assert _normalize_name("init_key") == ["init", "key"]++ def test_camel_case(self) -> None:+ assert _normalize_name("initKey") == ["init", "key"]++ def test_upper_camel_case(self) -> None:+ assert _normalize_name("InitKey") == ["init", "key"]++ def test_mixed(self) -> None:+ assert _normalize_name("my_initKey") == ["my", "init", "key"]++ def test_all_caps(self) -> None:+ tokens = _normalize_name("SSL")+ assert tokens == ["ssl"]+++class TestSymbolAccuracy:+ def test_exact_match(self) -> None:+ assert symbol_accuracy("init_key", "init_key") == 1.0++ def test_word_reorder(self) -> None:+ result = symbol_accuracy("init_key", "key_init")+ assert 0.7 <= result <= 0.9++ def test_partial_overlap(self) -> None:+ result = symbol_accuracy("init_key_value", "init_buffer_size")+ assert 0.1 <= result <= 0.6++ def test_no_match(self) -> None:+ assert symbol_accuracy("foo", "bar") == 0.0++ def test_case_insensitive(self) -> None:+ result = symbol_accuracy("InitKey", "init_key")+ assert result >= 0.8+++class TestParseSignature:+ def test_simple_signature(self) -> None:+ ret, params = _parse_signature("int add(int a, int b)")+ assert ret == "int"+ assert params == ["int", "int"]++ def test_void_return_void_params(self) -> None:+ ret, params = _parse_signature("void cleanup(void)")+ assert ret == "void"+ assert params == []++ def test_pointer_params(self) -> None:+ ret, params = _parse_signature("char * strdup(const char * src)")+ assert ret == "char *"+ assert params == ["const char *"]++ def test_no_params(self) -> None:+ ret, params = _parse_signature("int getpid()")+ assert ret == "int"+ assert params == []+++class TestTypeAccuracy:+ def test_exact_match(self) -> None:+ result = type_accuracy("int add(int a, int b)", "int add(int a, int b)")+ assert result == 1.0++ def test_return_type_match_params_differ(self) -> None:+ result = type_accuracy(+ "int foo(float x, float y)",+ "int bar(int a, int b)",+ )+ assert 0.4 <= result <= 0.8++ def test_param_count_and_types_match(self) -> None:+ result = type_accuracy(+ "int process(int x, char * buf)",+ "int handle(int count, char * data)",+ )+ assert result >= 0.8++ def test_void_params(self) -> None:+ result = type_accuracy("void init(void)", "void init(void)")+ assert result == 1.0++ def test_ignores_qualifiers(self) -> None:+ result = type_accuracy(+ "unsigned int get_count(const int x)",+ "int get_count(int x)",+ )+ assert result >= 0.8+++class TestExtractGroundTruth:+ def test_extracts_from_c_source(self, tmp_path: Path) -> None:+ c_source = tmp_path / "test.c"+ c_source.write_text(+ '#include <stdio.h>\n'+ '\n'+ 'int add(int a, int b) {\n'+ ' return a + b;\n'+ '}\n'+ '\n'+ 'void greet(const char *name) {\n'+ ' printf("Hello, %s\\n", name);\n'+ '}\n'+ )+ truth = extract_ground_truth(c_source)+ assert isinstance(truth, GroundTruth)+ assert len(truth.functions) == 2+ names = [f["name"] for f in truth.functions]+ assert "add" in names+ assert "greet" in names++ def test_extracts_static_functions(self, tmp_path: Path) -> None:+ c_source = tmp_path / "static.c"+ c_source.write_text(+ 'static int helper(int x) {\n'+ ' return x * 2;\n'+ '}\n'+ )+ truth = extract_ground_truth(c_source)+ assert len(truth.functions) == 1+ assert truth.functions[0]["name"] == "helper"++ def test_extracts_signature(self, tmp_path: Path) -> None:+ c_source = tmp_path / "sig.c"+ c_source.write_text(+ 'char * strdup(const char * src) {\n'+ ' return NULL;\n'+ '}\n'+ )+ truth = extract_ground_truth(c_source)+ assert len(truth.functions) == 1+ assert "char" in truth.functions[0]["signature"]+ assert "strdup" in truth.functions[0]["signature"]+++class TestMatchFunctions:+ def test_matches_by_best_score(self) -> None:+ predicted = [+ {"name": "init_key", "signature": "void init_key(void)"},+ {"name": "process_data", "signature": "int process_data(int x)"},+ ]+ truth = [+ {"name": "init_key", "signature": "void init_key(void)"},+ {"name": "process_data", "signature": "int process_data(int x)"},+ ]+ matches = match_functions(predicted, truth)+ assert len(matches) == 2+ for pred, tr, sc in matches:+ assert sc == 1.0++ def test_no_reuse_of_truth_entries(self) -> None:+ predicted = [+ {"name": "foo", "signature": "void foo(void)"},+ {"name": "foo", "signature": "void foo(void)"},+ ]+ truth = [+ {"name": "foo", "signature": "void foo(void)"},+ ]+ matches = match_functions(predicted, truth)+ assert len(matches) == 2+ matched_truths = [tr for _, tr, _ in matches if tr is not None]+ assert len(matched_truths) == 1+++class TestOverallScore:+ def test_returns_both_metrics(self) -> None:+ predicted = [+ {"name": "add", "signature": "int add(int a, int b)"},+ ]+ truth = [+ {"name": "add", "signature": "int add(int a, int b)"},+ ]+ result = overall_score(predicted, truth)+ assert "symbol_accuracy" in result+ assert "type_accuracy" in result+ assert result["symbol_accuracy"] == 1.0+ assert result["type_accuracy"] == 1.0+++class TestLoadAnalysis:+ def test_loads_analysis_json(self, tmp_path: Path) -> None:+ analysis = {+ "binary": {"name": "test"},+ "stats": {"llm_calls": 5, "duration_seconds": 10.0, "cost_usd": 0.5},+ "functions": [+ {+ "address": "0x00001000",+ "original_name": "FUN_00001000",+ "name": "add",+ "signature": "int add(int a, int b)",+ "confidence": 90,+ },+ ],+ }+ path = tmp_path / "analysis.json"+ path.write_text(json.dumps(analysis))+ entries = load_analysis(path)+ assert len(entries) == 1+ assert entries[0]["name"] == "add"+++class TestScorecard:+ def test_scorecard_structure(self, tmp_path: Path) -> None:+ analysis = {+ "binary": {"name": "test_binary"},+ "stats": {+ "total_functions": 2,+ "analyzed": 1,+ "llm_calls": 3,+ "duration_seconds": 5.0,+ "cost_usd": 0.25,+ },+ "functions": [+ {+ "address": "0x00001000",+ "original_name": "FUN_00001000",+ "name": "add",+ "signature": "int add(int a, int b)",+ "confidence": 95,+ },+ ],+ }+ analysis_path = tmp_path / "analysis.json"+ analysis_path.write_text(json.dumps(analysis))++ c_source = tmp_path / "source.c"+ c_source.write_text(+ 'int add(int a, int b) {\n'+ ' return a + b;\n'+ '}\n'+ '\n'+ 'int sub(int a, int b) {\n'+ ' return a - b;\n'+ '}\n'+ )++ result = score(analysis_path, c_source)+ assert isinstance(result, Scorecard)+ assert result.binary == "test_binary"+ assert result.total_functions == 2+ assert result.functions_analyzed == 1+ assert result.symbol_accuracy == 1.0+ assert result.type_accuracy == 1.0+ assert result.llm_calls == 3+ assert result.duration_seconds == 5.0+ assert result.cost_usd == 0.25+ assert len(result.per_function) == 1