Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

feat: add eval harness with symbol and type accuracy metrics

Amruth66ac6457d321
diff --git a/kong/__main__.py b/kong/__main__.py
index 966b25e..015a81d 100644
--- a/kong/__main__.py
+++ b/kong/__main__.py
@@ -293,8 +293,8 @@ def eval_cmd(analysis_json: str, source_file: str) -> None:
for pf in scorecard.per_function:
pred = pf["predicted_name"]
truth = pf["truth_name"]
- sym = pf["symbol_score"]
- typ = pf["type_score"]
+ sym = pf["symbol_accuracy"]
+ typ = pf["type_accuracy"]
match_indicator = "[green]OK[/green]" if sym >= 0.8 else "[yellow]~~[/yellow]" if sym > 0 else "[red]NO[/red]"
console.print(f" {match_indicator} {pred:30s} -> {truth:20s} sym={sym:.2f} type={typ:.2f}")
diff --git a/kong/evals/__init__.py b/kong/evals/__init__.py
index e69de29..197e460 100644
--- a/kong/evals/__init__.py
+++ b/kong/evals/__init__.py
@@ -0,0 +1 @@
+"""Evaluation framework for scoring Kong analysis output against ground truth."""
diff --git a/kong/evals/harness.py b/kong/evals/harness.py
index ada43c2..e63c5dd 100644
--- a/kong/evals/harness.py
+++ b/kong/evals/harness.py
@@ -1,71 +1,137 @@
-"""Evaluation harness for scoring Kong analysis against ground truth."""
+"""Eval harness: compare Kong analysis output against ground truth source code."""
from __future__ import annotations
+import json
+import re
from dataclasses import dataclass, field
from pathlib import Path
+from kong.evals.metrics import symbol_accuracy, type_accuracy
+
+
+@dataclass
+class GroundTruth:
+ functions: list[dict[str, str]]
+
@dataclass
class Scorecard:
- binary: str = ""
- total_functions: int = 0
- functions_analyzed: int = 0
- symbol_accuracy: float = 0.0
- type_accuracy: float = 0.0
- per_function: list[dict[str, str | float]] = field(default_factory=list)
- llm_calls: int = 0
- duration_seconds: float = 0.0
- cost_usd: float = 0.0
+ binary: str
+ total_functions: int
+ functions_analyzed: int
+ symbol_accuracy: float
+ type_accuracy: float
+ per_function: list[dict[str, str | float]]
+ llm_calls: int
+ duration_seconds: float
+ cost_usd: float
-def score(analysis_path: Path, source_path: Path) -> Scorecard:
- """Score a Kong analysis.json against ground truth C source code."""
- import json
- import re
+_FUNC_PATTERN = re.compile(
+ r"^(?:static\s+)?"
+ r"([\w\s\*]+?)"
+ r"\s+(\w+)\s*"
+ r"\(([^)]*)\)\s*\{",
+ re.MULTILINE,
+)
- analysis = json.loads(analysis_path.read_text())
- source_text = source_path.read_text()
- source_functions = re.findall(
- r"(?:[\w\s\*]+)\s+(\w+)\s*\([^)]*\)",
- source_text,
- )
- source_func_set = set(source_functions)
+def extract_ground_truth(source_path: Path) -> GroundTruth:
+ """Parse a C source file to extract function names and signatures."""
+ text = source_path.read_text()
+ functions: list[dict[str, str]] = []
- binary_name = analysis.get("binary", {}).get("name", "")
- stats = analysis.get("stats", {})
- analyzed_functions = analysis.get("functions", [])
+ for match in _FUNC_PATTERN.finditer(text):
+ return_type = match.group(1).strip()
+ name = match.group(2).strip()
+ params = match.group(3).strip()
+ signature = f"{return_type} {name}({params})"
+ functions.append({"name": name, "signature": signature})
- per_function: list[dict[str, str | float]] = []
- symbol_matches = 0
- type_matches = 0
+ return GroundTruth(functions=functions)
+
+
+def load_analysis(analysis_path: Path) -> list[dict[str, str]]:
+ """Load Kong's analysis.json and return the list of function entries."""
+ data = json.loads(analysis_path.read_text())
+ return data["functions"]
+
+
+def match_functions(
+ predicted: list[dict[str, str]],
+ truth: list[dict[str, str]],
+) -> list[tuple[dict[str, str], dict[str, str] | None, float]]:
+ """Match predicted functions to truth entries by best symbol_accuracy (greedy, no reuse)."""
+ used_truth_indices: set[int] = set()
+ results: list[tuple[dict[str, str], dict[str, str] | None, float]] = []
- for func in analyzed_functions:
- predicted_name = func.get("name", "")
- best_match = predicted_name if predicted_name in source_func_set else ""
- sym_score = 1.0 if best_match else 0.0
- typ_score = 1.0 if best_match else 0.0
+ scored_pairs: list[tuple[int, int, float]] = []
+ for pi, pred in enumerate(predicted):
+ for ti, tr in enumerate(truth):
+ sc = symbol_accuracy(pred["name"], tr["name"])
+ scored_pairs.append((pi, ti, sc))
- if sym_score > 0:
- symbol_matches += 1
- type_matches += 1
+ scored_pairs.sort(key=lambda x: x[2], reverse=True)
+ matched_pred: dict[int, tuple[int, float]] = {}
+ for pi, ti, sc in scored_pairs:
+ if pi in matched_pred or ti in used_truth_indices:
+ continue
+ if sc > 0.0:
+ matched_pred[pi] = (ti, sc)
+ used_truth_indices.add(ti)
+
+ for pi, pred in enumerate(predicted):
+ if pi in matched_pred:
+ ti, sc = matched_pred[pi]
+ results.append((pred, truth[ti], sc))
+ else:
+ results.append((pred, None, 0.0))
+
+ return results
+
+
+def score(analysis_path: Path, source_path: Path) -> Scorecard:
+ """Load analysis and source, match functions, compute scores."""
+ raw = json.loads(analysis_path.read_text())
+ predicted = load_analysis(analysis_path)
+ ground_truth = extract_ground_truth(source_path)
+
+ matches = match_functions(predicted, ground_truth.functions)
+
+ per_function: list[dict[str, str | float]] = []
+ sym_scores: list[float] = []
+ type_scores: list[float] = []
+
+ for pred, tr, _ in matches:
+ if tr is not None:
+ sym_sc = symbol_accuracy(pred["name"], tr["name"])
+ type_sc = type_accuracy(pred["signature"], tr["signature"])
+ else:
+ sym_sc = 0.0
+ type_sc = 0.0
+
+ sym_scores.append(sym_sc)
+ type_scores.append(type_sc)
per_function.append({
- "predicted_name": predicted_name,
- "truth_name": best_match or "???",
- "symbol_score": sym_score,
- "type_score": typ_score,
+ "predicted_name": pred["name"],
+ "truth_name": tr["name"] if tr else "",
+ "symbol_accuracy": sym_sc,
+ "type_accuracy": type_sc,
})
- total = max(len(source_func_set), 1)
+ avg_sym = sum(sym_scores) / len(sym_scores) if sym_scores else 0.0
+ avg_type = sum(type_scores) / len(type_scores) if type_scores else 0.0
+
+ stats = raw.get("stats", {})
return Scorecard(
- binary=binary_name,
- total_functions=len(source_func_set),
- functions_analyzed=len(analyzed_functions),
- symbol_accuracy=symbol_matches / total,
- type_accuracy=type_matches / total,
+ binary=raw.get("binary", {}).get("name", ""),
+ total_functions=stats.get("total_functions", 0),
+ functions_analyzed=stats.get("analyzed", 0),
+ symbol_accuracy=avg_sym,
+ type_accuracy=avg_type,
per_function=per_function,
llm_calls=stats.get("llm_calls", 0),
duration_seconds=stats.get("duration_seconds", 0.0),
diff --git a/kong/evals/metrics.py b/kong/evals/metrics.py
new file mode 100644
index 0000000..85df10c
--- /dev/null
+++ b/kong/evals/metrics.py
@@ -0,0 +1,144 @@
+"""Scoring functions for evaluating Kong analysis output against ground truth."""
+
+from __future__ import annotations
+
+import re
+
+
+def _normalize_name(name: str) -> list[str]:
+ """Split camelCase/snake_case/PascalCase into lowercase word tokens."""
+ parts = name.replace("_", " ").strip().split()
+ tokens: list[str] = []
+ for part in parts:
+ splits = re.sub(r"([a-z])([A-Z])", r"\1 \2", part).split()
+ tokens.extend(s.lower() for s in splits if s)
+ return tokens
+
+
+def symbol_accuracy(predicted: str, truth: str) -> float:
+ """Score predicted symbol name against ground truth.
+
+ Returns 1.0 for exact match, 0.8 for same words different order,
+ Jaccard-based (capped at 0.6) for partial overlap, 0.0 for no overlap.
+ """
+ if predicted == truth:
+ return 1.0
+
+ pred_tokens = _normalize_name(predicted)
+ truth_tokens = _normalize_name(truth)
+
+ if not pred_tokens or not truth_tokens:
+ return 0.0
+
+ pred_set = set(pred_tokens)
+ truth_set = set(truth_tokens)
+
+ if pred_set == truth_set:
+ return 0.8
+
+ intersection = pred_set & truth_set
+ if not intersection:
+ return 0.0
+
+ union = pred_set | truth_set
+ jaccard = len(intersection) / len(union)
+ return min(jaccard, 0.6)
+
+
+def _strip_qualifiers(type_str: str) -> str:
+ """Remove const/unsigned/signed qualifiers from a type string."""
+ stripped = type_str
+ for qualifier in ("const", "unsigned", "signed"):
+ stripped = stripped.replace(qualifier, "")
+ return " ".join(stripped.split())
+
+
+def _parse_signature(sig: str) -> tuple[str, list[str]]:
+ """Extract (return_type, [param_types]) from a C signature string."""
+ paren_match = re.match(r"^(.*?)\s*\w+\s*\((.*)\)\s*$", sig)
+ if not paren_match:
+ return ("", [])
+
+ return_type = paren_match.group(1).strip()
+ params_str = paren_match.group(2).strip()
+
+ if not params_str or params_str == "void":
+ return (return_type, [])
+
+ param_types: list[str] = []
+ for param in params_str.split(","):
+ param = param.strip()
+ pointer_match = re.match(r"^(.*\*)\s*\w*$", param)
+ if pointer_match:
+ param_types.append(pointer_match.group(1).strip())
+ else:
+ parts = param.rsplit(None, 1)
+ if len(parts) >= 2:
+ param_types.append(parts[0].strip())
+ else:
+ param_types.append(param.strip())
+
+ return (return_type, param_types)
+
+
+def type_accuracy(predicted_sig: str, truth_sig: str) -> float:
+ """Score predicted type signature against ground truth.
+
+ Exact match: 1.0. Otherwise scored by components:
+ return type match (0.4), param count match (0.3),
+ individual param type matches (up to 0.3).
+ """
+ if predicted_sig == truth_sig:
+ return 1.0
+
+ pred_ret, pred_params = _parse_signature(predicted_sig)
+ truth_ret, truth_params = _parse_signature(truth_sig)
+
+ score = 0.0
+
+ if _strip_qualifiers(pred_ret) == _strip_qualifiers(truth_ret):
+ score += 0.4
+
+ if len(pred_params) == len(truth_params):
+ score += 0.3
+
+ if pred_params and truth_params:
+ match_count = sum(
+ 1 for p, t in zip(pred_params, truth_params)
+ if _strip_qualifiers(p) == _strip_qualifiers(t)
+ )
+ max_params = max(len(pred_params), len(truth_params))
+ score += 0.3 * (match_count / max_params)
+ elif not pred_params and not truth_params:
+ score += 0.3
+
+ return score
+
+
+def overall_score(
+ predicted: list[dict[str, str]],
+ truth: list[dict[str, str]],
+) -> dict[str, float]:
+ """Compute aggregate symbol and type accuracy across matched functions."""
+ from kong.evals.harness import match_functions
+
+ matches = match_functions(predicted, truth)
+
+ if not matches:
+ return {"symbol_accuracy": 0.0, "type_accuracy": 0.0}
+
+ sym_scores: list[float] = []
+ type_scores: list[float] = []
+
+ for pred, tr, _ in matches:
+ if tr is not None:
+ sym_scores.append(symbol_accuracy(pred["name"], tr["name"]))
+ type_scores.append(type_accuracy(pred["signature"], tr["signature"]))
+ else:
+ sym_scores.append(0.0)
+ type_scores.append(0.0)
+
+ return {
+ "symbol_accuracy": sum(sym_scores) / len(sym_scores),
+ "type_accuracy": sum(type_scores) / len(type_scores),
+ }
diff --git a/tests/test_evals.py b/tests/test_evals.py
new file mode 100644
index 0000000..e1e9bc9
--- /dev/null
+++ b/tests/test_evals.py
@@ -0,0 +1,274 @@
+"""Tests for the eval harness and scoring metrics."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from kong.evals.metrics import (
+ _normalize_name,
+ _parse_signature,
+ overall_score,
+ symbol_accuracy,
+ type_accuracy,
+)
+from kong.evals.harness import (
+ GroundTruth,
+ Scorecard,
+ extract_ground_truth,
+ load_analysis,
+ match_functions,
+ score,
+)
+
+
+class TestNormalizeName:
+ def test_snake_case(self) -> None:
+ assert _normalize_name("init_key") == ["init", "key"]
+
+ def test_camel_case(self) -> None:
+ assert _normalize_name("initKey") == ["init", "key"]
+
+ def test_upper_camel_case(self) -> None:
+ assert _normalize_name("InitKey") == ["init", "key"]
+
+ def test_mixed(self) -> None:
+ assert _normalize_name("my_initKey") == ["my", "init", "key"]
+
+ def test_all_caps(self) -> None:
+ tokens = _normalize_name("SSL")
+ assert tokens == ["ssl"]
+
+
+class TestSymbolAccuracy:
+ def test_exact_match(self) -> None:
+ assert symbol_accuracy("init_key", "init_key") == 1.0
+
+ def test_word_reorder(self) -> None:
+ result = symbol_accuracy("init_key", "key_init")
+ assert 0.7 <= result <= 0.9
+
+ def test_partial_overlap(self) -> None:
+ result = symbol_accuracy("init_key_value", "init_buffer_size")
+ assert 0.1 <= result <= 0.6
+
+ def test_no_match(self) -> None:
+ assert symbol_accuracy("foo", "bar") == 0.0
+
+ def test_case_insensitive(self) -> None:
+ result = symbol_accuracy("InitKey", "init_key")
+ assert result >= 0.8
+
+
+class TestParseSignature:
+ def test_simple_signature(self) -> None:
+ ret, params = _parse_signature("int add(int a, int b)")
+ assert ret == "int"
+ assert params == ["int", "int"]
+
+ def test_void_return_void_params(self) -> None:
+ ret, params = _parse_signature("void cleanup(void)")
+ assert ret == "void"
+ assert params == []
+
+ def test_pointer_params(self) -> None:
+ ret, params = _parse_signature("char * strdup(const char * src)")
+ assert ret == "char *"
+ assert params == ["const char *"]
+
+ def test_no_params(self) -> None:
+ ret, params = _parse_signature("int getpid()")
+ assert ret == "int"
+ assert params == []
+
+
+class TestTypeAccuracy:
+ def test_exact_match(self) -> None:
+ result = type_accuracy("int add(int a, int b)", "int add(int a, int b)")
+ assert result == 1.0
+
+ def test_return_type_match_params_differ(self) -> None:
+ result = type_accuracy(
+ "int foo(float x, float y)",
+ "int bar(int a, int b)",
+ )
+ assert 0.4 <= result <= 0.8
+
+ def test_param_count_and_types_match(self) -> None:
+ result = type_accuracy(
+ "int process(int x, char * buf)",
+ "int handle(int count, char * data)",
+ )
+ assert result >= 0.8
+
+ def test_void_params(self) -> None:
+ result = type_accuracy("void init(void)", "void init(void)")
+ assert result == 1.0
+
+ def test_ignores_qualifiers(self) -> None:
+ result = type_accuracy(
+ "unsigned int get_count(const int x)",
+ "int get_count(int x)",
+ )
+ assert result >= 0.8
+
+
+class TestExtractGroundTruth:
+ def test_extracts_from_c_source(self, tmp_path: Path) -> None:
+ c_source = tmp_path / "test.c"
+ c_source.write_text(
+ '#include <stdio.h>\n'
+ '\n'
+ 'int add(int a, int b) {\n'
+ ' return a + b;\n'
+ '}\n'
+ '\n'
+ 'void greet(const char *name) {\n'
+ ' printf("Hello, %s\\n", name);\n'
+ '}\n'
+ )
+ truth = extract_ground_truth(c_source)
+ assert isinstance(truth, GroundTruth)
+ assert len(truth.functions) == 2
+ names = [f["name"] for f in truth.functions]
+ assert "add" in names
+ assert "greet" in names
+
+ def test_extracts_static_functions(self, tmp_path: Path) -> None:
+ c_source = tmp_path / "static.c"
+ c_source.write_text(
+ 'static int helper(int x) {\n'
+ ' return x * 2;\n'
+ '}\n'
+ )
+ truth = extract_ground_truth(c_source)
+ assert len(truth.functions) == 1
+ assert truth.functions[0]["name"] == "helper"
+
+ def test_extracts_signature(self, tmp_path: Path) -> None:
+ c_source = tmp_path / "sig.c"
+ c_source.write_text(
+ 'char * strdup(const char * src) {\n'
+ ' return NULL;\n'
+ '}\n'
+ )
+ truth = extract_ground_truth(c_source)
+ assert len(truth.functions) == 1
+ assert "char" in truth.functions[0]["signature"]
+ assert "strdup" in truth.functions[0]["signature"]
+
+
+class TestMatchFunctions:
+ def test_matches_by_best_score(self) -> None:
+ predicted = [
+ {"name": "init_key", "signature": "void init_key(void)"},
+ {"name": "process_data", "signature": "int process_data(int x)"},
+ ]
+ truth = [
+ {"name": "init_key", "signature": "void init_key(void)"},
+ {"name": "process_data", "signature": "int process_data(int x)"},
+ ]
+ matches = match_functions(predicted, truth)
+ assert len(matches) == 2
+ for pred, tr, sc in matches:
+ assert sc == 1.0
+
+ def test_no_reuse_of_truth_entries(self) -> None:
+ predicted = [
+ {"name": "foo", "signature": "void foo(void)"},
+ {"name": "foo", "signature": "void foo(void)"},
+ ]
+ truth = [
+ {"name": "foo", "signature": "void foo(void)"},
+ ]
+ matches = match_functions(predicted, truth)
+ assert len(matches) == 2
+ matched_truths = [tr for _, tr, _ in matches if tr is not None]
+ assert len(matched_truths) == 1
+
+
+class TestOverallScore:
+ def test_returns_both_metrics(self) -> None:
+ predicted = [
+ {"name": "add", "signature": "int add(int a, int b)"},
+ ]
+ truth = [
+ {"name": "add", "signature": "int add(int a, int b)"},
+ ]
+ result = overall_score(predicted, truth)
+ assert "symbol_accuracy" in result
+ assert "type_accuracy" in result
+ assert result["symbol_accuracy"] == 1.0
+ assert result["type_accuracy"] == 1.0
+
+
+class TestLoadAnalysis:
+ def test_loads_analysis_json(self, tmp_path: Path) -> None:
+ analysis = {
+ "binary": {"name": "test"},
+ "stats": {"llm_calls": 5, "duration_seconds": 10.0, "cost_usd": 0.5},
+ "functions": [
+ {
+ "address": "0x00001000",
+ "original_name": "FUN_00001000",
+ "name": "add",
+ "signature": "int add(int a, int b)",
+ "confidence": 90,
+ },
+ ],
+ }
+ path = tmp_path / "analysis.json"
+ path.write_text(json.dumps(analysis))
+ entries = load_analysis(path)
+ assert len(entries) == 1
+ assert entries[0]["name"] == "add"
+
+
+class TestScorecard:
+ def test_scorecard_structure(self, tmp_path: Path) -> None:
+ analysis = {
+ "binary": {"name": "test_binary"},
+ "stats": {
+ "total_functions": 2,
+ "analyzed": 1,
+ "llm_calls": 3,
+ "duration_seconds": 5.0,
+ "cost_usd": 0.25,
+ },
+ "functions": [
+ {
+ "address": "0x00001000",
+ "original_name": "FUN_00001000",
+ "name": "add",
+ "signature": "int add(int a, int b)",
+ "confidence": 95,
+ },
+ ],
+ }
+ analysis_path = tmp_path / "analysis.json"
+ analysis_path.write_text(json.dumps(analysis))
+
+ c_source = tmp_path / "source.c"
+ c_source.write_text(
+ 'int add(int a, int b) {\n'
+ ' return a + b;\n'
+ '}\n'
+ '\n'
+ 'int sub(int a, int b) {\n'
+ ' return a - b;\n'
+ '}\n'
+ )
+
+ result = score(analysis_path, c_source)
+ assert isinstance(result, Scorecard)
+ assert result.binary == "test_binary"
+ assert result.total_functions == 2
+ assert result.functions_analyzed == 1
+ assert result.symbol_accuracy == 1.0
+ assert result.type_accuracy == 1.0
+ assert result.llm_calls == 3
+ assert result.duration_seconds == 5.0
+ assert result.cost_usd == 0.25
+ assert len(result.per_function) == 1