The world's first agentic reverse engineer.
fix: improve eval metrics with recall-weighted scoring and Ghidra type normalization
Amruth026770becf6a
diff --git a/kong/evals/harness.py b/kong/evals/harness.pyindex e63c5dd..1b21030 100644--- a/kong/evals/harness.py+++ b/kong/evals/harness.py@@ -30,8 +30,8 @@ class Scorecard:_FUNC_PATTERN = re.compile(r"^(?:static\s+)?"- r"([\w\s\*]+?)"- r"\s+(\w+)\s*"+ r"((?:\w+[\s*]+)+)"+ r"(\w+)\s*"r"\(([^)]*)\)\s*\{",re.MULTILINE,)diff --git a/kong/evals/metrics.py b/kong/evals/metrics.pyindex 85df10c..6ddc726 100644--- a/kong/evals/metrics.py+++ b/kong/evals/metrics.py@@ -5,6 +5,41 @@ from __future__ import annotationsimport re+_SYNONYM_GROUPS: list[set[str]] = [+ {"search", "find", "lookup"},+ {"node", "entry", "element"},+ {"buffer", "buf"},+ {"encode", "encrypt"},+ {"decode", "decrypt"},+ {"delete", "remove"},+ {"create", "make", "new", "alloc"},+ {"insert", "add", "push"},+ {"string", "str"},+ {"print", "display", "show"},+ {"count", "num", "size", "length"},+]++_SYNONYM_MAP: dict[str, set[str]] = {}+for _group in _SYNONYM_GROUPS:+ for _word in _group:+ _SYNONYM_MAP[_word] = _group++_NOISE_WORDS = {"or", "from", "the", "a", "an", "of", "to", "and", "is", "in"}++_TYPE_ALIASES: dict[str, str] = {+ "uint": "int",+ "byte": "char",+ "undefined4": "int",+ "undefined8": "long",+ "undefined2": "short",+ "undefined1": "char",+ "bool": "int",+ "dword": "int",+ "qword": "long",+ "word": "short",+}++def _normalize_name(name: str) -> list[str]:"""Split camelCase/snake_case/PascalCase into lowercase word tokens."""parts = name.replace("_", " ").strip().split()@@ -12,14 +47,40 @@ def _normalize_name(name: str) -> list[str]:for part in parts:splits = re.sub(r"([a-z])([A-Z])", r"\1 \2", part).split()tokens.extend(s.lower() for s in splits if s)- return tokens+ return [t for t in tokens if t not in _NOISE_WORDS]+++def _is_synonym(a: str, b: str) -> bool:+ if a == b:+ return True+ group = _SYNONYM_MAP.get(a)+ return group is not None and b in group+++def _synonym_recall(pred_set: set[str], truth_set: set[str]) -> float:+ """Fraction of truth tokens matched by a pred token (exact or synonym)."""+ if not truth_set:+ return 0.0+ matched = 0+ for t in truth_set:+ if any(_is_synonym(t, p) for p in pred_set):+ matched += 1+ return matched / len(truth_set)def symbol_accuracy(predicted: str, truth: str) -> float:"""Score predicted symbol name against ground truth.- Returns 1.0 for exact match, 0.8 for same words different order,- Jaccard-based (capped at 0.6) for partial overlap, 0.0 for no overlap.+ Recall-weighted: measures how well the prediction captures the ground+ truth concepts. Extra descriptive words in the prediction are not+ penalized heavily — they add context in RE output.++ Returns:+ 1.0 exact string match+ 0.9 same word set, different order+ 0.8 all truth words present (superset) — exact or synonym+ recall * 0.7 partial overlap, scaled by truth coverage+ 0.0 no overlap even after synonym expansion"""if predicted == truth:return 1.0@@ -34,23 +95,29 @@ def symbol_accuracy(predicted: str, truth: str) -> float:truth_set = set(truth_tokens)if pred_set == truth_set:- return 0.8+ return 0.9- intersection = pred_set & truth_set- if not intersection:- return 0.0+ recall = _synonym_recall(pred_set, truth_set)- union = pred_set | truth_set- jaccard = len(intersection) / len(union)- return min(jaccard, 0.6)+ if recall >= 1.0:+ return 0.8+ if recall > 0.0:+ return recall * 0.7+ return 0.0-def _strip_qualifiers(type_str: str) -> str:- """Remove const/unsigned/signed qualifiers from a type string."""+def _normalize_type(type_str: str) -> str:+ """Normalize a C type string: strip qualifiers and resolve Ghidra aliases."""stripped = type_strfor qualifier in ("const", "unsigned", "signed"):stripped = stripped.replace(qualifier, "")- return " ".join(stripped.split())+ stripped = " ".join(stripped.split())++ base = stripped.rstrip("*").rstrip()+ stars = stripped[len(base):]++ resolved = _TYPE_ALIASES.get(base, base)+ return (resolved + stars).strip()def _parse_signature(sig: str) -> tuple[str, list[str]]:@@ -87,6 +154,8 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:Exact match: 1.0. Otherwise scored by components:return type match (0.4), param count match (0.3),individual param type matches (up to 0.3).++ Types are normalized through Ghidra alias resolution before comparison."""if predicted_sig == truth_sig:return 1.0@@ -96,7 +165,7 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:score = 0.0- if _strip_qualifiers(pred_ret) == _strip_qualifiers(truth_ret):+ if _normalize_type(pred_ret) == _normalize_type(truth_ret):score += 0.4if len(pred_params) == len(truth_params):@@ -105,7 +174,7 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:if pred_params and truth_params:match_count = sum(1 for p, t in zip(pred_params, truth_params)- if _strip_qualifiers(p) == _strip_qualifiers(t)+ if _normalize_type(p) == _normalize_type(t))max_params = max(len(pred_params), len(truth_params))score += 0.3 * (match_count / max_params)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex e1e9bc9..bbcfebc 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -9,7 +9,9 @@ import pytestfrom kong.evals.metrics import (_normalize_name,+ _normalize_type,_parse_signature,+ _synonym_recall,overall_score,symbol_accuracy,type_accuracy,@@ -41,6 +43,26 @@ class TestNormalizeName:tokens = _normalize_name("SSL")assert tokens == ["ssl"]+ def test_noise_words_removed(self) -> None:+ assert _normalize_name("remove_from_list") == ["remove", "list"]+ assert _normalize_name("create_or_update") == ["create", "update"]+++class TestSynonymRecall:+ def test_exact_overlap(self) -> None:+ assert _synonym_recall({"insert", "node"}, {"insert", "node"}) == 1.0++ def test_synonym_match(self) -> None:+ assert _synonym_recall({"search", "list"}, {"find"}) == 1.0+ assert _synonym_recall({"entry"}, {"node"}) == 1.0++ def test_no_overlap(self) -> None:+ assert _synonym_recall({"foo"}, {"bar"}) == 0.0++ def test_partial(self) -> None:+ result = _synonym_recall({"xor"}, {"xor", "encode"})+ assert result == 0.5+class TestSymbolAccuracy:def test_exact_match(self) -> None:@@ -48,11 +70,23 @@ class TestSymbolAccuracy:def test_word_reorder(self) -> None:result = symbol_accuracy("init_key", "key_init")- assert 0.7 <= result <= 0.9+ assert result == 0.9++ def test_superset_full_recall(self) -> None:+ result = symbol_accuracy("linked_list_insert_head", "insert")+ assert result == 0.8++ def test_superset_with_synonyms(self) -> None:+ result = symbol_accuracy("remove_entry_from_list", "remove_node")+ assert result == 0.8++ def test_synonym_only(self) -> None:+ result = symbol_accuracy("linked_list_search", "find")+ assert result == 0.8def test_partial_overlap(self) -> None:- result = symbol_accuracy("init_key_value", "init_buffer_size")- assert 0.1 <= result <= 0.6+ result = symbol_accuracy("xor_buffer", "xor_encode")+ assert 0.2 <= result <= 0.5def test_no_match(self) -> None:assert symbol_accuracy("foo", "bar") == 0.0@@ -62,6 +96,26 @@ class TestSymbolAccuracy:assert result >= 0.8+class TestNormalizeType:+ def test_ghidra_uint(self) -> None:+ assert _normalize_type("uint") == "int"++ def test_ghidra_byte_pointer(self) -> None:+ assert _normalize_type("byte *") == "char *"++ def test_ghidra_undefined4(self) -> None:+ assert _normalize_type("undefined4") == "int"++ def test_unsigned_int_stripped(self) -> None:+ assert _normalize_type("unsigned int") == "int"++ def test_plain_type_unchanged(self) -> None:+ assert _normalize_type("void") == "void"++ def test_pointer_preserved(self) -> None:+ assert _normalize_type("int*") == "int*"++class TestParseSignature:def test_simple_signature(self) -> None:ret, params = _parse_signature("int add(int a, int b)")@@ -114,6 +168,20 @@ class TestTypeAccuracy:)assert result >= 0.8+ def test_ghidra_uint_matches_unsigned_int(self) -> None:+ result = type_accuracy(+ "uint hash(byte *str)",+ "unsigned int hash(const char *s)",+ )+ assert result >= 0.8++ def test_ghidra_undefined4_matches_int(self) -> None:+ result = type_accuracy(+ "void foo(undefined4 x)",+ "void foo(int x)",+ )+ assert result == 1.0+class TestExtractGroundTruth:def test_extracts_from_c_source(self, tmp_path: Path) -> None:@@ -147,6 +215,42 @@ class TestExtractGroundTruth:assert len(truth.functions) == 1assert truth.functions[0]["name"] == "helper"+ def test_extracts_pointer_return_types(self, tmp_path: Path) -> None:+ c_source = tmp_path / "ptr.c"+ c_source.write_text(+ 'char *reverse_string(const char *s) {\n'+ ' return NULL;\n'+ '}\n'+ '\n'+ 'node_t *find(int key) {\n'+ ' return NULL;\n'+ '}\n'+ )+ truth = extract_ground_truth(c_source)+ assert len(truth.functions) == 2+ names = [f["name"] for f in truth.functions]+ assert "reverse_string" in names+ assert "find" in names++ def test_does_not_match_control_flow(self, tmp_path: Path) -> None:+ c_source = tmp_path / "control.c"+ c_source.write_text(+ 'void foo(void) {\n'+ ' while(1) {\n'+ ' break;\n'+ ' }\n'+ ' for(int i = 0; i < 10; i++) {\n'+ ' }\n'+ ' if(x) {\n'+ ' }\n'+ '}\n'+ )+ truth = extract_ground_truth(c_source)+ names = [f["name"] for f in truth.functions]+ assert "while" not in names+ assert "for" not in names+ assert "if" not in names+def test_extracts_signature(self, tmp_path: Path) -> None:c_source = tmp_path / "sig.c"c_source.write_text(