Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

fix: improve eval metrics with recall-weighted scoring and Ghidra type normalization

Amruth026770becf6a
diff --git a/kong/evals/harness.py b/kong/evals/harness.py
index e63c5dd..1b21030 100644
--- a/kong/evals/harness.py
+++ b/kong/evals/harness.py
@@ -30,8 +30,8 @@ class Scorecard:
_FUNC_PATTERN = re.compile(
r"^(?:static\s+)?"
- r"([\w\s\*]+?)"
- r"\s+(\w+)\s*"
+ r"((?:\w+[\s*]+)+)"
+ r"(\w+)\s*"
r"\(([^)]*)\)\s*\{",
re.MULTILINE,
)
diff --git a/kong/evals/metrics.py b/kong/evals/metrics.py
index 85df10c..6ddc726 100644
--- a/kong/evals/metrics.py
+++ b/kong/evals/metrics.py
@@ -5,6 +5,41 @@ from __future__ import annotations
import re
+_SYNONYM_GROUPS: list[set[str]] = [
+ {"search", "find", "lookup"},
+ {"node", "entry", "element"},
+ {"buffer", "buf"},
+ {"encode", "encrypt"},
+ {"decode", "decrypt"},
+ {"delete", "remove"},
+ {"create", "make", "new", "alloc"},
+ {"insert", "add", "push"},
+ {"string", "str"},
+ {"print", "display", "show"},
+ {"count", "num", "size", "length"},
+]
+
+_SYNONYM_MAP: dict[str, set[str]] = {}
+for _group in _SYNONYM_GROUPS:
+ for _word in _group:
+ _SYNONYM_MAP[_word] = _group
+
+_NOISE_WORDS = {"or", "from", "the", "a", "an", "of", "to", "and", "is", "in"}
+
+_TYPE_ALIASES: dict[str, str] = {
+ "uint": "int",
+ "byte": "char",
+ "undefined4": "int",
+ "undefined8": "long",
+ "undefined2": "short",
+ "undefined1": "char",
+ "bool": "int",
+ "dword": "int",
+ "qword": "long",
+ "word": "short",
+}
+
+
def _normalize_name(name: str) -> list[str]:
"""Split camelCase/snake_case/PascalCase into lowercase word tokens."""
parts = name.replace("_", " ").strip().split()
@@ -12,14 +47,40 @@ def _normalize_name(name: str) -> list[str]:
for part in parts:
splits = re.sub(r"([a-z])([A-Z])", r"\1 \2", part).split()
tokens.extend(s.lower() for s in splits if s)
- return tokens
+ return [t for t in tokens if t not in _NOISE_WORDS]
+
+
+def _is_synonym(a: str, b: str) -> bool:
+ if a == b:
+ return True
+ group = _SYNONYM_MAP.get(a)
+ return group is not None and b in group
+
+
+def _synonym_recall(pred_set: set[str], truth_set: set[str]) -> float:
+ """Fraction of truth tokens matched by a pred token (exact or synonym)."""
+ if not truth_set:
+ return 0.0
+ matched = 0
+ for t in truth_set:
+ if any(_is_synonym(t, p) for p in pred_set):
+ matched += 1
+ return matched / len(truth_set)
def symbol_accuracy(predicted: str, truth: str) -> float:
"""Score predicted symbol name against ground truth.
- Returns 1.0 for exact match, 0.8 for same words different order,
- Jaccard-based (capped at 0.6) for partial overlap, 0.0 for no overlap.
+ Recall-weighted: measures how well the prediction captures the ground
+ truth concepts. Extra descriptive words in the prediction are not
+ penalized heavily — they add context in RE output.
+
+ Returns:
+ 1.0 exact string match
+ 0.9 same word set, different order
+ 0.8 all truth words present (superset) — exact or synonym
+ recall * 0.7 partial overlap, scaled by truth coverage
+ 0.0 no overlap even after synonym expansion
"""
if predicted == truth:
return 1.0
@@ -34,23 +95,29 @@ def symbol_accuracy(predicted: str, truth: str) -> float:
truth_set = set(truth_tokens)
if pred_set == truth_set:
- return 0.8
+ return 0.9
- intersection = pred_set & truth_set
- if not intersection:
- return 0.0
+ recall = _synonym_recall(pred_set, truth_set)
- union = pred_set | truth_set
- jaccard = len(intersection) / len(union)
- return min(jaccard, 0.6)
+ if recall >= 1.0:
+ return 0.8
+ if recall > 0.0:
+ return recall * 0.7
+ return 0.0
-def _strip_qualifiers(type_str: str) -> str:
- """Remove const/unsigned/signed qualifiers from a type string."""
+def _normalize_type(type_str: str) -> str:
+ """Normalize a C type string: strip qualifiers and resolve Ghidra aliases."""
stripped = type_str
for qualifier in ("const", "unsigned", "signed"):
stripped = stripped.replace(qualifier, "")
- return " ".join(stripped.split())
+ stripped = " ".join(stripped.split())
+
+ base = stripped.rstrip("*").rstrip()
+ stars = stripped[len(base):]
+
+ resolved = _TYPE_ALIASES.get(base, base)
+ return (resolved + stars).strip()
def _parse_signature(sig: str) -> tuple[str, list[str]]:
@@ -87,6 +154,8 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:
Exact match: 1.0. Otherwise scored by components:
return type match (0.4), param count match (0.3),
individual param type matches (up to 0.3).
+
+ Types are normalized through Ghidra alias resolution before comparison.
"""
if predicted_sig == truth_sig:
return 1.0
@@ -96,7 +165,7 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:
score = 0.0
- if _strip_qualifiers(pred_ret) == _strip_qualifiers(truth_ret):
+ if _normalize_type(pred_ret) == _normalize_type(truth_ret):
score += 0.4
if len(pred_params) == len(truth_params):
@@ -105,7 +174,7 @@ def type_accuracy(predicted_sig: str, truth_sig: str) -> float:
if pred_params and truth_params:
match_count = sum(
1 for p, t in zip(pred_params, truth_params)
- if _strip_qualifiers(p) == _strip_qualifiers(t)
+ if _normalize_type(p) == _normalize_type(t)
)
max_params = max(len(pred_params), len(truth_params))
score += 0.3 * (match_count / max_params)
diff --git a/tests/test_evals.py b/tests/test_evals.py
index e1e9bc9..bbcfebc 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -9,7 +9,9 @@ import pytest
from kong.evals.metrics import (
_normalize_name,
+ _normalize_type,
_parse_signature,
+ _synonym_recall,
overall_score,
symbol_accuracy,
type_accuracy,
@@ -41,6 +43,26 @@ class TestNormalizeName:
tokens = _normalize_name("SSL")
assert tokens == ["ssl"]
+ def test_noise_words_removed(self) -> None:
+ assert _normalize_name("remove_from_list") == ["remove", "list"]
+ assert _normalize_name("create_or_update") == ["create", "update"]
+
+
+class TestSynonymRecall:
+ def test_exact_overlap(self) -> None:
+ assert _synonym_recall({"insert", "node"}, {"insert", "node"}) == 1.0
+
+ def test_synonym_match(self) -> None:
+ assert _synonym_recall({"search", "list"}, {"find"}) == 1.0
+ assert _synonym_recall({"entry"}, {"node"}) == 1.0
+
+ def test_no_overlap(self) -> None:
+ assert _synonym_recall({"foo"}, {"bar"}) == 0.0
+
+ def test_partial(self) -> None:
+ result = _synonym_recall({"xor"}, {"xor", "encode"})
+ assert result == 0.5
+
class TestSymbolAccuracy:
def test_exact_match(self) -> None:
@@ -48,11 +70,23 @@ class TestSymbolAccuracy:
def test_word_reorder(self) -> None:
result = symbol_accuracy("init_key", "key_init")
- assert 0.7 <= result <= 0.9
+ assert result == 0.9
+
+ def test_superset_full_recall(self) -> None:
+ result = symbol_accuracy("linked_list_insert_head", "insert")
+ assert result == 0.8
+
+ def test_superset_with_synonyms(self) -> None:
+ result = symbol_accuracy("remove_entry_from_list", "remove_node")
+ assert result == 0.8
+
+ def test_synonym_only(self) -> None:
+ result = symbol_accuracy("linked_list_search", "find")
+ assert result == 0.8
def test_partial_overlap(self) -> None:
- result = symbol_accuracy("init_key_value", "init_buffer_size")
- assert 0.1 <= result <= 0.6
+ result = symbol_accuracy("xor_buffer", "xor_encode")
+ assert 0.2 <= result <= 0.5
def test_no_match(self) -> None:
assert symbol_accuracy("foo", "bar") == 0.0
@@ -62,6 +96,26 @@ class TestSymbolAccuracy:
assert result >= 0.8
+class TestNormalizeType:
+ def test_ghidra_uint(self) -> None:
+ assert _normalize_type("uint") == "int"
+
+ def test_ghidra_byte_pointer(self) -> None:
+ assert _normalize_type("byte *") == "char *"
+
+ def test_ghidra_undefined4(self) -> None:
+ assert _normalize_type("undefined4") == "int"
+
+ def test_unsigned_int_stripped(self) -> None:
+ assert _normalize_type("unsigned int") == "int"
+
+ def test_plain_type_unchanged(self) -> None:
+ assert _normalize_type("void") == "void"
+
+ def test_pointer_preserved(self) -> None:
+ assert _normalize_type("int*") == "int*"
+
+
class TestParseSignature:
def test_simple_signature(self) -> None:
ret, params = _parse_signature("int add(int a, int b)")
@@ -114,6 +168,20 @@ class TestTypeAccuracy:
)
assert result >= 0.8
+ def test_ghidra_uint_matches_unsigned_int(self) -> None:
+ result = type_accuracy(
+ "uint hash(byte *str)",
+ "unsigned int hash(const char *s)",
+ )
+ assert result >= 0.8
+
+ def test_ghidra_undefined4_matches_int(self) -> None:
+ result = type_accuracy(
+ "void foo(undefined4 x)",
+ "void foo(int x)",
+ )
+ assert result == 1.0
+
class TestExtractGroundTruth:
def test_extracts_from_c_source(self, tmp_path: Path) -> None:
@@ -147,6 +215,42 @@ class TestExtractGroundTruth:
assert len(truth.functions) == 1
assert truth.functions[0]["name"] == "helper"
+ def test_extracts_pointer_return_types(self, tmp_path: Path) -> None:
+ c_source = tmp_path / "ptr.c"
+ c_source.write_text(
+ 'char *reverse_string(const char *s) {\n'
+ ' return NULL;\n'
+ '}\n'
+ '\n'
+ 'node_t *find(int key) {\n'
+ ' return NULL;\n'
+ '}\n'
+ )
+ truth = extract_ground_truth(c_source)
+ assert len(truth.functions) == 2
+ names = [f["name"] for f in truth.functions]
+ assert "reverse_string" in names
+ assert "find" in names
+
+ def test_does_not_match_control_flow(self, tmp_path: Path) -> None:
+ c_source = tmp_path / "control.c"
+ c_source.write_text(
+ 'void foo(void) {\n'
+ ' while(1) {\n'
+ ' break;\n'
+ ' }\n'
+ ' for(int i = 0; i < 10; i++) {\n'
+ ' }\n'
+ ' if(x) {\n'
+ ' }\n'
+ '}\n'
+ )
+ truth = extract_ground_truth(c_source)
+ names = [f["name"] for f in truth.functions]
+ assert "while" not in names
+ assert "for" not in names
+ assert "if" not in names
+
def test_extracts_signature(self, tmp_path: Path) -> None:
c_source = tmp_path / "sig.c"
c_source.write_text(