The world's first agentic reverse engineer.
feat: add syntactic normalizer for Ghidra decompilation cleanup
Amrutha4852edd710b
diff --git a/kong/normalizer/__init__.py b/kong/normalizer/__init__.pynew file mode 100644index 0000000..3a84057--- /dev/null+++ b/kong/normalizer/__init__.py@@ -0,0 +1,3 @@+"""Syntactic normalization of Ghidra decompilation output."""++from __future__ import annotationsdiff --git a/kong/normalizer/syntactic.py b/kong/normalizer/syntactic.pynew file mode 100644index 0000000..4c06138--- /dev/null+++ b/kong/normalizer/syntactic.py@@ -0,0 +1,187 @@+from __future__ import annotations++import re+++def normalize(code: str) -> str:+ if not code:+ return code+ code = _clean_negative_literals(code)+ code = _recover_modulo(code)+ code = _infer_undefined_types(code)+ code = _remove_dead_null_assignments(code)+ return code+++def _clean_negative_literals(code: str) -> str:+ return re.sub(r'\+ -(0x[0-9a-fA-F]+|\d+)', r'- \1', code)+++def _recover_modulo(code: str) -> str:+ _NUM = r'(?:0x[0-9a-fA-F]+|\d+)'++ # Find (EXPR / N) * -N or (EXPR / N) * N patterns, then verify+ # the preceding text contains EXPR with + or - operator.+ # Strategy: locate the division-multiply tail, extract the inner+ # expression, then search backwards for a matching leading expression.++ tail_pattern = re.compile(+ r'\((' # open paren + start capture of inner EXPR+ r'[^()]*' # simple content+ r'(?:\([^()]*\)[^()]*)*' # allow one level of nested parens+ r')' # end capture of inner EXPR+ r' / (' + _NUM + r')' # / N+ r'\)' # close paren+ r' \* -?(' + _NUM + r')' # * N or * -N+ )++ code = _recover_modulo_cast_wrapped(code, _NUM)++ changed = True+ while changed:+ changed = False+ for match in tail_pattern.finditer(code):+ inner_expr = match.group(1)+ divisor = match.group(2)+ multiplier = match.group(3)++ if divisor != multiplier:+ continue++ tail_start = match.start()+ prefix = code[:tail_start]++ # Check for EXPR - (EXPR / N) * N form+ suffix_sub = ' - '+ if prefix.endswith(suffix_sub) and prefix[:-len(suffix_sub)].endswith(inner_expr):+ expr_start = tail_start - len(suffix_sub) - len(inner_expr)+ code = code[:expr_start] + inner_expr + ' % ' + divisor + code[match.end():]+ changed = True+ break++ # Check for EXPR + (EXPR / N) * -N form+ suffix_add = ' + '+ has_neg_mult = match.group(0).find('* -') != -1+ if has_neg_mult and prefix.endswith(suffix_add) and prefix[:-len(suffix_add)].endswith(inner_expr):+ expr_start = tail_start - len(suffix_add) - len(inner_expr)+ code = code[:expr_start] + inner_expr + ' % ' + divisor + code[match.end():]+ changed = True+ break++ return code+++def _recover_modulo_cast_wrapped(code: str, _NUM: str) -> str:+ # Handles: (CAST)(EXPR) + (CAST)((EXPR) / N) * -N -> (CAST)((EXPR) % N)+ # Also: (CAST)(EXPR) - (CAST)((EXPR) / N) * N -> (CAST)((EXPR) % N)+ cast_tail = re.compile(+ r'\((\w+)\)' # (CAST) before division group+ r'\(\(' # ((+ r'([^()]*(?:\([^()]*\)[^()]*)*)' # EXPR inside+ r'\) / (' + _NUM + r')\)' # ) / N)+ r' \* (-?' + _NUM + r')' # * N or * -N+ )++ changed = True+ while changed:+ changed = False+ for match in cast_tail.finditer(code):+ cast = match.group(1)+ inner_expr = match.group(2)+ divisor = match.group(3)+ raw_multiplier = match.group(4)++ multiplier = raw_multiplier.lstrip('-')+ if divisor != multiplier:+ continue++ is_neg_mult = raw_multiplier.startswith('-')++ tail_start = match.start()+ prefix = code[:tail_start]++ # Look for (CAST)(EXPR) + or (CAST)(EXPR) - before+ leading_with_add = f'({cast})({inner_expr}) + '+ leading_with_sub = f'({cast})({inner_expr}) - '++ if is_neg_mult and prefix.endswith(leading_with_add):+ expr_start = tail_start - len(leading_with_add)+ replacement = f'({cast})(({inner_expr}) % {divisor})'+ code = code[:expr_start] + replacement + code[match.end():]+ changed = True+ break+ elif not is_neg_mult and prefix.endswith(leading_with_sub):+ expr_start = tail_start - len(leading_with_sub)+ replacement = f'({cast})(({inner_expr}) % {divisor})'+ code = code[:expr_start] + replacement + code[match.end():]+ changed = True+ break++ return code+++def _infer_undefined_types(code: str) -> str:+ undef4_vars = re.findall(r'undefined4\s+(\w+)\s*[;,]', code)+ undef8_vars = re.findall(r'undefined8\s+(\w+)\s*[;,]', code)++ for var in undef4_vars:+ if _is_loop_counter(code, var) or _is_accumulator(code, var):+ code = re.sub(+ r'undefined4(\s+' + re.escape(var) + r')',+ r'int\1',+ code,+ )++ for var in undef8_vars:+ if _is_pointer_like(code, var):+ code = re.sub(+ r'undefined8(\s+' + re.escape(var) + r')',+ r'long\1',+ code,+ )++ return code+++def _is_loop_counter(code: str, var: str) -> bool:+ v = re.escape(var)+ loop_pattern = re.compile(+ r'for\s*\(\s*' + v + r'\s*=\s*0\s*;\s*' + v + r'\s*<\s*\w+\s*;\s*'+ + v + r'\s*(?:=\s*' + v + r'\s*\+\s*1|\+\+)\s*\)'+ )+ return bool(loop_pattern.search(code))+++def _is_accumulator(code: str, var: str) -> bool:+ v = re.escape(var)+ init_zero = re.compile(v + r'\s*=\s*0\s*;')+ add_assign = re.compile(v + r'\s*\+=\s*\w+\s*;')+ add_self = re.compile(v + r'\s*=\s*' + v + r'\s*\+\s*\w+\s*;')+ return bool(init_zero.search(code) and (add_assign.search(code) or add_self.search(code)))+++def _is_pointer_like(code: str, var: str) -> bool:+ v = re.escape(var)+ null_cmp = re.compile(v + r'\s*==\s*(?:0\b|NULL\b)')+ dat_assign = re.compile(v + r'\s*=\s*DAT_\w+')+ ptr_cast_deref = re.compile(r'\*\s*\([^)]*\*\s*\)\s*' + v)+ return bool(null_cmp.search(code) or dat_assign.search(code) or ptr_cast_deref.search(code))+++def _remove_dead_null_assignments(code: str) -> str:+ def _remove_dead_in_block(m: re.Match[str]) -> str:+ var = m.group(1)+ type_cast = m.group(2)+ body = m.group(3)+ assignment_pattern = re.compile(+ r'\s*' + re.escape(var) + r'\s*=\s*\(' + re.escape(type_cast) + r'\)0x0\s*;\n?'+ )+ cleaned_body = assignment_pattern.sub('', body)+ return f'if ({var} == ({type_cast})0x0) {{{cleaned_body}}}'++ result = re.sub(+ r'if\s*\((\w+)\s*==\s*\(([^)]+\*)\)0x0\)\s*\{([^}]*)\}',+ _remove_dead_in_block,+ code,+ )+ return resultdiff --git a/tests/test_normalizer.py b/tests/test_normalizer.pynew file mode 100644index 0000000..63b57a9--- /dev/null+++ b/tests/test_normalizer.py@@ -0,0 +1,178 @@+from __future__ import annotations++import pytest++from kong.normalizer.syntactic import normalize+++class TestEmptyAndPassthrough:+ def test_empty_string(self) -> None:+ assert normalize("") == ""++ def test_clean_code_unchanged(self) -> None:+ code = "int x = a % 5;\nreturn x + 1;"+ assert normalize(code) == code++ def test_whitespace_only(self) -> None:+ assert normalize(" \n\n ") == " \n\n "+++class TestNegativeLiteralCleanup:+ def test_simple_plus_negative(self) -> None:+ assert normalize("x + -5") == "x - 5"++ def test_hex_negative(self) -> None:+ assert normalize("*local_8 + -0x54") == "*local_8 - 0x54"++ def test_multiple_occurrences(self) -> None:+ code = "a + -1 + b + -2"+ assert normalize(code) == "a - 1 + b - 2"++ def test_no_false_positive_minus_minus(self) -> None:+ code = "x - -5"+ assert normalize(code) == "x - -5"++ def test_negative_in_parenthesized_expression(self) -> None:+ assert normalize("(*local_8 + -0x54)") == "(*local_8 - 0x54)"++ def test_negative_literal_with_cast(self) -> None:+ code = "(char)(*local_8 + -0x54)"+ assert normalize(code) == "(char)(*local_8 - 0x54)"+++class TestModuloRecovery:+ def test_simple_modulo_subtraction_form(self) -> None:+ assert normalize("x - (x / 5) * 5") == "x % 5"++ def test_modulo_negative_multiply_form(self) -> None:+ assert normalize("x + (x / 10) * -10") == "x % 10"++ def test_modulo_hex_values(self) -> None:+ assert normalize("x + (x / 0x1a) * -0x1a") == "x % 0x1a"++ def test_no_false_positive_different_variables(self) -> None:+ code = "x - (y / 5) * 5"+ assert normalize(code) == "x - (y / 5) * 5"++ def test_no_false_positive_different_divisor_multiplier(self) -> None:+ code = "x - (x / 5) * 6"+ assert normalize(code) == "x - (x / 5) * 6"++ def test_complex_expression_modulo(self) -> None:+ code = "(*local_8 - 0x54) + ((*local_8 - 0x54) / 0x1a) * -0x1a"+ result = normalize(code)+ assert "% 0x1a" in result+ assert "* -0x1a" not in result++ def test_modulo_with_surrounding_code(self) -> None:+ code = "(char)((*local_8 - 0x54) % 0x1a) + 'a'"+ assert normalize(code) == code++ def test_rot13_full_line(self) -> None:+ code = "*local_8 = (char)(*local_8 + -0x54) + (char)((*local_8 + -0x54) / 0x1a) * -0x1a + 'a';"+ result = normalize(code)+ assert "+ -" not in result+ assert "% 0x1a" in result+++class TestUndefinedTypeInference:+ def test_loop_counter_for_loop(self) -> None:+ code = "undefined4 local_14;\nfor (local_14 = 0; local_14 < length; local_14 = local_14 + 1)"+ result = normalize(code)+ assert "int local_14" in result+ assert "undefined4" not in result++ def test_loop_counter_increment_pp(self) -> None:+ code = "undefined4 i;\nfor (i = 0; i < n; i++)"+ result = normalize(code)+ assert "int i" in result++ def test_accumulator_pattern(self) -> None:+ code = "undefined4 sum;\nsum = 0;\nsum += x;"+ result = normalize(code)+ assert "int sum" in result++ def test_accumulator_init_zero_and_add(self) -> None:+ code = "undefined4 total;\ntotal = 0;\ntotal = total + item;"+ result = normalize(code)+ assert "int total" in result++ def test_undefined8_null_comparison(self) -> None:+ code = "undefined8 ptr;\nif (ptr == 0)"+ result = normalize(code)+ assert "long ptr" in result++ def test_undefined8_null_literal(self) -> None:+ code = "undefined8 ptr;\nif (ptr == NULL)"+ result = normalize(code)+ assert "long ptr" in result++ def test_undefined8_dat_global(self) -> None:+ code = "undefined8 val;\nval = DAT_00100040;"+ result = normalize(code)+ assert "long val" in result++ def test_undefined8_pointer_cast(self) -> None:+ code = "undefined8 mem;\n*(int *)mem = 5;"+ result = normalize(code)+ assert "long mem" in result++ def test_no_inference_without_usage_pattern(self) -> None:+ code = "undefined4 mystery;\nfoo(mystery);"+ result = normalize(code)+ assert "undefined4 mystery" in result++ def test_multiple_variables_inferred(self) -> None:+ code = "undefined4 i;\nundefined4 sum;\nfor (i = 0; i < n; i++)\nsum = 0;\nsum += val;"+ result = normalize(code)+ assert "int i" in result+ assert "int sum" in result+++class TestDeadNullAssignmentRemoval:+ def test_simple_dead_null_assignment(self) -> None:+ code = "if (local_18 == (undefined4 *)0x0) {\n local_18 = (undefined4 *)0x0;\n}"+ result = normalize(code)+ assert "local_18 = (undefined4 *)0x0;" not in result+ assert "if (local_18 == (undefined4 *)0x0) {" in result++ def test_dead_null_different_type(self) -> None:+ code = "if (ptr == (char *)0x0) {\n ptr = (char *)0x0;\n}"+ result = normalize(code)+ assert "ptr = (char *)0x0;" not in result++ def test_non_dead_assignment_preserved(self) -> None:+ code = "if (ptr == (char *)0x0) {\n ptr = malloc(10);\n}"+ result = normalize(code)+ assert "ptr = malloc(10);" in result++ def test_dead_null_with_other_statements(self) -> None:+ code = "if (var == (int *)0x0) {\n var = (int *)0x0;\n printf(\"null\");\n}"+ result = normalize(code)+ assert "var = (int *)0x0;" not in result+ assert 'printf("null");' in result+++class TestPipelineOrdering:+ def test_negative_literal_before_modulo(self) -> None:+ code = "x + (x / 0x1a) * -0x1a"+ result = normalize(code)+ assert "% 0x1a" in result++ def test_full_rot13_pipeline(self) -> None:+ code = (+ "*local_8 = (char)(*local_8 + -0x54) + "+ "(char)((*local_8 + -0x54) / 0x1a) * -0x1a + 'a';"+ )+ result = normalize(code)+ assert "+ -" not in result+ assert "% 0x1a" in result++ def test_create_node_dead_assignment(self) -> None:+ code = (+ "if (local_18 == (undefined4 *)0x0) {\n"+ " local_18 = (undefined4 *)0x0;\n"+ "}"+ )+ result = normalize(code)+ assert "local_18 = (undefined4 *)0x0;" not in result