Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

feat: add syntactic normalizer for Ghidra decompilation cleanup

Amrutha4852edd710b
diff --git a/kong/normalizer/__init__.py b/kong/normalizer/__init__.py
new file mode 100644
index 0000000..3a84057
--- /dev/null
+++ b/kong/normalizer/__init__.py
@@ -0,0 +1,3 @@
+"""Syntactic normalization of Ghidra decompilation output."""
+
+from __future__ import annotations
diff --git a/kong/normalizer/syntactic.py b/kong/normalizer/syntactic.py
new file mode 100644
index 0000000..4c06138
--- /dev/null
+++ b/kong/normalizer/syntactic.py
@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import re
+
+
+def normalize(code: str) -> str:
+ if not code:
+ return code
+ code = _clean_negative_literals(code)
+ code = _recover_modulo(code)
+ code = _infer_undefined_types(code)
+ code = _remove_dead_null_assignments(code)
+ return code
+
+
+def _clean_negative_literals(code: str) -> str:
+ return re.sub(r'\+ -(0x[0-9a-fA-F]+|\d+)', r'- \1', code)
+
+
+def _recover_modulo(code: str) -> str:
+ _NUM = r'(?:0x[0-9a-fA-F]+|\d+)'
+
+ # Find (EXPR / N) * -N or (EXPR / N) * N patterns, then verify
+ # the preceding text contains EXPR with + or - operator.
+ # Strategy: locate the division-multiply tail, extract the inner
+ # expression, then search backwards for a matching leading expression.
+
+ tail_pattern = re.compile(
+ r'\((' # open paren + start capture of inner EXPR
+ r'[^()]*' # simple content
+ r'(?:\([^()]*\)[^()]*)*' # allow one level of nested parens
+ r')' # end capture of inner EXPR
+ r' / (' + _NUM + r')' # / N
+ r'\)' # close paren
+ r' \* -?(' + _NUM + r')' # * N or * -N
+ )
+
+ code = _recover_modulo_cast_wrapped(code, _NUM)
+
+ changed = True
+ while changed:
+ changed = False
+ for match in tail_pattern.finditer(code):
+ inner_expr = match.group(1)
+ divisor = match.group(2)
+ multiplier = match.group(3)
+
+ if divisor != multiplier:
+ continue
+
+ tail_start = match.start()
+ prefix = code[:tail_start]
+
+ # Check for EXPR - (EXPR / N) * N form
+ suffix_sub = ' - '
+ if prefix.endswith(suffix_sub) and prefix[:-len(suffix_sub)].endswith(inner_expr):
+ expr_start = tail_start - len(suffix_sub) - len(inner_expr)
+ code = code[:expr_start] + inner_expr + ' % ' + divisor + code[match.end():]
+ changed = True
+ break
+
+ # Check for EXPR + (EXPR / N) * -N form
+ suffix_add = ' + '
+ has_neg_mult = match.group(0).find('* -') != -1
+ if has_neg_mult and prefix.endswith(suffix_add) and prefix[:-len(suffix_add)].endswith(inner_expr):
+ expr_start = tail_start - len(suffix_add) - len(inner_expr)
+ code = code[:expr_start] + inner_expr + ' % ' + divisor + code[match.end():]
+ changed = True
+ break
+
+ return code
+
+
+def _recover_modulo_cast_wrapped(code: str, _NUM: str) -> str:
+ # Handles: (CAST)(EXPR) + (CAST)((EXPR) / N) * -N -> (CAST)((EXPR) % N)
+ # Also: (CAST)(EXPR) - (CAST)((EXPR) / N) * N -> (CAST)((EXPR) % N)
+ cast_tail = re.compile(
+ r'\((\w+)\)' # (CAST) before division group
+ r'\(\(' # ((
+ r'([^()]*(?:\([^()]*\)[^()]*)*)' # EXPR inside
+ r'\) / (' + _NUM + r')\)' # ) / N)
+ r' \* (-?' + _NUM + r')' # * N or * -N
+ )
+
+ changed = True
+ while changed:
+ changed = False
+ for match in cast_tail.finditer(code):
+ cast = match.group(1)
+ inner_expr = match.group(2)
+ divisor = match.group(3)
+ raw_multiplier = match.group(4)
+
+ multiplier = raw_multiplier.lstrip('-')
+ if divisor != multiplier:
+ continue
+
+ is_neg_mult = raw_multiplier.startswith('-')
+
+ tail_start = match.start()
+ prefix = code[:tail_start]
+
+ # Look for (CAST)(EXPR) + or (CAST)(EXPR) - before
+ leading_with_add = f'({cast})({inner_expr}) + '
+ leading_with_sub = f'({cast})({inner_expr}) - '
+
+ if is_neg_mult and prefix.endswith(leading_with_add):
+ expr_start = tail_start - len(leading_with_add)
+ replacement = f'({cast})(({inner_expr}) % {divisor})'
+ code = code[:expr_start] + replacement + code[match.end():]
+ changed = True
+ break
+ elif not is_neg_mult and prefix.endswith(leading_with_sub):
+ expr_start = tail_start - len(leading_with_sub)
+ replacement = f'({cast})(({inner_expr}) % {divisor})'
+ code = code[:expr_start] + replacement + code[match.end():]
+ changed = True
+ break
+
+ return code
+
+
+def _infer_undefined_types(code: str) -> str:
+ undef4_vars = re.findall(r'undefined4\s+(\w+)\s*[;,]', code)
+ undef8_vars = re.findall(r'undefined8\s+(\w+)\s*[;,]', code)
+
+ for var in undef4_vars:
+ if _is_loop_counter(code, var) or _is_accumulator(code, var):
+ code = re.sub(
+ r'undefined4(\s+' + re.escape(var) + r')',
+ r'int\1',
+ code,
+ )
+
+ for var in undef8_vars:
+ if _is_pointer_like(code, var):
+ code = re.sub(
+ r'undefined8(\s+' + re.escape(var) + r')',
+ r'long\1',
+ code,
+ )
+
+ return code
+
+
+def _is_loop_counter(code: str, var: str) -> bool:
+ v = re.escape(var)
+ loop_pattern = re.compile(
+ r'for\s*\(\s*' + v + r'\s*=\s*0\s*;\s*' + v + r'\s*<\s*\w+\s*;\s*'
+ + v + r'\s*(?:=\s*' + v + r'\s*\+\s*1|\+\+)\s*\)'
+ )
+ return bool(loop_pattern.search(code))
+
+
+def _is_accumulator(code: str, var: str) -> bool:
+ v = re.escape(var)
+ init_zero = re.compile(v + r'\s*=\s*0\s*;')
+ add_assign = re.compile(v + r'\s*\+=\s*\w+\s*;')
+ add_self = re.compile(v + r'\s*=\s*' + v + r'\s*\+\s*\w+\s*;')
+ return bool(init_zero.search(code) and (add_assign.search(code) or add_self.search(code)))
+
+
+def _is_pointer_like(code: str, var: str) -> bool:
+ v = re.escape(var)
+ null_cmp = re.compile(v + r'\s*==\s*(?:0\b|NULL\b)')
+ dat_assign = re.compile(v + r'\s*=\s*DAT_\w+')
+ ptr_cast_deref = re.compile(r'\*\s*\([^)]*\*\s*\)\s*' + v)
+ return bool(null_cmp.search(code) or dat_assign.search(code) or ptr_cast_deref.search(code))
+
+
+def _remove_dead_null_assignments(code: str) -> str:
+ def _remove_dead_in_block(m: re.Match[str]) -> str:
+ var = m.group(1)
+ type_cast = m.group(2)
+ body = m.group(3)
+ assignment_pattern = re.compile(
+ r'\s*' + re.escape(var) + r'\s*=\s*\(' + re.escape(type_cast) + r'\)0x0\s*;\n?'
+ )
+ cleaned_body = assignment_pattern.sub('', body)
+ return f'if ({var} == ({type_cast})0x0) {{{cleaned_body}}}'
+
+ result = re.sub(
+ r'if\s*\((\w+)\s*==\s*\(([^)]+\*)\)0x0\)\s*\{([^}]*)\}',
+ _remove_dead_in_block,
+ code,
+ )
+ return result
diff --git a/tests/test_normalizer.py b/tests/test_normalizer.py
new file mode 100644
index 0000000..63b57a9
--- /dev/null
+++ b/tests/test_normalizer.py
@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import pytest
+
+from kong.normalizer.syntactic import normalize
+
+
+class TestEmptyAndPassthrough:
+ def test_empty_string(self) -> None:
+ assert normalize("") == ""
+
+ def test_clean_code_unchanged(self) -> None:
+ code = "int x = a % 5;\nreturn x + 1;"
+ assert normalize(code) == code
+
+ def test_whitespace_only(self) -> None:
+ assert normalize(" \n\n ") == " \n\n "
+
+
+class TestNegativeLiteralCleanup:
+ def test_simple_plus_negative(self) -> None:
+ assert normalize("x + -5") == "x - 5"
+
+ def test_hex_negative(self) -> None:
+ assert normalize("*local_8 + -0x54") == "*local_8 - 0x54"
+
+ def test_multiple_occurrences(self) -> None:
+ code = "a + -1 + b + -2"
+ assert normalize(code) == "a - 1 + b - 2"
+
+ def test_no_false_positive_minus_minus(self) -> None:
+ code = "x - -5"
+ assert normalize(code) == "x - -5"
+
+ def test_negative_in_parenthesized_expression(self) -> None:
+ assert normalize("(*local_8 + -0x54)") == "(*local_8 - 0x54)"
+
+ def test_negative_literal_with_cast(self) -> None:
+ code = "(char)(*local_8 + -0x54)"
+ assert normalize(code) == "(char)(*local_8 - 0x54)"
+
+
+class TestModuloRecovery:
+ def test_simple_modulo_subtraction_form(self) -> None:
+ assert normalize("x - (x / 5) * 5") == "x % 5"
+
+ def test_modulo_negative_multiply_form(self) -> None:
+ assert normalize("x + (x / 10) * -10") == "x % 10"
+
+ def test_modulo_hex_values(self) -> None:
+ assert normalize("x + (x / 0x1a) * -0x1a") == "x % 0x1a"
+
+ def test_no_false_positive_different_variables(self) -> None:
+ code = "x - (y / 5) * 5"
+ assert normalize(code) == "x - (y / 5) * 5"
+
+ def test_no_false_positive_different_divisor_multiplier(self) -> None:
+ code = "x - (x / 5) * 6"
+ assert normalize(code) == "x - (x / 5) * 6"
+
+ def test_complex_expression_modulo(self) -> None:
+ code = "(*local_8 - 0x54) + ((*local_8 - 0x54) / 0x1a) * -0x1a"
+ result = normalize(code)
+ assert "% 0x1a" in result
+ assert "* -0x1a" not in result
+
+ def test_modulo_with_surrounding_code(self) -> None:
+ code = "(char)((*local_8 - 0x54) % 0x1a) + 'a'"
+ assert normalize(code) == code
+
+ def test_rot13_full_line(self) -> None:
+ code = "*local_8 = (char)(*local_8 + -0x54) + (char)((*local_8 + -0x54) / 0x1a) * -0x1a + 'a';"
+ result = normalize(code)
+ assert "+ -" not in result
+ assert "% 0x1a" in result
+
+
+class TestUndefinedTypeInference:
+ def test_loop_counter_for_loop(self) -> None:
+ code = "undefined4 local_14;\nfor (local_14 = 0; local_14 < length; local_14 = local_14 + 1)"
+ result = normalize(code)
+ assert "int local_14" in result
+ assert "undefined4" not in result
+
+ def test_loop_counter_increment_pp(self) -> None:
+ code = "undefined4 i;\nfor (i = 0; i < n; i++)"
+ result = normalize(code)
+ assert "int i" in result
+
+ def test_accumulator_pattern(self) -> None:
+ code = "undefined4 sum;\nsum = 0;\nsum += x;"
+ result = normalize(code)
+ assert "int sum" in result
+
+ def test_accumulator_init_zero_and_add(self) -> None:
+ code = "undefined4 total;\ntotal = 0;\ntotal = total + item;"
+ result = normalize(code)
+ assert "int total" in result
+
+ def test_undefined8_null_comparison(self) -> None:
+ code = "undefined8 ptr;\nif (ptr == 0)"
+ result = normalize(code)
+ assert "long ptr" in result
+
+ def test_undefined8_null_literal(self) -> None:
+ code = "undefined8 ptr;\nif (ptr == NULL)"
+ result = normalize(code)
+ assert "long ptr" in result
+
+ def test_undefined8_dat_global(self) -> None:
+ code = "undefined8 val;\nval = DAT_00100040;"
+ result = normalize(code)
+ assert "long val" in result
+
+ def test_undefined8_pointer_cast(self) -> None:
+ code = "undefined8 mem;\n*(int *)mem = 5;"
+ result = normalize(code)
+ assert "long mem" in result
+
+ def test_no_inference_without_usage_pattern(self) -> None:
+ code = "undefined4 mystery;\nfoo(mystery);"
+ result = normalize(code)
+ assert "undefined4 mystery" in result
+
+ def test_multiple_variables_inferred(self) -> None:
+ code = "undefined4 i;\nundefined4 sum;\nfor (i = 0; i < n; i++)\nsum = 0;\nsum += val;"
+ result = normalize(code)
+ assert "int i" in result
+ assert "int sum" in result
+
+
+class TestDeadNullAssignmentRemoval:
+ def test_simple_dead_null_assignment(self) -> None:
+ code = "if (local_18 == (undefined4 *)0x0) {\n local_18 = (undefined4 *)0x0;\n}"
+ result = normalize(code)
+ assert "local_18 = (undefined4 *)0x0;" not in result
+ assert "if (local_18 == (undefined4 *)0x0) {" in result
+
+ def test_dead_null_different_type(self) -> None:
+ code = "if (ptr == (char *)0x0) {\n ptr = (char *)0x0;\n}"
+ result = normalize(code)
+ assert "ptr = (char *)0x0;" not in result
+
+ def test_non_dead_assignment_preserved(self) -> None:
+ code = "if (ptr == (char *)0x0) {\n ptr = malloc(10);\n}"
+ result = normalize(code)
+ assert "ptr = malloc(10);" in result
+
+ def test_dead_null_with_other_statements(self) -> None:
+ code = "if (var == (int *)0x0) {\n var = (int *)0x0;\n printf(\"null\");\n}"
+ result = normalize(code)
+ assert "var = (int *)0x0;" not in result
+ assert 'printf("null");' in result
+
+
+class TestPipelineOrdering:
+ def test_negative_literal_before_modulo(self) -> None:
+ code = "x + (x / 0x1a) * -0x1a"
+ result = normalize(code)
+ assert "% 0x1a" in result
+
+ def test_full_rot13_pipeline(self) -> None:
+ code = (
+ "*local_8 = (char)(*local_8 + -0x54) + "
+ "(char)((*local_8 + -0x54) / 0x1a) * -0x1a + 'a';"
+ )
+ result = normalize(code)
+ assert "+ -" not in result
+ assert "% 0x1a" in result
+
+ def test_create_node_dead_assignment(self) -> None:
+ code = (
+ "if (local_18 == (undefined4 *)0x0) {\n"
+ " local_18 = (undefined4 *)0x0;\n"
+ "}"
+ )
+ result = normalize(code)
+ assert "local_18 = (undefined4 *)0x0;" not in result