Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

debug agent, update tests/gitignore

Amruth86833dcf1d6a
diff --git a/.gitignore b/.gitignore
index 8177e76..100d49f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -217,4 +217,4 @@ __marimo__/
.plist
#internal testing
-internal_testing/
\ No newline at end of file
+internal-testing/
\ No newline at end of file
diff --git a/kong/__main__.py b/kong/__main__.py
index 5fea1d8..87f4ecb 100644
--- a/kong/__main__.py
+++ b/kong/__main__.py
@@ -127,12 +127,21 @@ def analyze(
finally:
stats = supervisor.stats
console.print()
- console.print(f"[bold]Results:[/bold] {stats.named}/{stats.total_functions} functions named")
+ console.print(
+ f"[bold]Results:[/bold] {stats.named}/{stats.total_functions} functions named "
+ f"({stats.renamed} renamed, {stats.confirmed} confirmed)"
+ )
console.print(
f"[bold]Confidence:[/bold] {stats.high_confidence} high, "
f"{stats.medium_confidence} med, {stats.low_confidence} low"
)
console.print(f"[bold]LLM calls:[/bold] {stats.llm_calls}")
+ for model_name, mu in llm_client.usage.by_model.items():
+ short_name = model_name.split("-")[1] if "-" in model_name else model_name
+ console.print(
+ f" {short_name}: {mu.calls} calls, "
+ f"${mu.cost_usd(model_name):.4f}"
+ )
console.print(f"[bold]Cost:[/bold] ${llm_client.total_cost_usd:.4f}")
console.print(f"[bold]Duration:[/bold] {stats.duration_seconds:.1f}s")
client.close()
diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.py
index 5d9311b..b1fed84 100644
--- a/kong/agent/analyzer.py
+++ b/kong/agent/analyzer.py
@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
class LLMClient(Protocol):
"""Protocol for LLM interaction."""
- def analyze_function(self, prompt: str) -> LLMResponse: ...
+ def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse: ...
def analyze_with_tools(
self,
@@ -128,6 +128,7 @@ class Analyzer:
known_results: dict[int, FunctionResult],
strings: list[StringEntry],
known_types: list[StructDefinition] | None = None,
+ model: str | None = None,
) -> FunctionResult:
"""Full analysis pipeline for one function."""
from kong.agent.deobfuscator import Deobfuscator, classify_obfuscation
@@ -157,7 +158,7 @@ class Analyzer:
)
prompt = self._build_prompt(context)
- response = self.llm.analyze_function(prompt)
+ response = self.llm.analyze_function(prompt, model=model)
sig_applied = self._write_back(func.address, response)
return FunctionResult(
@@ -363,7 +364,7 @@ class Analyzer:
try:
self.client.set_function_signature(addr, response.signature)
except Exception as e:
- logger.warning("Failed to set signature at 0x%08x: %s", addr, e)
+ logger.debug("Signature deferred for 0x%08x (will retry in cleanup): %s", addr, e)
signature_ok = False
if response.comments:
@@ -408,20 +409,24 @@ class Analyzer:
struct_proposals = []
for sp in data.get("struct_proposals", []):
- if "name" not in sp or "total_size" not in sp:
+ if not sp.get("name") or not sp.get("total_size"):
+ continue
+ try:
+ total_size = int(sp["total_size"])
+ except (ValueError, TypeError):
continue
fields = [
StructFieldProposal(
name=f.get("name", f"field_{i}"),
data_type=f.get("data_type", "undefined"),
- offset=f.get("offset", 0),
- size=f.get("size", 4),
+ offset=int(f.get("offset", 0)),
+ size=int(f.get("size", 4)),
)
for i, f in enumerate(sp.get("fields", []))
]
struct_proposals.append(StructProposal(
name=sp["name"],
- total_size=sp["total_size"],
+ total_size=total_size,
fields=fields,
used_by_param=sp.get("used_by_param", ""),
))
diff --git a/kong/agent/queue.py b/kong/agent/queue.py
index c98b492..8aac767 100644
--- a/kong/agent/queue.py
+++ b/kong/agent/queue.py
@@ -164,5 +164,14 @@ class WorkQueue:
def get_by_address(self, addr: int) -> WorkItem | None:
return self._by_address.get(addr)
+ def items_by_depth(self) -> list[list[WorkItem]]:
+ """Return items grouped by depth level, sorted ascending (leaves first)."""
+ if not self._items:
+ return []
+ depth_map: dict[int, list[WorkItem]] = {}
+ for item in self._items:
+ depth_map.setdefault(item.depth, []).append(item)
+ return [depth_map[d] for d in sorted(depth_map)]
+
def all_items(self) -> list[WorkItem]:
return list(self._items)
diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.py
index 6fbd4bd..34fb7f1 100644
--- a/kong/agent/supervisor.py
+++ b/kong/agent/supervisor.py
@@ -8,10 +8,11 @@ from __future__ import annotations
import logging
import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
-from kong.agent.analyzer import Analyzer, LLMClient
-from kong.agent.deobfuscator import Deobfuscator
+from kong.agent.analyzer import Analyzer, LLMClient, LLMResponse
+from kong.agent.deobfuscator import Deobfuscator, classify_obfuscation
from kong.agent.events import Event, EventCallback, EventType, Phase
from kong.agent.models import FunctionResult
from kong.agent.queue import WorkItem, WorkQueue
@@ -24,13 +25,20 @@ from kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntry
logger = logging.getLogger(__name__)
+HAIKU_MODEL = "claude-haiku-4-5-20251001"
+SONNET_MODEL = "claude-sonnet-4-20250514"
+
+MAX_PARALLEL_LLM = 4
+REQUEST_STAGGER_SECONDS = 0.5
+
@dataclass
class AnalysisStats:
"""Aggregate statistics for the full run."""
total_functions: int = 0
analyzed: int = 0
- named: int = 0
+ renamed: int = 0
+ confirmed: int = 0
high_confidence: int = 0 # >= 80
medium_confidence: int = 0 # 50-79
low_confidence: int = 0 # < 50
@@ -41,6 +49,10 @@ class AnalysisStats:
end_time: float = 0.0
signature_matches: int = 0
+ @property
+ def named(self) -> int:
+ return self.renamed + self.confirmed
+
@property
def duration_seconds(self) -> float:
end = self.end_time or time.time()
@@ -59,8 +71,11 @@ class AnalysisStats:
return
self.analyzed += 1
self.llm_calls += result.llm_calls
- if result.name and result.name != result.original_name:
- self.named += 1
+ if result.name:
+ if result.name != result.original_name:
+ self.renamed += 1
+ else:
+ self.confirmed += 1
# TODO: calibrate these eventually. these buckets are arbitrary, need eval data to
# determine meaningful confidence tiers for the LLM's self-reported scores.
if result.confidence >= 80:
@@ -219,73 +234,100 @@ class Supervisor:
def _run_analysis(self) -> None:
- """Analyze each function bottom-up via LLM."""
+ """Analyze functions bottom-up via LLM, parallelized within each depth level."""
self._emit(Event(
type=EventType.PHASE_START,
phase=Phase.ANALYSIS,
message="Starting bottom-up analysis...",
))
- while self.queue:
+ depth_batches = self.queue.items_by_depth()
+ completed_count = 0
+
+ for batch in depth_batches:
if self._paused:
time.sleep(0.1)
continue
- item = self.queue.next()
- if item is None:
- break
+ parallel_items: list[tuple[WorkItem, str, str]] = []
+ sequential_items: list[WorkItem] = []
+ analyzer = Analyzer(self.client, self.llm_client) if self.llm_client else None
- func = item.function
- self._emit(Event(
- type=EventType.FUNCTION_START,
- phase=Phase.ANALYSIS,
- message=f"Analyzing {func.name} ({func.address_hex})...",
- data={
- "address": func.address,
- "name": func.name,
- "size": func.size,
- "depth": item.depth,
- "progress": f"{self.queue.completed}/{self.queue.total}",
- },
- ))
+ for item in batch:
+ func = item.function
- try:
- result = self._analyze_function(item)
- except Exception as e:
- result = FunctionResult(
- address=func.address,
- original_name=func.name,
- error=str(e),
- )
- self._emit(Event(
- type=EventType.FUNCTION_ERROR,
- phase=Phase.ANALYSIS,
- message=f"Error analyzing {func.name}: {e}",
- data={"address": func.address, "error": str(e)},
- ))
+ if func.classification and func.classification.value == "trivial":
+ result = FunctionResult(
+ address=func.address,
+ original_name=func.name,
+ skipped=True,
+ skip_reason="trivial",
+ )
+ self.results[func.address] = result
+ self.stats.record_result(result)
+ completed_count += 1
+ continue
+
+ if analyzer is None:
+ result = FunctionResult(
+ address=func.address,
+ original_name=func.name,
+ name=func.name,
+ confidence=0,
+ )
+ self.results[func.address] = result
+ self.stats.record_result(result)
+ completed_count += 1
+ continue
- self.results[func.address] = result
- self.stats.record_result(result)
+ decompilation = self.client.get_decompilation(func.address)
+ techniques = classify_obfuscation(decompilation)
- if result.struct_proposals:
- self.struct_accumulator.add_proposals(func.address, result.struct_proposals)
+ if techniques:
+ sequential_items.append(item)
+ continue
- if not result.error and not result.skipped:
+ model = self._pick_model(item, decompilation)
+ context = analyzer._build_context(
+ item, self.binary_info, self.results, self.strings,
+ )
+ prompt = analyzer._build_prompt(context)
+ parallel_items.append((item, prompt, model))
+
+ if parallel_items:
+ self._analyze_parallel(parallel_items, completed_count)
+ completed_count += len(parallel_items)
+
+ for item in sequential_items:
+ completed_count += 1
+ func = item.function
self._emit(Event(
- type=EventType.FUNCTION_COMPLETE,
+ type=EventType.FUNCTION_START,
phase=Phase.ANALYSIS,
- message=(
- f"{func.address_hex} → {result.name} "
- f"(confidence: {result.confidence}%)"
- ),
+ message=f"Analyzing {func.name} ({func.address_hex})...",
data={
"address": func.address,
- "original_name": result.original_name,
- "name": result.name,
- "confidence": result.confidence,
- "classification": result.classification,
+ "name": func.name,
+ "size": func.size,
+ "depth": item.depth,
+ "progress": f"{completed_count}/{self.queue.total}",
},
))
+ try:
+ result = self._analyze_function_sequential(item)
+ except Exception as e:
+ result = FunctionResult(
+ address=func.address,
+ original_name=func.name,
+ error=str(e),
+ )
+ self._emit(Event(
+ type=EventType.FUNCTION_ERROR,
+ phase=Phase.ANALYSIS,
+ message=f"Error analyzing {func.name}: {e}",
+ data={"address": func.address, "error": str(e)},
+ ))
+ self._record_analysis_result(func, result)
self._emit(Event(
type=EventType.PHASE_COMPLETE,
@@ -296,26 +338,86 @@ class Supervisor:
),
))
- def _analyze_function(self, item: WorkItem) -> FunctionResult:
- """Analyze a single function. Delegates to the Analyzer agent."""
- func = item.function
+ def _pick_model(self, item: WorkItem, decompilation: str) -> str:
+ """Choose Haiku for small/simple functions, Sonnet for complex ones."""
+ line_count = decompilation.count("\n")
+ if item.function.size <= 200 and len(item.callees) <= 2 and line_count <= 30:
+ return HAIKU_MODEL
+ return SONNET_MODEL
+
+ def _analyze_parallel(
+ self,
+ items: list[tuple[WorkItem, str, str]],
+ completed_base: int,
+ ) -> None:
+ """Send LLM calls in parallel for a batch of non-obfuscated functions."""
+ for idx, (item, _, model) in enumerate(items):
+ func = item.function
+ self._emit(Event(
+ type=EventType.FUNCTION_START,
+ phase=Phase.ANALYSIS,
+ message=f"Analyzing {func.name} ({func.address_hex})...",
+ data={
+ "address": func.address,
+ "name": func.name,
+ "size": func.size,
+ "depth": item.depth,
+ "model": model.split("-")[1] if "-" in model else model,
+ "progress": f"{completed_base + idx + 1}/{self.queue.total}",
+ },
+ ))
+
+ def _call_llm(prompt: str, model: str) -> LLMResponse:
+ assert self.llm_client is not None
+ return self.llm_client.analyze_function(prompt, model=model)
+
+ futures_map: dict[object, WorkItem] = {}
+
+ with ThreadPoolExecutor(max_workers=MAX_PARALLEL_LLM) as executor:
+ for i, (item, prompt, model) in enumerate(items):
+ if i > 0 and i % MAX_PARALLEL_LLM == 0:
+ time.sleep(REQUEST_STAGGER_SECONDS)
+ future = executor.submit(_call_llm, prompt, model)
+ futures_map[future] = item
+
+ for future in as_completed(futures_map):
+ item = futures_map[future]
+ func = item.function
+ try:
+ response = future.result()
+ sig_applied = self._write_back_response(func.address, response)
+ result = FunctionResult(
+ address=func.address,
+ original_name=func.name,
+ name=response.name,
+ signature=response.signature,
+ confidence=response.confidence,
+ classification=response.classification,
+ comments=response.comments,
+ reasoning=response.reasoning,
+ llm_calls=1,
+ signature_applied=sig_applied,
+ struct_proposals=response.struct_proposals,
+ )
+ except Exception as e:
+ result = FunctionResult(
+ address=func.address,
+ original_name=func.name,
+ error=str(e),
+ )
+ self._emit(Event(
+ type=EventType.FUNCTION_ERROR,
+ phase=Phase.ANALYSIS,
+ message=f"Error analyzing {func.name}: {e}",
+ data={"address": func.address, "error": str(e)},
+ ))
- if func.classification and func.classification.value == "trivial":
- return FunctionResult(
- address=func.address,
- original_name=func.name,
- skipped=True,
- skip_reason="trivial",
- )
-
- if self.llm_client is None:
- return FunctionResult(
- address=func.address,
- original_name=func.name,
- name=func.name,
- confidence=0,
- )
+ self._record_analysis_result(func, result)
+ def _analyze_function_sequential(self, item: WorkItem) -> FunctionResult:
+ """Analyze a single function sequentially (used for obfuscated functions)."""
+ assert self.llm_client is not None
+ func = item.function
deobfuscator = Deobfuscator(self.client, self.llm_client)
analyzer = Analyzer(self.client, self.llm_client, deobfuscator=deobfuscator)
result = analyzer.analyze(
@@ -323,6 +425,7 @@ class Supervisor:
binary_info=self.binary_info,
known_results=self.results,
strings=self.strings,
+ model=SONNET_MODEL,
)
if result.obfuscation_techniques:
@@ -342,6 +445,55 @@ class Supervisor:
return result
+ def _write_back_response(self, addr: int, response: LLMResponse) -> bool:
+ """Write LLM response back to Ghidra (rename, signature, comments)."""
+ sig_ok = True
+ if response.name:
+ try:
+ self.client.rename_function(addr, response.name)
+ except Exception as e:
+ logger.warning("Failed to rename 0x%08x to %s: %s", addr, response.name, e)
+
+ if response.signature:
+ try:
+ self.client.set_function_signature(addr, response.signature)
+ except Exception as e:
+ logger.debug("Signature deferred for 0x%08x (will retry in cleanup): %s", addr, e)
+ sig_ok = False
+
+ if response.comments:
+ try:
+ self.client.add_comment(addr, response.comments)
+ except Exception as e:
+ logger.debug("Failed to add comment at 0x%08x: %s", addr, e)
+
+ return sig_ok
+
+ def _record_analysis_result(self, func: FunctionInfo, result: FunctionResult) -> None:
+ """Record a function result into the results dict and stats."""
+ self.results[func.address] = result
+ self.stats.record_result(result)
+
+ if result.struct_proposals:
+ self.struct_accumulator.add_proposals(func.address, result.struct_proposals)
+
+ if not result.error and not result.skipped:
+ self._emit(Event(
+ type=EventType.FUNCTION_COMPLETE,
+ phase=Phase.ANALYSIS,
+ message=(
+ f"{func.address_hex} → {result.name} "
+ f"(confidence: {result.confidence}%)"
+ ),
+ data={
+ "address": func.address,
+ "original_name": result.original_name,
+ "name": result.name,
+ "confidence": result.confidence,
+ "classification": result.classification,
+ },
+ ))
+
def _run_cleanup(self) -> None:
"""Unify struct types, apply to Ghidra, re-analyze affected and low-confidence functions."""
self._emit(Event(
@@ -536,6 +688,8 @@ class Supervisor:
"total_functions": s.total_functions,
"analyzed": s.analyzed,
"named": s.named,
+ "renamed": s.renamed,
+ "confirmed": s.confirmed,
"high_confidence": s.high_confidence,
"medium_confidence": s.medium_confidence,
"low_confidence": s.low_confidence,
diff --git a/kong/agent/type_recovery.py b/kong/agent/type_recovery.py
index 1d21f6d..9ae79d9 100644
--- a/kong/agent/type_recovery.py
+++ b/kong/agent/type_recovery.py
@@ -147,7 +147,7 @@ class StructAccumulator:
size=best.size,
))
- max_size = max(p.total_size for p in group)
+ max_size = max((p.total_size for p in group if p.total_size is not None), default=0)
last_field_end = max(
(f.offset + f.size for f in merged_fields),
default=0,
diff --git a/tests/test_agent_supervisor.py b/tests/test_agent_supervisor.py
index 300fd4e..3290428 100644
--- a/tests/test_agent_supervisor.py
+++ b/tests/test_agent_supervisor.py
@@ -127,42 +127,50 @@ class TestSupervisorAnalysis:
assert addr in results
def test_function_start_and_complete_events(self, tmp_path):
+ from kong.agent.analyzer import LLMResponse
+
funcs = [_func(0x1000, "a")]
client = _make_client(functions=funcs)
+ client.get_decompilation.return_value = "void a(void) { return; }"
config = KongConfig(output=OutputConfig(directory=tmp_path / "out"))
- sup = Supervisor(client, config)
+
+ mock_llm = MagicMock()
+ mock_llm.analyze_function.return_value = LLMResponse(
+ name="init", confidence=80, raw='{"name":"init","confidence":80}',
+ )
+
+ sup = Supervisor(client, config, llm_client=mock_llm)
events = []
sup.on_event(events.append)
sup.run()
starts = [e for e in events if e.type == EventType.FUNCTION_START]
- assert len(starts) == 1
- assert starts[0].data["address"] == 0x1000
+ assert len(starts) >= 1
+ assert any(e.data["address"] == 0x1000 for e in starts)
def test_handles_analysis_error(self, tmp_path):
funcs = [_func(0x1000, "a")]
client = _make_client(functions=funcs)
+ client.get_decompilation.return_value = "void a(void) { return; }"
config = KongConfig(output=OutputConfig(directory=tmp_path / "out"))
- sup = Supervisor(client, config)
- # Force an error in _analyze_function
- def explode(item):
- raise RuntimeError("LLM timeout")
+ mock_llm = MagicMock()
+ mock_llm.analyze_function.side_effect = RuntimeError("LLM timeout")
- sup._analyze_function = explode
+ sup = Supervisor(client, config, llm_client=mock_llm)
events = []
sup.on_event(events.append)
results = sup.run()
- assert results[0x1000].error == "LLM timeout"
+ assert results[0x1000].error
error_events = [e for e in events if e.type == EventType.FUNCTION_ERROR]
assert len(error_events) == 1
class TestAnalysisStats:
- def test_record_result_named(self):
+ def test_record_result_renamed(self):
stats = AnalysisStats(total_functions=10)
result = FunctionResult(
address=0x1000, original_name="FUN_1000",
@@ -171,6 +179,7 @@ class TestAnalysisStats:
stats.record_result(result)
assert stats.analyzed == 1
+ assert stats.renamed == 1
assert stats.named == 1
assert stats.high_confidence == 1
assert stats.llm_calls == 2
@@ -197,7 +206,7 @@ class TestAnalysisStats:
assert stats.errors == 1
assert stats.analyzed == 0
- def test_record_result_unchanged_name_not_counted(self):
+ def test_record_result_unchanged_name_is_confirmed(self):
stats = AnalysisStats(total_functions=10)
result = FunctionResult(
address=0x1000, original_name="FUN_1000",
@@ -206,7 +215,9 @@ class TestAnalysisStats:
stats.record_result(result)
assert stats.analyzed == 1
- assert stats.named == 0 # name unchanged
+ assert stats.renamed == 0
+ assert stats.confirmed == 1
+ assert stats.named == 1
assert stats.low_confidence == 1
def test_confidence_buckets(self):
@@ -222,7 +233,8 @@ class TestAnalysisStats:
def test_name_rate(self):
stats = AnalysisStats(total_functions=4)
- stats.named = 3
+ stats.renamed = 2
+ stats.confirmed = 1
assert stats.name_rate == 0.75
def test_name_rate_zero_functions(self):
diff --git a/tests/test_type_recovery.py b/tests/test_type_recovery.py
index 99464c7..47796c3 100644
--- a/tests/test_type_recovery.py
+++ b/tests/test_type_recovery.py
@@ -597,6 +597,8 @@ class TestSupervisorCleanupIntegration:
client.get_strings.return_value = []
client.get_callers.return_value = []
client.get_callees.return_value = []
+ client.get_decompilation.return_value = "void f() {}"
+ client.get_xrefs_from.return_value = []
response = LLMResponse(name="helper", confidence=80)
llm = MagicMock()
@@ -640,7 +642,7 @@ class TestSupervisorCleanupIntegration:
call_count = 0
- def varying_response(prompt: str) -> LLMResponse:
+ def varying_response(prompt: str, **kwargs: object) -> LLMResponse:
nonlocal call_count
call_count += 1
if call_count == 1: