The world's first agentic reverse engineer.
debug agent, update tests/gitignore
Amruth86833dcf1d6a
diff --git a/.gitignore b/.gitignoreindex 8177e76..100d49f 100644--- a/.gitignore+++ b/.gitignore@@ -217,4 +217,4 @@ __marimo__/.plist#internal testing-internal_testing/\ No newline at end of file+internal-testing/\ No newline at end of filediff --git a/kong/__main__.py b/kong/__main__.pyindex 5fea1d8..87f4ecb 100644--- a/kong/__main__.py+++ b/kong/__main__.py@@ -127,12 +127,21 @@ def analyze(finally:stats = supervisor.statsconsole.print()- console.print(f"[bold]Results:[/bold] {stats.named}/{stats.total_functions} functions named")+ console.print(+ f"[bold]Results:[/bold] {stats.named}/{stats.total_functions} functions named "+ f"({stats.renamed} renamed, {stats.confirmed} confirmed)"+ )console.print(f"[bold]Confidence:[/bold] {stats.high_confidence} high, "f"{stats.medium_confidence} med, {stats.low_confidence} low")console.print(f"[bold]LLM calls:[/bold] {stats.llm_calls}")+ for model_name, mu in llm_client.usage.by_model.items():+ short_name = model_name.split("-")[1] if "-" in model_name else model_name+ console.print(+ f" {short_name}: {mu.calls} calls, "+ f"${mu.cost_usd(model_name):.4f}"+ )console.print(f"[bold]Cost:[/bold] ${llm_client.total_cost_usd:.4f}")console.print(f"[bold]Duration:[/bold] {stats.duration_seconds:.1f}s")client.close()diff --git a/kong/agent/analyzer.py b/kong/agent/analyzer.pyindex 5d9311b..b1fed84 100644--- a/kong/agent/analyzer.py+++ b/kong/agent/analyzer.py@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)class LLMClient(Protocol):"""Protocol for LLM interaction."""- def analyze_function(self, prompt: str) -> LLMResponse: ...+ def analyze_function(self, prompt: str, *, model: str | None = None) -> LLMResponse: ...def analyze_with_tools(self,@@ -128,6 +128,7 @@ class Analyzer:known_results: dict[int, FunctionResult],strings: list[StringEntry],known_types: list[StructDefinition] | None = None,+ model: str | None = None,) -> FunctionResult:"""Full analysis pipeline for one function."""from kong.agent.deobfuscator import Deobfuscator, classify_obfuscation@@ -157,7 +158,7 @@ class Analyzer:)prompt = self._build_prompt(context)- response = self.llm.analyze_function(prompt)+ response = self.llm.analyze_function(prompt, model=model)sig_applied = self._write_back(func.address, response)return FunctionResult(@@ -363,7 +364,7 @@ class Analyzer:try:self.client.set_function_signature(addr, response.signature)except Exception as e:- logger.warning("Failed to set signature at 0x%08x: %s", addr, e)+ logger.debug("Signature deferred for 0x%08x (will retry in cleanup): %s", addr, e)signature_ok = Falseif response.comments:@@ -408,20 +409,24 @@ class Analyzer:struct_proposals = []for sp in data.get("struct_proposals", []):- if "name" not in sp or "total_size" not in sp:+ if not sp.get("name") or not sp.get("total_size"):+ continue+ try:+ total_size = int(sp["total_size"])+ except (ValueError, TypeError):continuefields = [StructFieldProposal(name=f.get("name", f"field_{i}"),data_type=f.get("data_type", "undefined"),- offset=f.get("offset", 0),- size=f.get("size", 4),+ offset=int(f.get("offset", 0)),+ size=int(f.get("size", 4)),)for i, f in enumerate(sp.get("fields", []))]struct_proposals.append(StructProposal(name=sp["name"],- total_size=sp["total_size"],+ total_size=total_size,fields=fields,used_by_param=sp.get("used_by_param", ""),))diff --git a/kong/agent/queue.py b/kong/agent/queue.pyindex c98b492..8aac767 100644--- a/kong/agent/queue.py+++ b/kong/agent/queue.py@@ -164,5 +164,14 @@ class WorkQueue:def get_by_address(self, addr: int) -> WorkItem | None:return self._by_address.get(addr)+ def items_by_depth(self) -> list[list[WorkItem]]:+ """Return items grouped by depth level, sorted ascending (leaves first)."""+ if not self._items:+ return []+ depth_map: dict[int, list[WorkItem]] = {}+ for item in self._items:+ depth_map.setdefault(item.depth, []).append(item)+ return [depth_map[d] for d in sorted(depth_map)]+def all_items(self) -> list[WorkItem]:return list(self._items)diff --git a/kong/agent/supervisor.py b/kong/agent/supervisor.pyindex 6fbd4bd..34fb7f1 100644--- a/kong/agent/supervisor.py+++ b/kong/agent/supervisor.py@@ -8,10 +8,11 @@ from __future__ import annotationsimport loggingimport time+from concurrent.futures import ThreadPoolExecutor, as_completedfrom dataclasses import dataclass-from kong.agent.analyzer import Analyzer, LLMClient-from kong.agent.deobfuscator import Deobfuscator+from kong.agent.analyzer import Analyzer, LLMClient, LLMResponse+from kong.agent.deobfuscator import Deobfuscator, classify_obfuscationfrom kong.agent.events import Event, EventCallback, EventType, Phasefrom kong.agent.models import FunctionResultfrom kong.agent.queue import WorkItem, WorkQueue@@ -24,13 +25,20 @@ from kong.ghidra.types import BinaryInfo, FunctionInfo, StringEntrylogger = logging.getLogger(__name__)+HAIKU_MODEL = "claude-haiku-4-5-20251001"+SONNET_MODEL = "claude-sonnet-4-20250514"++MAX_PARALLEL_LLM = 4+REQUEST_STAGGER_SECONDS = 0.5+@dataclassclass AnalysisStats:"""Aggregate statistics for the full run."""total_functions: int = 0analyzed: int = 0- named: int = 0+ renamed: int = 0+ confirmed: int = 0high_confidence: int = 0 # >= 80medium_confidence: int = 0 # 50-79low_confidence: int = 0 # < 50@@ -41,6 +49,10 @@ class AnalysisStats:end_time: float = 0.0signature_matches: int = 0+ @property+ def named(self) -> int:+ return self.renamed + self.confirmed+@propertydef duration_seconds(self) -> float:end = self.end_time or time.time()@@ -59,8 +71,11 @@ class AnalysisStats:returnself.analyzed += 1self.llm_calls += result.llm_calls- if result.name and result.name != result.original_name:- self.named += 1+ if result.name:+ if result.name != result.original_name:+ self.renamed += 1+ else:+ self.confirmed += 1# TODO: calibrate these eventually. these buckets are arbitrary, need eval data to# determine meaningful confidence tiers for the LLM's self-reported scores.if result.confidence >= 80:@@ -219,73 +234,100 @@ class Supervisor:def _run_analysis(self) -> None:- """Analyze each function bottom-up via LLM."""+ """Analyze functions bottom-up via LLM, parallelized within each depth level."""self._emit(Event(type=EventType.PHASE_START,phase=Phase.ANALYSIS,message="Starting bottom-up analysis...",))- while self.queue:+ depth_batches = self.queue.items_by_depth()+ completed_count = 0++ for batch in depth_batches:if self._paused:time.sleep(0.1)continue- item = self.queue.next()- if item is None:- break+ parallel_items: list[tuple[WorkItem, str, str]] = []+ sequential_items: list[WorkItem] = []+ analyzer = Analyzer(self.client, self.llm_client) if self.llm_client else None- func = item.function- self._emit(Event(- type=EventType.FUNCTION_START,- phase=Phase.ANALYSIS,- message=f"Analyzing {func.name} ({func.address_hex})...",- data={- "address": func.address,- "name": func.name,- "size": func.size,- "depth": item.depth,- "progress": f"{self.queue.completed}/{self.queue.total}",- },- ))+ for item in batch:+ func = item.function- try:- result = self._analyze_function(item)- except Exception as e:- result = FunctionResult(- address=func.address,- original_name=func.name,- error=str(e),- )- self._emit(Event(- type=EventType.FUNCTION_ERROR,- phase=Phase.ANALYSIS,- message=f"Error analyzing {func.name}: {e}",- data={"address": func.address, "error": str(e)},- ))+ if func.classification and func.classification.value == "trivial":+ result = FunctionResult(+ address=func.address,+ original_name=func.name,+ skipped=True,+ skip_reason="trivial",+ )+ self.results[func.address] = result+ self.stats.record_result(result)+ completed_count += 1+ continue++ if analyzer is None:+ result = FunctionResult(+ address=func.address,+ original_name=func.name,+ name=func.name,+ confidence=0,+ )+ self.results[func.address] = result+ self.stats.record_result(result)+ completed_count += 1+ continue- self.results[func.address] = result- self.stats.record_result(result)+ decompilation = self.client.get_decompilation(func.address)+ techniques = classify_obfuscation(decompilation)- if result.struct_proposals:- self.struct_accumulator.add_proposals(func.address, result.struct_proposals)+ if techniques:+ sequential_items.append(item)+ continue- if not result.error and not result.skipped:+ model = self._pick_model(item, decompilation)+ context = analyzer._build_context(+ item, self.binary_info, self.results, self.strings,+ )+ prompt = analyzer._build_prompt(context)+ parallel_items.append((item, prompt, model))++ if parallel_items:+ self._analyze_parallel(parallel_items, completed_count)+ completed_count += len(parallel_items)++ for item in sequential_items:+ completed_count += 1+ func = item.functionself._emit(Event(- type=EventType.FUNCTION_COMPLETE,+ type=EventType.FUNCTION_START,phase=Phase.ANALYSIS,- message=(- f"{func.address_hex} → {result.name} "- f"(confidence: {result.confidence}%)"- ),+ message=f"Analyzing {func.name} ({func.address_hex})...",data={"address": func.address,- "original_name": result.original_name,- "name": result.name,- "confidence": result.confidence,- "classification": result.classification,+ "name": func.name,+ "size": func.size,+ "depth": item.depth,+ "progress": f"{completed_count}/{self.queue.total}",},))+ try:+ result = self._analyze_function_sequential(item)+ except Exception as e:+ result = FunctionResult(+ address=func.address,+ original_name=func.name,+ error=str(e),+ )+ self._emit(Event(+ type=EventType.FUNCTION_ERROR,+ phase=Phase.ANALYSIS,+ message=f"Error analyzing {func.name}: {e}",+ data={"address": func.address, "error": str(e)},+ ))+ self._record_analysis_result(func, result)self._emit(Event(type=EventType.PHASE_COMPLETE,@@ -296,26 +338,86 @@ class Supervisor:),))- def _analyze_function(self, item: WorkItem) -> FunctionResult:- """Analyze a single function. Delegates to the Analyzer agent."""- func = item.function+ def _pick_model(self, item: WorkItem, decompilation: str) -> str:+ """Choose Haiku for small/simple functions, Sonnet for complex ones."""+ line_count = decompilation.count("\n")+ if item.function.size <= 200 and len(item.callees) <= 2 and line_count <= 30:+ return HAIKU_MODEL+ return SONNET_MODEL++ def _analyze_parallel(+ self,+ items: list[tuple[WorkItem, str, str]],+ completed_base: int,+ ) -> None:+ """Send LLM calls in parallel for a batch of non-obfuscated functions."""+ for idx, (item, _, model) in enumerate(items):+ func = item.function+ self._emit(Event(+ type=EventType.FUNCTION_START,+ phase=Phase.ANALYSIS,+ message=f"Analyzing {func.name} ({func.address_hex})...",+ data={+ "address": func.address,+ "name": func.name,+ "size": func.size,+ "depth": item.depth,+ "model": model.split("-")[1] if "-" in model else model,+ "progress": f"{completed_base + idx + 1}/{self.queue.total}",+ },+ ))++ def _call_llm(prompt: str, model: str) -> LLMResponse:+ assert self.llm_client is not None+ return self.llm_client.analyze_function(prompt, model=model)++ futures_map: dict[object, WorkItem] = {}++ with ThreadPoolExecutor(max_workers=MAX_PARALLEL_LLM) as executor:+ for i, (item, prompt, model) in enumerate(items):+ if i > 0 and i % MAX_PARALLEL_LLM == 0:+ time.sleep(REQUEST_STAGGER_SECONDS)+ future = executor.submit(_call_llm, prompt, model)+ futures_map[future] = item++ for future in as_completed(futures_map):+ item = futures_map[future]+ func = item.function+ try:+ response = future.result()+ sig_applied = self._write_back_response(func.address, response)+ result = FunctionResult(+ address=func.address,+ original_name=func.name,+ name=response.name,+ signature=response.signature,+ confidence=response.confidence,+ classification=response.classification,+ comments=response.comments,+ reasoning=response.reasoning,+ llm_calls=1,+ signature_applied=sig_applied,+ struct_proposals=response.struct_proposals,+ )+ except Exception as e:+ result = FunctionResult(+ address=func.address,+ original_name=func.name,+ error=str(e),+ )+ self._emit(Event(+ type=EventType.FUNCTION_ERROR,+ phase=Phase.ANALYSIS,+ message=f"Error analyzing {func.name}: {e}",+ data={"address": func.address, "error": str(e)},+ ))- if func.classification and func.classification.value == "trivial":- return FunctionResult(- address=func.address,- original_name=func.name,- skipped=True,- skip_reason="trivial",- )-- if self.llm_client is None:- return FunctionResult(- address=func.address,- original_name=func.name,- name=func.name,- confidence=0,- )+ self._record_analysis_result(func, result)+ def _analyze_function_sequential(self, item: WorkItem) -> FunctionResult:+ """Analyze a single function sequentially (used for obfuscated functions)."""+ assert self.llm_client is not None+ func = item.functiondeobfuscator = Deobfuscator(self.client, self.llm_client)analyzer = Analyzer(self.client, self.llm_client, deobfuscator=deobfuscator)result = analyzer.analyze(@@ -323,6 +425,7 @@ class Supervisor:binary_info=self.binary_info,known_results=self.results,strings=self.strings,+ model=SONNET_MODEL,)if result.obfuscation_techniques:@@ -342,6 +445,55 @@ class Supervisor:return result+ def _write_back_response(self, addr: int, response: LLMResponse) -> bool:+ """Write LLM response back to Ghidra (rename, signature, comments)."""+ sig_ok = True+ if response.name:+ try:+ self.client.rename_function(addr, response.name)+ except Exception as e:+ logger.warning("Failed to rename 0x%08x to %s: %s", addr, response.name, e)++ if response.signature:+ try:+ self.client.set_function_signature(addr, response.signature)+ except Exception as e:+ logger.debug("Signature deferred for 0x%08x (will retry in cleanup): %s", addr, e)+ sig_ok = False++ if response.comments:+ try:+ self.client.add_comment(addr, response.comments)+ except Exception as e:+ logger.debug("Failed to add comment at 0x%08x: %s", addr, e)++ return sig_ok++ def _record_analysis_result(self, func: FunctionInfo, result: FunctionResult) -> None:+ """Record a function result into the results dict and stats."""+ self.results[func.address] = result+ self.stats.record_result(result)++ if result.struct_proposals:+ self.struct_accumulator.add_proposals(func.address, result.struct_proposals)++ if not result.error and not result.skipped:+ self._emit(Event(+ type=EventType.FUNCTION_COMPLETE,+ phase=Phase.ANALYSIS,+ message=(+ f"{func.address_hex} → {result.name} "+ f"(confidence: {result.confidence}%)"+ ),+ data={+ "address": func.address,+ "original_name": result.original_name,+ "name": result.name,+ "confidence": result.confidence,+ "classification": result.classification,+ },+ ))+def _run_cleanup(self) -> None:"""Unify struct types, apply to Ghidra, re-analyze affected and low-confidence functions."""self._emit(Event(@@ -536,6 +688,8 @@ class Supervisor:"total_functions": s.total_functions,"analyzed": s.analyzed,"named": s.named,+ "renamed": s.renamed,+ "confirmed": s.confirmed,"high_confidence": s.high_confidence,"medium_confidence": s.medium_confidence,"low_confidence": s.low_confidence,diff --git a/kong/agent/type_recovery.py b/kong/agent/type_recovery.pyindex 1d21f6d..9ae79d9 100644--- a/kong/agent/type_recovery.py+++ b/kong/agent/type_recovery.py@@ -147,7 +147,7 @@ class StructAccumulator:size=best.size,))- max_size = max(p.total_size for p in group)+ max_size = max((p.total_size for p in group if p.total_size is not None), default=0)last_field_end = max((f.offset + f.size for f in merged_fields),default=0,diff --git a/tests/test_agent_supervisor.py b/tests/test_agent_supervisor.pyindex 300fd4e..3290428 100644--- a/tests/test_agent_supervisor.py+++ b/tests/test_agent_supervisor.py@@ -127,42 +127,50 @@ class TestSupervisorAnalysis:assert addr in resultsdef test_function_start_and_complete_events(self, tmp_path):+ from kong.agent.analyzer import LLMResponse+funcs = [_func(0x1000, "a")]client = _make_client(functions=funcs)+ client.get_decompilation.return_value = "void a(void) { return; }"config = KongConfig(output=OutputConfig(directory=tmp_path / "out"))- sup = Supervisor(client, config)++ mock_llm = MagicMock()+ mock_llm.analyze_function.return_value = LLMResponse(+ name="init", confidence=80, raw='{"name":"init","confidence":80}',+ )++ sup = Supervisor(client, config, llm_client=mock_llm)events = []sup.on_event(events.append)sup.run()starts = [e for e in events if e.type == EventType.FUNCTION_START]- assert len(starts) == 1- assert starts[0].data["address"] == 0x1000+ assert len(starts) >= 1+ assert any(e.data["address"] == 0x1000 for e in starts)def test_handles_analysis_error(self, tmp_path):funcs = [_func(0x1000, "a")]client = _make_client(functions=funcs)+ client.get_decompilation.return_value = "void a(void) { return; }"config = KongConfig(output=OutputConfig(directory=tmp_path / "out"))- sup = Supervisor(client, config)- # Force an error in _analyze_function- def explode(item):- raise RuntimeError("LLM timeout")+ mock_llm = MagicMock()+ mock_llm.analyze_function.side_effect = RuntimeError("LLM timeout")- sup._analyze_function = explode+ sup = Supervisor(client, config, llm_client=mock_llm)events = []sup.on_event(events.append)results = sup.run()- assert results[0x1000].error == "LLM timeout"+ assert results[0x1000].errorerror_events = [e for e in events if e.type == EventType.FUNCTION_ERROR]assert len(error_events) == 1class TestAnalysisStats:- def test_record_result_named(self):+ def test_record_result_renamed(self):stats = AnalysisStats(total_functions=10)result = FunctionResult(address=0x1000, original_name="FUN_1000",@@ -171,6 +179,7 @@ class TestAnalysisStats:stats.record_result(result)assert stats.analyzed == 1+ assert stats.renamed == 1assert stats.named == 1assert stats.high_confidence == 1assert stats.llm_calls == 2@@ -197,7 +206,7 @@ class TestAnalysisStats:assert stats.errors == 1assert stats.analyzed == 0- def test_record_result_unchanged_name_not_counted(self):+ def test_record_result_unchanged_name_is_confirmed(self):stats = AnalysisStats(total_functions=10)result = FunctionResult(address=0x1000, original_name="FUN_1000",@@ -206,7 +215,9 @@ class TestAnalysisStats:stats.record_result(result)assert stats.analyzed == 1- assert stats.named == 0 # name unchanged+ assert stats.renamed == 0+ assert stats.confirmed == 1+ assert stats.named == 1assert stats.low_confidence == 1def test_confidence_buckets(self):@@ -222,7 +233,8 @@ class TestAnalysisStats:def test_name_rate(self):stats = AnalysisStats(total_functions=4)- stats.named = 3+ stats.renamed = 2+ stats.confirmed = 1assert stats.name_rate == 0.75def test_name_rate_zero_functions(self):diff --git a/tests/test_type_recovery.py b/tests/test_type_recovery.pyindex 99464c7..47796c3 100644--- a/tests/test_type_recovery.py+++ b/tests/test_type_recovery.py@@ -597,6 +597,8 @@ class TestSupervisorCleanupIntegration:client.get_strings.return_value = []client.get_callers.return_value = []client.get_callees.return_value = []+ client.get_decompilation.return_value = "void f() {}"+ client.get_xrefs_from.return_value = []response = LLMResponse(name="helper", confidence=80)llm = MagicMock()@@ -640,7 +642,7 @@ class TestSupervisorCleanupIntegration:call_count = 0- def varying_response(prompt: str) -> LLMResponse:+ def varying_response(prompt: str, **kwargs: object) -> LLMResponse:nonlocal call_countcall_count += 1if call_count == 1: