Ghidrust provides a hyper fast, AI native reverse engineering platform built in Rust, inspired by Ghidra. Includes experimental GPU kernels to rapidly decompile large files, a decryption suite, and network monitor for packet to binary reverse engineering.
Update Ghidrust CLI to enhance `ghidra-headtohead` command with new options for spawn timeout and function cap. Modify documentation to clarify usage and improve error handling in Ghidra integration. Add new entries to .gitignore for head-to-head output files.
oofz9b940d4163ab
diff --git a/.gitignore b/.gitignoreindex f22c32d..d8ef8a2 100644--- a/.gitignore+++ b/.gitignore@@ -18,6 +18,8 @@ Thumbs.db*.gdecomp*_metrics.logre_metrics.log+headtohead_*.json+headtohead_*.txt# Local development notes (parity tracking, roadmaps, private benches)# Not published — research/metrics stay in docs/diff --git a/crates/ghidrust-cli/src/main.rs b/crates/ghidrust-cli/src/main.rsindex f9ffdcf..e8ded84 100644--- a/crates/ghidrust-cli/src/main.rs+++ b/crates/ghidrust-cli/src/main.rs@@ -68,7 +68,7 @@ fn print_help() {ghidrust bulk-bench <path> [--json] # seq vs parallel vs GPU/fallback timings\n\ghidrust decompile <path> [--addr HEX] [--count N] [--stage05|--stage1] [--json]\n\ghidrust decompile-bench <path> [--functions N] [--count N] [--out FILE] [--json]\n\- ghidrust ghidra-headtohead <path> [--functions N] [--count N] [--ghidra DIR] [--captured JSON] [--out FILE] [--json]\n\+ ghidrust ghidra-headtohead <path> [--functions N] [--count N] [--ghidra DIR] [--captured JSON] [--out FILE] [--spawn-timeout SECS] [--ghidra-fn-cap N] [--json]\n\ghidrust gpu-decompile <path> [--out FILE] [--metrics FILE] [--json]\n\ghidrust re-bench <path> [--out FILE] [--json] # CPU then GPU metrics (decomp+bulk)\n\ghidrust analyzer-bench <path> [--large] [--out FILE] [--json]\n\@@ -1145,7 +1145,7 @@ fn cmd_ghidra_headtohead(args: &[String], json: bool) -> ExitCode {Some(p) => PathBuf::from(p),None => {eprintln!(- "usage: ghidrust ghidra-headtohead <path> [--functions N] [--count N] \\\n [--ghidra DIR] [--captured JSON] [--out FILE] [--json]"+ "usage: ghidrust ghidra-headtohead <path> [--functions N] [--count N] \\\n [--ghidra DIR] [--captured JSON] [--out FILE] \\\n [--spawn-timeout SECS] [--ghidra-fn-cap N] [--json]");return ExitCode::from(2);}@@ -1155,6 +1155,8 @@ fn cmd_ghidra_headtohead(args: &[String], json: bool) -> ExitCode {let mut out_path: Option<PathBuf> = None;let mut ghidra_dir: Option<PathBuf> = None;let mut captured_path: Option<PathBuf> = None;+ let mut spawn_timeout_secs: Option<u64> = None;+ let mut ghidra_fn_cap: Option<usize> = None;let mut i = 1;while i < args.len() {match args[i].as_str() {@@ -1178,6 +1180,14 @@ fn cmd_ghidra_headtohead(args: &[String], json: bool) -> ExitCode {captured_path = Some(PathBuf::from(&args[i + 1]));i += 2;}+ "--spawn-timeout" if i + 1 < args.len() => {+ spawn_timeout_secs = args[i + 1].parse().ok();+ i += 2;+ }+ "--ghidra-fn-cap" if i + 1 < args.len() => {+ ghidra_fn_cap = args[i + 1].parse().ok();+ i += 2;+ }_ => i += 1,}}@@ -1212,9 +1222,9 @@ fn cmd_ghidra_headtohead(args: &[String], json: bool) -> ExitCode {max_functions,max_insns_per_fn: count,captured_ghidra_decompiles: captured,- // Forward the binary path so `--ghidra DIR` can auto-spawn- // analyzeHeadless against the same bytes Ghidrust just benched.binary_path: Some(path.clone()),+ spawn_timeout_secs,+ ghidra_fn_cap,..Default::default()};let report = ghidrust_decomp::ghidra_headtohead(&prog, &cfg);diff --git a/crates/ghidrust-decomp/src/ghidra_oracle.rs b/crates/ghidrust-decomp/src/ghidra_oracle.rsindex 034cdb2..5b6171a 100644--- a/crates/ghidrust-decomp/src/ghidra_oracle.rs+++ b/crates/ghidrust-decomp/src/ghidra_oracle.rs@@ -72,6 +72,15 @@ pub struct GhidraOracleConfig {/// back to a 5-minute cap when unspecified so runaway JVMs don't/// deadlock the harness.pub spawn_timeout_secs: Option<u64>,+ /// Optional cap on how many functions the Ghidra post-script will+ /// decompile. The Java loop walks `getFunctions(true)` and stops after+ /// this many rows. `None` = decompile every function in the imported+ /// binary (only tractable on small fixtures).+ ///+ /// Wired through as the second `-postScript` argument+ /// (`DecompileAndReport.java <out.json> <max_fn>`); the Java script+ /// treats a missing or non-numeric value as "no cap".+ pub ghidra_fn_cap: Option<usize>,}impl GhidraOracleConfig {@@ -337,7 +346,15 @@ pub enum GhidraSpawnError {NonZeroExit { code: Option<i32>, stderr: String },Timeout(Duration),ParseCapture(String),- NoCaptureEmitted(PathBuf),+ /// analyzeHeadless exited 0 but never wrote our capture JSON (typical+ /// causes: post-script compile error, wrong output path, script threw).+ /// The scratch dir is intentionally left on disk so callers can inspect+ /// the source, compiled classes, and any partial output.+ NoCaptureEmitted {+ path: PathBuf,+ scratch: PathBuf,+ stderr_tail: String,+ },}impl std::fmt::Display for GhidraSpawnError {@@ -354,7 +371,15 @@ impl std::fmt::Display for GhidraSpawnError {}Self::Timeout(d) => write!(f, "analyzeHeadless timed out after {}s", d.as_secs()),Self::ParseCapture(e) => write!(f, "capture JSON parse failed: {e}"),- Self::NoCaptureEmitted(p) => write!(f, "no capture file written at {}", p.display()),+ Self::NoCaptureEmitted { path, scratch, stderr_tail } => {+ write!(+ f,+ "no capture file written at {} (scratch kept at {} for inspection); stderr tail: {}",+ path.display(),+ scratch.display(),+ if stderr_tail.is_empty() { "<empty>" } else { stderr_tail.as_str() }+ )+ }}}}@@ -395,18 +420,57 @@ import ghidra.program.model.listing.Function;import java.io.PrintWriter;public class DecompileAndReport extends GhidraScript {+ // Escape a Java string for embedding inside a JSON string literal.+ // Handles backslash, double-quote, and ALL control characters+ // (0x00-0x1F) that a strict JSON parser (serde_json) rejects unless+ // escaped. NOTE: this comment intentionally avoids the six-character+ // sequence "backslash+u+four-hex" because the Java lexer processes+ // that sequence BEFORE comments are recognized and would raise+ // "illegal unicode escape" at compile time.+ private static String jsonEscape(String s) {+ if (s == null) return "";+ StringBuilder sb = new StringBuilder(s.length() + 16);+ for (int i = 0; i < s.length(); i++) {+ char c = s.charAt(i);+ switch (c) {+ case '\\': sb.append("\\\\"); break;+ case '"': sb.append("\\\""); break;+ case '\b': sb.append("\\b"); break;+ case '\f': sb.append("\\f"); break;+ case '\n': sb.append("\\n"); break;+ case '\r': sb.append("\\r"); break;+ case '\t': sb.append("\\t"); break;+ default:+ if (c < 0x20) {+ sb.append(String.format("\\u%04x", (int) c));+ } else {+ sb.append(c);+ }+ }+ }+ return sb.toString();+ }+public void run() throws Exception {String[] argv = getScriptArgs();if (argv.length < 1) {throw new IllegalArgumentException("expected output json path as script arg");}+ // Optional second arg: max function count. Missing/non-numeric = no cap.+ int maxFn = Integer.MAX_VALUE;+ if (argv.length >= 2) {+ try { maxFn = Integer.parseInt(argv[1]); } catch (NumberFormatException ignored) {}+ if (maxFn <= 0) maxFn = Integer.MAX_VALUE;+ }DecompInterface dc = new DecompInterface();dc.setOptions(new DecompileOptions());dc.openProgram(currentProgram);try (PrintWriter out = new PrintWriter(argv[0])) {out.println("[");boolean first = true;+ int emitted = 0;for (Function f : currentProgram.getFunctionManager().getFunctions(true)) {+ if (emitted >= maxFn) break;long t0 = System.nanoTime();DecompileResults r = dc.decompileFunction(f, 30, monitor);long t1 = System.nanoTime();@@ -414,14 +478,14 @@ public class DecompileAndReport extends GhidraScript {first = false;String src = "";if (r != null && r.getDecompiledFunction() != null) {- src = r.getDecompiledFunction().getC()- .replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n");+ src = jsonEscape(r.getDecompiledFunction().getC());}out.printf("{\"name\":\"%s\",\"entry\":%d,\"wall_us\":%d,\"c_source\":\"%s\"}",- f.getName().replace("\"","\\\""),+ jsonEscape(f.getName()),f.getEntryPoint().getOffset(),(t1 - t0) / 1000L,src);+ emitted++;}out.println("]");}@@ -480,8 +544,15 @@ pub fn spawn_ghidra_headless(.arg(&scratch).arg("-postScript").arg("DecompileAndReport.java")- .arg(&out_json)- .arg("-deleteProject");+ .arg(&out_json);+ if let Some(cap) = cfg.ghidra_fn_cap {+ // Second post-script arg is an optional function cap. Any positive+ // integer here makes Ghidra stop after decompiling that many+ // functions instead of every function in the binary — essential+ // for keeping large-image spawns under the wall-clock timeout.+ cmd.arg(cap.to_string());+ }+ cmd.arg("-deleteProject");let output = run_with_timeout(cmd, timeout).map_err(|e| match e {RunErr::Spawn(msg) => GhidraSpawnError::Spawn(msg),@@ -497,8 +568,22 @@ pub fn spawn_ghidra_headless(}if !out_json.is_file() {- let _ = std::fs::remove_dir_all(&scratch);- return Err(GhidraSpawnError::NoCaptureEmitted(out_json));+ // Keep the scratch dir around for post-mortem — Ghidra headless may+ // have written the capture at an unexpected path, or the post-script+ // may have thrown. Surface a stderr tail so callers don't have to+ // fish through logs.+ let stderr_txt = String::from_utf8_lossy(&output.stderr).to_string();+ let mut lines: Vec<&str> = stderr_txt.lines().collect();+ let n = lines.len();+ if n > 30 {+ lines = lines.split_off(n - 30);+ }+ let stderr_tail = lines.join("\n");+ return Err(GhidraSpawnError::NoCaptureEmitted {+ path: out_json,+ scratch: scratch.clone(),+ stderr_tail,+ });}let text = std::fs::read_to_string(&out_json).map_err(|e| GhidraSpawnError::ParseCapture(e.to_string()))?;diff --git a/docs/GHIDRA_HEADTOHEAD.md b/docs/GHIDRA_HEADTOHEAD.mdindex 6ae747d..280d0b6 100644--- a/docs/GHIDRA_HEADTOHEAD.md+++ b/docs/GHIDRA_HEADTOHEAD.md@@ -8,14 +8,12 @@ with the [decompiler superiority roadmap](../../.cursor/plans/decompiler_superio— specifically **Phase C** ("Faster than Ghidra") which requires capturedhead-to-head numbers, never invented ones.-`ghidra-headtohead --ghidra <DIR>` now auto-spawns `analyzeHeadless` on the-supplied binary using the embedded `DecompileAndReport.java` post-script-(source lives in `crates/ghidrust-decomp/src/ghidra_oracle.rs :: DECOMPILE_AND_REPORT_JAVA`).-When spawn succeeds the report includes real per-function `wall_us`-timings and structural matches. When spawn fails (missing install,-unreadable binary, non-zero exit, JVM timeout, parse error), the report-records the exact failure reason via [`GhidraSpawnError`] and leaves the-Ghidra column blank — no fabricated numbers, ever.+`ghidra-headtohead --ghidra <DIR>` auto-spawns `analyzeHeadless` with an+embedded `DecompileAndReport.java` post-script (see+`crates/ghidrust-decomp/src/ghidra_oracle.rs`). Spawn failures leave the+Ghidra column blank — no fabricated numbers. **Do not treat current+ledger rows as a quality or speed win over Ghidra** until both sides+share the same function-entry list and a real quality metric (see §5).---@@ -41,7 +39,7 @@ Ghidra column blank — no fabricated numbers, ever.## Runbook — capturing Ghidra headless output-### 1. Install Ghidra 11.x+### 1. Install Ghidra 11.x / 12.x- Linux/macOS: unpack the release, ensure `<ghidraDir>/support/analyzeHeadless`is executable.- Windows: use `<ghidraDir>\support\analyzeHeadless.bat`.@@ -150,17 +148,41 @@ The report includes:* `methodology` — this runbook, embedded so external readers can reproducethe numbers.-### 5. Publish tables+### 5. What this harness is (and is not) for-Landed timings live under `docs/headtohead/` (git-tracked) so every claim is-reproducible. Add a new row each time the harness runs on a new machine:+**Useful for:** wiring a reproducible spawn/capture path, locking Ghidrust+baselines (`--json` without Ghidra), and future *fair* comparisons once both+sides decompile the **same entry addresses**.-| Date | Host | Binary | Ghidrust wall (Stage-0) | Ghidrust wall (Stage-0.5) | Ghidra wall | Notes |-|------|------|--------|------------------------|---------------------------|-------------|-------|-| _(fill in from actual capture)_ | | | | | | |+**Not useful for marketing “we beat Ghidra” today.** Early live runs showed+large `MissingGhidra` counts because each tool took its own first *N*+functions — different work sets, so summed timings are not comparable. The+`Similar` label is only a coarse block-count proxy (`floor(ir_ops/8)` vs `{`+count in Ghidra C), not “same C.” Those run tables were **removed** from this+doc so we do not publish an inaccurate head-to-head.-The row **must** cite the captured `headtohead.json` and Ghidra version. No-row without a captured file — the roadmap forbids fabricated numbers.+**Fair comparison (future):** fix a shared address list (same entries both+sides), measure Stage-1 vs Ghidra C on those entries, and compare quality with+a real structural/AST metric — then publish tables under `docs/headtohead/`.++### Why Ghidrust Stage-0 / 0.5 looks “impossibly fast”++Ghidra’s decompiler runs the heavy pipeline: pcode lift, SSA/dataflow, type+propagation, high-level structuring, and pretty-printed C. That is milliseconds+per function for a reason.++Ghidrust Stage-0 / Stage-0.5 (what the harness times today) does **much less**:++| Stage | Work | Roughly like… |+|-------|------|----------------|+| Stage-0 | Decode + CFG → goto scaffolding | Fast structured listing |+| Stage-0.5 | + x86→IR lift + assignment-ish emit | Lift, not full decompile |+| Stage-1 | + SSA / structure / types (partial) | Closer, still not Ghidra C |+| Ghidra | Full DecompInterface C | The expensive product |++So a μs vs ms gap mostly means **we are not doing the same job yet**, not that+we finished Ghidra’s work faster. Native Rust and no JVM help, but they are+secondary next to the work-product difference.---