Ghidrust provides a hyper fast, AI native reverse engineering platform built in Rust, inspired by Ghidra. Includes experimental GPU kernels to rapidly decompile large files, a decryption suite, and network monitor for packet to binary reverse engineering.
//! Decompiler token model — `` for cross-highlight and nav.
//!
//! **Design note.** Stage-1 may supply emit-time tokens via
//! [`from_emit_tokens`]; otherwise this module tokenises the **pseudo-C
//! string** into a `Vec<Token>` with kinds and (optionally) a machine address
//! so the GUI can:
//!
//! * cross-highlight the Listing line when a token is clicked;
//! * navigate to a function/label on double-click;
//! * middle-click a variable to highlight every occurrence in the pane.
//!
//! When the decompiler eventually emits real token spans, replace [`tokenize`]
//! with a shim that just adopts those spans — the GUI consumer surface stays
//! the same.
/// One rendered token in the decompiler pane.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub text: String,
/// Machine address this token maps to, when known.
pub va: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
/// Keyword — `if`, `else`, `goto`, `return`, `void`, primitive types, …
Keyword,
/// Function name (declaration or call).
Function,
/// Local variable / parameter / register name.
Variable,
/// Block label — `block_3:` on the left; `block_3` in `goto block_3`.
Label,
/// Numeric literal — either a scalar or an address (`0x140001000`).
Constant,
/// Hex address literal (subclass of Constant with a resolved VA).
Address,
/// Comment (`// …` or `/* … */`).
Comment,
/// Punctuation / operator / whitespace glue.
Syntax,
/// Whitespace-only chunk (preserved so we can round-trip render).
Whitespace,
}
/// One decompiler line, with its (optional) source instruction address and its tokens.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DecompLine {
/// 0-based line index in the pseudo-C output.
pub line: usize,
/// Machine address the line maps to (first address token found on the line, if any).
pub machine_addr: Option<u64>,
/// Block label this line lives in, when detectable.
pub block_label: Option<String>,
pub tokens: Vec<Token>,
}
/// C-ish keywords used by the Stage-0 / Stage-0.5 emitter.
const C_KEYWORDS: &[&str] = &[
"if", "else", "goto", "return", "while", "for", "do", "switch", "case", "break", "continue",
"default", "void", "int", "char", "short", "long", "unsigned", "signed", "float", "double",
"bool", "true", "false", "null", "sizeof", "struct", "union", "enum",
];
/// Register names we always classify as `Variable` (x86-64 GPRs + segment/xmm subset).
const REGISTER_NAMES: &[&str] = &[
"rax", "rbx", "rcx", "rdx", "rsi", "rdi", "rbp", "rsp", "r8", "r9", "r10", "r11", "r12", "r13",
"r14", "r15", "eax", "ebx", "ecx", "edx", "esi", "edi", "ebp", "esp", "ax", "bx", "cx", "dx",
"si", "di", "bp", "sp", "al", "bl", "cl", "dl", "ah", "bh", "ch", "dh", "sil", "dil", "bpl",
"spl", "r8d", "r9d", "r10d", "r11d", "r12d", "r13d", "r14d", "r15d", "r8w", "r9w", "r10w",
"r11w", "r12w", "r13w", "r14w", "r15w", "r8b", "r9b", "r10b", "r11b", "r12b", "r13b", "r14b",
"r15b", "rip", "eip", "ip", "cs", "ds", "es", "fs", "gs", "ss",
];
/// Tokenise a full pseudo-C source blob into per-line spans.
pub fn tokenize(source: &str) -> Vec<DecompLine> {
let mut out = Vec::new();
let mut current_block: Option<String> = None;
for (idx, raw) in source.split('\n').enumerate() {
let mut tokens = Vec::new();
let mut machine_addr = None;
tokenize_line(raw, &mut tokens);
// Resolve line-level machine addr: first Address token wins, else scrape a
// hex literal out of any comment on the line.
for t in &tokens {
if matches!(t.kind, TokenKind::Address) {
if let Some(v) = t.va {
machine_addr = Some(v);
break;
}
}
}
if machine_addr.is_none() {
for t in &tokens {
if matches!(t.kind, TokenKind::Comment) {
if let Some(v) = t.va {
machine_addr = Some(v);
break;
}
}
}
}
// Track "block_N:" label state so downstream renderers can group.
if let Some(label) = block_label_from_line(raw) {
current_block = Some(label);
}
out.push(DecompLine {
line: idx,
machine_addr,
block_label: current_block.clone(),
tokens,
});
}
out
}
/// Adopt Stage-1 emit-time tokens (R5) into GUI [`DecompLine`]s.
///
/// Groups on `\n` text tokens. Falls back to string tokenisation when the
/// stream is empty.
pub fn from_emit_tokens(emit: &[(String, Option<u64>, &'static str)]) -> Vec<DecompLine> {
// Simplified adapter: callers map EmitToken → (text, va, kind_name).
if emit.is_empty() {
return Vec::new();
}
let mut lines: Vec<DecompLine> = Vec::new();
let mut cur = DecompLine {
line: 0,
machine_addr: None,
block_label: None,
tokens: Vec::new(),
};
for (text, va, kind) in emit {
let tk = match *kind {
"keyword" | "type" => TokenKind::Keyword,
"function" => TokenKind::Function,
"variable" => TokenKind::Variable,
"number" => TokenKind::Constant,
"comment" => TokenKind::Comment,
"address" => TokenKind::Address,
"label" => TokenKind::Label,
_ => TokenKind::Syntax,
};
if text == "\n" {
if cur.machine_addr.is_none() {
cur.machine_addr = cur.tokens.iter().find_map(|t| t.va);
}
lines.push(std::mem::replace(
&mut cur,
DecompLine {
line: lines.len() + 1,
machine_addr: None,
block_label: None,
tokens: Vec::new(),
},
));
continue;
}
if text.contains('\n') {
for (i, part) in text.split('\n').enumerate() {
if i > 0 {
if cur.machine_addr.is_none() {
cur.machine_addr = cur.tokens.iter().find_map(|t| t.va);
}
lines.push(std::mem::replace(
&mut cur,
DecompLine {
line: lines.len() + 1,
machine_addr: None,
block_label: None,
tokens: Vec::new(),
},
));
}
if !part.is_empty() {
cur.tokens.push(Token {
kind: tk,
text: part.to_string(),
va: *va,
});
}
}
continue;
}
cur.tokens.push(Token {
kind: tk,
text: text.clone(),
va: *va,
});
}
if !cur.tokens.is_empty() {
if cur.machine_addr.is_none() {
cur.machine_addr = cur.tokens.iter().find_map(|t| t.va);
}
lines.push(cur);
}
lines
}
/// Map `ghidrust_decomp::EmitToken` kinds into the adapter used by
/// [`from_emit_tokens`].
pub fn emit_token_triples(
tokens: &[ghidrust_decomp::EmitToken],
) -> Vec<(String, Option<u64>, &'static str)> {
tokens
.iter()
.map(|t| {
let kind = match t.kind {
ghidrust_decomp::EmitTokenKind::Keyword => "keyword",
ghidrust_decomp::EmitTokenKind::Type => "type",
ghidrust_decomp::EmitTokenKind::Function => "function",
ghidrust_decomp::EmitTokenKind::Variable => "variable",
ghidrust_decomp::EmitTokenKind::Number => "number",
ghidrust_decomp::EmitTokenKind::String => "number",
ghidrust_decomp::EmitTokenKind::Comment => "comment",
ghidrust_decomp::EmitTokenKind::Operator => "syntax",
ghidrust_decomp::EmitTokenKind::Punct => "syntax",
ghidrust_decomp::EmitTokenKind::Label => "label",
ghidrust_decomp::EmitTokenKind::Address => "address",
ghidrust_decomp::EmitTokenKind::Text => "syntax",
};
(t.text.clone(), t.va, kind)
})
.collect()
}
/// Parse a `block_N:` label at the start of a stripped line, else `None`.
fn block_label_from_line(line: &str) -> Option<String> {
let t = line.trim_start();
if !t.starts_with("block_") {
return None;
}
let end = t.find(':')?;
let label = &t[..end];
if !label
.strip_prefix("block_")
.map(|rest| rest.chars().all(|c| c.is_ascii_digit()))
.unwrap_or(false)
{
return None;
}
Some(label.to_string())
}
fn tokenize_line(line: &str, out: &mut Vec<Token>) {
let bytes = line.as_bytes();
let mut i = 0;
while i < bytes.len() {
let c = bytes[i] as char;
// Line comment: consume rest of line.
if c == '/' && i + 1 < bytes.len() && bytes[i + 1] == b'/' {
let text = &line[i..];
let va = first_hex_in(text);
out.push(Token {
kind: TokenKind::Comment,
text: text.to_string(),
va,
});
return;
}
// Block comment: consume to `*/` or end.
if c == '/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' {
let mut j = i + 2;
while j + 1 < bytes.len() && !(bytes[j] == b'*' && bytes[j + 1] == b'/') {
j += 1;
}
let end = (j + 2).min(bytes.len());
// Scrape any 0x… addresses from inside the comment.
let text = &line[i..end];
let addr_va = first_hex_in(text);
out.push(Token {
kind: TokenKind::Comment,
text: text.to_string(),
va: addr_va,
});
i = end;
continue;
}
// Whitespace run.
if c.is_ascii_whitespace() {
let start = i;
while i < bytes.len() && (bytes[i] as char).is_ascii_whitespace() {
i += 1;
}
out.push(Token {
kind: TokenKind::Whitespace,
text: line[start..i].to_string(),
va: None,
});
continue;
}
// Hex / numeric literal.
if c == '0' && i + 1 < bytes.len() && (bytes[i + 1] == b'x' || bytes[i + 1] == b'X') {
let start = i;
i += 2;
while i < bytes.len() && (bytes[i] as char).is_ascii_hexdigit() {
i += 1;
}
let text = line[start..i].to_string();
let va = u64::from_str_radix(&text[2..], 16).ok();
out.push(Token {
kind: TokenKind::Address,
text,
va,
});
continue;
}
if c.is_ascii_digit() {
let start = i;
while i < bytes.len() && (bytes[i] as char).is_ascii_digit() {
i += 1;
}
let text = line[start..i].to_string();
let va = text.parse::<u64>().ok();
out.push(Token {
kind: TokenKind::Constant,
text,
va,
});
continue;
}
// Identifier / keyword.
if is_ident_start(c) {
let start = i;
while i < bytes.len() && is_ident_part(bytes[i] as char) {
i += 1;
}
let text = &line[start..i];
let kind = classify_ident(text, &line[i..]);
let va = if text.starts_with("FUN_") {
u64::from_str_radix(&text[4..], 16).ok()
} else if let Some(rest) = text.strip_prefix("block_") {
// block_N is a label, not an address, but keep the id as VA-ish so
// consumers can uniquely key on it.
rest.parse::<u64>().ok()
} else {
None
};
out.push(Token {
kind,
text: text.to_string(),
va,
});
continue;
}
// Fallback: single punctuation/operator character.
let start = i;
// Group common multi-char operators.
let two = if i + 1 < bytes.len() {
&line[i..i + 2]
} else {
""
};
let n = match two {
"==" | "!=" | "<=" | ">=" | "&&" | "||" | "->" | "::" | "<<" | ">>" | "+=" | "-="
| "*=" | "/=" => 2,
_ => 1,
};
i += n;
out.push(Token {
kind: TokenKind::Syntax,
text: line[start..i].to_string(),
va: None,
});
}
}
fn is_ident_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_'
}
fn is_ident_part(c: char) -> bool {
c.is_ascii_alphanumeric() || c == '_'
}
fn classify_ident(text: &str, rest_of_line: &str) -> TokenKind {
if C_KEYWORDS.contains(&text) {
return TokenKind::Keyword;
}
if REGISTER_NAMES.contains(&text) {
return TokenKind::Variable;
}
if text.starts_with("block_") {
return TokenKind::Label;
}
if text.starts_with("FUN_") {
return TokenKind::Function;
}
// Heuristic: if the identifier is followed by `(`, treat as function call/decl.
if rest_of_line.trim_start().starts_with('(') {
return TokenKind::Function;
}
// Everything else: variable-ish.
TokenKind::Variable
}
/// First `0x…` in a slice, parsed as u64.
fn first_hex_in(s: &str) -> Option<u64> {
let idx = s.find("0x")?;
let rest = &s[idx + 2..];
let end = rest
.find(|c: char| !c.is_ascii_hexdigit())
.unwrap_or(rest.len());
u64::from_str_radix(&rest[..end], 16).ok()
}
/// Return every distinct occurrence of `text` in the given line set (for middle-click highlight).
#[cfg(test)]
pub fn occurrences_of<'a>(lines: &'a [DecompLine], text: &str) -> Vec<(usize, usize)> {
let mut hits = Vec::new();
for line in lines {
for (ti, tok) in line.tokens.iter().enumerate() {
if tok.text == text
&& !matches!(tok.kind, TokenKind::Whitespace | TokenKind::Syntax)
{
hits.push((line.line, ti));
}
}
}
hits
}
/// Return the line index (if any) whose `machine_addr == Some(va)` — used to
/// cross-highlight the decompiler when the Listing cursor moves.
pub fn line_for_va(lines: &[DecompLine], va: u64) -> Option<usize> {
// Prefer exact match, else nearest prior.
let mut best = None;
for line in lines {
if let Some(a) = line.machine_addr {
if a == va {
return Some(line.line);
}
if a < va {
best = Some(line.line);
}
if a > va {
break;
}
}
}
best
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keywords_functions_addresses_and_registers() {
let src = "// hi 0x1000\nvoid FUN_140001000(void) {\n block_0:\n if (eax == 0) {\n goto block_1;\n }\n return;\n}\n";
let lines = tokenize(src);
assert_eq!(lines.len(), 9, "expected 9 split-by-newline lines");
// Line 0: comment with a hex inside.
assert!(lines[0]
.tokens
.iter()
.any(|t| matches!(t.kind, TokenKind::Comment)));
assert_eq!(lines[0].machine_addr, Some(0x1000));
// Line 1: keyword `void`, function `FUN_…`.
let kinds: Vec<TokenKind> = lines[1].tokens.iter().map(|t| t.kind).collect();
assert!(kinds.contains(&TokenKind::Keyword), "void keyword missing");
assert!(kinds.contains(&TokenKind::Function), "FUN_ missing");
// Line 2: block_0: label.
assert_eq!(lines[2].block_label.as_deref(), Some("block_0"));
assert!(lines[2]
.tokens
.iter()
.any(|t| matches!(t.kind, TokenKind::Label)));
// Line 3: keyword if + register eax.
assert!(lines[3]
.tokens
.iter()
.any(|t| matches!(t.kind, TokenKind::Keyword) && t.text == "if"));
assert!(lines[3]
.tokens
.iter()
.any(|t| matches!(t.kind, TokenKind::Variable) && t.text == "eax"));
}
#[test]
fn line_for_va_finds_exact_and_nearest() {
let src = "// 0x1000\n// 0x1010\n// 0x1020\n";
let lines = tokenize(src);
assert_eq!(line_for_va(&lines, 0x1010), Some(1));
assert_eq!(line_for_va(&lines, 0x1015), Some(1)); // nearest prior
assert_eq!(line_for_va(&lines, 0x0FFF), None);
}
#[test]
fn occurrences_of_matches_only_meaningful_tokens() {
let src = "eax + eax; // eax\n";
let lines = tokenize(src);
let hits = occurrences_of(&lines, "eax");
// Two Variable hits — the "eax" inside the comment must NOT count.
assert_eq!(hits.len(), 2);
}
#[test]
fn ida_style_fun_extract_va() {
let src = "void FUN_140001000(void) {}\n";
let lines = tokenize(src);
let f = lines[0]
.tokens
.iter()
.find(|t| t.text == "FUN_140001000")
.unwrap();
assert_eq!(f.va, Some(0x140001000));
assert_eq!(f.kind, TokenKind::Function);
}
}