The world's first agentic reverse engineer.
add pretraining/training files
Amruth772be466f2d9
diff --git a/tools/xda/configs/finetune.yaml b/tools/xda/configs/finetune.yamlnew file mode 100644index 0000000..b209436--- /dev/null+++ b/tools/xda/configs/finetune.yaml@@ -0,0 +1,23 @@+model:+ hidden_size: 768+ num_hidden_layers: 12+ num_attention_heads: 12+ intermediate_size: 3072++training:+ epochs: 5+ batch_size: 64+ learning_rate: 2.0e-5+ warmup_ratio: 0.1+ weight_decay: 0.01+ fp16: true++data:+ window_size: 512+ stride: 256++output:+ checkpoint_dir: checkpoints/finetune+ save_best: true++pretrained_checkpoint: nulldiff --git a/tools/xda/configs/pretrain.yaml b/tools/xda/configs/pretrain.yamlnew file mode 100644index 0000000..75221e5--- /dev/null+++ b/tools/xda/configs/pretrain.yaml@@ -0,0 +1,24 @@+model:+ hidden_size: 768+ num_hidden_layers: 12+ num_attention_heads: 12+ intermediate_size: 3072++training:+ epochs: 10+ batch_size: 64+ learning_rate: 1.0e-4+ warmup_ratio: 0.1+ weight_decay: 0.01+ mask_probability: 0.20+ mask_token_ratio: 0.50+ max_length: 514+ fp16: true++data:+ window_size: 512+ stride: 256++output:+ save_every_n_epochs: 2+ checkpoint_dir: checkpoints/pretraindiff --git a/tools/xda/scripts/build_corpus.sh b/tools/xda/scripts/build_corpus.shindex edef77c..49c0e8c 100755--- a/tools/xda/scripts/build_corpus.sh+++ b/tools/xda/scripts/build_corpus.sh@@ -9,17 +9,25 @@ set -euo pipefailARCH="${1:?Usage: build_corpus.sh <arch> <output_dir>}"OUT="${2:?Usage: build_corpus.sh <arch> <output_dir>}"+ZIG_TARGET=""case "$ARCH" in- x86_64) CC="gcc"; STRIP="strip" ;;- aarch64) CC="aarch64-linux-gnu-gcc"; STRIP="aarch64-linux-gnu-strip" ;;- arm) CC="arm-linux-gnueabihf-gcc"; STRIP="arm-linux-gnueabihf-strip" ;;- mips) CC="mips-linux-gnu-gcc"; STRIP="mips-linux-gnu-strip" ;;- riscv64) CC="riscv64-linux-gnu-gcc"; STRIP="riscv64-linux-gnu-strip" ;;+ x86_64) CC="x86_64-linux-gnu-gcc"; STRIP="x86_64-linux-gnu-strip"; ZIG_TARGET="x86_64-linux-gnu" ;;+ aarch64) CC="aarch64-linux-gnu-gcc"; STRIP="aarch64-linux-gnu-strip"; ZIG_TARGET="aarch64-linux-gnu" ;;+ arm) CC="arm-linux-gnueabihf-gcc"; STRIP="arm-linux-gnueabihf-strip"; ZIG_TARGET="arm-linux-gnueabihf" ;;+ mips) CC="mips-linux-gnu-gcc"; STRIP="mips-linux-gnu-strip"; ZIG_TARGET="mips-linux-gnu" ;;+ riscv64) CC="riscv64-linux-gnu-gcc"; STRIP="riscv64-linux-gnu-strip"; ZIG_TARGET="riscv64-linux-gnu" ;;*) echo "Unknown arch: $ARCH"; exit 1 ;;esac-# Verify compiler exists-command -v "$CC" >/dev/null 2>&1 || { echo "$CC not found. Install cross-compiler."; exit 1; }+USE_ZIG=0+if command -v "$CC" >/dev/null 2>&1; then+ : # native cross-compiler found+elif command -v zig >/dev/null 2>&1 && [ -n "$ZIG_TARGET" ]; then+ USE_ZIG=1+ echo "Using zig cc -target $ZIG_TARGET (native $CC not found)"+else+ echo "$CC not found and zig not available. Install a cross-compiler or zig."; exit 1+fimkdir -p "$OUT"@@ -85,9 +93,15 @@ for src in "$SOURCES_DIR"/*.c; doout_debug="${OUT}/${name}_${opt}_debug"out_stripped="${OUT}/${name}_${opt}_stripped"- "$CC" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue- cp "$out_debug" "$out_stripped"- "$STRIP" "$out_stripped"+ if [ "$USE_ZIG" -eq 1 ]; then+ zig cc -target "$ZIG_TARGET" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue+ cp "$out_debug" "$out_stripped"+ zig cc -target "$ZIG_TARGET" -s -o "$out_stripped" "$src" 2>/dev/null || continue+ else+ "$CC" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue+ cp "$out_debug" "$out_stripped"+ "$STRIP" "$out_stripped"+ fiecho "Built: ${name}_${opt}"donediff --git a/tools/xda/src/__init__.py b/tools/xda/src/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tools/xda/src/build_dataset.py b/tools/xda/src/build_dataset.pynew file mode 100644index 0000000..4565f80--- /dev/null+++ b/tools/xda/src/build_dataset.py@@ -0,0 +1,132 @@+from __future__ import annotations++import json+import random+from pathlib import Path++from tqdm import tqdm++from .dataset import chunk_labeled_bytes+from .extract_labels import extract_function_boundaries, generate_byte_labels+++def _serialize_chunks(chunks: list[dict], path: Path) -> None:+ serialized = [{"bytes": list(c["bytes"]), "labels": c["labels"]} for c in chunks]+ path.write_text(json.dumps(serialized))+++def _split_and_save(+ all_chunks: list[dict],+ output_path: Path,+ val_split: float,+ stats: dict,+) -> dict:+ random.shuffle(all_chunks)+ split_idx = int(len(all_chunks) * (1 - val_split))+ train_chunks = all_chunks[:split_idx]+ val_chunks = all_chunks[split_idx:]++ output_path.mkdir(parents=True, exist_ok=True)+ _serialize_chunks(train_chunks, output_path / "train.json")+ _serialize_chunks(val_chunks, output_path / "val.json")++ stats["train_chunks"] = len(train_chunks)+ stats["val_chunks"] = len(val_chunks)+ (output_path / "stats.json").write_text(json.dumps(stats, indent=2))++ return stats+++def build_from_corpus_dir(+ corpus_dir: Path,+ output_path: Path,+ window_size: int = 512,+ stride: int = 256,+ val_split: float = 0.1,+) -> dict:+ """Process debug binaries from build_corpus.sh output.++ Expects corpus_dir to contain *_debug and *_stripped files.+ Uses debug files for DWARF labels and .text bytes.+ """+ all_chunks: list[dict] = []+ stats: dict = {"binaries": 0, "functions": 0, "chunks": 0, "errors": []}++ debug_files = sorted(corpus_dir.glob("*_debug"))+ if not debug_files:+ debug_files = sorted(+ f for f in corpus_dir.iterdir()+ if f.is_file() and not f.name.startswith(".") and "stripped" not in f.name+ )++ for debug_file in tqdm(debug_files, desc="Processing corpus"):+ if debug_file.is_dir():+ continue++ try:+ boundaries = extract_function_boundaries(str(debug_file))+ if not boundaries:+ continue++ text_bytes, labels = generate_byte_labels(str(debug_file), boundaries)+ if not text_bytes:+ continue++ chunks = chunk_labeled_bytes(text_bytes, labels, window_size, stride)+ all_chunks.extend(chunks)++ stats["binaries"] += 1+ stats["functions"] += len(boundaries)+ stats["chunks"] += len(chunks)+ except Exception as e:+ stats["errors"].append({"file": str(debug_file), "error": str(e)})++ return _split_and_save(all_chunks, output_path, val_split, stats)+++def build_from_pairs_dir(+ pairs_dir: Path,+ output_path: Path,+ window_size: int = 512,+ stride: int = 256,+ val_split: float = 0.1,+) -> dict:+ """Process binary/debug pairs from Debian package extraction.++ Expects pairs_dir to contain subdirectories, each with 'binary' and 'debug' files.+ """+ all_chunks: list[dict] = []+ stats: dict = {"binaries": 0, "functions": 0, "chunks": 0, "errors": []}++ for pair_dir in tqdm(sorted(pairs_dir.iterdir()), desc="Processing binaries"):+ if not pair_dir.is_dir():+ continue++ debug_file = pair_dir / "debug"+ binary_file = pair_dir / "binary"++ if not debug_file.exists():+ continue++ label_source = str(debug_file)+ byte_source = str(binary_file) if binary_file.exists() else label_source++ try:+ boundaries = extract_function_boundaries(label_source)+ if not boundaries:+ continue++ text_bytes, labels = generate_byte_labels(byte_source, boundaries)+ if not text_bytes:+ continue++ chunks = chunk_labeled_bytes(text_bytes, labels, window_size, stride)+ all_chunks.extend(chunks)++ stats["binaries"] += 1+ stats["functions"] += len(boundaries)+ stats["chunks"] += len(chunks)+ except Exception as e:+ stats["errors"].append({"file": str(pair_dir), "error": str(e)})++ return _split_and_save(all_chunks, output_path, val_split, stats)diff --git a/tools/xda/src/export.py b/tools/xda/src/export.pynew file mode 100644index 0000000..b49dac8--- /dev/null+++ b/tools/xda/src/export.py@@ -0,0 +1,50 @@+from __future__ import annotations++import argparse+import json+from pathlib import Path++import torch+from safetensors.torch import save_file++from .model import XdaConfig, XdaModel+++def export(checkpoint_path: str, config_path: str, output_dir: str, arch: str) -> None:+ out = Path(output_dir)+ out.mkdir(parents=True, exist_ok=True)++ with open(config_path) as f:+ config_dict = json.load(f)+ config = XdaConfig(**config_dict)++ model = XdaModel(config)+ state = torch.load(checkpoint_path, map_location="cpu", weights_only=True)+ model.load_state_dict(state)++ flat_state = {}+ for key, tensor in model.state_dict().items():+ flat_state[key] = tensor.float()++ weights_path = out / f"xda_{arch}.safetensors"+ save_file(flat_state, str(weights_path))++ config_out = out / f"xda_{arch}_config.json"+ config_out.write_text(json.dumps(config_dict, indent=2))++ total_params = sum(t.numel() for t in flat_state.values())+ file_size_mb = weights_path.stat().st_size / (1024 * 1024)+ print(f"Exported {arch} model:")+ print(f" Weights: {weights_path} ({file_size_mb:.1f} MB)")+ print(f" Config: {config_out}")+ print(f" Parameters: {total_params:,}")+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("--checkpoint", required=True, help="Path to .pt checkpoint")+ parser.add_argument("--config", required=True, help="Path to config.json from training")+ parser.add_argument("--output", required=True, help="Output directory")+ parser.add_argument("--arch", required=True, help="Architecture name (x86_64, aarch64, etc.)")+ args = parser.parse_args()+ export(args.checkpoint, args.config, args.output, args.arch)diff --git a/tools/xda/src/finetune.py b/tools/xda/src/finetune.pynew file mode 100644index 0000000..f517eb5--- /dev/null+++ b/tools/xda/src/finetune.py@@ -0,0 +1,176 @@+from __future__ import annotations++import argparse+import json+from pathlib import Path++import torch+import yaml+from torch import nn+from torch.utils.data import DataLoader+from tqdm import tqdm++from .dataset import ByteDataset+from .model import XdaConfig, XdaModel+from .tokenizer import ByteTokenizer+++def evaluate(model: XdaModel, loader: DataLoader, device: torch.device) -> dict:+ model.eval()+ tp = [0, 0, 0]+ fp = [0, 0, 0]+ fn = [0, 0, 0]++ with torch.no_grad():+ for batch in loader:+ input_ids = batch["input_ids"].to(device)+ attention_mask = batch["attention_mask"].to(device)+ labels = batch["labels"].to(device)++ logits = model(input_ids, attention_mask)+ preds = logits.argmax(dim=-1)++ mask = labels != -100+ preds = preds[mask]+ labels = labels[mask]++ for cls in range(3):+ tp[cls] += ((preds == cls) & (labels == cls)).sum().item()+ fp[cls] += ((preds == cls) & (labels != cls)).sum().item()+ fn[cls] += ((preds != cls) & (labels == cls)).sum().item()++ results = {}+ for cls, name in enumerate(["non_function", "function_start", "function_body"]):+ precision = tp[cls] / (tp[cls] + fp[cls]) if tp[cls] + fp[cls] > 0 else 0+ recall = tp[cls] / (tp[cls] + fn[cls]) if tp[cls] + fn[cls] > 0 else 0+ f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0+ results[name] = {"precision": precision, "recall": recall, "f1": f1}++ return results+++def finetune(config_path: str, data_path: str) -> None:+ with open(config_path) as f:+ cfg = yaml.safe_load(f)++ device = torch.device(+ "cuda" if torch.cuda.is_available()+ else "mps" if torch.backends.mps.is_available()+ else "cpu"+ )+ print(f"Using device: {device}")++ train_chunks = json.loads(Path(data_path, "train.json").read_text())+ val_chunks = json.loads(Path(data_path, "val.json").read_text())+ print(f"Train: {len(train_chunks)} chunks, Val: {len(val_chunks)} chunks")++ tokenizer = ByteTokenizer()+ train_ds = ByteDataset(train_chunks, tokenizer)+ val_ds = ByteDataset(val_chunks, tokenizer)++ train_loader = DataLoader(+ train_ds, batch_size=cfg["training"]["batch_size"],+ shuffle=True, num_workers=4, pin_memory=True,+ )+ val_loader = DataLoader(+ val_ds, batch_size=cfg["training"]["batch_size"], num_workers=4,+ )++ model_cfg = XdaConfig(+ hidden_size=cfg["model"]["hidden_size"],+ num_hidden_layers=cfg["model"]["num_hidden_layers"],+ num_attention_heads=cfg["model"]["num_attention_heads"],+ intermediate_size=cfg["model"]["intermediate_size"],+ num_labels=3,+ )+ model = XdaModel(model_cfg)++ pretrained_path = cfg.get("pretrained_checkpoint")+ if pretrained_path and Path(pretrained_path).exists():+ print(f"Loading pre-trained weights from {pretrained_path}")+ state = torch.load(pretrained_path, map_location="cpu", weights_only=True)+ encoder_state = {k: v for k, v in state.items() if k.startswith("encoder.")}+ model.load_state_dict(encoder_state, strict=False)+ print(f"Loaded {len(encoder_state)} encoder weight tensors")++ model = model.to(device)++ optimizer = torch.optim.AdamW(+ model.parameters(),+ lr=cfg["training"]["learning_rate"],+ weight_decay=cfg["training"]["weight_decay"],+ )+ total_steps = len(train_loader) * cfg["training"]["epochs"]+ warmup_steps = int(total_steps * cfg["training"]["warmup_ratio"])+ scheduler = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,+ )++ loss_fn = nn.CrossEntropyLoss(ignore_index=-100)+ scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and device.type == "cuda")++ checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])+ checkpoint_dir.mkdir(parents=True, exist_ok=True)++ best_f1 = 0.0++ for epoch in range(cfg["training"]["epochs"]):+ model.train()+ total_loss = 0.0+ steps = 0++ pbar = tqdm(train_loader, desc=f"Epoch {epoch + 1}/{cfg['training']['epochs']}")+ for batch in pbar:+ input_ids = batch["input_ids"].to(device)+ attention_mask = batch["attention_mask"].to(device)+ labels = batch["labels"].to(device)++ with torch.amp.autocast(device.type, enabled=cfg["training"]["fp16"]):+ logits = model(input_ids, attention_mask)+ loss = loss_fn(logits.view(-1, 3), labels.view(-1))++ scaler.scale(loss).backward()+ scaler.step(optimizer)+ scaler.update()+ optimizer.zero_grad()+ scheduler.step()++ total_loss += loss.item()+ steps += 1+ pbar.set_postfix(loss=f"{total_loss / steps:.4f}")++ results = evaluate(model, val_loader, device)+ fs_f1 = results["function_start"]["f1"]++ print(f"Epoch {epoch + 1}: loss={total_loss / steps:.4f}")+ for name, metrics in results.items():+ print(f" {name}: P={metrics['precision']:.4f} R={metrics['recall']:.4f} F1={metrics['f1']:.4f}")++ if cfg["output"]["save_best"] and fs_f1 > best_f1:+ best_f1 = fs_f1+ torch.save(model.state_dict(), checkpoint_dir / "best.pt")+ print(f" New best function_start F1: {fs_f1:.4f}")++ torch.save(model.state_dict(), checkpoint_dir / f"epoch{epoch + 1}.pt")++ config_out = {+ "vocab_size": model_cfg.vocab_size,+ "hidden_size": model_cfg.hidden_size,+ "num_hidden_layers": model_cfg.num_hidden_layers,+ "num_attention_heads": model_cfg.num_attention_heads,+ "intermediate_size": model_cfg.intermediate_size,+ "max_position_embeddings": model_cfg.max_position_embeddings,+ "num_labels": model_cfg.num_labels,+ "classifier_hidden": model_cfg.classifier_hidden,+ }+ (checkpoint_dir / "config.json").write_text(json.dumps(config_out, indent=2))++ print(f"Fine-tuning complete. Best F1: {best_f1:.4f}")+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("--config", required=True)+ parser.add_argument("--data", required=True)+ args = parser.parse_args()+ finetune(args.config, args.data)diff --git a/tools/xda/src/pretrain.py b/tools/xda/src/pretrain.pynew file mode 100644index 0000000..5a94709--- /dev/null+++ b/tools/xda/src/pretrain.py@@ -0,0 +1,155 @@+from __future__ import annotations++import argparse+import json+import random+from pathlib import Path++import torch+import yaml+from torch import nn+from torch.utils.data import DataLoader, Dataset+from tqdm import tqdm++from .model import XdaConfig, XdaModel+from .tokenizer import ByteTokenizer+++class MaskedByteDataset(Dataset):++ def __init__(+ self,+ chunks: list[dict],+ tokenizer: ByteTokenizer,+ mask_prob: float = 0.2,+ mask_token_ratio: float = 0.5,+ ) -> None:+ self.chunks = chunks+ self.tokenizer = tokenizer+ self.mask_prob = mask_prob+ self.mask_token_ratio = mask_token_ratio++ def __len__(self) -> int:+ return len(self.chunks)++ def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:+ raw_bytes = bytes(self.chunks[idx]["bytes"])+ input_ids = self.tokenizer.encode(raw_bytes)+ labels = list(input_ids)++ for i in range(1, len(input_ids) - 1):+ if random.random() < self.mask_prob:+ if random.random() < self.mask_token_ratio:+ input_ids[i] = self.tokenizer.mask_id+ else:+ input_ids[i] = random.randint(0, 255)+ else:+ labels[i] = -100++ labels[0] = -100+ labels[-1] = -100++ return {+ "input_ids": torch.tensor(input_ids, dtype=torch.long),+ "attention_mask": torch.ones(len(input_ids), dtype=torch.long),+ "labels": torch.tensor(labels, dtype=torch.long),+ }+++def pretrain(config_path: str, data_path: str) -> None:+ with open(config_path) as f:+ cfg = yaml.safe_load(f)++ device = torch.device(+ "cuda" if torch.cuda.is_available()+ else "mps" if torch.backends.mps.is_available()+ else "cpu"+ )+ print(f"Using device: {device}")++ train_chunks = json.loads(Path(data_path, "train.json").read_text())+ print(f"Loaded {len(train_chunks)} training chunks")++ tokenizer = ByteTokenizer()+ dataset = MaskedByteDataset(+ train_chunks,+ tokenizer,+ mask_prob=cfg["training"]["mask_probability"],+ mask_token_ratio=cfg["training"]["mask_token_ratio"],+ )+ loader = DataLoader(+ dataset,+ batch_size=cfg["training"]["batch_size"],+ shuffle=True,+ num_workers=4,+ pin_memory=True,+ )++ model_cfg = XdaConfig(+ hidden_size=cfg["model"]["hidden_size"],+ num_hidden_layers=cfg["model"]["num_hidden_layers"],+ num_attention_heads=cfg["model"]["num_attention_heads"],+ intermediate_size=cfg["model"]["intermediate_size"],+ num_labels=tokenizer.vocab_size,+ )+ model = XdaModel(model_cfg).to(device)++ optimizer = torch.optim.AdamW(+ model.parameters(),+ lr=cfg["training"]["learning_rate"],+ weight_decay=cfg["training"]["weight_decay"],+ )++ total_steps = len(loader) * cfg["training"]["epochs"]+ warmup_steps = int(total_steps * cfg["training"]["warmup_ratio"])+ scheduler = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,+ )++ loss_fn = nn.CrossEntropyLoss(ignore_index=-100)+ scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and device.type == "cuda")++ checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])+ checkpoint_dir.mkdir(parents=True, exist_ok=True)++ for epoch in range(cfg["training"]["epochs"]):+ model.train()+ total_loss = 0.0+ steps = 0++ pbar = tqdm(loader, desc=f"Epoch {epoch + 1}/{cfg['training']['epochs']}")+ for batch in pbar:+ input_ids = batch["input_ids"].to(device)+ attention_mask = batch["attention_mask"].to(device)+ labels = batch["labels"].to(device)++ with torch.amp.autocast(device.type, enabled=cfg["training"]["fp16"]):+ logits = model(input_ids, attention_mask)+ loss = loss_fn(logits.view(-1, model_cfg.num_labels), labels.view(-1))++ scaler.scale(loss).backward()+ scaler.step(optimizer)+ scaler.update()+ optimizer.zero_grad()+ scheduler.step()++ total_loss += loss.item()+ steps += 1+ pbar.set_postfix(loss=f"{total_loss / steps:.4f}")++ avg_loss = total_loss / steps+ print(f"Epoch {epoch + 1}: avg_loss={avg_loss:.4f}")++ if (epoch + 1) % cfg["output"]["save_every_n_epochs"] == 0:+ torch.save(model.state_dict(), checkpoint_dir / f"pretrain_epoch{epoch + 1}.pt")++ torch.save(model.state_dict(), checkpoint_dir / "pretrain_final.pt")+ print(f"Pre-training complete. Final checkpoint: {checkpoint_dir / 'pretrain_final.pt'}")+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("--config", required=True)+ parser.add_argument("--data", required=True)+ args = parser.parse_args()+ pretrain(args.config, args.data)