The world's first agentic reverse engineer.
pretraining fixes
Amruth126486ca40fe
diff --git a/tools/xda/configs/finetune.yaml b/tools/xda/configs/finetune.yamlindex b209436..379aaa8 100644--- a/tools/xda/configs/finetune.yaml+++ b/tools/xda/configs/finetune.yaml@@ -5,7 +5,7 @@ model:intermediate_size: 3072training:- epochs: 5+ epochs: 15batch_size: 64learning_rate: 2.0e-5warmup_ratio: 0.1diff --git a/tools/xda/scripts/export_all.sh b/tools/xda/scripts/export_all.shnew file mode 100755index 0000000..567421b--- /dev/null+++ b/tools/xda/scripts/export_all.sh@@ -0,0 +1,44 @@+#!/usr/bin/env bash+# tools/xda/scripts/export_all.sh+# Export fine-tuned checkpoints to safetensors for Rust/candle inference.+# Run from tools/xda/ on a GPU VM (or locally).+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"+cd "$XDA_DIR"++ARCHS=(x86_64 aarch64 arm riscv64)++mkdir -p models++echo "=== Exporting models ==="++for arch in "${ARCHS[@]}"; do+ ckpt_dir="checkpoints/finetune_${arch}"++ if [ -f "models/xda_${arch}.safetensors" ]; then+ echo "--- $arch: already exported, skipping ---"+ continue+ fi++ if [ ! -f "$ckpt_dir/best.pt" ] || [ ! -f "$ckpt_dir/config.json" ]; then+ echo "--- $arch: no checkpoint at $ckpt_dir, skipping ---"+ continue+ fi++ echo "--- $arch ---"+ uv run python -m src.export \+ --checkpoint "$ckpt_dir/best.pt" \+ --config "$ckpt_dir/config.json" \+ --output ./models/ \+ --arch "$arch"+done++echo ""+echo "=== Export complete ==="+echo "Models:"+ls -lh models/*.safetensors 2>/dev/null || echo " (none)"+echo ""+echo "Copy to local machine:"+echo " scp -P <port> root@<host>:$(pwd)/models/* ~/Development/kong/tools/xda/models/"diff --git a/tools/xda/scripts/finetune_all.sh b/tools/xda/scripts/finetune_all.shnew file mode 100755index 0000000..fed7dd4--- /dev/null+++ b/tools/xda/scripts/finetune_all.sh@@ -0,0 +1,69 @@+#!/usr/bin/env bash+# tools/xda/scripts/finetune_all.sh+# Fine-tune all architectures for function boundary detection.+# x86_64: from CUMLSec checkpoint or from scratch.+# Others: from pretrain checkpoint (run pretrain_all.sh first).+# Run from tools/xda/ on a GPU VM.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"+cd "$XDA_DIR"++ARCHS=(x86_64 aarch64 arm riscv64)++# Set this to the path of the CUMLSec pretrained checkpoint for x86_64.+# Leave empty to train x86_64 from scratch.+CUMLSEC_CHECKPOINT="${CUMLSEC_CHECKPOINT:-}"++echo "=== Fine-tuning: ${ARCHS[*]} ==="+if [ -n "$CUMLSEC_CHECKPOINT" ]; then+ echo " x86_64 pretrained: $CUMLSEC_CHECKPOINT"+else+ echo " x86_64: from scratch (set CUMLSEC_CHECKPOINT to use pretrained)"+fi++for arch in "${ARCHS[@]}"; do+ ckpt_dir="checkpoints/finetune_${arch}"++ if [ -f "$ckpt_dir/best.pt" ]; then+ echo "--- $arch: checkpoint already exists, skipping ---"+ continue+ fi++ if [ ! -f "data/${arch}/train.json" ]; then+ echo "--- $arch: no training data at data/${arch}/train.json, skipping ---"+ continue+ fi++ echo "--- $arch ---"+ tmp=$(mktemp /tmp/ft_XXXX.yaml)+ sed "s|checkpoint_dir:.*|checkpoint_dir: ${ckpt_dir}|" configs/finetune.yaml > "$tmp"++ if [ "$arch" = "x86_64" ]; then+ if [ -n "$CUMLSEC_CHECKPOINT" ]; then+ sed -i "s|pretrained_checkpoint:.*|pretrained_checkpoint: ${CUMLSEC_CHECKPOINT}|" "$tmp"+ fi+ else+ pretrain_ckpt="checkpoints/pretrain_${arch}/pretrain_final.pt"+ if [ -f "$pretrain_ckpt" ]; then+ sed -i "s|pretrained_checkpoint:.*|pretrained_checkpoint: ${pretrain_ckpt}|" "$tmp"+ else+ echo " WARNING: no pretrain checkpoint at $pretrain_ckpt, training from scratch"+ fi+ fi++ uv run python -m src.finetune --config "$tmp" --data "./data/${arch}" --arch "$arch"+ rm -f "$tmp"+done++echo ""+echo "=== Fine-tuning complete ==="+for arch in "${ARCHS[@]}"; do+ ckpt="checkpoints/finetune_${arch}/best.pt"+ if [ -f "$ckpt" ]; then+ echo " $arch: $ckpt"+ else+ echo " $arch: MISSING"+ fi+donediff --git a/tools/xda/scripts/pretrain_all.sh b/tools/xda/scripts/pretrain_all.shnew file mode 100755index 0000000..554fe4e--- /dev/null+++ b/tools/xda/scripts/pretrain_all.sh@@ -0,0 +1,46 @@+#!/usr/bin/env bash+# tools/xda/scripts/pretrain_all.sh+# Masked byte modeling pre-training for non-x86 architectures.+# Uses the training data (only reads bytes, ignores labels).+# Run from tools/xda/ on a GPU VM.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"+cd "$XDA_DIR"++ARCHS=(aarch64 arm riscv64)++echo "=== Pre-training: ${ARCHS[*]} ==="++for arch in "${ARCHS[@]}"; do+ ckpt_dir="checkpoints/pretrain_${arch}"++ if [ -f "$ckpt_dir/pretrain_final.pt" ]; then+ echo "--- $arch: pretrain checkpoint exists, skipping ---"+ continue+ fi++ if [ ! -f "data/${arch}/train.json" ]; then+ echo "--- $arch: no training data at data/${arch}/train.json, skipping ---"+ continue+ fi++ echo "--- $arch: pre-training ---"+ tmp=$(mktemp /tmp/pt_XXXX.yaml)+ sed "s|checkpoint_dir:.*|checkpoint_dir: ${ckpt_dir}|" configs/pretrain.yaml > "$tmp"++ uv run python -m src.pretrain --config "$tmp" --data "./data/${arch}"+ rm -f "$tmp"+done++echo ""+echo "=== Pre-training complete ==="+for arch in "${ARCHS[@]}"; do+ ckpt="checkpoints/pretrain_${arch}/pretrain_final.pt"+ if [ -f "$ckpt" ]; then+ echo " $arch: $ckpt"+ else+ echo " $arch: MISSING"+ fi+donediff --git a/tools/xda/src/finetune.py b/tools/xda/src/finetune.pyindex 827eb3d..8e9e126 100644--- a/tools/xda/src/finetune.py+++ b/tools/xda/src/finetune.py@@ -66,6 +66,19 @@ def finetune(config_path: str, data_path: str, arch: str = "unknown") -> None:val_chunks = json.loads(Path(data_path, "val.json").read_text())print(f"Train: {len(train_chunks)} chunks, Val: {len(val_chunks)} chunks")+ label_counts = [0, 0, 0]+ for chunk in train_chunks:+ for label in chunk["labels"]:+ if 0 <= label <= 2:+ label_counts[label] += 1+ total = sum(label_counts)+ class_weights = torch.tensor(+ [total / (3 * c) if c > 0 else 1.0 for c in label_counts],+ dtype=torch.float32,+ ).to(device)+ print(f"Label counts: non_func={label_counts[0]}, func_start={label_counts[1]}, func_body={label_counts[2]}")+ print(f"Class weights: {class_weights.tolist()}")+tokenizer = ByteTokenizer()train_ds = ByteDataset(train_chunks, tokenizer)val_ds = ByteDataset(val_chunks, tokenizer)@@ -110,7 +123,7 @@ def finetune(config_path: str, data_path: str, arch: str = "unknown") -> None:optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,)- loss_fn = nn.CrossEntropyLoss(ignore_index=-100)+ loss_fn = nn.CrossEntropyLoss(weight=class_weights, ignore_index=-100)scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and use_cuda)checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])diff --git a/tools/xda/src/pretrain.py b/tools/xda/src/pretrain.pyindex 5a94709..d6e65d5 100644--- a/tools/xda/src/pretrain.py+++ b/tools/xda/src/pretrain.py@@ -65,6 +65,7 @@ def pretrain(config_path: str, data_path: str) -> None:else "mps" if torch.backends.mps.is_available()else "cpu")+ use_cuda = device.type == "cuda"print(f"Using device: {device}")train_chunks = json.loads(Path(data_path, "train.json").read_text())@@ -81,8 +82,8 @@ def pretrain(config_path: str, data_path: str) -> None:dataset,batch_size=cfg["training"]["batch_size"],shuffle=True,- num_workers=4,- pin_memory=True,+ num_workers=4 if use_cuda else 0,+ pin_memory=use_cuda,)model_cfg = XdaConfig(