Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

pretraining fixes

Amruth126486ca40fe
diff --git a/tools/xda/configs/finetune.yaml b/tools/xda/configs/finetune.yaml
index b209436..379aaa8 100644
--- a/tools/xda/configs/finetune.yaml
+++ b/tools/xda/configs/finetune.yaml
@@ -5,7 +5,7 @@ model:
intermediate_size: 3072
training:
- epochs: 5
+ epochs: 15
batch_size: 64
learning_rate: 2.0e-5
warmup_ratio: 0.1
diff --git a/tools/xda/scripts/export_all.sh b/tools/xda/scripts/export_all.sh
new file mode 100755
index 0000000..567421b
--- /dev/null
+++ b/tools/xda/scripts/export_all.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# tools/xda/scripts/export_all.sh
+# Export fine-tuned checkpoints to safetensors for Rust/candle inference.
+# Run from tools/xda/ on a GPU VM (or locally).
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$XDA_DIR"
+
+ARCHS=(x86_64 aarch64 arm riscv64)
+
+mkdir -p models
+
+echo "=== Exporting models ==="
+
+for arch in "${ARCHS[@]}"; do
+ ckpt_dir="checkpoints/finetune_${arch}"
+
+ if [ -f "models/xda_${arch}.safetensors" ]; then
+ echo "--- $arch: already exported, skipping ---"
+ continue
+ fi
+
+ if [ ! -f "$ckpt_dir/best.pt" ] || [ ! -f "$ckpt_dir/config.json" ]; then
+ echo "--- $arch: no checkpoint at $ckpt_dir, skipping ---"
+ continue
+ fi
+
+ echo "--- $arch ---"
+ uv run python -m src.export \
+ --checkpoint "$ckpt_dir/best.pt" \
+ --config "$ckpt_dir/config.json" \
+ --output ./models/ \
+ --arch "$arch"
+done
+
+echo ""
+echo "=== Export complete ==="
+echo "Models:"
+ls -lh models/*.safetensors 2>/dev/null || echo " (none)"
+echo ""
+echo "Copy to local machine:"
+echo " scp -P <port> root@<host>:$(pwd)/models/* ~/Development/kong/tools/xda/models/"
diff --git a/tools/xda/scripts/finetune_all.sh b/tools/xda/scripts/finetune_all.sh
new file mode 100755
index 0000000..fed7dd4
--- /dev/null
+++ b/tools/xda/scripts/finetune_all.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# tools/xda/scripts/finetune_all.sh
+# Fine-tune all architectures for function boundary detection.
+# x86_64: from CUMLSec checkpoint or from scratch.
+# Others: from pretrain checkpoint (run pretrain_all.sh first).
+# Run from tools/xda/ on a GPU VM.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$XDA_DIR"
+
+ARCHS=(x86_64 aarch64 arm riscv64)
+
+# Set this to the path of the CUMLSec pretrained checkpoint for x86_64.
+# Leave empty to train x86_64 from scratch.
+CUMLSEC_CHECKPOINT="${CUMLSEC_CHECKPOINT:-}"
+
+echo "=== Fine-tuning: ${ARCHS[*]} ==="
+if [ -n "$CUMLSEC_CHECKPOINT" ]; then
+ echo " x86_64 pretrained: $CUMLSEC_CHECKPOINT"
+else
+ echo " x86_64: from scratch (set CUMLSEC_CHECKPOINT to use pretrained)"
+fi
+
+for arch in "${ARCHS[@]}"; do
+ ckpt_dir="checkpoints/finetune_${arch}"
+
+ if [ -f "$ckpt_dir/best.pt" ]; then
+ echo "--- $arch: checkpoint already exists, skipping ---"
+ continue
+ fi
+
+ if [ ! -f "data/${arch}/train.json" ]; then
+ echo "--- $arch: no training data at data/${arch}/train.json, skipping ---"
+ continue
+ fi
+
+ echo "--- $arch ---"
+ tmp=$(mktemp /tmp/ft_XXXX.yaml)
+ sed "s|checkpoint_dir:.*|checkpoint_dir: ${ckpt_dir}|" configs/finetune.yaml > "$tmp"
+
+ if [ "$arch" = "x86_64" ]; then
+ if [ -n "$CUMLSEC_CHECKPOINT" ]; then
+ sed -i "s|pretrained_checkpoint:.*|pretrained_checkpoint: ${CUMLSEC_CHECKPOINT}|" "$tmp"
+ fi
+ else
+ pretrain_ckpt="checkpoints/pretrain_${arch}/pretrain_final.pt"
+ if [ -f "$pretrain_ckpt" ]; then
+ sed -i "s|pretrained_checkpoint:.*|pretrained_checkpoint: ${pretrain_ckpt}|" "$tmp"
+ else
+ echo " WARNING: no pretrain checkpoint at $pretrain_ckpt, training from scratch"
+ fi
+ fi
+
+ uv run python -m src.finetune --config "$tmp" --data "./data/${arch}" --arch "$arch"
+ rm -f "$tmp"
+done
+
+echo ""
+echo "=== Fine-tuning complete ==="
+for arch in "${ARCHS[@]}"; do
+ ckpt="checkpoints/finetune_${arch}/best.pt"
+ if [ -f "$ckpt" ]; then
+ echo " $arch: $ckpt"
+ else
+ echo " $arch: MISSING"
+ fi
+done
diff --git a/tools/xda/scripts/pretrain_all.sh b/tools/xda/scripts/pretrain_all.sh
new file mode 100755
index 0000000..554fe4e
--- /dev/null
+++ b/tools/xda/scripts/pretrain_all.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# tools/xda/scripts/pretrain_all.sh
+# Masked byte modeling pre-training for non-x86 architectures.
+# Uses the training data (only reads bytes, ignores labels).
+# Run from tools/xda/ on a GPU VM.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+XDA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$XDA_DIR"
+
+ARCHS=(aarch64 arm riscv64)
+
+echo "=== Pre-training: ${ARCHS[*]} ==="
+
+for arch in "${ARCHS[@]}"; do
+ ckpt_dir="checkpoints/pretrain_${arch}"
+
+ if [ -f "$ckpt_dir/pretrain_final.pt" ]; then
+ echo "--- $arch: pretrain checkpoint exists, skipping ---"
+ continue
+ fi
+
+ if [ ! -f "data/${arch}/train.json" ]; then
+ echo "--- $arch: no training data at data/${arch}/train.json, skipping ---"
+ continue
+ fi
+
+ echo "--- $arch: pre-training ---"
+ tmp=$(mktemp /tmp/pt_XXXX.yaml)
+ sed "s|checkpoint_dir:.*|checkpoint_dir: ${ckpt_dir}|" configs/pretrain.yaml > "$tmp"
+
+ uv run python -m src.pretrain --config "$tmp" --data "./data/${arch}"
+ rm -f "$tmp"
+done
+
+echo ""
+echo "=== Pre-training complete ==="
+for arch in "${ARCHS[@]}"; do
+ ckpt="checkpoints/pretrain_${arch}/pretrain_final.pt"
+ if [ -f "$ckpt" ]; then
+ echo " $arch: $ckpt"
+ else
+ echo " $arch: MISSING"
+ fi
+done
diff --git a/tools/xda/src/finetune.py b/tools/xda/src/finetune.py
index 827eb3d..8e9e126 100644
--- a/tools/xda/src/finetune.py
+++ b/tools/xda/src/finetune.py
@@ -66,6 +66,19 @@ def finetune(config_path: str, data_path: str, arch: str = "unknown") -> None:
val_chunks = json.loads(Path(data_path, "val.json").read_text())
print(f"Train: {len(train_chunks)} chunks, Val: {len(val_chunks)} chunks")
+ label_counts = [0, 0, 0]
+ for chunk in train_chunks:
+ for label in chunk["labels"]:
+ if 0 <= label <= 2:
+ label_counts[label] += 1
+ total = sum(label_counts)
+ class_weights = torch.tensor(
+ [total / (3 * c) if c > 0 else 1.0 for c in label_counts],
+ dtype=torch.float32,
+ ).to(device)
+ print(f"Label counts: non_func={label_counts[0]}, func_start={label_counts[1]}, func_body={label_counts[2]}")
+ print(f"Class weights: {class_weights.tolist()}")
+
tokenizer = ByteTokenizer()
train_ds = ByteDataset(train_chunks, tokenizer)
val_ds = ByteDataset(val_chunks, tokenizer)
@@ -110,7 +123,7 @@ def finetune(config_path: str, data_path: str, arch: str = "unknown") -> None:
optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,
)
- loss_fn = nn.CrossEntropyLoss(ignore_index=-100)
+ loss_fn = nn.CrossEntropyLoss(weight=class_weights, ignore_index=-100)
scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and use_cuda)
checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])
diff --git a/tools/xda/src/pretrain.py b/tools/xda/src/pretrain.py
index 5a94709..d6e65d5 100644
--- a/tools/xda/src/pretrain.py
+++ b/tools/xda/src/pretrain.py
@@ -65,6 +65,7 @@ def pretrain(config_path: str, data_path: str) -> None:
else "mps" if torch.backends.mps.is_available()
else "cpu"
)
+ use_cuda = device.type == "cuda"
print(f"Using device: {device}")
train_chunks = json.loads(Path(data_path, "train.json").read_text())
@@ -81,8 +82,8 @@ def pretrain(config_path: str, data_path: str) -> None:
dataset,
batch_size=cfg["training"]["batch_size"],
shuffle=True,
- num_workers=4,
- pin_memory=True,
+ num_workers=4 if use_cuda else 0,
+ pin_memory=use_cuda,
)
model_cfg = XdaConfig(