Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

add pretraining/training files

Amruth772be466f2d9
diff --git a/tools/xda/configs/finetune.yaml b/tools/xda/configs/finetune.yaml
new file mode 100644
index 0000000..b209436
--- /dev/null
+++ b/tools/xda/configs/finetune.yaml
@@ -0,0 +1,23 @@
+model:
+ hidden_size: 768
+ num_hidden_layers: 12
+ num_attention_heads: 12
+ intermediate_size: 3072
+
+training:
+ epochs: 5
+ batch_size: 64
+ learning_rate: 2.0e-5
+ warmup_ratio: 0.1
+ weight_decay: 0.01
+ fp16: true
+
+data:
+ window_size: 512
+ stride: 256
+
+output:
+ checkpoint_dir: checkpoints/finetune
+ save_best: true
+
+pretrained_checkpoint: null
diff --git a/tools/xda/configs/pretrain.yaml b/tools/xda/configs/pretrain.yaml
new file mode 100644
index 0000000..75221e5
--- /dev/null
+++ b/tools/xda/configs/pretrain.yaml
@@ -0,0 +1,24 @@
+model:
+ hidden_size: 768
+ num_hidden_layers: 12
+ num_attention_heads: 12
+ intermediate_size: 3072
+
+training:
+ epochs: 10
+ batch_size: 64
+ learning_rate: 1.0e-4
+ warmup_ratio: 0.1
+ weight_decay: 0.01
+ mask_probability: 0.20
+ mask_token_ratio: 0.50
+ max_length: 514
+ fp16: true
+
+data:
+ window_size: 512
+ stride: 256
+
+output:
+ save_every_n_epochs: 2
+ checkpoint_dir: checkpoints/pretrain
diff --git a/tools/xda/scripts/build_corpus.sh b/tools/xda/scripts/build_corpus.sh
index edef77c..49c0e8c 100755
--- a/tools/xda/scripts/build_corpus.sh
+++ b/tools/xda/scripts/build_corpus.sh
@@ -9,17 +9,25 @@ set -euo pipefail
ARCH="${1:?Usage: build_corpus.sh <arch> <output_dir>}"
OUT="${2:?Usage: build_corpus.sh <arch> <output_dir>}"
+ZIG_TARGET=""
case "$ARCH" in
- x86_64) CC="gcc"; STRIP="strip" ;;
- aarch64) CC="aarch64-linux-gnu-gcc"; STRIP="aarch64-linux-gnu-strip" ;;
- arm) CC="arm-linux-gnueabihf-gcc"; STRIP="arm-linux-gnueabihf-strip" ;;
- mips) CC="mips-linux-gnu-gcc"; STRIP="mips-linux-gnu-strip" ;;
- riscv64) CC="riscv64-linux-gnu-gcc"; STRIP="riscv64-linux-gnu-strip" ;;
+ x86_64) CC="x86_64-linux-gnu-gcc"; STRIP="x86_64-linux-gnu-strip"; ZIG_TARGET="x86_64-linux-gnu" ;;
+ aarch64) CC="aarch64-linux-gnu-gcc"; STRIP="aarch64-linux-gnu-strip"; ZIG_TARGET="aarch64-linux-gnu" ;;
+ arm) CC="arm-linux-gnueabihf-gcc"; STRIP="arm-linux-gnueabihf-strip"; ZIG_TARGET="arm-linux-gnueabihf" ;;
+ mips) CC="mips-linux-gnu-gcc"; STRIP="mips-linux-gnu-strip"; ZIG_TARGET="mips-linux-gnu" ;;
+ riscv64) CC="riscv64-linux-gnu-gcc"; STRIP="riscv64-linux-gnu-strip"; ZIG_TARGET="riscv64-linux-gnu" ;;
*) echo "Unknown arch: $ARCH"; exit 1 ;;
esac
-# Verify compiler exists
-command -v "$CC" >/dev/null 2>&1 || { echo "$CC not found. Install cross-compiler."; exit 1; }
+USE_ZIG=0
+if command -v "$CC" >/dev/null 2>&1; then
+ : # native cross-compiler found
+elif command -v zig >/dev/null 2>&1 && [ -n "$ZIG_TARGET" ]; then
+ USE_ZIG=1
+ echo "Using zig cc -target $ZIG_TARGET (native $CC not found)"
+else
+ echo "$CC not found and zig not available. Install a cross-compiler or zig."; exit 1
+fi
mkdir -p "$OUT"
@@ -85,9 +93,15 @@ for src in "$SOURCES_DIR"/*.c; do
out_debug="${OUT}/${name}_${opt}_debug"
out_stripped="${OUT}/${name}_${opt}_stripped"
- "$CC" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue
- cp "$out_debug" "$out_stripped"
- "$STRIP" "$out_stripped"
+ if [ "$USE_ZIG" -eq 1 ]; then
+ zig cc -target "$ZIG_TARGET" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue
+ cp "$out_debug" "$out_stripped"
+ zig cc -target "$ZIG_TARGET" -s -o "$out_stripped" "$src" 2>/dev/null || continue
+ else
+ "$CC" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue
+ cp "$out_debug" "$out_stripped"
+ "$STRIP" "$out_stripped"
+ fi
echo "Built: ${name}_${opt}"
done
diff --git a/tools/xda/src/__init__.py b/tools/xda/src/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/xda/src/build_dataset.py b/tools/xda/src/build_dataset.py
new file mode 100644
index 0000000..4565f80
--- /dev/null
+++ b/tools/xda/src/build_dataset.py
@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import json
+import random
+from pathlib import Path
+
+from tqdm import tqdm
+
+from .dataset import chunk_labeled_bytes
+from .extract_labels import extract_function_boundaries, generate_byte_labels
+
+
+def _serialize_chunks(chunks: list[dict], path: Path) -> None:
+ serialized = [{"bytes": list(c["bytes"]), "labels": c["labels"]} for c in chunks]
+ path.write_text(json.dumps(serialized))
+
+
+def _split_and_save(
+ all_chunks: list[dict],
+ output_path: Path,
+ val_split: float,
+ stats: dict,
+) -> dict:
+ random.shuffle(all_chunks)
+ split_idx = int(len(all_chunks) * (1 - val_split))
+ train_chunks = all_chunks[:split_idx]
+ val_chunks = all_chunks[split_idx:]
+
+ output_path.mkdir(parents=True, exist_ok=True)
+ _serialize_chunks(train_chunks, output_path / "train.json")
+ _serialize_chunks(val_chunks, output_path / "val.json")
+
+ stats["train_chunks"] = len(train_chunks)
+ stats["val_chunks"] = len(val_chunks)
+ (output_path / "stats.json").write_text(json.dumps(stats, indent=2))
+
+ return stats
+
+
+def build_from_corpus_dir(
+ corpus_dir: Path,
+ output_path: Path,
+ window_size: int = 512,
+ stride: int = 256,
+ val_split: float = 0.1,
+) -> dict:
+ """Process debug binaries from build_corpus.sh output.
+
+ Expects corpus_dir to contain *_debug and *_stripped files.
+ Uses debug files for DWARF labels and .text bytes.
+ """
+ all_chunks: list[dict] = []
+ stats: dict = {"binaries": 0, "functions": 0, "chunks": 0, "errors": []}
+
+ debug_files = sorted(corpus_dir.glob("*_debug"))
+ if not debug_files:
+ debug_files = sorted(
+ f for f in corpus_dir.iterdir()
+ if f.is_file() and not f.name.startswith(".") and "stripped" not in f.name
+ )
+
+ for debug_file in tqdm(debug_files, desc="Processing corpus"):
+ if debug_file.is_dir():
+ continue
+
+ try:
+ boundaries = extract_function_boundaries(str(debug_file))
+ if not boundaries:
+ continue
+
+ text_bytes, labels = generate_byte_labels(str(debug_file), boundaries)
+ if not text_bytes:
+ continue
+
+ chunks = chunk_labeled_bytes(text_bytes, labels, window_size, stride)
+ all_chunks.extend(chunks)
+
+ stats["binaries"] += 1
+ stats["functions"] += len(boundaries)
+ stats["chunks"] += len(chunks)
+ except Exception as e:
+ stats["errors"].append({"file": str(debug_file), "error": str(e)})
+
+ return _split_and_save(all_chunks, output_path, val_split, stats)
+
+
+def build_from_pairs_dir(
+ pairs_dir: Path,
+ output_path: Path,
+ window_size: int = 512,
+ stride: int = 256,
+ val_split: float = 0.1,
+) -> dict:
+ """Process binary/debug pairs from Debian package extraction.
+
+ Expects pairs_dir to contain subdirectories, each with 'binary' and 'debug' files.
+ """
+ all_chunks: list[dict] = []
+ stats: dict = {"binaries": 0, "functions": 0, "chunks": 0, "errors": []}
+
+ for pair_dir in tqdm(sorted(pairs_dir.iterdir()), desc="Processing binaries"):
+ if not pair_dir.is_dir():
+ continue
+
+ debug_file = pair_dir / "debug"
+ binary_file = pair_dir / "binary"
+
+ if not debug_file.exists():
+ continue
+
+ label_source = str(debug_file)
+ byte_source = str(binary_file) if binary_file.exists() else label_source
+
+ try:
+ boundaries = extract_function_boundaries(label_source)
+ if not boundaries:
+ continue
+
+ text_bytes, labels = generate_byte_labels(byte_source, boundaries)
+ if not text_bytes:
+ continue
+
+ chunks = chunk_labeled_bytes(text_bytes, labels, window_size, stride)
+ all_chunks.extend(chunks)
+
+ stats["binaries"] += 1
+ stats["functions"] += len(boundaries)
+ stats["chunks"] += len(chunks)
+ except Exception as e:
+ stats["errors"].append({"file": str(pair_dir), "error": str(e)})
+
+ return _split_and_save(all_chunks, output_path, val_split, stats)
diff --git a/tools/xda/src/export.py b/tools/xda/src/export.py
new file mode 100644
index 0000000..b49dac8
--- /dev/null
+++ b/tools/xda/src/export.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import torch
+from safetensors.torch import save_file
+
+from .model import XdaConfig, XdaModel
+
+
+def export(checkpoint_path: str, config_path: str, output_dir: str, arch: str) -> None:
+ out = Path(output_dir)
+ out.mkdir(parents=True, exist_ok=True)
+
+ with open(config_path) as f:
+ config_dict = json.load(f)
+ config = XdaConfig(**config_dict)
+
+ model = XdaModel(config)
+ state = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
+ model.load_state_dict(state)
+
+ flat_state = {}
+ for key, tensor in model.state_dict().items():
+ flat_state[key] = tensor.float()
+
+ weights_path = out / f"xda_{arch}.safetensors"
+ save_file(flat_state, str(weights_path))
+
+ config_out = out / f"xda_{arch}_config.json"
+ config_out.write_text(json.dumps(config_dict, indent=2))
+
+ total_params = sum(t.numel() for t in flat_state.values())
+ file_size_mb = weights_path.stat().st_size / (1024 * 1024)
+ print(f"Exported {arch} model:")
+ print(f" Weights: {weights_path} ({file_size_mb:.1f} MB)")
+ print(f" Config: {config_out}")
+ print(f" Parameters: {total_params:,}")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--checkpoint", required=True, help="Path to .pt checkpoint")
+ parser.add_argument("--config", required=True, help="Path to config.json from training")
+ parser.add_argument("--output", required=True, help="Output directory")
+ parser.add_argument("--arch", required=True, help="Architecture name (x86_64, aarch64, etc.)")
+ args = parser.parse_args()
+ export(args.checkpoint, args.config, args.output, args.arch)
diff --git a/tools/xda/src/finetune.py b/tools/xda/src/finetune.py
new file mode 100644
index 0000000..f517eb5
--- /dev/null
+++ b/tools/xda/src/finetune.py
@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import torch
+import yaml
+from torch import nn
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+
+from .dataset import ByteDataset
+from .model import XdaConfig, XdaModel
+from .tokenizer import ByteTokenizer
+
+
+def evaluate(model: XdaModel, loader: DataLoader, device: torch.device) -> dict:
+ model.eval()
+ tp = [0, 0, 0]
+ fp = [0, 0, 0]
+ fn = [0, 0, 0]
+
+ with torch.no_grad():
+ for batch in loader:
+ input_ids = batch["input_ids"].to(device)
+ attention_mask = batch["attention_mask"].to(device)
+ labels = batch["labels"].to(device)
+
+ logits = model(input_ids, attention_mask)
+ preds = logits.argmax(dim=-1)
+
+ mask = labels != -100
+ preds = preds[mask]
+ labels = labels[mask]
+
+ for cls in range(3):
+ tp[cls] += ((preds == cls) & (labels == cls)).sum().item()
+ fp[cls] += ((preds == cls) & (labels != cls)).sum().item()
+ fn[cls] += ((preds != cls) & (labels == cls)).sum().item()
+
+ results = {}
+ for cls, name in enumerate(["non_function", "function_start", "function_body"]):
+ precision = tp[cls] / (tp[cls] + fp[cls]) if tp[cls] + fp[cls] > 0 else 0
+ recall = tp[cls] / (tp[cls] + fn[cls]) if tp[cls] + fn[cls] > 0 else 0
+ f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0
+ results[name] = {"precision": precision, "recall": recall, "f1": f1}
+
+ return results
+
+
+def finetune(config_path: str, data_path: str) -> None:
+ with open(config_path) as f:
+ cfg = yaml.safe_load(f)
+
+ device = torch.device(
+ "cuda" if torch.cuda.is_available()
+ else "mps" if torch.backends.mps.is_available()
+ else "cpu"
+ )
+ print(f"Using device: {device}")
+
+ train_chunks = json.loads(Path(data_path, "train.json").read_text())
+ val_chunks = json.loads(Path(data_path, "val.json").read_text())
+ print(f"Train: {len(train_chunks)} chunks, Val: {len(val_chunks)} chunks")
+
+ tokenizer = ByteTokenizer()
+ train_ds = ByteDataset(train_chunks, tokenizer)
+ val_ds = ByteDataset(val_chunks, tokenizer)
+
+ train_loader = DataLoader(
+ train_ds, batch_size=cfg["training"]["batch_size"],
+ shuffle=True, num_workers=4, pin_memory=True,
+ )
+ val_loader = DataLoader(
+ val_ds, batch_size=cfg["training"]["batch_size"], num_workers=4,
+ )
+
+ model_cfg = XdaConfig(
+ hidden_size=cfg["model"]["hidden_size"],
+ num_hidden_layers=cfg["model"]["num_hidden_layers"],
+ num_attention_heads=cfg["model"]["num_attention_heads"],
+ intermediate_size=cfg["model"]["intermediate_size"],
+ num_labels=3,
+ )
+ model = XdaModel(model_cfg)
+
+ pretrained_path = cfg.get("pretrained_checkpoint")
+ if pretrained_path and Path(pretrained_path).exists():
+ print(f"Loading pre-trained weights from {pretrained_path}")
+ state = torch.load(pretrained_path, map_location="cpu", weights_only=True)
+ encoder_state = {k: v for k, v in state.items() if k.startswith("encoder.")}
+ model.load_state_dict(encoder_state, strict=False)
+ print(f"Loaded {len(encoder_state)} encoder weight tensors")
+
+ model = model.to(device)
+
+ optimizer = torch.optim.AdamW(
+ model.parameters(),
+ lr=cfg["training"]["learning_rate"],
+ weight_decay=cfg["training"]["weight_decay"],
+ )
+ total_steps = len(train_loader) * cfg["training"]["epochs"]
+ warmup_steps = int(total_steps * cfg["training"]["warmup_ratio"])
+ scheduler = torch.optim.lr_scheduler.LinearLR(
+ optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,
+ )
+
+ loss_fn = nn.CrossEntropyLoss(ignore_index=-100)
+ scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and device.type == "cuda")
+
+ checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])
+ checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+ best_f1 = 0.0
+
+ for epoch in range(cfg["training"]["epochs"]):
+ model.train()
+ total_loss = 0.0
+ steps = 0
+
+ pbar = tqdm(train_loader, desc=f"Epoch {epoch + 1}/{cfg['training']['epochs']}")
+ for batch in pbar:
+ input_ids = batch["input_ids"].to(device)
+ attention_mask = batch["attention_mask"].to(device)
+ labels = batch["labels"].to(device)
+
+ with torch.amp.autocast(device.type, enabled=cfg["training"]["fp16"]):
+ logits = model(input_ids, attention_mask)
+ loss = loss_fn(logits.view(-1, 3), labels.view(-1))
+
+ scaler.scale(loss).backward()
+ scaler.step(optimizer)
+ scaler.update()
+ optimizer.zero_grad()
+ scheduler.step()
+
+ total_loss += loss.item()
+ steps += 1
+ pbar.set_postfix(loss=f"{total_loss / steps:.4f}")
+
+ results = evaluate(model, val_loader, device)
+ fs_f1 = results["function_start"]["f1"]
+
+ print(f"Epoch {epoch + 1}: loss={total_loss / steps:.4f}")
+ for name, metrics in results.items():
+ print(f" {name}: P={metrics['precision']:.4f} R={metrics['recall']:.4f} F1={metrics['f1']:.4f}")
+
+ if cfg["output"]["save_best"] and fs_f1 > best_f1:
+ best_f1 = fs_f1
+ torch.save(model.state_dict(), checkpoint_dir / "best.pt")
+ print(f" New best function_start F1: {fs_f1:.4f}")
+
+ torch.save(model.state_dict(), checkpoint_dir / f"epoch{epoch + 1}.pt")
+
+ config_out = {
+ "vocab_size": model_cfg.vocab_size,
+ "hidden_size": model_cfg.hidden_size,
+ "num_hidden_layers": model_cfg.num_hidden_layers,
+ "num_attention_heads": model_cfg.num_attention_heads,
+ "intermediate_size": model_cfg.intermediate_size,
+ "max_position_embeddings": model_cfg.max_position_embeddings,
+ "num_labels": model_cfg.num_labels,
+ "classifier_hidden": model_cfg.classifier_hidden,
+ }
+ (checkpoint_dir / "config.json").write_text(json.dumps(config_out, indent=2))
+
+ print(f"Fine-tuning complete. Best F1: {best_f1:.4f}")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--config", required=True)
+ parser.add_argument("--data", required=True)
+ args = parser.parse_args()
+ finetune(args.config, args.data)
diff --git a/tools/xda/src/pretrain.py b/tools/xda/src/pretrain.py
new file mode 100644
index 0000000..5a94709
--- /dev/null
+++ b/tools/xda/src/pretrain.py
@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+import torch
+import yaml
+from torch import nn
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+
+from .model import XdaConfig, XdaModel
+from .tokenizer import ByteTokenizer
+
+
+class MaskedByteDataset(Dataset):
+
+ def __init__(
+ self,
+ chunks: list[dict],
+ tokenizer: ByteTokenizer,
+ mask_prob: float = 0.2,
+ mask_token_ratio: float = 0.5,
+ ) -> None:
+ self.chunks = chunks
+ self.tokenizer = tokenizer
+ self.mask_prob = mask_prob
+ self.mask_token_ratio = mask_token_ratio
+
+ def __len__(self) -> int:
+ return len(self.chunks)
+
+ def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
+ raw_bytes = bytes(self.chunks[idx]["bytes"])
+ input_ids = self.tokenizer.encode(raw_bytes)
+ labels = list(input_ids)
+
+ for i in range(1, len(input_ids) - 1):
+ if random.random() < self.mask_prob:
+ if random.random() < self.mask_token_ratio:
+ input_ids[i] = self.tokenizer.mask_id
+ else:
+ input_ids[i] = random.randint(0, 255)
+ else:
+ labels[i] = -100
+
+ labels[0] = -100
+ labels[-1] = -100
+
+ return {
+ "input_ids": torch.tensor(input_ids, dtype=torch.long),
+ "attention_mask": torch.ones(len(input_ids), dtype=torch.long),
+ "labels": torch.tensor(labels, dtype=torch.long),
+ }
+
+
+def pretrain(config_path: str, data_path: str) -> None:
+ with open(config_path) as f:
+ cfg = yaml.safe_load(f)
+
+ device = torch.device(
+ "cuda" if torch.cuda.is_available()
+ else "mps" if torch.backends.mps.is_available()
+ else "cpu"
+ )
+ print(f"Using device: {device}")
+
+ train_chunks = json.loads(Path(data_path, "train.json").read_text())
+ print(f"Loaded {len(train_chunks)} training chunks")
+
+ tokenizer = ByteTokenizer()
+ dataset = MaskedByteDataset(
+ train_chunks,
+ tokenizer,
+ mask_prob=cfg["training"]["mask_probability"],
+ mask_token_ratio=cfg["training"]["mask_token_ratio"],
+ )
+ loader = DataLoader(
+ dataset,
+ batch_size=cfg["training"]["batch_size"],
+ shuffle=True,
+ num_workers=4,
+ pin_memory=True,
+ )
+
+ model_cfg = XdaConfig(
+ hidden_size=cfg["model"]["hidden_size"],
+ num_hidden_layers=cfg["model"]["num_hidden_layers"],
+ num_attention_heads=cfg["model"]["num_attention_heads"],
+ intermediate_size=cfg["model"]["intermediate_size"],
+ num_labels=tokenizer.vocab_size,
+ )
+ model = XdaModel(model_cfg).to(device)
+
+ optimizer = torch.optim.AdamW(
+ model.parameters(),
+ lr=cfg["training"]["learning_rate"],
+ weight_decay=cfg["training"]["weight_decay"],
+ )
+
+ total_steps = len(loader) * cfg["training"]["epochs"]
+ warmup_steps = int(total_steps * cfg["training"]["warmup_ratio"])
+ scheduler = torch.optim.lr_scheduler.LinearLR(
+ optimizer, start_factor=0.01, end_factor=1.0, total_iters=warmup_steps,
+ )
+
+ loss_fn = nn.CrossEntropyLoss(ignore_index=-100)
+ scaler = torch.amp.GradScaler(enabled=cfg["training"]["fp16"] and device.type == "cuda")
+
+ checkpoint_dir = Path(cfg["output"]["checkpoint_dir"])
+ checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+ for epoch in range(cfg["training"]["epochs"]):
+ model.train()
+ total_loss = 0.0
+ steps = 0
+
+ pbar = tqdm(loader, desc=f"Epoch {epoch + 1}/{cfg['training']['epochs']}")
+ for batch in pbar:
+ input_ids = batch["input_ids"].to(device)
+ attention_mask = batch["attention_mask"].to(device)
+ labels = batch["labels"].to(device)
+
+ with torch.amp.autocast(device.type, enabled=cfg["training"]["fp16"]):
+ logits = model(input_ids, attention_mask)
+ loss = loss_fn(logits.view(-1, model_cfg.num_labels), labels.view(-1))
+
+ scaler.scale(loss).backward()
+ scaler.step(optimizer)
+ scaler.update()
+ optimizer.zero_grad()
+ scheduler.step()
+
+ total_loss += loss.item()
+ steps += 1
+ pbar.set_postfix(loss=f"{total_loss / steps:.4f}")
+
+ avg_loss = total_loss / steps
+ print(f"Epoch {epoch + 1}: avg_loss={avg_loss:.4f}")
+
+ if (epoch + 1) % cfg["output"]["save_every_n_epochs"] == 0:
+ torch.save(model.state_dict(), checkpoint_dir / f"pretrain_epoch{epoch + 1}.pt")
+
+ torch.save(model.state_dict(), checkpoint_dir / "pretrain_final.pt")
+ print(f"Pre-training complete. Final checkpoint: {checkpoint_dir / 'pretrain_final.pt'}")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--config", required=True)
+ parser.add_argument("--data", required=True)
+ args = parser.parse_args()
+ pretrain(args.config, args.data)