The world's first agentic reverse engineer.
add dataset builder + model arch v1
Amruth7b37432b672c
diff --git a/tools/xda/src/dataset.py b/tools/xda/src/dataset.pynew file mode 100644index 0000000..3db1eb3--- /dev/null+++ b/tools/xda/src/dataset.py@@ -0,0 +1,54 @@+import torch+from torch.utils.data import Dataset++from .tokenizer import ByteTokenizer+++def chunk_labeled_bytes(+ text_bytes: bytes,+ labels: list[int],+ window_size: int = 512,+ stride: int = 256,+) -> list[dict]:+ """Chunk labeled bytes into fixed-size windows with overlap."""+ chunks = []+ for i in range(0, max(1, len(text_bytes) - window_size + 1), stride):+ end = i + window_size+ if end > len(text_bytes):+ # Pad the last chunk+ b = text_bytes[i:] + bytes(end - len(text_bytes))+ l = labels[i:] + [0] * (end - len(text_bytes))+ else:+ b = text_bytes[i:end]+ l = labels[i:end]+ chunks.append({"bytes": b, "labels": l})+ return chunks+++class ByteDataset(Dataset):+ """PyTorch dataset for XDA training. Each item is a fixed-size byte window."""++ def __init__(self, chunks: list[dict], tokenizer: ByteTokenizer) -> None:+ self.chunks = chunks+ self.tokenizer = tokenizer++ def __len__(self) -> int:+ return len(self.chunks)++ def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:+ chunk = self.chunks[idx]+ raw_bytes = chunk["bytes"]+ raw_labels = chunk["labels"]++ # Tokenize: [CLS] + bytes + [SEP]+ input_ids = self.tokenizer.encode(raw_bytes)+ attention_mask = self.tokenizer.attention_mask(input_ids)++ # Labels: -100 for special tokens (ignored by CrossEntropyLoss)+ labels = [-100] + list(raw_labels) + [-100]++ return {+ "input_ids": torch.tensor(input_ids, dtype=torch.long),+ "attention_mask": torch.tensor(attention_mask, dtype=torch.long),+ "labels": torch.tensor(labels, dtype=torch.long),+ }\ No newline at end of filediff --git a/tools/xda/src/model.py b/tools/xda/src/model.pynew file mode 100644index 0000000..caa9f38--- /dev/null+++ b/tools/xda/src/model.py@@ -0,0 +1,55 @@+from dataclasses import dataclass++import torch+from torch import nn+from transformers import BertConfig, BertModel+++@dataclass+class XdaConfig:+ vocab_size: int = 261 # 256 bytes + 5 special tokens+ hidden_size: int = 768+ num_hidden_layers: int = 12+ num_attention_heads: int = 12+ intermediate_size: int = 3072+ max_position_embeddings: int = 514 # 512 byte window + [CLS] + [SEP]+ num_labels: int = 3 # non-function, function_start, function_body+ classifier_hidden: int = 256 # MLP hidden dimension+ hidden_dropout_prob: float = 0.1+ attention_probs_dropout_prob: float = 0.1++ def to_bert_config(self) -> BertConfig:+ return BertConfig(+ vocab_size=self.vocab_size,+ hidden_size=self.hidden_size,+ num_hidden_layers=self.num_hidden_layers,+ num_attention_heads=self.num_attention_heads,+ intermediate_size=self.intermediate_size,+ max_position_embeddings=self.max_position_embeddings,+ hidden_dropout_prob=self.hidden_dropout_prob,+ attention_probs_dropout_prob=self.attention_probs_dropout_prob,+ )+++class XdaModel(nn.Module):+ """XDA: BERT encoder with a 2-layer MLP token classification head."""++ def __init__(self, config: XdaConfig) -> None:+ super().__init__()+ self.config = config+ self.encoder = BertModel(config.to_bert_config(), add_pooling_layer=False)+ self.classifier = nn.Sequential(+ nn.Linear(config.hidden_size, config.classifier_hidden),+ nn.GELU(),+ nn.Dropout(config.hidden_dropout_prob),+ nn.Linear(config.classifier_hidden, config.num_labels),+ )++ def forward(+ self,+ input_ids: torch.Tensor,+ attention_mask: torch.Tensor,+ ) -> torch.Tensor:+ """Returns logits of shape (batch, seq_len, num_labels)."""+ outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)+ return self.classifier(outputs.last_hidden_state)\ No newline at end of filediff --git a/tools/xda/tests/test_dataset.py b/tools/xda/tests/test_dataset.pynew file mode 100644index 0000000..9f2cf72--- /dev/null+++ b/tools/xda/tests/test_dataset.py@@ -0,0 +1,50 @@+import torch+from src.dataset import ByteDataset, chunk_labeled_bytes+from src.tokenizer import ByteTokenizer+++def test_chunk_labeled_bytes():+ text_bytes = bytes(range(256)) * 4 # 1024 bytes+ labels = [0] * 1024+ labels[0] = 1 # function start at byte 0+ labels[512] = 1 # function start at byte 512++ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=512, stride=256)++ assert len(chunks) > 0+ for chunk in chunks:+ assert len(chunk["bytes"]) == 512+ assert len(chunk["labels"]) == 512+++def test_byte_dataset_shapes():+ text_bytes = bytes(range(256)) * 4+ labels = [0] * 1024+ labels[0] = 1++ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=512, stride=256)+ ds = ByteDataset(chunks, ByteTokenizer())++ item = ds[0]+ assert item["input_ids"].shape == (514,) # 512 + [CLS] + [SEP]+ assert item["attention_mask"].shape == (514,)+ assert item["labels"].shape == (514,)+ assert item["input_ids"][0].item() == 257 # [CLS]+ assert item["input_ids"][-1].item() == 258 # [SEP]+ # Labels for [CLS] and [SEP] should be -100 (ignored in loss)+ assert item["labels"][0].item() == -100+ assert item["labels"][-1].item() == -100+++def test_byte_dataset_label_values():+ text_bytes = bytes([0x55, 0x48, 0x89, 0xe5, 0xc3, 0x00, 0x00, 0x00])+ labels = [1, 2, 2, 2, 2, 0, 0, 0]++ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=8, stride=8)+ ds = ByteDataset(chunks, ByteTokenizer())++ item = ds[0]+ # Labels offset by 1 for [CLS] prefix+ assert item["labels"][1].item() == 1 # function_start+ assert item["labels"][2].item() == 2 # function_body+ assert item["labels"][6].item() == 0 # non-functiondiff --git a/tools/xda/tests/test_model.py b/tools/xda/tests/test_model.pynew file mode 100644index 0000000..a69451d--- /dev/null+++ b/tools/xda/tests/test_model.py@@ -0,0 +1,40 @@+import torch+from src.model import XdaModel, XdaConfig+++def test_model_config_defaults():+ config = XdaConfig()+ assert config.vocab_size == 261+ assert config.hidden_size == 768+ assert config.num_hidden_layers == 12+ assert config.num_attention_heads == 12+ assert config.intermediate_size == 3072+ assert config.num_labels == 3+ assert config.max_position_embeddings == 514 # 512 + [CLS] + [SEP]+++def test_model_forward_shape():+ config = XdaConfig(+ num_hidden_layers=2, # smaller for test speed+ hidden_size=128,+ num_attention_heads=4,+ intermediate_size=256,+ )+ model = XdaModel(config)++ batch_size = 4+ seq_len = 514+ input_ids = torch.randint(0, 261, (batch_size, seq_len))+ attention_mask = torch.ones(batch_size, seq_len, dtype=torch.long)++ logits = model(input_ids, attention_mask)+ assert logits.shape == (batch_size, seq_len, 3)+++def test_model_parameter_count():+ """Sanity check that full-size model has ~87M params."""+ config = XdaConfig()+ model = XdaModel(config)+ total = sum(p.numel() for p in model.parameters())+ # Allow some tolerance — should be in the 80-95M range+ assert 80_000_000 < total < 100_000_000, f"Param count {total} outside expected range"\ No newline at end of file