Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

add dataset builder + model arch v1

Amruth7b37432b672c
diff --git a/tools/xda/src/dataset.py b/tools/xda/src/dataset.py
new file mode 100644
index 0000000..3db1eb3
--- /dev/null
+++ b/tools/xda/src/dataset.py
@@ -0,0 +1,54 @@
+import torch
+from torch.utils.data import Dataset
+
+from .tokenizer import ByteTokenizer
+
+
+def chunk_labeled_bytes(
+ text_bytes: bytes,
+ labels: list[int],
+ window_size: int = 512,
+ stride: int = 256,
+) -> list[dict]:
+ """Chunk labeled bytes into fixed-size windows with overlap."""
+ chunks = []
+ for i in range(0, max(1, len(text_bytes) - window_size + 1), stride):
+ end = i + window_size
+ if end > len(text_bytes):
+ # Pad the last chunk
+ b = text_bytes[i:] + bytes(end - len(text_bytes))
+ l = labels[i:] + [0] * (end - len(text_bytes))
+ else:
+ b = text_bytes[i:end]
+ l = labels[i:end]
+ chunks.append({"bytes": b, "labels": l})
+ return chunks
+
+
+class ByteDataset(Dataset):
+ """PyTorch dataset for XDA training. Each item is a fixed-size byte window."""
+
+ def __init__(self, chunks: list[dict], tokenizer: ByteTokenizer) -> None:
+ self.chunks = chunks
+ self.tokenizer = tokenizer
+
+ def __len__(self) -> int:
+ return len(self.chunks)
+
+ def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
+ chunk = self.chunks[idx]
+ raw_bytes = chunk["bytes"]
+ raw_labels = chunk["labels"]
+
+ # Tokenize: [CLS] + bytes + [SEP]
+ input_ids = self.tokenizer.encode(raw_bytes)
+ attention_mask = self.tokenizer.attention_mask(input_ids)
+
+ # Labels: -100 for special tokens (ignored by CrossEntropyLoss)
+ labels = [-100] + list(raw_labels) + [-100]
+
+ return {
+ "input_ids": torch.tensor(input_ids, dtype=torch.long),
+ "attention_mask": torch.tensor(attention_mask, dtype=torch.long),
+ "labels": torch.tensor(labels, dtype=torch.long),
+ }
\ No newline at end of file
diff --git a/tools/xda/src/model.py b/tools/xda/src/model.py
new file mode 100644
index 0000000..caa9f38
--- /dev/null
+++ b/tools/xda/src/model.py
@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+from transformers import BertConfig, BertModel
+
+
+@dataclass
+class XdaConfig:
+ vocab_size: int = 261 # 256 bytes + 5 special tokens
+ hidden_size: int = 768
+ num_hidden_layers: int = 12
+ num_attention_heads: int = 12
+ intermediate_size: int = 3072
+ max_position_embeddings: int = 514 # 512 byte window + [CLS] + [SEP]
+ num_labels: int = 3 # non-function, function_start, function_body
+ classifier_hidden: int = 256 # MLP hidden dimension
+ hidden_dropout_prob: float = 0.1
+ attention_probs_dropout_prob: float = 0.1
+
+ def to_bert_config(self) -> BertConfig:
+ return BertConfig(
+ vocab_size=self.vocab_size,
+ hidden_size=self.hidden_size,
+ num_hidden_layers=self.num_hidden_layers,
+ num_attention_heads=self.num_attention_heads,
+ intermediate_size=self.intermediate_size,
+ max_position_embeddings=self.max_position_embeddings,
+ hidden_dropout_prob=self.hidden_dropout_prob,
+ attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+ )
+
+
+class XdaModel(nn.Module):
+ """XDA: BERT encoder with a 2-layer MLP token classification head."""
+
+ def __init__(self, config: XdaConfig) -> None:
+ super().__init__()
+ self.config = config
+ self.encoder = BertModel(config.to_bert_config(), add_pooling_layer=False)
+ self.classifier = nn.Sequential(
+ nn.Linear(config.hidden_size, config.classifier_hidden),
+ nn.GELU(),
+ nn.Dropout(config.hidden_dropout_prob),
+ nn.Linear(config.classifier_hidden, config.num_labels),
+ )
+
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ attention_mask: torch.Tensor,
+ ) -> torch.Tensor:
+ """Returns logits of shape (batch, seq_len, num_labels)."""
+ outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
+ return self.classifier(outputs.last_hidden_state)
\ No newline at end of file
diff --git a/tools/xda/tests/test_dataset.py b/tools/xda/tests/test_dataset.py
new file mode 100644
index 0000000..9f2cf72
--- /dev/null
+++ b/tools/xda/tests/test_dataset.py
@@ -0,0 +1,50 @@
+import torch
+from src.dataset import ByteDataset, chunk_labeled_bytes
+from src.tokenizer import ByteTokenizer
+
+
+def test_chunk_labeled_bytes():
+ text_bytes = bytes(range(256)) * 4 # 1024 bytes
+ labels = [0] * 1024
+ labels[0] = 1 # function start at byte 0
+ labels[512] = 1 # function start at byte 512
+
+ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=512, stride=256)
+
+ assert len(chunks) > 0
+ for chunk in chunks:
+ assert len(chunk["bytes"]) == 512
+ assert len(chunk["labels"]) == 512
+
+
+def test_byte_dataset_shapes():
+ text_bytes = bytes(range(256)) * 4
+ labels = [0] * 1024
+ labels[0] = 1
+
+ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=512, stride=256)
+ ds = ByteDataset(chunks, ByteTokenizer())
+
+ item = ds[0]
+ assert item["input_ids"].shape == (514,) # 512 + [CLS] + [SEP]
+ assert item["attention_mask"].shape == (514,)
+ assert item["labels"].shape == (514,)
+ assert item["input_ids"][0].item() == 257 # [CLS]
+ assert item["input_ids"][-1].item() == 258 # [SEP]
+ # Labels for [CLS] and [SEP] should be -100 (ignored in loss)
+ assert item["labels"][0].item() == -100
+ assert item["labels"][-1].item() == -100
+
+
+def test_byte_dataset_label_values():
+ text_bytes = bytes([0x55, 0x48, 0x89, 0xe5, 0xc3, 0x00, 0x00, 0x00])
+ labels = [1, 2, 2, 2, 2, 0, 0, 0]
+
+ chunks = chunk_labeled_bytes(text_bytes, labels, window_size=8, stride=8)
+ ds = ByteDataset(chunks, ByteTokenizer())
+
+ item = ds[0]
+ # Labels offset by 1 for [CLS] prefix
+ assert item["labels"][1].item() == 1 # function_start
+ assert item["labels"][2].item() == 2 # function_body
+ assert item["labels"][6].item() == 0 # non-function
diff --git a/tools/xda/tests/test_model.py b/tools/xda/tests/test_model.py
new file mode 100644
index 0000000..a69451d
--- /dev/null
+++ b/tools/xda/tests/test_model.py
@@ -0,0 +1,40 @@
+import torch
+from src.model import XdaModel, XdaConfig
+
+
+def test_model_config_defaults():
+ config = XdaConfig()
+ assert config.vocab_size == 261
+ assert config.hidden_size == 768
+ assert config.num_hidden_layers == 12
+ assert config.num_attention_heads == 12
+ assert config.intermediate_size == 3072
+ assert config.num_labels == 3
+ assert config.max_position_embeddings == 514 # 512 + [CLS] + [SEP]
+
+
+def test_model_forward_shape():
+ config = XdaConfig(
+ num_hidden_layers=2, # smaller for test speed
+ hidden_size=128,
+ num_attention_heads=4,
+ intermediate_size=256,
+ )
+ model = XdaModel(config)
+
+ batch_size = 4
+ seq_len = 514
+ input_ids = torch.randint(0, 261, (batch_size, seq_len))
+ attention_mask = torch.ones(batch_size, seq_len, dtype=torch.long)
+
+ logits = model(input_ids, attention_mask)
+ assert logits.shape == (batch_size, seq_len, 3)
+
+
+def test_model_parameter_count():
+ """Sanity check that full-size model has ~87M params."""
+ config = XdaConfig()
+ model = XdaModel(config)
+ total = sum(p.numel() for p in model.parameters())
+ # Allow some tolerance — should be in the 80-95M range
+ assert 80_000_000 < total < 100_000_000, f"Param count {total} outside expected range"
\ No newline at end of file