Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Change tokenizer
HenryNdubuakub616b4e849fb
diff --git a/src/data.py b/src/data.pyindex 59c149b..1c1bb31 100644--- a/src/data.py+++ b/src/data.py@@ -14,8 +14,9 @@ CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cachdef get_tokenizer():- tokenizer = AutoTokenizer.from_pretrained("gpt2")- tokenizer.pad_token = tokenizer.eos_token+ tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-1.2B")+ if tokenizer.pad_token is None:+ tokenizer.pad_token = tokenizer.eos_tokenreturn tokenizer@@ -26,8 +27,8 @@ def load_tinystories(split="train", max_samples=None):return ds-def _cache_key(n_samples, max_enc_len, max_dec_len):- key = f"tinystories_{n_samples}_{max_enc_len}_{max_dec_len}"+def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="lfm2"):+ key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"return hashlib.md5(key.encode()).hexdigest()[:12]diff --git a/src/model.py b/src/model.pyindex 1956e19..cc7b4bf 100644--- a/src/model.py+++ b/src/model.py@@ -31,7 +31,7 @@ class ZCRMSNorm(nn.Module):@dataclassclass TransformerConfig:- vocab_size: int = 50257+ vocab_size: int = 65536d_model: int = 128num_heads: int = 4num_kv_heads: int = 2@@ -40,7 +40,7 @@ class TransformerConfig:d_ff: int = 512max_seq_len: int = 128dropout_rate: float = 0.1- pad_token_id: int = 50256+ pad_token_id: int = 0rope_theta: float = 10000.0dtype: str = "bfloat16"diff --git a/src/train.py b/src/train.pyindex 1029914..2df2eca 100644--- a/src/train.py+++ b/src/train.py@@ -131,7 +131,7 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):- pad_id = 50256+ pad_id = 0def loss_fn(params):src_mask = make_padding_mask(src, pad_id)