Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Change tokenizer

HenryNdubuakub616b4e849fb
diff --git a/src/data.py b/src/data.py
index 59c149b..1c1bb31 100644
--- a/src/data.py
+++ b/src/data.py
@@ -14,8 +14,9 @@ CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cach
def get_tokenizer():
- tokenizer = AutoTokenizer.from_pretrained("gpt2")
- tokenizer.pad_token = tokenizer.eos_token
+ tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-1.2B")
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
return tokenizer
@@ -26,8 +27,8 @@ def load_tinystories(split="train", max_samples=None):
return ds
-def _cache_key(n_samples, max_enc_len, max_dec_len):
- key = f"tinystories_{n_samples}_{max_enc_len}_{max_dec_len}"
+def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="lfm2"):
+ key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"
return hashlib.md5(key.encode()).hexdigest()[:12]
diff --git a/src/model.py b/src/model.py
index 1956e19..cc7b4bf 100644
--- a/src/model.py
+++ b/src/model.py
@@ -31,7 +31,7 @@ class ZCRMSNorm(nn.Module):
@dataclass
class TransformerConfig:
- vocab_size: int = 50257
+ vocab_size: int = 65536
d_model: int = 128
num_heads: int = 4
num_kv_heads: int = 2
@@ -40,7 +40,7 @@ class TransformerConfig:
d_ff: int = 512
max_seq_len: int = 128
dropout_rate: float = 0.1
- pad_token_id: int = 50256
+ pad_token_id: int = 0
rope_theta: float = 10000.0
dtype: str = "bfloat16"
diff --git a/src/train.py b/src/train.py
index 1029914..2df2eca 100644
--- a/src/train.py
+++ b/src/train.py
@@ -131,7 +131,7 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_
def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
- pad_id = 50256
+ pad_id = 0
def loss_fn(params):
src_mask = make_padding_mask(src, pad_id)