Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

reduce decoder size

HenryNdubuakub60241b5e459
diff --git a/src/cli.py b/src/cli.py
index b3c1f3b..98d56bf 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -74,6 +74,7 @@ TOY_CONFIG = {
"d_model": 128,
"num_heads": 4,
"num_layers": 2,
+ "num_dec_layers": 2,
"max_enc_len": 128,
"max_dec_len": 128,
"max_samples": 10000,
@@ -87,9 +88,10 @@ BASE_CONFIG = {
"batch_size": 32,
"lr": 3e-4,
"muon_lr": 0.02,
- "d_model": 256,
+ "d_model": 512,
"num_heads": 4,
"num_layers": 4,
+ "num_dec_layers": 2,
"max_enc_len": 256,
"max_dec_len": 256,
"max_samples": 500000,
@@ -124,6 +126,7 @@ def main():
p.add_argument("--d-model", type=int, default=None)
p.add_argument("--num-heads", type=int, default=None)
p.add_argument("--num-layers", type=int, default=None)
+ p.add_argument("--num-dec-layers", type=int, default=None)
p.add_argument("--max-enc-len", type=int, default=None)
p.add_argument("--max-dec-len", type=int, default=None)
p.add_argument("--max-samples", type=int, default=None)
diff --git a/src/data.py b/src/data.py
index 4e0f1ed..a2c7471 100644
--- a/src/data.py
+++ b/src/data.py
@@ -1,9 +1,12 @@
import hashlib
import os
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
+os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
+os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
import logging
logging.getLogger("transformers").setLevel(logging.ERROR)
+logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
from datasets import load_dataset
diff --git a/src/train.py b/src/train.py
index a422122..ee9a098 100644
--- a/src/train.py
+++ b/src/train.py
@@ -391,7 +391,7 @@ def train(args):
d_model=args.d_model,
num_heads=args.num_heads,
num_encoder_layers=args.num_layers,
- num_decoder_layers=args.num_layers,
+ num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),
d_ff=args.d_model * 4,
max_seq_len=max(args.max_enc_len, args.max_dec_len),
dtype=args.dtype,