Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
reduce decoder size
HenryNdubuakub60241b5e459
diff --git a/src/cli.py b/src/cli.pyindex b3c1f3b..98d56bf 100644--- a/src/cli.py+++ b/src/cli.py@@ -74,6 +74,7 @@ TOY_CONFIG = {"d_model": 128,"num_heads": 4,"num_layers": 2,+ "num_dec_layers": 2,"max_enc_len": 128,"max_dec_len": 128,"max_samples": 10000,@@ -87,9 +88,10 @@ BASE_CONFIG = {"batch_size": 32,"lr": 3e-4,"muon_lr": 0.02,- "d_model": 256,+ "d_model": 512,"num_heads": 4,"num_layers": 4,+ "num_dec_layers": 2,"max_enc_len": 256,"max_dec_len": 256,"max_samples": 500000,@@ -124,6 +126,7 @@ def main():p.add_argument("--d-model", type=int, default=None)p.add_argument("--num-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=None)+ p.add_argument("--num-dec-layers", type=int, default=None)p.add_argument("--max-enc-len", type=int, default=None)p.add_argument("--max-dec-len", type=int, default=None)p.add_argument("--max-samples", type=int, default=None)diff --git a/src/data.py b/src/data.pyindex 4e0f1ed..a2c7471 100644--- a/src/data.py+++ b/src/data.py@@ -1,9 +1,12 @@import hashlibimport osos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"+os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"+os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"import logginglogging.getLogger("transformers").setLevel(logging.ERROR)+logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as npfrom datasets import load_datasetdiff --git a/src/train.py b/src/train.pyindex a422122..ee9a098 100644--- a/src/train.py+++ b/src/train.py@@ -391,7 +391,7 @@ def train(args):d_model=args.d_model,num_heads=args.num_heads,num_encoder_layers=args.num_layers,- num_decoder_layers=args.num_layers,+ num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),d_ff=args.d_model * 4,max_seq_len=max(args.max_enc_len, args.max_dec_len),dtype=args.dtype,