Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuakuee904382dd12
diff --git a/src/cli.py b/src/cli.pyindex 20e2964..eed0cbb 100644--- a/src/cli.py+++ b/src/cli.py@@ -19,7 +19,7 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--d-model", type=int, default=1536)p.add_argument("--num-heads", type=int, default=4)p.add_argument("--num-kv-heads", type=int, default=4)p.add_argument("--num-layers", type=int, default=6)@@ -46,7 +46,7 @@ def main():p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--num-memory-slots", type=int, default=128,help="(DEPRECATED — ignored, kept for checkpoint compat)")- p.add_argument("--mat-factors", type=int, nargs="*", default=[],+ p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4, 8],help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")p.add_argument("--dropout", type=float, default=0.0,help="Dropout rate for residual connections (default: 0.0)")