Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuakuf1966660818f
diff --git a/notes/architecture.txt b/notes/architecture.txtindex a8e4a91..bda9733 100644--- a/notes/architecture.txt+++ b/notes/architecture.txt@@ -13,7 +13,7 @@ The architecture choices are interdependent:- We do not need MLPs for tiny models- Structured loss weighting targets the actual error distribution- QAT provides regularisation eliminating the need for dropout- - Attention is a nonlinearity in the absence of MLP+ - softmax is a nonlinearity, so scaled dot-product attention (softmax(QK^T/√d) · V) is nonlinear.----diff --git a/src/cli.py b/src/cli.pyindex 91a55e4..b41f59c 100644--- a/src/cli.py+++ b/src/cli.py@@ -19,7 +19,7 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--d-model", type=int, default=768)p.add_argument("--num-heads", type=int, default=4)p.add_argument("--num-kv-heads", type=int, default=4)p.add_argument("--num-layers", type=int, default=6)diff --git a/src/train.py b/src/train.pyindex e321bb7..9761520 100644--- a/src/train.py+++ b/src/train.py@@ -1104,7 +1104,10 @@ def train(args):_best_metric = pool_metrics.get("single", {}).get("call_f1", 0)if _best_metric > best_call_f1:best_call_f1 = _best_metric- best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_{config.d_ff}_best.pkl")+ if config.no_feedforward:+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_best.pkl")+ else:+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_{config.d_ff}_best.pkl")import shutil as _shutil_shutil.copy2(ckpt_path, best_ckpt_path)print(f" ** New best single call_f1={best_call_f1:.1%} → {best_ckpt_path}")