Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakuf1966660818f
diff --git a/notes/architecture.txt b/notes/architecture.txt
index a8e4a91..bda9733 100644
--- a/notes/architecture.txt
+++ b/notes/architecture.txt
@@ -13,7 +13,7 @@ The architecture choices are interdependent:
- We do not need MLPs for tiny models
- Structured loss weighting targets the actual error distribution
- QAT provides regularisation eliminating the need for dropout
- - Attention is a nonlinearity in the absence of MLP
+ - softmax is a nonlinearity, so scaled dot-product attention (softmax(QK^T/√d) · V) is nonlinear.
----
diff --git a/src/cli.py b/src/cli.py
index 91a55e4..b41f59c 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -19,7 +19,7 @@ def main():
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=512)
+ p.add_argument("--d-model", type=int, default=768)
p.add_argument("--num-heads", type=int, default=4)
p.add_argument("--num-kv-heads", type=int, default=4)
p.add_argument("--num-layers", type=int, default=6)
diff --git a/src/train.py b/src/train.py
index e321bb7..9761520 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1104,7 +1104,10 @@ def train(args):
_best_metric = pool_metrics.get("single", {}).get("call_f1", 0)
if _best_metric > best_call_f1:
best_call_f1 = _best_metric
- best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_{config.d_ff}_best.pkl")
+ if config.no_feedforward:
+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_best.pkl")
+ else:
+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_{config.d_ff}_best.pkl")
import shutil as _shutil
_shutil.copy2(ckpt_path, best_ckpt_path)
print(f" ** New best single call_f1={best_call_f1:.1%} → {best_ckpt_path}")