Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Changes
HenryNdubuakud735046c97bd
diff --git a/needle/training/optim.py b/needle/training/optim.pyindex 4ca0690..be8890b 100644--- a/needle/training/optim.py+++ b/needle/training/optim.py@@ -4,7 +4,7 @@ import jaximport jax.numpy as jnpimport optaxfrom flax.training import train_state-from ..model.architecture import EncoderDecoderTransformer+from ..model.architecture import SimpleAttentionNetworkdef _newton_schulz(G, steps=5):@@ -82,7 +82,7 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):- model = EncoderDecoderTransformer(config)+ model = SimpleAttentionNetwork(config)rng, init_rng = jax.random.split(rng)dummy_src = jnp.ones((1, 128), dtype=jnp.int32)diff --git a/needle/training/train.py b/needle/training/train.pyindex 9f680fa..657559f 100644--- a/needle/training/train.py+++ b/needle/training/train.py@@ -16,7 +16,7 @@ from ..dataset.dataset import (get_contrastive_batches,)from ..model.architecture import (- EncoderDecoderTransformer,+ SimpleAttentionNetwork,TransformerConfig,make_packing_mask,make_causal_packing_mask,@@ -325,7 +325,7 @@ def train(args):muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)tokens_per_batch = global_batch_size * (args.max_enc_len + args.max_dec_len)- eval_model = EncoderDecoderTransformer(config)+ eval_model = SimpleAttentionNetwork(config)last_val_ppl = None# Dummy contrastive arrays — used when contrastive is inactive