Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Changes

HenryNdubuakud735046c97bd
diff --git a/needle/training/optim.py b/needle/training/optim.py
index 4ca0690..be8890b 100644
--- a/needle/training/optim.py
+++ b/needle/training/optim.py
@@ -4,7 +4,7 @@ import jax
import jax.numpy as jnp
import optax
from flax.training import train_state
-from ..model.architecture import EncoderDecoderTransformer
+from ..model.architecture import SimpleAttentionNetwork
def _newton_schulz(G, steps=5):
@@ -82,7 +82,7 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):
- model = EncoderDecoderTransformer(config)
+ model = SimpleAttentionNetwork(config)
rng, init_rng = jax.random.split(rng)
dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
diff --git a/needle/training/train.py b/needle/training/train.py
index 9f680fa..657559f 100644
--- a/needle/training/train.py
+++ b/needle/training/train.py
@@ -16,7 +16,7 @@ from ..dataset.dataset import (
get_contrastive_batches,
)
from ..model.architecture import (
- EncoderDecoderTransformer,
+ SimpleAttentionNetwork,
TransformerConfig,
make_packing_mask,
make_causal_packing_mask,
@@ -325,7 +325,7 @@ def train(args):
muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
tokens_per_batch = global_batch_size * (args.max_enc_len + args.max_dec_len)
- eval_model = EncoderDecoderTransformer(config)
+ eval_model = SimpleAttentionNetwork(config)
last_val_ppl = None
# Dummy contrastive arrays — used when contrastive is inactive