Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix: default precompute_rope_freqs theta to 100000
jakmro2e7d5751f46e
diff --git a/needle/model/architecture.py b/needle/model/architecture.pyindex 368cc5a..ae9efde 100644--- a/needle/model/architecture.py+++ b/needle/model/architecture.py@@ -348,7 +348,7 @@ def width_slice(params, config, w):-def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):+def precompute_rope_freqs(head_dim, seq_len, theta=100000.0):freqs = 1.0 / (theta ** (jnp.arange(0, head_dim, 2).astype(jnp.float32) / head_dim))t = jnp.arange(seq_len).astype(jnp.float32)angles = jnp.outer(t, freqs)diff --git a/needle/model/architecture_mlx.py b/needle/model/architecture_mlx.pyindex c174689..f2ce647 100644--- a/needle/model/architecture_mlx.py+++ b/needle/model/architecture_mlx.py@@ -66,7 +66,7 @@ def _dense(x, kernel, dtype):return x.astype(dtype) @ kernel.astype(dtype)-def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):+def precompute_rope_freqs(head_dim, seq_len, theta=100000.0):freqs = 1.0 / (theta ** (mx.arange(0, head_dim, 2).astype(mx.float32) / head_dim))angles = mx.arange(seq_len).astype(mx.float32)[:, None] * freqs[None, :]return mx.cos(angles), mx.sin(angles)