Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

fix: default precompute_rope_freqs theta to 100000

jakmro2e7d5751f46e
diff --git a/needle/model/architecture.py b/needle/model/architecture.py
index 368cc5a..ae9efde 100644
--- a/needle/model/architecture.py
+++ b/needle/model/architecture.py
@@ -348,7 +348,7 @@ def width_slice(params, config, w):
-def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):
+def precompute_rope_freqs(head_dim, seq_len, theta=100000.0):
freqs = 1.0 / (theta ** (jnp.arange(0, head_dim, 2).astype(jnp.float32) / head_dim))
t = jnp.arange(seq_len).astype(jnp.float32)
angles = jnp.outer(t, freqs)
diff --git a/needle/model/architecture_mlx.py b/needle/model/architecture_mlx.py
index c174689..f2ce647 100644
--- a/needle/model/architecture_mlx.py
+++ b/needle/model/architecture_mlx.py
@@ -66,7 +66,7 @@ def _dense(x, kernel, dtype):
return x.astype(dtype) @ kernel.astype(dtype)
-def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):
+def precompute_rope_freqs(head_dim, seq_len, theta=100000.0):
freqs = 1.0 / (theta ** (mx.arange(0, head_dim, 2).astype(mx.float32) / head_dim))
angles = mx.arange(seq_len).astype(mx.float32)[:, None] * freqs[None, :]
return mx.cos(angles), mx.sin(angles)