Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku9047ae43ea49
diff --git a/src/cli.py b/src/cli.py
index b41f59c..d108120 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -19,11 +19,11 @@ def main():
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=768)
- p.add_argument("--num-heads", type=int, default=4)
+ p.add_argument("--d-model", type=int, default=512)
+ p.add_argument("--num-heads", type=int, default=8)
p.add_argument("--num-kv-heads", type=int, default=4)
- p.add_argument("--num-layers", type=int, default=6)
- p.add_argument("--num-dec-layers", type=int, default=6)
+ p.add_argument("--num-layers", type=int, default=12)
+ p.add_argument("--num-dec-layers", type=int, default=12)
p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
@@ -56,8 +56,6 @@ def main():
help="Weight for CLIP-style contrastive loss (default: 0.1)")
p.add_argument("--contrastive-dim", type=int, default=128,
help="Dimension of contrastive projection head (default: 128)")
- p.add_argument("--conv-kernel-size", type=int, default=0,
- help="Conformer conv kernel size in encoder (0=disabled, 15=default when enabled)")
p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,
help="Remove feedforward layers entirely (default: True)")
diff --git a/src/model.py b/src/model.py
index 9e34ea3..731b8c0 100644
--- a/src/model.py
+++ b/src/model.py
@@ -48,7 +48,6 @@ class TransformerConfig:
n_mels: int = 80
dropout_rate: float = 0.1
contrastive_dim: int = 128
- conv_kernel_size: int = 0
enable_speech: bool = False
no_feedforward: bool = True
@@ -194,33 +193,6 @@ class SpeechSubsampling(nn.Module):
return x
-class ConvModule(nn.Module):
- """Conformer-style convolution module: pointwise up -> GLU -> depthwise conv -> norm -> pointwise down."""
- d_model: int
- kernel_size: int
- num_layers: int
- dtype: jnp.dtype = jnp.bfloat16
-
- @nn.compact
- def __call__(self, x):
-
- x = nn.Dense(self.d_model * 2, dtype=self.dtype, use_bias=False,
- kernel_init=default_init(), name="pw_up")(x)
-
- x1, x2 = jnp.split(x, 2, axis=-1)
- x = x1 * nn.sigmoid(x2)
-
- x = nn.Conv(features=self.d_model, kernel_size=(self.kernel_size,),
- padding='SAME', feature_group_count=self.d_model,
- dtype=self.dtype, use_bias=False, name="dw_conv")(x)
-
- x = ZCRMSNorm(dtype=self.dtype, name="conv_norm")(x)
-
- x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
- kernel_init=residual_init(self.num_layers), name="pw_down")(x)
- return x
-
-
class EncoderBlock(nn.Module):
"""Pre-norm self-attention + pre-norm FFN with gated residual connections."""
num_heads: int
@@ -231,7 +203,6 @@ class EncoderBlock(nn.Module):
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
dropout_rate: float = 0.0
- conv_kernel_size: int = 0
no_feedforward: bool = True
@nn.compact
@@ -244,14 +215,6 @@ class EncoderBlock(nn.Module):
)
x = residual + gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
- if self.conv_kernel_size > 0:
- conv_gate = nn.sigmoid(self.param("conv_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = ConvModule(self.d_model, self.conv_kernel_size, self.num_layers,
- self.dtype, name="conv_module")(x)
- x = residual + conv_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
-
if not self.no_feedforward:
ffn_gate = nn.sigmoid(self.param("ffn_gate", jinit.zeros, ())).astype(self.dtype)
residual = x
@@ -272,7 +235,6 @@ class _EncoderScanBody(nn.Module):
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
dropout_rate: float = 0.0
- conv_kernel_size: int = 0
no_feedforward: bool = True
deterministic: bool = True
@@ -282,7 +244,7 @@ class _EncoderScanBody(nn.Module):
x = EncoderBlock(
self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,
self.num_layers, self.dtype, self.activation, self.dropout_rate,
- self.conv_kernel_size, self.no_feedforward,
+ self.no_feedforward,
)(x, mask, rope, ffn_mask, self.deterministic)
return (x, mask, rope, ffn_mask), None
@@ -306,7 +268,7 @@ class Encoder(nn.Module):
(x, _, _, _), _ = ScanBlock(
cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
- cfg.conv_kernel_size, cfg.no_feedforward, deterministic, name="layers",
+ cfg.no_feedforward, deterministic, name="layers",
)((x, mask, rope, ffn_mask), None)
x = ZCRMSNorm(dtype=dt, name="final_norm")(x)
diff --git a/src/train.py b/src/train.py
index 9761520..61e97e5 100644
--- a/src/train.py
+++ b/src/train.py
@@ -441,17 +441,13 @@ def _estimate_mat_params(config, matryoshka_factor):
emb = v * d
attn = d * d + d * kv_dim * 2 + d * d
ffn = d * d_ff * 3
- conv = 0
- if config.conv_kernel_size > 0:
- K = config.conv_kernel_size
- conv = d * 2 * d + d * K + d + d * d
speech_sub = 0
if getattr(config, "enable_speech", True):
C = 256
n_mels = getattr(config, "n_mels", 80)
freq_out = n_mels // 8
speech_sub = (1 * C * 9) + (C * 9 + C * C) + (C * 9 + C * C) + (C * freq_out * d)
- enc_block = attn + conv + ffn
+ enc_block = attn + ffn
dec_block = attn * 2 + ffn
total = emb + speech_sub + n_enc * enc_block + n_dec * dec_block
return int(total)
@@ -529,7 +525,6 @@ def train(args):
activation=getattr(args, "activation", "drelu"),
num_memory_slots=getattr(args, "num_memory_slots", 64),
contrastive_dim=getattr(args, "contrastive_dim", 128),
- conv_kernel_size=getattr(args, "conv_kernel_size", 0),
enable_speech=getattr(args, "enable_speech", False),
no_feedforward=getattr(args, "no_feedforward", True),
)