Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku9047ae43ea49
diff --git a/src/cli.py b/src/cli.pyindex b41f59c..d108120 100644--- a/src/cli.py+++ b/src/cli.py@@ -19,11 +19,11 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=768)- p.add_argument("--num-heads", type=int, default=4)+ p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--num-heads", type=int, default=8)p.add_argument("--num-kv-heads", type=int, default=4)- p.add_argument("--num-layers", type=int, default=6)- p.add_argument("--num-dec-layers", type=int, default=6)+ p.add_argument("--num-layers", type=int, default=12)+ p.add_argument("--num-dec-layers", type=int, default=12)p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)@@ -56,8 +56,6 @@ def main():help="Weight for CLIP-style contrastive loss (default: 0.1)")p.add_argument("--contrastive-dim", type=int, default=128,help="Dimension of contrastive projection head (default: 128)")- p.add_argument("--conv-kernel-size", type=int, default=0,- help="Conformer conv kernel size in encoder (0=disabled, 15=default when enabled)")p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,help="Remove feedforward layers entirely (default: True)")diff --git a/src/model.py b/src/model.pyindex 9e34ea3..731b8c0 100644--- a/src/model.py+++ b/src/model.py@@ -48,7 +48,6 @@ class TransformerConfig:n_mels: int = 80dropout_rate: float = 0.1contrastive_dim: int = 128- conv_kernel_size: int = 0enable_speech: bool = Falseno_feedforward: bool = True@@ -194,33 +193,6 @@ class SpeechSubsampling(nn.Module):return x-class ConvModule(nn.Module):- """Conformer-style convolution module: pointwise up -> GLU -> depthwise conv -> norm -> pointwise down."""- d_model: int- kernel_size: int- num_layers: int- dtype: jnp.dtype = jnp.bfloat16-- @nn.compact- def __call__(self, x):-- x = nn.Dense(self.d_model * 2, dtype=self.dtype, use_bias=False,- kernel_init=default_init(), name="pw_up")(x)-- x1, x2 = jnp.split(x, 2, axis=-1)- x = x1 * nn.sigmoid(x2)-- x = nn.Conv(features=self.d_model, kernel_size=(self.kernel_size,),- padding='SAME', feature_group_count=self.d_model,- dtype=self.dtype, use_bias=False, name="dw_conv")(x)-- x = ZCRMSNorm(dtype=self.dtype, name="conv_norm")(x)-- x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,- kernel_init=residual_init(self.num_layers), name="pw_down")(x)- return x--class EncoderBlock(nn.Module):"""Pre-norm self-attention + pre-norm FFN with gated residual connections."""num_heads: int@@ -231,7 +203,6 @@ class EncoderBlock(nn.Module):dtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"dropout_rate: float = 0.0- conv_kernel_size: int = 0no_feedforward: bool = True@nn.compact@@ -244,14 +215,6 @@ class EncoderBlock(nn.Module):)x = residual + gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)- if self.conv_kernel_size > 0:- conv_gate = nn.sigmoid(self.param("conv_gate", jinit.zeros, ())).astype(self.dtype)- residual = x- x = ZCRMSNorm(dtype=self.dtype)(x)- x = ConvModule(self.d_model, self.conv_kernel_size, self.num_layers,- self.dtype, name="conv_module")(x)- x = residual + conv_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)-if not self.no_feedforward:ffn_gate = nn.sigmoid(self.param("ffn_gate", jinit.zeros, ())).astype(self.dtype)residual = x@@ -272,7 +235,6 @@ class _EncoderScanBody(nn.Module):dtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"dropout_rate: float = 0.0- conv_kernel_size: int = 0no_feedforward: bool = Truedeterministic: bool = True@@ -282,7 +244,7 @@ class _EncoderScanBody(nn.Module):x = EncoderBlock(self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,self.num_layers, self.dtype, self.activation, self.dropout_rate,- self.conv_kernel_size, self.no_feedforward,+ self.no_feedforward,)(x, mask, rope, ffn_mask, self.deterministic)return (x, mask, rope, ffn_mask), None@@ -306,7 +268,7 @@ class Encoder(nn.Module):(x, _, _, _), _ = ScanBlock(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,- cfg.conv_kernel_size, cfg.no_feedforward, deterministic, name="layers",+ cfg.no_feedforward, deterministic, name="layers",)((x, mask, rope, ffn_mask), None)x = ZCRMSNorm(dtype=dt, name="final_norm")(x)diff --git a/src/train.py b/src/train.pyindex 9761520..61e97e5 100644--- a/src/train.py+++ b/src/train.py@@ -441,17 +441,13 @@ def _estimate_mat_params(config, matryoshka_factor):emb = v * dattn = d * d + d * kv_dim * 2 + d * dffn = d * d_ff * 3- conv = 0- if config.conv_kernel_size > 0:- K = config.conv_kernel_size- conv = d * 2 * d + d * K + d + d * dspeech_sub = 0if getattr(config, "enable_speech", True):C = 256n_mels = getattr(config, "n_mels", 80)freq_out = n_mels // 8speech_sub = (1 * C * 9) + (C * 9 + C * C) + (C * 9 + C * C) + (C * freq_out * d)- enc_block = attn + conv + ffn+ enc_block = attn + ffndec_block = attn * 2 + ffntotal = emb + speech_sub + n_enc * enc_block + n_dec * dec_blockreturn int(total)@@ -529,7 +525,6 @@ def train(args):activation=getattr(args, "activation", "drelu"),num_memory_slots=getattr(args, "num_memory_slots", 64),contrastive_dim=getattr(args, "contrastive_dim", 128),- conv_kernel_size=getattr(args, "conv_kernel_size", 0),enable_speech=getattr(args, "enable_speech", False),no_feedforward=getattr(args, "no_feedforward", True),)