Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
uodates
HenryNdubuaku153afbbf363c
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex 7c6db82..97974fa 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -37,7 +37,7 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"MODEL_POOL = ["google/gemini-3.1-flash-lite-preview",- "google/gemini-3-flash-preview",+ "google/gemini-2.5-flash-lite",]DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"diff --git a/src/train.py b/src/train.pyindex 078f4e1..969be93 100644--- a/src/train.py+++ b/src/train.py@@ -322,18 +322,40 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_maskreturn state, ema_params, loss, grad_norm-def _train_step_contrastive(state, ema_params, query_tokens, tool_tokens, rng):- """Separate contrastive training step using CLIP loss."""+def _train_step_fused(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):+ """Fused text + contrastive training step — single backward pass & optimizer step."""ema_decay = 0.999- cl_loss, grads = jax.value_and_grad(- lambda p: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, rng)- )(state.params)+ def combined_loss(p):+ text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)+ cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)+ return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss+ (loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")- cl_loss = jax.lax.pmean(cl_loss, axis_name="batch")- grads = jax.tree.map(lambda g: g * _CONTRASTIVE_WEIGHT, grads)+ text_loss = jax.lax.pmean(text_loss, axis_name="batch")+ grad_norm = optax.global_norm(grads)state = state.apply_gradients(grads=grads)ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)- return state, ema_params, cl_loss+ return state, ema_params, text_loss, grad_norm+++def _train_step_fused_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):+ """Fused text + contrastive training step with prune mask application."""+ ema_decay = 0.999++ def combined_loss(p):+ text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)+ cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)+ return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss++ (loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ text_loss = jax.lax.pmean(text_loss, axis_name="batch")+ grad_norm = optax.global_norm(grads)+ state = state.apply_gradients(grads=grads)+ masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)+ state = state.replace(params=masked_params)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)+ return state, ema_params, text_loss, grad_norm@@ -346,8 +368,12 @@ def _make_p_train_step_masked():return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))-def _make_p_train_step_contrastive():- return jax.pmap(_train_step_contrastive, axis_name="batch", donate_argnums=(0, 1))+def _make_p_train_step_fused():+ return jax.pmap(_train_step_fused, axis_name="batch", donate_argnums=(0, 1))+++def _make_p_train_step_fused_masked():+ return jax.pmap(_train_step_fused_masked, axis_name="batch", donate_argnums=(0, 1))def _make_val_loss_fn(apply_fn):@@ -490,7 +516,8 @@ def train(args):n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1p_train_step = _make_p_train_step()p_train_step_masked = _make_p_train_step_masked()- p_train_step_contrastive = _make_p_train_step_contrastive()+ p_train_step_fused = _make_p_train_step_fused()+ p_train_step_fused_masked = _make_p_train_step_fused_masked()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)@@ -625,22 +652,28 @@ def train(args):rng, text_rng = jax.random.split(rng)text_rngs = jax.random.split(text_rng, num_devices)- if prune_mask is not None:- state, ema_params, loss, grad_norm = p_train_step_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,- )- else:- state, ema_params, loss, grad_norm = p_train_step(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,- )-- # Separate contrastive step (if data available)if cl_q_b is not None and cl_t_b is not None:+ # Fused text + contrastive: single backward pass & optimizer steprng, cl_rng = jax.random.split(rng)cl_rngs = jax.random.split(cl_rng, num_devices)- state, ema_params, cl_loss = p_train_step_contrastive(- state, ema_params, cl_q_b, cl_t_b, cl_rngs,- )+ if prune_mask is not None:+ state, ema_params, loss, grad_norm = p_train_step_fused_masked(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,+ )+ else:+ state, ema_params, loss, grad_norm = p_train_step_fused(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,+ )+ else:+ # Text-only step (no contrastive data this iteration)+ if prune_mask is not None:+ state, ema_params, loss, grad_norm = p_train_step_masked(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,+ )+ else:+ state, ema_params, loss, grad_norm = p_train_step(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,+ )text_loss_val = float(loss[0])text_losses.append(text_loss_val)