Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
optimise
HenryNdubuakuaf75c4b79cda
diff --git a/src/model.py b/src/model.pyindex a60fa03..af99328 100644--- a/src/model.py+++ b/src/model.py@@ -242,7 +242,7 @@ class MemoryMixerEncoder(nn.Module):mask = mask[..., :T_new]for i in range(cfg.num_encoder_layers):- x, s = MemoryMixerBlock(+ x, s = nn.remat(MemoryMixerBlock)(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}")(x, s, mask=mask, rope=rope)@@ -295,7 +295,7 @@ class Decoder(nn.Module):x = x.astype(dt)for i in range(cfg.num_decoder_layers):- x = DecoderBlock(+ x = nn.remat(DecoderBlock)(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}")(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)diff --git a/src/train.py b/src/train.pyindex 3fafe3d..1da0d7e 100644--- a/src/train.py+++ b/src/train.py@@ -159,52 +159,6 @@ def _fake_quantize_int4(w, group_size=32):return w + jax.lax.stop_gradient(w_q - w)-def _block_prune_params(params, prune_ratio=0.25, group_size=32):- """Block magnitude pruning aligned to quantization groups.-- All 2D weight matrices (Dense kernels + embeddings) are divided into- (group_size, 1) blocks along axis 0. Blocks are scored by L1 norm,- and the bottom `prune_ratio` fraction (globally) are zeroed out.- """- all_scores = []- for path, leaf in jax.tree_util.tree_leaves_with_path(params):- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if leaf.ndim != 2:- continue- w = np.array(leaf)- in_feat, out_feat = w.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- if pad:- w = np.pad(w, ((0, pad), (0, 0)))- w_grouped = w.reshape(-1, gs, out_feat)- scores = np.sum(np.abs(w_grouped), axis=1).ravel()- all_scores.append(scores)-- if not all_scores:- return params-- threshold = np.percentile(np.concatenate(all_scores), prune_ratio * 100)-- def _prune_weight(path, leaf):- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if leaf.ndim != 2:- return leaf- w = np.array(leaf)- in_feat, out_feat = w.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- if pad:- w = np.pad(w, ((0, pad), (0, 0)))- w_grouped = w.reshape(-1, gs, out_feat)- scores = np.sum(np.abs(w_grouped), axis=1, keepdims=True)- mask = (scores > threshold).astype(np.float32)- w_pruned = (w_grouped * mask).reshape(-1, out_feat)[:in_feat]- return jnp.array(w_pruned)-- return jax.tree_util.tree_map_with_path(_prune_weight, params)--def _cubic_sparsity_schedule(step, t_start, t_end, s_final):"""Cubic sparsity ramp (Zhu & Gupta 2017). Returns target sparsity at *step*."""if step < t_start:@@ -266,48 +220,50 @@ _GROUP_SIZE = 32_MRL_DIMS = ()-def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):+def _loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):pad_id = 0- ema_decay = 0.999-- def loss_fn(params):- src_mask = make_padding_mask(src, pad_id)- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)- cross_mask = src_mask-- logits, slot_div, mrl_logits = state.apply_fn(- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},- src,- tgt_in,- src_mask=src_mask,- tgt_mask=tgt_mask,- cross_mask=cross_mask,- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,- method="forward_with_aux",- )+ src_mask = make_padding_mask(src, pad_id)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)+ cross_mask = src_mask++ logits, slot_div, mrl_logits = state.apply_fn(+ {"params": _quantize_params(params, group_size=_GROUP_SIZE)},+ src,+ tgt_in,+ src_mask=src_mask,+ tgt_mask=tgt_mask,+ cross_mask=cross_mask,+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,+ method="forward_with_aux",+ )- logits_f32 = logits.astype(jnp.float32)- mask = (tgt_out != pad_id).astype(jnp.float32)- ce_loss = jnp.sum(- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask+ logits_f32 = logits.astype(jnp.float32)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ ce_loss = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask+ ) / jnp.maximum(jnp.sum(mask), 1.0)++ total_loss = ce_loss+ num_scales = 1+ for mrl_log in mrl_logits:+ mrl_log_f32 = mrl_log.astype(jnp.float32)+ aux_loss = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask) / jnp.maximum(jnp.sum(mask), 1.0)+ total_loss = total_loss + aux_loss+ num_scales = num_scales + 1+ total_loss = total_loss / num_scales- total_loss = ce_loss- num_scales = 1- for mrl_log in mrl_logits:- mrl_log_f32 = mrl_log.astype(jnp.float32)- aux_loss = jnp.sum(- optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask- ) / jnp.maximum(jnp.sum(mask), 1.0)- total_loss = total_loss + aux_loss- num_scales = num_scales + 1- total_loss = total_loss / num_scales-- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- div_loss = 1e-4 * slot_div- return total_loss + z_loss + div_loss-- loss, grads = jax.value_and_grad(loss_fn)(state.params)+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)+ div_loss = 1e-4 * slot_div+ return total_loss + z_loss + div_loss+++def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):+ ema_decay = 0.999+ loss, grads = jax.value_and_grad(+ lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ )(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")grad_norm = optax.global_norm(grads)@@ -316,10 +272,30 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):return state, ema_params, loss, grad_norm+def _train_step_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask):+ """Training step with fused mask application — avoids post-pmap copies."""+ ema_decay = 0.999+ loss, grads = jax.value_and_grad(+ lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ )(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ loss = jax.lax.pmean(loss, axis_name="batch")+ grad_norm = optax.global_norm(grads)+ state = state.apply_gradients(grads=grads)+ masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)+ state = state.replace(params=masked_params)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)+ return state, ema_params, loss, grad_norm++def _make_p_train_step():return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))+def _make_p_train_step_masked():+ return jax.pmap(_train_step_masked, axis_name="batch", donate_argnums=(0, 1))++def _make_val_loss_fn(apply_fn):@jax.jitdef val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):@@ -456,6 +432,7 @@ def train(args):else:_MRL_DIMS = ()p_train_step = _make_p_train_step()+ p_train_step_masked = _make_p_train_step_masked()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)@@ -540,15 +517,18 @@ def train(args):for src, tgt_in, tgt_out in pbar:t0 = time.perf_counter()+ src_b = shard_batch(src, num_devices)+ tgt_in_b = shard_batch(tgt_in, num_devices)+ tgt_out_b = shard_batch(tgt_out, num_devices)- state, ema_params, loss, grad_norm = p_train_step(- state,- ema_params,- shard_batch(src, num_devices),- shard_batch(tgt_in, num_devices),- shard_batch(tgt_out, num_devices),- causal_mask,- )+ if prune_mask is not None:+ state, ema_params, loss, grad_norm = p_train_step_masked(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask,+ )+ else:+ state, ema_params, loss, grad_norm = p_train_step(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask,+ )# --- Gradual magnitude sparsification (epoch 1 only) ---if epoch == weight_prune_epoch and not gradual_sparsify_done:@@ -561,10 +541,6 @@ def train(args):prune_mask = jax_utils.replicate(mask)del mask- if prune_mask is not None:- state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))- ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)-loss_val = float(loss[0])losses.append(loss_val)global_step += 1@@ -612,7 +588,9 @@ def train(args):del ema_unrprune_mask = jax_utils.replicate(mask)del mask- state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))+ # Apply mask once via the fused step (reuses donated buffers)+ state = state.replace(+ params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)final_pruned = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))total_p = sum(x.size for x in jax.tree.leaves(final_pruned))