Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
QAT
HenryNdubuaku1fa0059cab2f
diff --git a/src/train.py b/src/train.pyindex 5297e02..b1d36b0 100644--- a/src/train.py+++ b/src/train.py@@ -133,6 +133,43 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_)+def _fake_quantize_int4(w, group_size=32):+ """Symmetric group-wise INT4 fake quantization with STE.++ Divides the input dimension (axis 0) into groups of `group_size` elements,+ each with its own scale factor. Falls back to per-channel if in_features < group_size.+ """+ in_feat, out_feat = w.shape+ gs = min(group_size, in_feat)++ pad = (gs - in_feat % gs) % gs+ if pad > 0:+ w_padded = jnp.pad(w, ((0, pad), (0, 0)))+ else:+ w_padded = w++ num_groups = w_padded.shape[0] // gs+ w_grouped = w_padded.reshape(num_groups, gs, out_feat)++ scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0+ scale = jnp.maximum(scale, 1e-8)+ w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale++ w_q = w_q.reshape(-1, out_feat)[:in_feat]++ return w + jax.lax.stop_gradient(w_q - w)+++def _quantize_params(params):+ """Fake-quantize all Dense kernels in the param tree."""+ def _maybe_quantize(path, leaf):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel":+ return _fake_quantize_int4(leaf)+ return leaf+ return jax.tree_util.tree_map_with_path(_maybe_quantize, params)++def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rng):pad_id = 0ema_decay = 0.999@@ -143,7 +180,7 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rncross_mask = src_masklogits = state.apply_fn(- {"params": params},+ {"params": _quantize_params(params)},src,tgt_in,src_mask=src_mask,@@ -247,14 +284,11 @@ def train(args):total_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))- # Scale LRs for data parallelism: linear for Adam, sqrt for Muon- # (Muon's Newton-Schulz orthogonalization is sensitive to large LRs).scaled_lr = args.lr * num_devicesmuon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)val_loss_fn = _make_val_loss_fn(state.apply_fn)- # Replicate state and EMA params across all devicesema_params = jax.tree.map(jnp.copy, state.params)state = jax_utils.replicate(state)ema_params = jax_utils.replicate(ema_params)