Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
optimise
HenryNdubuakuddedc4da8e6b
diff --git a/src/model.py b/src/model.pyindex 81b3134..2e77085 100644--- a/src/model.py+++ b/src/model.py@@ -49,6 +49,7 @@ class TransformerConfig:dropout_rate: float = 0.1contrastive_dim: int = 128conv_kernel_size: int = 0+ enable_speech: bool = False@propertydef jax_dtype(self):@@ -429,9 +430,10 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)- self.spec_augment = SpecAugment()- self.speech_subsample = SpeechSubsampling(- self.config.d_model, dtype=self.config.jax_dtype, name="speech_subsample")+ if self.config.enable_speech:+ self.spec_augment = SpecAugment()+ self.speech_subsample = SpeechSubsampling(+ self.config.d_model, dtype=self.config.jax_dtype, name="speech_subsample")self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)self.contrastive_proj = nn.Dense(@@ -454,6 +456,7 @@ class EncoderDecoderTransformer(nn.Module):return self.encode_text(src, src_mask=src_mask)def encode_speech(self, mel, src_mask=None, ffn_mask=None, deterministic=True):+ assert self.config.enable_speech, "Speech is disabled in config"mel = self.spec_augment(mel, deterministic=deterministic)# FastConformer-style 8x subsampling: (B, T, 80) -> (B, T//8, d_model)x = self.speech_subsample(mel) * self.embed_scale@@ -574,15 +577,16 @@ class EncoderDecoderTransformer(nn.Module):return logits, 0.0, mat_logits- def init_all(self, src, tgt, mel):- """Dummy forward through both text and speech pathways to initialize all params."""+ def init_all(self, src, tgt, mel=None):+ """Dummy forward through text (and optionally speech) pathways to initialize all params."""src_mask = make_padding_mask(src, self.config.pad_token_id)tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)text_out, text_enc_mask = self.encode_text(src, src_mask=src_mask)- mel_mask = make_mel_padding_mask(mel)- speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)_ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)- _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)+ if self.config.enable_speech and mel is not None:+ mel_mask = make_mel_padding_mask(mel)+ speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)_ = self.encode_contrastive(src)return jnp.zeros(())diff --git a/src/train.py b/src/train.pyindex 4947c64..9762ab1 100644--- a/src/train.py+++ b/src/train.py@@ -132,12 +132,12 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_rng, init_rng = jax.random.split(rng)dummy_src = jnp.ones((1, 128), dtype=jnp.int32)dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)- dummy_mel = jnp.ones((1, 128, config.n_mels), dtype=jnp.float32)+ init_args = [dummy_src, dummy_tgt]+ if config.enable_speech:+ init_args.append(jnp.ones((1, 128, config.n_mels), dtype=jnp.float32))variables = model.init({"params": init_rng},- dummy_src,- dummy_tgt,- dummy_mel,+ *init_args,method="init_all",)@@ -438,10 +438,12 @@ def _estimate_mat_params(config, matryoshka_factor):if config.conv_kernel_size > 0:K = config.conv_kernel_sizeconv = d * 2 * d + d * K + d + d * d- C = 256- n_mels = getattr(config, "n_mels", 80)- freq_out = n_mels // 8- speech_sub = (1 * C * 9) + (C * 9 + C * C) + (C * 9 + C * C) + (C * freq_out * d)+ speech_sub = 0+ if getattr(config, "enable_speech", True):+ C = 256+ n_mels = getattr(config, "n_mels", 80)+ freq_out = n_mels // 8+ speech_sub = (1 * C * 9) + (C * 9 + C * C) + (C * 9 + C * C) + (C * freq_out * d)enc_block = attn + conv + ffndec_block = attn * 2 + ffntotal = emb + speech_sub + n_enc * enc_block + n_dec * dec_block@@ -482,6 +484,8 @@ def train(args):val_loss_mask = val_data["loss_mask"]print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")+ val_ds = load_tool_calls("val", max_samples=args.max_samples)+cl_query_tokens = train_data.get("query_only")cl_tool_tokens = train_data.get("tool_individual")cl_tool_ex_idx = train_data.get("tool_ex_idx")@@ -514,6 +518,7 @@ def train(args):num_memory_slots=getattr(args, "num_memory_slots", 64),contrastive_dim=getattr(args, "contrastive_dim", 128),conv_kernel_size=getattr(args, "conv_kernel_size", 0),+ enable_speech=getattr(args, "enable_speech", False),)global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF, _CONTRASTIVE_WEIGHT@@ -703,7 +708,9 @@ def train(args):_eval_params = jax_utils.unreplicate(state).paramsval_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):+ _val_n = count_batches(len(val_enc), args.batch_size)+ for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask),+ total=_val_n, desc=" val loss", leave=False):vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])total_loss += float(vl)total_toks += float(vt)@@ -777,8 +784,11 @@ def train(args):q_loss, q_toks = 0.0, 0.0mat_accum = {f: [0.0, 0.0] for f in _MAT_FACTORS}- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,- shuffle=False, loss_mask=val_loss_mask):+ _val_n = count_batches(len(val_enc), args.batch_size)+ _val_bar = tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,+ shuffle=False, loss_mask=val_loss_mask),+ total=_val_n, desc=" eval val loss", leave=False)+ for vb in _val_bar:src, dec_in, dec_tgt, lm = vb[0], vb[1], vb[2], vb[3]vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)full_loss += float(vl); full_toks += float(vt)@@ -810,14 +820,10 @@ def train(args):pickle.dump({"params": params_np, "config": config.__dict__}, f)del params_np- from .eval import measure_throughputfrom .run import generate_batch- tp = measure_throughput(eval_model, eval_params, tokenizer, num_runs=5)val_kept = val_data["kept_indices"]- val_ds = load_tool_calls("val", max_samples=args.max_samples)- # Pick 5 display samples (shuffled for diversity): 4 with tool calls, 1 withoutsample_rng = np.random.RandomState(epoch + 7)sample_pool = sample_rng.permutation(len(val_kept))display_with, display_without = [], []@@ -833,23 +839,32 @@ def train(args):display_without.append(ex)display_pairs = display_with + display_without- # Collect eval examples (random sample, up to 500)import json as _json_modtc_eval_n = 500tc_rng = np.random.RandomState(epoch + 42)tc_pool = tc_rng.permutation(len(val_kept))[:tc_eval_n]tc_eval_pairs = [val_ds[int(val_kept[k])] for k in tc_pool]- # Batch-generate: display samples + tc eval samples in one pass+ # Batch-generate: display samples + tc eval samples in chunksall_eval_examples = display_pairs + tc_eval_pairseval_gen_len = min(args.max_dec_len, 512)- all_preds = generate_batch(- eval_model, eval_params, tokenizer,- [ex["query"] for ex in all_eval_examples],- [ex["tools"] for ex in all_eval_examples],- max_gen_len=eval_gen_len,- max_enc_len=args.max_enc_len,- )+ _EVAL_BATCH = 32+ _n_chunks = (len(all_eval_examples) + _EVAL_BATCH - 1) // _EVAL_BATCH+ all_preds = []+ _gen_t0 = time.perf_counter()+ for _ei in tqdm(range(0, len(all_eval_examples), _EVAL_BATCH),+ total=_n_chunks, desc=" eval generate", leave=False):+ _chunk = all_eval_examples[_ei:_ei + _EVAL_BATCH]+ all_preds.extend(generate_batch(+ eval_model, eval_params, tokenizer,+ [ex["query"] for ex in _chunk],+ [ex["tools"] for ex in _chunk],+ max_gen_len=eval_gen_len,+ max_enc_len=args.max_enc_len,+ ))+ _gen_elapsed = time.perf_counter() - _gen_t0+ _gen_total_toks = sum(len(tokenizer.encode(p)) for p in all_preds)+ _gen_tok_per_sec = _gen_total_toks / max(_gen_elapsed, 1e-6)display_preds = all_preds[:len(display_pairs)]tc_preds = all_preds[len(display_pairs):]@@ -982,10 +997,8 @@ def train(args):if tc_metrics and tc_metrics["call_f1"] > best_call_f1:best_call_f1 = tc_metrics["call_f1"]best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_best.pkl")- params_best = jax.tree.map(np.array, jax_utils.unreplicate(state).params)- with open(best_ckpt_path, "wb") as f:- pickle.dump({"params": params_best, "config": config.__dict__}, f)- del params_best+ import shutil as _shutil+ _shutil.copy2(ckpt_path, best_ckpt_path)print(f" ** New best call_f1={best_call_f1:.1%} → {best_ckpt_path}")_upload_checkpoint(best_ckpt_path)@@ -1049,8 +1062,7 @@ def train(args):print(f" Recall@{k:<3} {v:>10.1%}")print(f" MRR {rm['mrr']:>10.3f}")print(f" ─────────────────────────────────────")- print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")- print(f" Latency {tp['avg_latency_s']:>11.3f}s")+ print(f" Throughput {_gen_tok_per_sec:>10.1f} tok/s ({len(all_eval_examples)} samples, {_gen_elapsed:.1f}s)")if unified_samples:print(f" ─── Samples ({len(unified_samples)}) ───────────────────")for j, s in enumerate(unified_samples):