Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuakudad24d07b342
diff --git a/src/calibrate.py b/src/calibrate.pyindex 4041b82..f457969 100644--- a/src/calibrate.py+++ b/src/calibrate.py@@ -124,13 +124,13 @@ def compute_per_example_perplexity(model, params, train_data, batch_size, max_sareturn perplexities-def perplexity_to_confidence(perplexities, mu=None, k=3.0):+def perplexity_to_confidence(perplexities, mu=None, k=5.0):"""Map perplexity values to [0, 1] confidence via sigmoid calibration.confidence = sigmoid(-k * (log(ppl) - log(mu)))"""if mu is None:- mu = float(np.median(perplexities))+ mu = float(np.percentile(perplexities, 80))log_ppl = np.log(np.maximum(perplexities, 1.0))log_mu = np.log(mu)@@ -314,7 +314,9 @@ def main(args):with open(args.checkpoint, "rb") as f:data = pickle.load(f)params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])- config = TransformerConfig(**data["config"])+ config_dict = {k: v for k, v in data["config"].items()+ if k in TransformerConfig.__dataclass_fields__}+ config = TransformerConfig(**config_dict)model = EncoderDecoderTransformer(config)param_count = sum(x.size for x in jax.tree.leaves(params))@@ -322,15 +324,26 @@ def main(args):print(f" Config: d={config.d_model}, heads={config.num_heads}, "f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")- # Check if confidence head params exist; if not, initialize them+ # Check if confidence head params need (re)initialization+ rng = jax.random.PRNGKey(0)+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)+ init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")+ init_params = init_vars["params"]+flat_keys = str(jax.tree.map(lambda x: x.shape, params))- if "confidence_hidden" not in flat_keys:- print(" Confidence head not found in checkpoint — initializing...")- rng = jax.random.PRNGKey(0)- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)- init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")- init_params = init_vars["params"]+ needs_reinit = "confidence_hidden" not in flat_keys+ if not needs_reinit:+ # Check shape match+ for key in ("confidence_hidden", "confidence_out"):+ old_shapes = jax.tree.map(lambda x: x.shape, params[key])+ new_shapes = jax.tree.map(lambda x: x.shape, init_params[key])+ if str(old_shapes) != str(new_shapes):+ needs_reinit = True+ break++ if needs_reinit:+ print(" Confidence head shape mismatch or missing — reinitializing...")params = {**params, "confidence_hidden": init_params["confidence_hidden"],"confidence_out": init_params["confidence_out"]}print(" Confidence head initialized.")diff --git a/src/cli.py b/src/cli.pyindex d79b903..342e4d8 100644--- a/src/cli.py+++ b/src/cli.py@@ -60,14 +60,14 @@ def main():help="Loss weight for argument key tokens (default: 1.5)")p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,help="Remove feedforward layers entirely (default: True)")- p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=True,- help="Run confidence head calibration after training (default: True)")+ p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=False,+ help="Run confidence head calibration after training (default: False)")p.add_argument("--calibrate-epochs", type=int, default=1,help="Epochs for confidence head calibration (default: 1)")p.add_argument("--calibrate-lr", type=float, default=1e-3,help="Learning rate for confidence head calibration (default: 1e-3)")- p.add_argument("--calibrate-k", type=float, default=3.0,- help="Sigmoid steepness for PPL→confidence mapping (default: 3.0)")+ p.add_argument("--calibrate-k", type=float, default=5.0,+ help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,@@ -111,7 +111,7 @@ def main():p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")p.add_argument("--epochs", type=int, default=1)p.add_argument("--lr", type=float, default=1e-3)- p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")+ p.add_argument("--k", type=float, default=5.0, help="Sigmoid steepness for PPL→confidence mapping")p = sub.add_parser("generate-data", add_help=False)p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")diff --git a/src/model.py b/src/model.pyindex b555c91..94e02c6 100644--- a/src/model.py+++ b/src/model.py@@ -422,14 +422,14 @@ class EncoderDecoderTransformer(nn.Module):)self.log_temp = self.param("log_temp", jinit.zeros, ())- self.confidence_hidden = nn.Dense(- self.config.d_model // 4, dtype=self.config.jax_dtype,- kernel_init=default_init(), name="confidence_hidden",- )- self.confidence_out = nn.Dense(- 1, dtype=jnp.float32,- kernel_init=jinit.zeros, name="confidence_out",- )+ # self.confidence_hidden = nn.Dense(+ # self.config.d_model, dtype=self.config.jax_dtype,+ # kernel_init=default_init(), name="confidence_hidden",+ # )+ # self.confidence_out = nn.Dense(+ # 1, dtype=jnp.float32,+ # kernel_init=jinit.zeros, name="confidence_out",+ # )def _rope(self, seq_len):head_dim = self.config.d_model // self.config.num_heads@@ -480,32 +480,17 @@ class EncoderDecoderTransformer(nn.Module):counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0)return summed / counts- def predict_confidence(self, encoder_out, enc_mask):- """Predict confidence score [0, 1] from encoder output.-- High confidence (→1.0) means the model can handle this query locally.- Low confidence (→0.0) means the query should be routed to cloud.-- Args:- encoder_out: (B, T, d_model) encoder hidden states- enc_mask: (B, 1, 1, T) padding mask+ # def predict_confidence(self, encoder_out, enc_mask):+ # """Predict confidence score [0, 1] from encoder output."""+ # pooled = self._mean_pool(encoder_out, enc_mask)+ # h = nn.relu(self.confidence_hidden(pooled))+ # return nn.sigmoid(self.confidence_out(h)).squeeze(-1)- Returns:- (B,) confidence scores in [0, 1]- """- pooled = self._mean_pool(encoder_out, enc_mask) # (B, d_model)- h = nn.relu(self.confidence_hidden(pooled)) # (B, d_model // 4)- return nn.sigmoid(self.confidence_out(h)).squeeze(-1) # (B,)-- def forward_confidence(self, src, src_mask=None):- """End-to-end confidence prediction from token input.-- Returns (confidence, encoder_out, enc_mask) so the caller can- reuse encoder output if confidence is high enough to decode.- """- encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)- confidence = self.predict_confidence(encoder_out, enc_mask)- return confidence, encoder_out, enc_mask+ # def forward_confidence(self, src, src_mask=None):+ # """End-to-end confidence prediction from token input."""+ # encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)+ # confidence = self.predict_confidence(encoder_out, enc_mask)+ # return confidence, encoder_out, enc_maskdef encode_contrastive(self, tokens, deterministic=True):"""Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""@@ -608,7 +593,6 @@ class EncoderDecoderTransformer(nn.Module):speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)_ = self.encode_contrastive(src)- _ = self.predict_confidence(text_out, text_enc_mask)return jnp.zeros(())