Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakudad24d07b342
diff --git a/src/calibrate.py b/src/calibrate.py
index 4041b82..f457969 100644
--- a/src/calibrate.py
+++ b/src/calibrate.py
@@ -124,13 +124,13 @@ def compute_per_example_perplexity(model, params, train_data, batch_size, max_sa
return perplexities
-def perplexity_to_confidence(perplexities, mu=None, k=3.0):
+def perplexity_to_confidence(perplexities, mu=None, k=5.0):
"""Map perplexity values to [0, 1] confidence via sigmoid calibration.
confidence = sigmoid(-k * (log(ppl) - log(mu)))
"""
if mu is None:
- mu = float(np.median(perplexities))
+ mu = float(np.percentile(perplexities, 80))
log_ppl = np.log(np.maximum(perplexities, 1.0))
log_mu = np.log(mu)
@@ -314,7 +314,9 @@ def main(args):
with open(args.checkpoint, "rb") as f:
data = pickle.load(f)
params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])
- config = TransformerConfig(**data["config"])
+ config_dict = {k: v for k, v in data["config"].items()
+ if k in TransformerConfig.__dataclass_fields__}
+ config = TransformerConfig(**config_dict)
model = EncoderDecoderTransformer(config)
param_count = sum(x.size for x in jax.tree.leaves(params))
@@ -322,15 +324,26 @@ def main(args):
print(f" Config: d={config.d_model}, heads={config.num_heads}, "
f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")
- # Check if confidence head params exist; if not, initialize them
+ # Check if confidence head params need (re)initialization
+ rng = jax.random.PRNGKey(0)
+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
+ init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")
+ init_params = init_vars["params"]
+
flat_keys = str(jax.tree.map(lambda x: x.shape, params))
- if "confidence_hidden" not in flat_keys:
- print(" Confidence head not found in checkpoint — initializing...")
- rng = jax.random.PRNGKey(0)
- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
- init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")
- init_params = init_vars["params"]
+ needs_reinit = "confidence_hidden" not in flat_keys
+ if not needs_reinit:
+ # Check shape match
+ for key in ("confidence_hidden", "confidence_out"):
+ old_shapes = jax.tree.map(lambda x: x.shape, params[key])
+ new_shapes = jax.tree.map(lambda x: x.shape, init_params[key])
+ if str(old_shapes) != str(new_shapes):
+ needs_reinit = True
+ break
+
+ if needs_reinit:
+ print(" Confidence head shape mismatch or missing — reinitializing...")
params = {**params, "confidence_hidden": init_params["confidence_hidden"],
"confidence_out": init_params["confidence_out"]}
print(" Confidence head initialized.")
diff --git a/src/cli.py b/src/cli.py
index d79b903..342e4d8 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -60,14 +60,14 @@ def main():
help="Loss weight for argument key tokens (default: 1.5)")
p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,
help="Remove feedforward layers entirely (default: True)")
- p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=True,
- help="Run confidence head calibration after training (default: True)")
+ p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=False,
+ help="Run confidence head calibration after training (default: False)")
p.add_argument("--calibrate-epochs", type=int, default=1,
help="Epochs for confidence head calibration (default: 1)")
p.add_argument("--calibrate-lr", type=float, default=1e-3,
help="Learning rate for confidence head calibration (default: 1e-3)")
- p.add_argument("--calibrate-k", type=float, default=3.0,
- help="Sigmoid steepness for PPL→confidence mapping (default: 3.0)")
+ p.add_argument("--calibrate-k", type=float, default=5.0,
+ help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
@@ -111,7 +111,7 @@ def main():
p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")
p.add_argument("--epochs", type=int, default=1)
p.add_argument("--lr", type=float, default=1e-3)
- p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")
+ p.add_argument("--k", type=float, default=5.0, help="Sigmoid steepness for PPL→confidence mapping")
p = sub.add_parser("generate-data", add_help=False)
p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")
diff --git a/src/model.py b/src/model.py
index b555c91..94e02c6 100644
--- a/src/model.py
+++ b/src/model.py
@@ -422,14 +422,14 @@ class EncoderDecoderTransformer(nn.Module):
)
self.log_temp = self.param("log_temp", jinit.zeros, ())
- self.confidence_hidden = nn.Dense(
- self.config.d_model // 4, dtype=self.config.jax_dtype,
- kernel_init=default_init(), name="confidence_hidden",
- )
- self.confidence_out = nn.Dense(
- 1, dtype=jnp.float32,
- kernel_init=jinit.zeros, name="confidence_out",
- )
+ # self.confidence_hidden = nn.Dense(
+ # self.config.d_model, dtype=self.config.jax_dtype,
+ # kernel_init=default_init(), name="confidence_hidden",
+ # )
+ # self.confidence_out = nn.Dense(
+ # 1, dtype=jnp.float32,
+ # kernel_init=jinit.zeros, name="confidence_out",
+ # )
def _rope(self, seq_len):
head_dim = self.config.d_model // self.config.num_heads
@@ -480,32 +480,17 @@ class EncoderDecoderTransformer(nn.Module):
counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0)
return summed / counts
- def predict_confidence(self, encoder_out, enc_mask):
- """Predict confidence score [0, 1] from encoder output.
-
- High confidence (→1.0) means the model can handle this query locally.
- Low confidence (→0.0) means the query should be routed to cloud.
-
- Args:
- encoder_out: (B, T, d_model) encoder hidden states
- enc_mask: (B, 1, 1, T) padding mask
+ # def predict_confidence(self, encoder_out, enc_mask):
+ # """Predict confidence score [0, 1] from encoder output."""
+ # pooled = self._mean_pool(encoder_out, enc_mask)
+ # h = nn.relu(self.confidence_hidden(pooled))
+ # return nn.sigmoid(self.confidence_out(h)).squeeze(-1)
- Returns:
- (B,) confidence scores in [0, 1]
- """
- pooled = self._mean_pool(encoder_out, enc_mask) # (B, d_model)
- h = nn.relu(self.confidence_hidden(pooled)) # (B, d_model // 4)
- return nn.sigmoid(self.confidence_out(h)).squeeze(-1) # (B,)
-
- def forward_confidence(self, src, src_mask=None):
- """End-to-end confidence prediction from token input.
-
- Returns (confidence, encoder_out, enc_mask) so the caller can
- reuse encoder output if confidence is high enough to decode.
- """
- encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
- confidence = self.predict_confidence(encoder_out, enc_mask)
- return confidence, encoder_out, enc_mask
+ # def forward_confidence(self, src, src_mask=None):
+ # """End-to-end confidence prediction from token input."""
+ # encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
+ # confidence = self.predict_confidence(encoder_out, enc_mask)
+ # return confidence, encoder_out, enc_mask
def encode_contrastive(self, tokens, deterministic=True):
"""Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""
@@ -608,7 +593,6 @@ class EncoderDecoderTransformer(nn.Module):
speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)
_ = self.encode_contrastive(src)
- _ = self.predict_confidence(text_out, text_enc_mask)
return jnp.zeros(())