Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updated
HenryNdubuakud76a96c7717f
diff --git a/src/cli.py b/src/cli.pyindex 0943516..abf2660 100644--- a/src/cli.py+++ b/src/cli.py@@ -27,8 +27,8 @@ def main():p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)- p.add_argument("--warmup-ratio", type=float, default=0.1)- p.add_argument("--decay-ratio", type=float, default=0.40)+ p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--decay-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")diff --git a/src/data.py b/src/data.pyindex 6ab4c9d..8076f2e 100644--- a/src/data.py+++ b/src/data.py@@ -513,7 +513,15 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]- # Track which samples have empty tools (will use <defer> token instead)+ def _compact(s):+ try:+ return _json.dumps(_json.loads(s), separators=(",", ":"))+ except (ValueError, TypeError):+ return s++ ans_texts = [_compact(a) for a in ans_texts]+ tools_texts = [_compact(t) for t in tools_texts]+empty_tools_mask = []for t in tools_texts:try:diff --git a/src/train.py b/src/train.pyindex 25c991e..bc98a48 100644--- a/src/train.py+++ b/src/train.py@@ -83,7 +83,7 @@ def _param_labels(params):return jax.tree_util.tree_map_with_path(_label, params)-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):+def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):"""Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""decay_steps = max(1, int(total_steps * decay_ratio))stable_steps = total_steps - warmup_steps - decay_steps@@ -97,7 +97,7 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):)-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.40):+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):model = EncoderDecoderTransformer(config)rng, init_rng = jax.random.split(rng)@@ -504,7 +504,7 @@ def train(args):scaled_lr = args.lr * num_devicesmuon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)- decay_ratio = getattr(args, "decay_ratio", 0.40)+ decay_ratio = getattr(args, "decay_ratio", 0.15)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)val_loss_fn = _make_val_loss_fn(state.apply_fn)