Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updated

HenryNdubuakud76a96c7717f
diff --git a/src/cli.py b/src/cli.py
index 0943516..abf2660 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -27,8 +27,8 @@ def main():
p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
- p.add_argument("--warmup-ratio", type=float, default=0.1)
- p.add_argument("--decay-ratio", type=float, default=0.40)
+ p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--decay-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
diff --git a/src/data.py b/src/data.py
index 6ab4c9d..8076f2e 100644
--- a/src/data.py
+++ b/src/data.py
@@ -513,7 +513,15 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
- # Track which samples have empty tools (will use <defer> token instead)
+ def _compact(s):
+ try:
+ return _json.dumps(_json.loads(s), separators=(",", ":"))
+ except (ValueError, TypeError):
+ return s
+
+ ans_texts = [_compact(a) for a in ans_texts]
+ tools_texts = [_compact(t) for t in tools_texts]
+
empty_tools_mask = []
for t in tools_texts:
try:
diff --git a/src/train.py b/src/train.py
index 25c991e..bc98a48 100644
--- a/src/train.py
+++ b/src/train.py
@@ -83,7 +83,7 @@ def _param_labels(params):
return jax.tree_util.tree_map_with_path(_label, params)
-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):
+def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
"""Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""
decay_steps = max(1, int(total_steps * decay_ratio))
stable_steps = total_steps - warmup_steps - decay_steps
@@ -97,7 +97,7 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):
)
-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.40):
+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):
model = EncoderDecoderTransformer(config)
rng, init_rng = jax.random.split(rng)
@@ -504,7 +504,7 @@ def train(args):
scaled_lr = args.lr * num_devices
muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)
- decay_ratio = getattr(args, "decay_ratio", 0.40)
+ decay_ratio = getattr(args, "decay_ratio", 0.15)
state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
val_loss_fn = _make_val_loss_fn(state.apply_fn)