Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

big head, small leg

HenryNdubuaku6838109115c3
diff --git a/src/cli.py b/src/cli.py
index fa90530..d79b903 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -23,7 +23,7 @@ def main():
p.add_argument("--num-heads", type=int, default=8)
p.add_argument("--num-kv-heads", type=int, default=4)
p.add_argument("--num-layers", type=int, default=16)
- p.add_argument("--num-dec-layers", type=int, default=16)
+ p.add_argument("--num-dec-layers", type=int, default=8)
p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
@@ -114,8 +114,8 @@ def main():
p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")
p = sub.add_parser("generate-data", add_help=False)
- p.add_argument("--num-samples", type=int, default=5000, help="Number of samples to generate")
- p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")
+ p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")
+ p.add_argument("--batch-size", type=int, default=25, help="Examples per Gemini call")
p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
p.add_argument("--model", type=str, default=None, help="Gemini model override")
p.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")
diff --git a/src/train.py b/src/train.py
index 14c6dcf..2d015cd 100644
--- a/src/train.py
+++ b/src/train.py
@@ -121,7 +121,7 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
[
optax.linear_schedule(0.0, peak_value, warmup_steps),
optax.constant_schedule(peak_value),
- optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),
+ optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.0),
],
boundaries=[warmup_steps, warmup_steps + stable_steps],
)