Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

refactor: simplify optimizer setup in finetune_main by removing ROWS_LR and consolidating learning rate scheduling

jakmro810ded9d83b1
diff --git a/needle/model/whistle_finetune.py b/needle/model/whistle_finetune.py
index 0e8c887..da409e1 100644
--- a/needle/model/whistle_finetune.py
+++ b/needle/model/whistle_finetune.py
@@ -14,7 +14,6 @@ DEFAULT_BASE = "checkpoints/whistle.safetensors"
SAMPLE_RATE = 16000
SECONDS = 30
SPACE = "▁"
-ROWS_LR = 10.0
class WhistleTokenizer:
@@ -331,13 +330,10 @@ def finetune_main(args):
total_steps = args.epochs * steps_per_epoch
warmup = min(max(1, total_steps // 20), total_steps - 1)
- def adamw(lr):
- return optax.adamw(optax.warmup_cosine_decay_schedule(
- init_value=0.0, peak_value=lr, warmup_steps=warmup, decay_steps=total_steps))
-
- labels = {"lora": jax.tree.map(lambda _: "lora", train["lora"]), "rows": "rows"}
- optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.multi_transform(
- {"lora": adamw(args.lr), "rows": adamw(args.lr * ROWS_LR)}, labels))
+ schedule = optax.warmup_cosine_decay_schedule(
+ init_value=0.0, peak_value=args.lr,
+ warmup_steps=warmup, decay_steps=total_steps)
+ optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.adamw(schedule))
opt_state = optimizer.init(train)
print(f" {'schedule':<9} {total_steps} steps warmup {warmup} cosine decay clip 1.0 (compiling...)")