Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
refactor: simplify optimizer setup in finetune_main by removing ROWS_LR and consolidating learning rate scheduling
jakmro810ded9d83b1
diff --git a/needle/model/whistle_finetune.py b/needle/model/whistle_finetune.pyindex 0e8c887..da409e1 100644--- a/needle/model/whistle_finetune.py+++ b/needle/model/whistle_finetune.py@@ -14,7 +14,6 @@ DEFAULT_BASE = "checkpoints/whistle.safetensors"SAMPLE_RATE = 16000SECONDS = 30SPACE = "▁"-ROWS_LR = 10.0class WhistleTokenizer:@@ -331,13 +330,10 @@ def finetune_main(args):total_steps = args.epochs * steps_per_epochwarmup = min(max(1, total_steps // 20), total_steps - 1)- def adamw(lr):- return optax.adamw(optax.warmup_cosine_decay_schedule(- init_value=0.0, peak_value=lr, warmup_steps=warmup, decay_steps=total_steps))-- labels = {"lora": jax.tree.map(lambda _: "lora", train["lora"]), "rows": "rows"}- optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.multi_transform(- {"lora": adamw(args.lr), "rows": adamw(args.lr * ROWS_LR)}, labels))+ schedule = optax.warmup_cosine_decay_schedule(+ init_value=0.0, peak_value=args.lr,+ warmup_steps=warmup, decay_steps=total_steps)+ optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.adamw(schedule))opt_state = optimizer.init(train)print(f" {'schedule':<9} {total_steps} steps warmup {warmup} cosine decay clip 1.0 (compiling...)")