Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix(finetune): avoid NaN and dead-weight LoRA groups on CPU (#59)
Lanny Boartsa87f2d88f510
diff --git a/needle/model/finetune.py b/needle/model/finetune.pyindex a624513..0e0137b 100644--- a/needle/model/finetune.py+++ b/needle/model/finetune.py@@ -227,9 +227,16 @@ def load_jsonl(path, tokenizer, max_len):def lora_target_paths(params):+ import jax.numpy as jnpfrom flax.traverse_util import flatten_dict- return [path for path in flatten_dict(params)- if path[-1] == "kernel" and any(t in path for t in LORA_TARGETS)]+ flat = flatten_dict(params)+ paths = [path for path in flat+ if path[-1] == "kernel"+ and "stack" in path+ and "layers" in path+ and any(t in path for t in LORA_TARGETS)]+ paths = [p for p in paths if jnp.max(jnp.abs(flat[p])) > 1e-6]+ return pathsdef init_lora(params, paths, rank, key):@@ -244,8 +251,8 @@ def init_lora(params, paths, rank, key):lead = weight.shape[:-2]key, sub = jax.random.split(key)lora[path] = {- "A": (jax.random.normal(sub, lead + (in_dim, rank), jnp.float32) / rank).astype(weight.dtype),- "B": jnp.zeros(lead + (rank, out_dim), weight.dtype),+ "A": jax.random.normal(sub, lead + (in_dim, rank), jnp.float32) / rank,+ "B": jnp.zeros(lead + (rank, out_dim), jnp.float32),}return lora@@ -255,7 +262,7 @@ def merge_lora(params, lora, scale):from flax.traverse_util import flatten_dict, unflatten_dictflat = dict(flatten_dict(params))for path, adapter in lora.items():- flat[path] = flat[path] + scale * jnp.matmul(adapter["A"], adapter["B"]).astype(flat[path].dtype)+ flat[path] = flat[path] + (scale * jnp.matmul(adapter["A"], adapter["B"])).astype(flat[path].dtype)return unflatten_dict(flat)