Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix: enhance logging messages for dataset generation, training, and tokenizer fetching
HenryNdubuaku6ae57d0609ff
diff --git a/README.md b/README.mdindex 9ac8de4..5b97d6a 100644--- a/README.md+++ b/README.md@@ -138,7 +138,8 @@ With a large catalogue, persist tool embeddings across runs with `needle.Needle("reasoning": "'living room' -> room; 'dim' -> on true, brightness 30","confidence": 0.94,"prefill_tps": 4300.0,- "decode_tps": 850.0+ "decode_tps": 850.0,+ "peak_ram_mb": 28.5}```@@ -184,7 +185,7 @@ receipt = [{}]agent = needle.Needle(tools=receipt)print(agent.complete("GreenMart receipt: oat milk 3.50, total 7.75 paid by visa")["function_calls"])-# -> [{"name": "receipt", "arguments": {"merchant": "GreenMart", "total": 7.75}}]+# -> [{'name': 'receipt', 'arguments': {'merchant': 'GreenMart', 'total': 7.75}}]```Because it is the same operation, everything else applies unchanged: `confidence` gates the extraction, unsupported input returns the empty call `[]`, and fine-tuning uses the same data format (the record as the tool, the passage as the query).@@ -232,11 +233,17 @@ Set `OPENROUTER_URL` to use an OpenAI-compatible gateway instead of the default**2. LoRA fine-tune.** The base checkpoint auto-downloads from Hugging Face if you do not pass `--checkpoint`. `--generate N` first synthesizes N more examples from the tools in your data (also needs `OPENROUTER_API_KEY`).```sh-needle finetune data.jsonl --epochs 3-needle finetune data.jsonl --epochs 3 --generate 300 --lora-rank 16 --lora-alpha 32+needle finetune data.jsonl --epochs 10+needle finetune data.jsonl --epochs 10 --generate 300 --lora-rank 16 --lora-alpha 32```-Key options: `--lora-rank` (default 16), `--lora-alpha` (32), `--lr` (1e-4), `--batch-size` (16), `--max-len` (1024), `--checkpoint <base.pkl>`, `--out <adapter.pkl>`. The adapter is written to `checkpoints/needle_lora.pkl`.+Key options: `--epochs` (default 3), `--lora-rank` (16), `--lora-alpha` (32), `--lr` (1e-4), `--batch-size` (16), `--max-len` (1024), `--val-split` (0.1), `--checkpoint <base.pkl>`, `--out <adapter.pkl>`. The adapter is written to `checkpoints/needle_lora.pkl`. A validation loss prints each epoch from the held out split.++Training is plain JAX and runs on any accelerator jax supports. On an NVIDIA machine install the CUDA build and the same command trains on the GPU:++```sh+pip install "cactus-needle[gpu]"+```**3. Build a tuned `.cact`.** Merge the adapter into the base and quantize. The base auto-downloads if absent.diff --git a/doc/finetuning.md b/doc/finetuning.mdindex 975cd71..75575b1 100644--- a/doc/finetuning.md+++ b/doc/finetuning.md@@ -41,6 +41,14 @@ needle generate-data --augment data.jsonl --num-samples 1000The playground button labelled Finetune on these tools runs the same pipeline from the browser.+Training is plain JAX, so it runs on any accelerator jax supports. On an NVIDIA machine install the CUDA build and the same command trains on the GPU, nothing else changes:++```sh+pip install "cactus-needle[gpu]"+```++Apple GPUs are not currently supported: the jax metal plugin (0.1.1) fails on basic operations with current jax. Apple Silicon trains on CPU, which handles small LoRA runs comfortably now that padding fits the data.+## Reading the lossThe loss covers only the target: the reasoning line plus the JSON call. Much of the call is boilerplate the base model already predicts (the tool name, the braces, the field names), so training starts near 1.0 rather than near random. Judge a run by its trend, not its level.diff --git a/needle/model/finetune.py b/needle/model/finetune.pyindex 48e8c13..977e8bd 100644--- a/needle/model/finetune.py+++ b/needle/model/finetune.py@@ -123,14 +123,14 @@ def generate_dataset(tools, num_samples, model=DEFAULT_MODEL, batch_size=25,rows.append(example)except Exception as exc:failed += 1- print(f" batch failed: {exc}", flush=True)+ print(f" {'failed':<9} {exc}", flush=True)if len(rows) < num_samples and submitted < max_submissions:_submit()done_count = min(len(rows), num_samples)if progress:progress(done_count, num_samples)else:- print(f" generated {done_count}/{num_samples} (failed={failed})", flush=True)+ print(f" {'generated':<9} {done_count}/{num_samples} failed {failed}", flush=True)finally:pool.shutdown(wait=False, cancel_futures=True)@@ -160,7 +160,7 @@ def augment_jsonl(path, num_samples, model=DEFAULT_MODEL, batch_size=25, out_patwith open(out_path, "w") as handle:for example in examples + generated:handle.write(json.dumps(example) + "\n")- print(f"wrote {len(examples) + len(generated)} examples -> {out_path}")+ print(f" {'wrote':<9} {len(examples) + len(generated)} examples {out_path}")return out_path@@ -176,7 +176,7 @@ def generate_main(args):with open(out, "w") as handle:for row in rows:handle.write(json.dumps(row) + "\n")- print(f"wrote {len(rows)} examples -> {out}")+ print(f" {'wrote':<9} {len(rows)} examples {out}")elif args.augment:augment_jsonl(args.augment, args.num_samples, model=model,batch_size=args.batch_size, out_path=args.output, workers=workers)@@ -309,13 +309,13 @@ def finetune_local(args, progress=None):seqs, masks = load_jsonl(data_path, tokenizer, max_len)if len(seqs) == 0:raise SystemExit("no usable examples in " + data_path)- emit(f"training on {len(seqs)} examples, seq_len {max_len} (cap {args.max_len})")+ emit(f" {'data':<9} {len(seqs)} examples seq_len {max_len} cap {args.max_len}")model = SimpleAttentionNetwork(config)paths = lora_target_paths(params)scale = args.lora_alpha / args.lora_ranklora = init_lora(params, paths, args.lora_rank, jax.random.PRNGKey(0))- emit(f"LoRA rank {args.lora_rank} on {len(paths)} weight groups (compiling...)")+ emit(f" {'lora':<9} rank {args.lora_rank} alpha {args.lora_alpha:g} {len(paths)} weight groups")n_val = min(int(len(seqs) * getattr(args, "val_split", 0.1)), len(seqs) - 1)if n_val > 0:@@ -323,17 +323,18 @@ def finetune_local(args, progress=None):seqs, masks = seqs[order], masks[order]val_seqs, val_masks = seqs[:n_val], masks[:n_val]seqs, masks = seqs[n_val:], masks[n_val:]- emit(f"holding out {n_val} examples for validation")+ emit(f" {'holdout':<9} {n_val} examples for validation")batch, count = args.batch_size, len(seqs)steps_per_epoch = -(-count // batch)total_steps = args.epochs * steps_per_epoch+ warmup = min(max(1, total_steps // 20), total_steps - 1)schedule = optax.warmup_cosine_decay_schedule(init_value=0.0, peak_value=args.lr,- warmup_steps=min(max(1, total_steps // 20), total_steps - 1),- decay_steps=total_steps)+ warmup_steps=warmup, decay_steps=total_steps)optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.adamw(schedule))opt_state = optimizer.init(lora)+ emit(f" {'schedule':<9} {total_steps} steps warmup {warmup} cosine decay clip 1.0 (compiling...)")def loss_fn(lora, ids, mask):logits = model.apply({"params": merge_lora(params, lora, scale)}, ids)@@ -361,14 +362,14 @@ def finetune_local(args, progress=None):last = float(loss)step_i += 1if step_i % every == 0:- emit(f"epoch {epoch + 1}/{args.epochs} step {step_i}/{total_steps} loss {last:.4f}")+ emit(f" {'step':<9} {step_i}/{total_steps} loss {last:.4f}")if n_val > 0:val = np.mean([float(eval_step(lora, jnp.asarray(val_seqs[i:i + batch]),jnp.asarray(val_masks[i:i + batch])))for i in range(0, n_val, batch)])- emit(f"epoch {epoch + 1}/{args.epochs} loss {last:.4f} val loss {val:.4f}")+ emit(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f} val {val:.4f}")else:- emit(f"epoch {epoch + 1}/{args.epochs} loss {last:.4f}")+ emit(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f}")os.makedirs(args.checkpoint_dir, exist_ok=True)out = args.out or os.path.join(args.checkpoint_dir, "needle_lora.pkl")@@ -380,10 +381,9 @@ def finetune_local(args, progress=None):"base": base_path,"rank": args.lora_rank,}, handle)- print(f"saved LoRA adapter -> {out}")- print(f"merge + export with: needle build {base_path} --lora {out}")- print("note: finetuning does not update the confidence head; "- "confidence is reported as None when running tuned weights")+ print(f" {'adapter':<9} {out}")+ print(f" {'next':<9} needle build {base_path} --lora {out}")+ print(f" {'note':<9} confidence reports None with tuned weights; the head is not tuned")def build_main(args):@@ -400,7 +400,7 @@ def build_main(args):lora = {tuple(key.split("/")): {"A": jnp.asarray(v["A"]), "B": jnp.asarray(v["B"])}for key, v in adapter["lora"].items()}params = merge_lora(params, lora, adapter["scale"])- print(f"merged LoRA adapter {args.lora} into {len(lora)} weight groups")+ print(f" {'merged':<9} {len(lora)} weight groups {args.lora}")bits = args.bitsbits_map = None if bits else (getattr(config, "weight_bits", "") or None)@@ -414,8 +414,8 @@ def build_main(args):tokenizer=get_tokenizer(config.vocab_size),kv_window=effective_kv_window(config))scheme = f"mixed[{bits_map}]" if bits_map else f"W{bits}"- print(f"exported {info['path']} ({info['bytes'] / 1e6:.2f} MB, {info['tensors']} tensors, {scheme}A8)")- print(f"run it with: needle.Needle(weights={out!r}, tools=[...])")+ print(f" {'wrote':<9} {info['path']} {info['bytes'] / 1e6:.2f} MB {info['tensors']} tensors {scheme}A8")+ print(f" {'next':<9} needle.Needle(weights={out!r}, tools=[...])")if args.upload:repo = os.environ.get("NEEDLE_HF_REPO")@@ -426,4 +426,4 @@ def build_main(args):api.create_repo(repo, repo_type="model", exist_ok=True)api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),repo_id=repo, repo_type="model")- print(f"uploaded {os.path.basename(out)} -> {repo}")+ print(f" {'uploaded':<9} {os.path.basename(out)} {repo}")diff --git a/needle/model/run.py b/needle/model/run.pyindex 11f690f..9cc974c 100644--- a/needle/model/run.py+++ b/needle/model/run.py@@ -63,7 +63,7 @@ def load_checkpoint(path, return_run=False):from huggingface_hub import hf_hub_downloadfrom huggingface_hub.errors import EntryNotFoundErrorfrom .tokenizer import HF_REPO- print(f"{path} not found locally, downloading from HF...", flush=True)+ print(f" {'fetch':<9} {path} downloading from Hugging Face", flush=True)rel = os.path.normpath(path).replace(os.sep, "/")names = []if not os.path.isabs(path):diff --git a/needle/model/tokenizer.py b/needle/model/tokenizer.pyindex 3261a2c..195599e 100644--- a/needle/model/tokenizer.py+++ b/needle/model/tokenizer.py@@ -103,7 +103,7 @@ def get_tokenizer(vocab_size=None):model_path = prefix + ".model"if not os.path.exists(model_path):try:- print(f"Downloading {os.path.basename(prefix)} from HuggingFace...")+ print(f" {'fetch':<9} {os.path.basename(prefix)} tokenizer downloading from Hugging Face")_download_tokenizer_from_hf(prefix)except Exception as e:raise RuntimeError(