Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Add Whistle fine-tuning functionality and related tests

jakmro482361d8d9f9
diff --git a/needle/agent/whistle.py b/needle/agent/whistle.py
index bd189f9..ea23173 100644
--- a/needle/agent/whistle.py
+++ b/needle/agent/whistle.py
@@ -6,6 +6,7 @@ import json
import os
import queue
import shutil
+import struct
import sys
import threading
import time
@@ -74,6 +75,31 @@ def _read_wav(path):
return soxr.resample(numpy.asarray(mono, numpy.float32), rate, SAMPLE_RATE, quality="HQ")
+def languages(weights):
+ try:
+ with open(weights, "rb") as handle:
+ raw = handle.read()
+ tag, count, codebook = struct.unpack_from("<III", raw)
+ except (OSError, struct.error):
+ return LANGUAGES
+ if tag != 0x05E12A84:
+ return LANGUAGES
+ record = 196 + codebook * 4
+ for _ in range(count):
+ if raw[record] == 4:
+ offset, size = struct.unpack_from("<QQ", raw, record + 20)
+ blob, found = raw[offset:offset + size], []
+ at = 24
+ for _ in range(struct.unpack_from("<I", blob)[0]):
+ length = struct.unpack_from("<H", blob, at + 5)[0]
+ piece = blob[at + 7:at + 7 + length].decode("utf-8", "replace")
+ at += 7 + length
+ found = found + [piece[2:-2]] if piece.startswith("<|") and piece.endswith("|>") else []
+ return tuple(found) or LANGUAGES
+ record += 44
+ return LANGUAGES
+
+
def _options(language, keywords):
if keywords is not None and not isinstance(keywords, str):
keywords = "\n".join(keywords)
@@ -357,6 +383,9 @@ def playground(args):
"stream": False}
print("whistle playground: downloading and initializing the model...", flush=True)
whistle = Whistle(weights=args.weights)
+ known = languages(whistle.weights)
+ if args.language and args.language not in known:
+ raise SystemExit("--language is one of " + " ".join(known))
if args.audio:
return show_transcript(whistle, args.audio, state)
print(PLAYGROUND_HELP)
@@ -367,8 +396,8 @@ def playground(args):
return
if command == "/language":
language = rest.strip()
- if language and language not in LANGUAGES:
- print(" language is one of", " ".join(LANGUAGES))
+ if language and language not in known:
+ print(" language is one of", " ".join(known))
continue
state["language"] = language or None
print(" language", state["language"] or "detected")
diff --git a/needle/cli.py b/needle/cli.py
index 71521c0..e9f5c64 100644
--- a/needle/cli.py
+++ b/needle/cli.py
@@ -14,7 +14,7 @@ HELP = """usage: needle <command> [options]
download needle3 | needle3.safetensors | <platform> | model-<id> | <org>/<repo>[/<file>.cact]
fetch fetch the engine library for this platform
playground serve the browser playground
- whistle speech to text: playground | compare
+ whistle speech to text: playground | compare | finetune | build
needle <command> --help for the options of one command.
Check the readme for the rest."""
@@ -271,8 +271,9 @@ def main():
v = verbs.add_parser("playground")
v.add_argument("audio", type=str, nargs="?", default=None,
help="A WAV file to transcribe once (default: press Enter to speak)")
- v.add_argument("--language", type=str, default=None, choices=LANGUAGES,
- help="Force the language (default: detect it)")
+ v.add_argument("--language", type=str, default=None,
+ help="Force the language, one the weights know: " + " ".join(LANGUAGES) + " for the published "
+ "weights (default: detect it)")
v.add_argument("--keywords", type=str, default="",
help="Words and phrases to favour, comma separated")
v.add_argument("--word-timestamps", action="store_true",
@@ -284,6 +285,39 @@ def main():
help="A WAV file to run through every model once (default: press Enter to speak)")
v.add_argument("--weights", type=str, default=None,
help="whistle.cact to load (default: the published weights)")
+ v = verbs.add_parser("finetune")
+ v.add_argument("jsonl_path", type=str,
+ help='JSONL of {"audio": WAV path, "text": transcript, "language": code}, one clip per line')
+ v.add_argument("--language", type=str, default=None,
+ help="Language of lines that do not name one; a new code adds that language")
+ v.add_argument("--tokens", type=str, default=None,
+ help="File of tokens to add to the vocabulary, one per line")
+ v.add_argument("--checkpoint", type=str, default=None,
+ help="Base checkpoint (default: checkpoints/whistle.safetensors, downloaded if missing)")
+ v.add_argument("--epochs", type=int, default=3)
+ v.add_argument("--batch-size", type=int, default=16)
+ v.add_argument("--lr", type=float, default=1e-4)
+ v.add_argument("--lora-rank", type=int, default=16, help="LoRA adapter rank (default: 16)")
+ v.add_argument("--lora-alpha", type=float, default=32.0, help="LoRA scaling alpha (default: 32)")
+ v.add_argument("--val-split", type=float, default=0.1,
+ help="Fraction of clips held out for validation (0 disables)")
+ v.add_argument("--seed", type=int, default=0,
+ help="Random seed for LoRA init, validation split, and epoch shuffling")
+ v.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ v.add_argument("--out", type=str, default=None,
+ help="Output adapter path (.safetensors)")
+ v = verbs.add_parser("build")
+ v.add_argument("checkpoint", type=str, nargs="?", default=None,
+ help="Base checkpoint (.safetensors); defaults to the adapter's "
+ "base, else the Whistle base (auto-downloads)")
+ v.add_argument("--lora", type=str, default=None, help="LoRA adapter to merge before export")
+ v.add_argument("--out", type=str, default=None, help="Output .cact path")
+ v.add_argument("--upload", action="store_true", help="Push the .cact to $NEEDLE_HF_REPO")
+ v.add_argument("--layers", type=int, default=None,
+ help="Export the N-layer decoder rung of the base (2..8); default the full 8")
+ v.add_argument("--platform", type=str, default=None, choices=fetch.PLATFORMS,
+ help="Also download that platform's engine and header, and place the "
+ "archive beside them as whistle.cact (--out is then a directory)")
args = parser.parse_args()
@@ -381,6 +415,10 @@ def main():
from .agent.whistle import playground as whistle_main
elif args.verb == "compare":
from .agent.whistle import compare as whistle_main
+ elif args.verb == "finetune":
+ from .model.whistle_finetune import finetune_main as whistle_main
+ elif args.verb == "build":
+ from .model.whistle_finetune import build_main as whistle_main
else:
- raise SystemExit("needle whistle playground | compare")
+ raise SystemExit("needle whistle playground | compare | finetune | build")
whistle_main(args)
diff --git a/needle/model/checkpoints.py b/needle/model/checkpoints.py
index 7de5f47..681e994 100644
--- a/needle/model/checkpoints.py
+++ b/needle/model/checkpoints.py
@@ -88,7 +88,7 @@ def write_checkpoint(path, checkpoint):
library.save_file(flatten(checkpoint["params"]), os.fspath(path), metadata=metadata)
-_ADAPTER_FIELDS = ("scale", "base", "rank", "seed")
+_ADAPTER_FIELDS = ("scale", "base", "rank", "seed", "pieces", "languages")
def write_adapter(path, adapter):
@@ -99,6 +99,8 @@ def write_adapter(path, adapter):
for name, value in adapter["lora"].items():
tensors[f"lora/{name}/A"] = _contiguous(value["A"])
tensors[f"lora/{name}/B"] = _contiguous(value["B"])
+ if adapter.get("rows") is not None:
+ tensors["rows"] = _contiguous(adapter["rows"])
metadata = {key: json.dumps(adapter.get(key), default=str) for key in _ADAPTER_FIELDS}
library.save_file(tensors, os.fspath(path), metadata=metadata)
@@ -111,11 +113,14 @@ def read_adapter(path):
with safe_open(os.fspath(path), framework="np") as handle:
metadata = handle.metadata() or {}
- lora = {}
+ lora, rows = {}, None
for name, value in library.load_file(os.fspath(path)).items():
+ if name == "rows":
+ rows = value
+ continue
_, key, matrix = name.rsplit("/", 2) if name.count("/") == 2 else (None, *name[5:].rsplit("/", 1))
lora.setdefault(key, {})[matrix] = value
- adapter = {"lora": lora}
+ adapter = {"lora": lora, "rows": rows}
for key in _ADAPTER_FIELDS:
adapter[key] = _json_or_default(metadata.get(key), None)
return adapter
diff --git a/needle/model/export.py b/needle/model/export.py
index faa2f8f..506a62e 100644
--- a/needle/model/export.py
+++ b/needle/model/export.py
@@ -233,7 +233,7 @@ def _q(name, mat, bits, group):
return _Tensor(name, CQ, mat.shape, packed.tobytes() + norms.tobytes(), group, bits)
-def _tensors(params, config, bits, group):
+def _tensors(params, config, bits, group, audio=None):
_, _, orders, heads, sub_dim, sites = _geometry(config)
num_tables = len(orders) * heads
embedding = np.asarray(_get(params, ("embedding", "embedding")))
@@ -308,6 +308,71 @@ def _tensors(params, config, bits, group):
ts.append(_fp16("final_norm", params["stack"]["final_norm"]["scale"]))
ts += _head_tensors(params, group)
+ if audio is not None:
+ ts += _audio_tensors(params, audio, bits, group)
+ return ts
+
+
+AUDIO_MANIFEST_MAGIC = 3246.0
+HADAMARD = ("d1", "d2", "b2", "d3", "d4", "w1a", "w1b", "w2a", "w2b", "w3a", "w3b", "cond_v", "cond_u")
+PROJ = ("q_proj", "k_proj", "v_proj", "gate_proj", "out_proj")
+
+
+def _audio_tensors(params, cfg, bits, group):
+ from .whistle import mel_matrix
+ ts = [_Tensor("audio.manifest", FP32, (13,), np.asarray([
+ AUDIO_MANIFEST_MAGIC, 1, cfg.text, cfg.enc_layers, cfg.enc_lanes, cfg.conv_kernel, cfg.stem,
+ cfg.stem_stages, cfg.mels, cfg.rate, cfg.fft, cfg.win, cfg.hop], np.float32).tobytes())]
+
+ def proj(prefix, p, i):
+ return [_q(f"{prefix}.{n}", np.asarray(p[n]["kernel"][i]).T, bits, group) for n in PROJ]
+
+ b = params["stack"]["layers"]["block"]
+ ca = b["cross_attn"]
+ for i in range(cfg.dec_layers):
+ ts.append(_fp16(f"layer{i:02d}.cross_norm", b["cross_norm"]["scale"][i]))
+ ts += proj(f"layer{i:02d}.cross", ca, i)
+ ts += [
+ _fp16(f"layer{i:02d}.cross_q_norm", ca["q_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.cross_k_norm", ca["k_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.post_cross_norm", b["post_cross_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.cross_gate", np.asarray(b["cross_gate"][i]).reshape(1)),
+ ]
+ ts.append(_fp16("pe_gate", np.asarray(params["pe_gate"]).reshape(1)))
+
+ e = params["encoder"]["layers"]["block"]
+ for i in range(cfg.enc_layers):
+ ts += [_fp16(f"encoder{i:02d}.{n}", e[n]["scale"][i]) for n in (
+ "pre_hada_norm_0", "ZCRMSNorm_0", "post_attn_norm", "conv_norm", "conv_out_norm", "pre_hada_norm")]
+ ts += proj(f"encoder{i:02d}", e["self_attn"], i)
+ ts += [
+ _fp16(f"encoder{i:02d}.q_norm", e["self_attn"]["q_norm"]["scale"][i]),
+ _fp16(f"encoder{i:02d}.k_norm", e["self_attn"]["k_norm"]["scale"][i]),
+ _fp16(f"encoder{i:02d}.attn_gate", np.asarray(e["attn_gate"][i]).reshape(1)),
+ _q(f"encoder{i:02d}.pw1", np.asarray(e["pw1"]["kernel"][i]).T, bits, group),
+ _q(f"encoder{i:02d}.pw2", np.asarray(e["pw2"]["kernel"][i]).T, bits, group),
+ _fp16(f"encoder{i:02d}.dw", np.asarray(e["dw"][i]).reshape(cfg.conv_kernel, -1)),
+ ]
+ ts += [_fp16(f"encoder{i:02d}.{mlp}.{k}", e[mlp][k][i])
+ for mlp in ("hadamard_mlp_0", "hadamard_mlp") for k in HADAMARD]
+
+ enc = params["encoder"]
+ for name in ("mhc_a_pre", "mhc_a_post", "mhc_a_res", "mhc_b_pre", "mhc_b_post", "mhc_b_res"):
+ ts.append(_fp16(f"encoder.{name}", np.asarray(enc[name])))
+ for name in ("mhc_phi_pre", "mhc_phi_post", "mhc_phi_res"):
+ phi = np.asarray(enc[name])
+ L, nC, lanes = phi.shape
+ ts.append(_q(f"encoder.{name}", phi.transpose(0, 2, 1).reshape(L * lanes, nC), bits, group))
+ ts.append(_fp16("encoder.final_norm", enc["final_norm"]["scale"]))
+
+ stem = params["stem"]
+ ts.append(_fp16("stem.w", np.asarray(stem["w"]).reshape(9, -1)))
+ for i in range(1, cfg.stem_stages):
+ ts += [_fp16(f"stem.dw_{i}", np.asarray(stem[f"dw_{i}"]).reshape(9, -1)),
+ _q(f"stem.pw_{i}", np.asarray(stem[f"pw_{i}"]["kernel"]).T, bits, group)]
+ ts.append(_q("stem.out", np.asarray(stem["out"]["kernel"]).T, bits, group))
+ mel = mel_matrix(cfg)
+ ts.append(_Tensor("mel", FP32, mel.shape, mel.tobytes()))
return ts
@@ -362,6 +427,22 @@ def _tokenizer_blob(tok):
return bytes(out)
+_PROTO_TYPES = {1: TK_NORMAL, 2: TK_UNKNOWN, 3: TK_CONTROL, 4: TK_USER_DEFINED, 6: TK_BYTE}
+
+
+def whistle_tokenizer_blob(proto, languages):
+ spec = proto.trainer_spec
+ out = bytearray(struct.pack(_TK_HDR, len(proto.pieces) + len(languages), spec.pad_id, spec.eos_id, spec.bos_id,
+ spec.unk_id, int(proto.normalizer_spec.add_dummy_prefix), int(spec.byte_fallback), 0))
+ for piece in proto.pieces:
+ b = piece.piece.encode("utf-8")
+ out += struct.pack(_TK_REC, piece.score, _PROTO_TYPES[piece.type], len(b)) + b
+ for language in languages:
+ b = f"<|{language}|>".encode("utf-8")
+ out += struct.pack(_TK_REC, 0.0, TK_USER_DEFINED, len(b)) + b
+ return bytes(out)
+
+
def _tokenizer_pieces(blob):
return struct.unpack_from(_TK_HDR, blob, 0)[0]
@@ -389,9 +470,9 @@ def read_tokenizer_blob(path):
raise ValueError(f"{path} carries no tokenizer")
-def _pack_cact(params, config, bits, group, tokenizer, kv_window=0):
+def _pack_cact(params, config, bits, group, tokenizer, kv_window=0, audio=None):
params = {k: v for k, v in params.items()}
- ts = _tensors(params, config, bits, group)
+ ts = _tensors(params, config, bits, group, audio)
vocab = int(config.vocab_size)
out_vocab = int(getattr(config, "out_vocab", 0) or 0)
if out_vocab > vocab:
@@ -447,8 +528,8 @@ def _pack_cact(params, config, bits, group, tokenizer, kv_window=0):
def write_export(params, config, path, bits=WEIGHT_BITS, group=128, tokenizer=None,
- kv_window=0):
- buf, n = _pack_cact(params, config, bits, group, tokenizer, kv_window)
+ kv_window=0, audio=None):
+ buf, n = _pack_cact(params, config, bits, group, tokenizer, kv_window, audio)
with open(path, "wb") as f:
f.write(buf)
return {"path": path, "bytes": len(buf), "tensors": n}
diff --git a/needle/model/finetune.py b/needle/model/finetune.py
index 7063fcb..a5d49f5 100644
--- a/needle/model/finetune.py
+++ b/needle/model/finetune.py
@@ -309,7 +309,6 @@ def lora_target_paths(params):
flat = flatten_dict(params)
paths = [path for path in flat
if path[-1] == "kernel"
- and "stack" in path
and "layers" in path
and any(t in path for t in LORA_TARGETS)]
paths = [p for p in paths if jnp.max(jnp.abs(flat[p])) > 1e-6]
@@ -538,6 +537,8 @@ def build_main(args):
print(f" {'wrote':<9} {out} {os.path.getsize(out) / 1e6:.2f} MB the published base archive")
else:
adapter = read_adapter(args.lora) if args.lora else None
+ if adapter and adapter.get("languages") is not None:
+ raise SystemExit(f"{args.lora} is a Whistle adapter; build it with needle whistle build")
checkpoint = args.checkpoint
if not checkpoint and adapter and adapter.get("base") and os.path.exists(adapter["base"]):
checkpoint = adapter["base"]
@@ -566,12 +567,16 @@ def build_main(args):
print(f" {'next':<9} needle.Needle(weights={out!r}, tools=[...])")
if args.upload:
- repo = os.environ.get("NEEDLE_HF_REPO")
- if not repo:
- raise SystemExit("set NEEDLE_HF_REPO=<you>/<model> to upload")
- from huggingface_hub import HfApi
- api = HfApi()
- api.create_repo(repo, repo_type="model", exist_ok=True)
- api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),
- repo_id=repo, repo_type="model")
- print(f" {'uploaded':<9} {os.path.basename(out)} {repo}")
+ upload_archive(out)
+
+
+def upload_archive(out):
+ repo = os.environ.get("NEEDLE_HF_REPO")
+ if not repo:
+ raise SystemExit("set NEEDLE_HF_REPO=<you>/<model> to upload")
+ from huggingface_hub import HfApi
+ api = HfApi()
+ api.create_repo(repo, repo_type="model", exist_ok=True)
+ api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),
+ repo_id=repo, repo_type="model")
+ print(f" {'uploaded':<9} {os.path.basename(out)} {repo}")
diff --git a/needle/model/whistle.py b/needle/model/whistle.py
new file mode 100644
index 0000000..94a0aa1
--- /dev/null
+++ b/needle/model/whistle.py
@@ -0,0 +1,375 @@
+import math
+from dataclasses import dataclass, fields
+from functools import partial
+
+import numpy as np
+import jax
+import jax.numpy as jnp
+import jax.nn.initializers as jinit
+import flax.linen as nn
+
+from . import quantize as _quantize
+from .architecture import (
+ DTYPE_MAP, ENGRAM_CONV_TAPS, Engram, HadamardMLP, MultiHeadAttention, TransformerConfig, ZCRMSNorm,
+ _aq, _mask_diag, _res_identity_init, _rms_unit, _sinkhorn, default_init, engram_geometry,
+ engram_indices, make_causal_mask, precompute_rope_freqs, residual_init,
+)
+
+LANGUAGES = ("en", "de", "fr", "es", "it", "nl", "pl")
+
+
+@dataclass(frozen=True)
+class WhistleConfig:
+ text: int = 8192
+ languages: tuple = LANGUAGES
+ dim: int = 512
+ kv_heads: int = 2
+ qk_dim: int = 48
+ v_dim: int = 64
+ enc_layers: int = 8
+ dec_layers: int = 8
+ conv_kernel: int = 9
+ taps: int = 3
+ lanes: int = 4
+ enc_lanes: int = 4
+ sites: tuple = (3, 7)
+ orders: tuple = (2, 3)
+ slots: int = 18432
+ hashes: int = 2
+ stem: int = 128
+ stem_stages: int = 3
+ rope: float = 100000.0
+ mels: int = 80
+ rate: int = 16000
+ fft: int = 512
+ win: int = 400
+ hop: int = 160
+ max_tokens: int = 320
+ pad: int = 0
+ eos: int = 1
+ bos: int = 2
+ dtype: str = "float32"
+ flash: bool = True
+ remat: bool = True
+ scan_unroll: int = 1
+
+ @classmethod
+ def from_saved(cls, saved):
+ known = {f.name for f in fields(cls)} - {"dtype", "flash", "remat", "scan_unroll"}
+ return cls(**{k: tuple(v) if isinstance(v, list) else v for k, v in saved.items() if k in known})
+
+ @property
+ def vocab(self):
+ return self.text + len(self.languages)
+
+ @property
+ def heads(self):
+ return self.dim // self.v_dim
+
+ @property
+ def jax_dtype(self):
+ return DTYPE_MAP[self.dtype]
+
+ def tag(self, language):
+ return self.text + self.languages.index(language)
+
+ def needle(self):
+ return TransformerConfig(
+ vocab_size=self.vocab, d_model=self.dim, num_heads=self.heads, num_kv_heads=self.kv_heads,
+ num_layers=self.dec_layers, qk_head_dim=self.qk_dim, v_head_dim=self.v_dim, rope_theta=self.rope,
+ max_seq_len=self.max_tokens, engram_orders=self.orders, engram_heads=self.hashes,
+ engram_slots=self.slots, engram_layers=self.sites, global_layers=(), sliding_window=0,
+ mhc_lanes=self.lanes, qkv_conv_taps=self.taps)
+
+
+def mel_matrix(cfg):
+ def to_mel(f):
+ return np.where(f < 1000.0, f * 3.0 / 200.0, 15.0 + np.log(np.maximum(f, 1e-9) / 1000.0) * 27.0 / np.log(6.4))
+
+ def to_hz(m):
+ return np.where(m < 15.0, m * 200.0 / 3.0, 1000.0 * np.exp((m - 15.0) * np.log(6.4) / 27.0))
+
+ points = to_hz(np.linspace(0.0, to_mel(np.array(cfg.rate / 2.0)), cfg.mels + 2))
+ freqs = np.linspace(0.0, cfg.rate / 2, cfg.fft // 2 + 1)[:, None]
+ lo, mid, hi = points[:-2], points[1:-1], points[2:]
+ bank = np.maximum(0.0, np.minimum((freqs - lo) / (mid - lo), (hi - freqs) / (hi - mid)))
+ return (bank * 2.0 / (hi - lo)).astype(np.float32)
+
+
+def features(cfg, wav, lengths):
+ wav = wav.astype(jnp.float32) * (jnp.arange(wav.shape[1]) < lengths[:, None])
+ window = np.hanning(cfg.win).astype(np.float32)
+ _, _, z = jax.scipy.signal.stft(
+ wav, window=window, nperseg=cfg.win, noverlap=cfg.win - cfg.hop, nfft=cfg.fft, boundary="zeros", padded=False)
+ power = jnp.swapaxes(jnp.abs(z * window.sum()) ** 2, 1, 2)
+ x = jnp.log(power @ mel_matrix(cfg) + 2.0 ** -24)
+ lengths = lengths // cfg.hop
+ mask = (jnp.arange(x.shape[1]) < lengths[:, None])[..., None]
+ n = jnp.maximum(lengths, 1)[:, None, None]
+ mean = (x * mask).sum(1, keepdims=True) / n
+ std = jnp.sqrt((((x - mean) * mask) ** 2).sum(1, keepdims=True) / jnp.maximum(n - 1, 1))
+ return (x - mean) / (std + 1e-5) * mask, lengths
+
+
+def sinusoids(cfg, frames):
+ ang = jnp.arange(frames)[:, None] * jnp.exp(-np.log(10000.0) / (cfg.dim // 2 - 1) * jnp.arange(cfg.dim // 2))
+ return jnp.concatenate([jnp.sin(ang), jnp.cos(ang)], -1)
+
+
+def _cross_attend(q, k, v, mem_mask):
+ s = jnp.einsum("bshd,bthd->bhst", q.astype(jnp.float32), k.astype(jnp.float32)) / math.sqrt(q.shape[-1])
+ s = jnp.where(mem_mask[:, None, None, :], s, jnp.finfo(jnp.float32).min)
+ return jnp.einsum("bhst,bthd->bshd", jax.nn.softmax(s, -1), v.astype(jnp.float32)).astype(v.dtype)
+
+
+class Stem(nn.Module):
+ cfg: WhistleConfig
+ dtype: jnp.dtype = jnp.float32
+
+ @nn.compact
+ def __call__(self, x, lengths, quant):
+ c = self.cfg
+
+ def down(x, w, lengths):
+ x = jax.lax.conv_general_dilated(
+ x, w.astype(x.dtype), (2, 2), ((1, 1), (1, 1)),
+ dimension_numbers=("NHWC", "HWIO", "NHWC"), feature_group_count=x.shape[-1])
+ lengths = (lengths + 1) // 2
+ return x * (jnp.arange(x.shape[1]) < lengths[:, None])[..., None, None], lengths
+
+ x, lengths = down(x[..., None], self.param("w", default_init(), (3, 3, 1, c.stem)), lengths)
+ for i in range(1, c.stem_stages):
+ x, lengths = down(jax.nn.silu(x), self.param(f"dw_{i}", default_init(), (3, 3, 1, c.stem)), lengths)
+ x = nn.Dense(c.stem, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name=f"pw_{i}")(x)
+ x = jnp.swapaxes(jax.nn.silu(x), 2, 3)
+ x = _aq(x.reshape(*x.shape[:2], -1), quant)
+ return nn.Dense(c.dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="out")(x), lengths
+
+
+class EncoderBlock(nn.Module):
+ cfg: WhistleConfig
+ dtype: jnp.dtype = jnp.float32
+
+ @nn.compact
+ def __call__(self, x, mask, valid, rope, quant):
+ c, dt = self.cfg, self.dtype
+ x = x + 0.5 * HadamardMLP(c.dim, dt, name="hadamard_mlp_0")(ZCRMSNorm(dtype=dt, name="pre_hada_norm_0")(x))
+
+ skip = x
+ h = ZCRMSNorm(dtype=dt, name="ZCRMSNorm_0")(x)
+ h = MultiHeadAttention(c.heads, c.kv_heads, c.dim, c.enc_layers, dt, c.flash, c.qk_dim, c.v_dim, 0,
+ name="self_attn")(h, mask=mask, rope=rope, quant=quant)
+ h = ZCRMSNorm(dtype=dt, name="post_attn_norm")(h)
+ x = skip + nn.sigmoid(self.param("attn_gate", jinit.zeros, ())).astype(dt) * h
+
+ h = ZCRMSNorm(dtype=dt, name="conv_norm")(x)
+ h = nn.Dense(2 * c.dim, dtype=dt, use_bias=False, kernel_init=default_init(), name="pw1")(_aq(h, quant))
+ h = jax.nn.glu(h) * valid
+ dw = self.param("dw", default_init(), (c.conv_kernel, 1, c.dim)).astype(h.dtype)
+ h = jax.lax.conv_general_dilated(h, dw, (1,), "SAME", dimension_numbers=("NWC", "WIO", "NWC"),
+ feature_group_count=c.dim)
+ h = jax.nn.silu(ZCRMSNorm(dtype=dt, name="conv_out_norm")(h))
+ pw2 = nn.Dense(c.dim, dtype=dt, use_bias=False, kernel_init=residual_init(c.enc_layers), name="pw2")
+ x = x + pw2(_aq(h, quant))
+
+ return x + 0.5 * HadamardMLP(c.dim, dt, name="hadamard_mlp")(ZCRMSNorm(dtype=dt, name="pre_hada_norm")(x))
+
+
+class CrossAttention(nn.Module):
+ cfg: WhistleConfig
+ dtype: jnp.dtype = jnp.float32
+
+ @nn.compact
+ def __call__(self, x, mem, mem_mask, quant=False):
+ c = self.cfg
+ heads, qk_hd, v_hd = c.heads, c.qk_dim, c.v_dim
+ B = x.shape[0]
+
+ x = _aq(x, quant)
+ dense = partial(nn.Dense, dtype=self.dtype, use_bias=False, kernel_init=default_init())
+ q = dense(heads * qk_hd, name="q_proj")(x).reshape(B, -1, heads, qk_hd)
+ k = dense(heads * qk_hd, name="k_proj")(mem).reshape(B, -1, heads, qk_hd)
+ v = dense(heads * v_hd, name="v_proj")(mem).reshape(B, -1, heads, v_hd)
+ q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q.transpose(0, 2, 1, 3))
+ k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k.transpose(0, 2, 1, 3))
+ v = v.transpose(0, 2, 1, 3)
+
+ q = _quantize.maybe_quant_query(q, quant)
+ k = _quantize.maybe_quant_kv(k, quant)
+ v = _quantize.maybe_quant_kv(v, quant)
+
+ if c.flash:
+ fused = max(qk_hd, v_hd)
+ qf = q * jnp.asarray(math.sqrt(fused / qk_hd), q.dtype)
+ if fused > qk_hd:
+ qf = jnp.pad(qf, ((0, 0), (0, 0), (0, 0), (0, fused - qk_hd)))
+ k = jnp.pad(k, ((0, 0), (0, 0), (0, 0), (0, fused - qk_hd)))
+ vf = jnp.pad(v, ((0, 0), (0, 0), (0, 0), (0, fused - v_hd))) if fused > v_hd else v
+ mask = jnp.broadcast_to(mem_mask[:, None, None, :], (B, 1, qf.shape[2], k.shape[2]))
+ out = jax.nn.dot_product_attention(qf.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3),
+ vf.transpose(0, 2, 1, 3), mask=mask)
+ out = out[..., :v_hd].reshape(B, -1, heads * v_hd)
+ else:
+ out = _cross_attend(q.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3), v.transpose(0, 2, 1, 3), mem_mask)
+ out = out.reshape(B, -1, heads * v_hd)
+
+ out = out * nn.sigmoid(dense(heads * v_hd, name="gate_proj")(x))
+ out = _aq(out, quant)
+ return nn.Dense(c.dim, dtype=self.dtype, use_bias=False, kernel_init=residual_init(c.dec_layers),
+ name="out_proj")(out)
+
+
+class DecoderBlock(nn.Module):
+ cfg: WhistleConfig
+ dtype: jnp.dtype = jnp.float32
+
+ def _gate(self, name):
+ return nn.sigmoid(self.param(name, jinit.zeros, ())).astype(self.dtype)
+
+ @nn.compact
+ def __call__(self, x, mask=None, rope=None, quant=False, engram_kv=None, site_flags=None, mem=None, mem_mask=None):
+ c, dt = self.cfg, self.dtype
+ if engram_kv is not None:
+ ek, ev = engram_kv
+ alpha = nn.sigmoid(jnp.einsum("btd,sbtd->sbt", _rms_unit(x), _rms_unit(ek)) / math.sqrt(c.dim))
+ x = x + jnp.einsum("s,sbt,sbtd->btd", site_flags.astype(jnp.float32), alpha,
+ ev.astype(jnp.float32)).astype(x.dtype)
+
+ skip = x
+ x = ZCRMSNorm(dtype=dt)(x)
+ x = MultiHeadAttention(c.heads, c.kv_heads, c.dim, c.dec_layers, dt, c.flash, qk_head_dim=c.qk_dim,
+ v_head_dim=c.v_dim, qkv_conv_taps=c.taps,
+ name="self_attn")(x, mask=mask, rope=rope, quant=quant)
+ x = ZCRMSNorm(dtype=dt, name="post_attn_norm")(x)
+ x = skip + self._gate("attn_gate") * x
+
+ skip = x
+ x = ZCRMSNorm(dtype=dt, name="cross_norm")(x)
+ x = CrossAttention(c, dt, name="cross_attn")(x, mem, mem_mask, quant)
+ x = ZCRMSNorm(dtype=dt, name="post_cross_norm")(x)
+ x = skip + self._gate("cross_gate") * x
+
+ skip = x
+ x = ZCRMSNorm(dtype=dt, name="pre_hada_norm")(x)
+ return skip + HadamardMLP(c.dim, dt, name="hadamard_mlp")(x)
+
+
+class _LaneBody(nn.Module):
+ make: object
+ dtype: jnp.dtype = jnp.float32
+
+ @nn.compact
+ def __call__(self, x, xs, quant, block_kwargs):
+ site_flags, hc = xs
+ block = self.make(name="block")
+ extra = {} if site_flags is None else {"site_flags": site_flags}
+ B, T, n, C = x.shape
+ xf = x.astype(jnp.float32)
+ nx = _aq(_rms_unit(x.reshape(B, T, n * C)), quant)
+ hpre = nn.sigmoid(hc["a_pre"] * (nx @ hc["phi_pre"].astype(jnp.float32)) + hc["b_pre"] + hc["pre_off"])
+ u = jnp.einsum("btn,btnc->btc", hpre, xf).astype(self.dtype)
+ y = block(u, quant=quant, **block_kwargs, **extra) - u
+ hpost = 2 * nn.sigmoid(hc["a_post"] * (nx @ hc["phi_post"].astype(jnp.float32)) + hc["b_post"] + hc["post_off"])
+ hres = _sinkhorn(hc["a_res"] * (nx @ hc["phi_res"].astype(jnp.float32)).reshape(B, T, n, n) + hc["b_res"])
+ out = jnp.einsum("btij,btjc->btic", hres, xf) + hpost[..., None] * y.astype(jnp.float32)[:, :, None, :]
+ return out.astype(self.dtype), None
+
+
+class LaneStack(nn.Module):
+ num_layers: int
+ lanes: int
+ d_model: int
+ make: object
+ engram_layers: tuple = ()
+ dtype: jnp.dtype = jnp.float32
+ remat: bool = True
+ unroll: int = 1
+
+ @nn.compact
+ def __call__(self, x, quant=False, **block_kwargs):
+ x = x.astype(self.dtype)
+ site_flags = None
+ if block_kwargs.get("engram_kv") is not None:
+ flags = np.zeros((self.num_layers, len(self.engram_layers)), np.float32)
+ for s, layer in enumerate(self.engram_layers):
+ flags[layer, s] = 1.0
+ site_flags = jnp.asarray(flags)
+
+ n, L, nC = self.lanes, self.num_layers, self.lanes * self.d_model
+ lane = np.eye(n, dtype=np.float32)[np.arange(L) % n]
+ hc = {
+ "phi_pre": self.param("mhc_phi_pre", default_init(), (L, nC, n)),
+ "phi_post": self.param("mhc_phi_post", default_init(), (L, nC, n)),
+ "phi_res": self.param("mhc_phi_res", default_init(), (L, nC, n * n)),
+ "b_pre": self.param("mhc_b_pre", jinit.zeros, (L, n)),
+ "b_post": self.param("mhc_b_post", jinit.zeros, (L, n)),
+ "b_res": self.param("mhc_b_res", _res_identity_init, (L, n, n)),
+ "a_pre": self.param("mhc_a_pre", jinit.constant(0.01), (L,)),
+ "a_post": self.param("mhc_a_post", jinit.constant(0.01), (L,)),
+ "a_res": self.param("mhc_a_res", jinit.constant(0.01), (L,)),
+ "pre_off": jnp.asarray(8 * lane - 4),
+ "post_off": jnp.asarray(-4 * (1 - lane)),
+ }
+ x = jnp.broadcast_to(x[:, :, None, :], (*x.shape[:2], n, x.shape[-1]))
+ scan = nn.scan(nn.remat(_LaneBody) if self.remat else _LaneBody, variable_axes={"params": 0},
+ split_rngs={"params": True}, length=L, in_axes=(0, nn.broadcast, nn.broadcast),
+ unroll=self.unroll)
+ x, _ = scan(self.make, self.dtype, name="layers")(x, (site_flags, hc), quant, block_kwargs)
+ return ZCRMSNorm(dtype=self.dtype, name="final_norm")(jnp.mean(x, axis=2))
+
+
+class Whistle(nn.Module):
+ cfg: WhistleConfig
+
+ def setup(self):
+ c, dt = self.cfg, self.cfg.jax_dtype
+ self.stem = Stem(c, dt)
+ self.encoder = LaneStack(c.enc_layers, c.enc_lanes, c.dim, partial(EncoderBlock, c, dt), dtype=dt,
+ remat=c.remat, unroll=c.scan_unroll)
+ self.embedding = nn.Embed(c.vocab, c.dim, embedding_init=jinit.normal(stddev=0.02))
+ self.stack = LaneStack(c.dec_layers, c.lanes, c.dim, partial(DecoderBlock, c, dt), c.sites, dtype=dt,
+ remat=c.remat, unroll=c.scan_unroll)
+ orders, heads, sub_dim = engram_geometry(c.needle())
+ self.engrams = [Engram(c.dim, len(orders) * heads, c.slots, sub_dim, c.dec_layers, max(orders), dt)
+ for _ in c.sites]
+ self.pe_gate = self.param("pe_gate", jinit.zeros, ())
+
+ def encode(self, wav, lengths, quant=False):
+ c = self.cfg
+ x, lengths = features(c, wav, lengths)
+ x, lengths = self.stem(x.astype(c.jax_dtype), lengths, quant)
+ x = jnp.pad(x, ((0, 0), (0, x.shape[1] % 2), (0, 0)))
+ seen = jnp.arange(x.shape[1]) < lengths[:, None]
+ mask = jnp.broadcast_to(seen[:, None, None, :], (len(seen), 1, seen.shape[1], seen.shape[1]))
+ rope = precompute_rope_freqs(c.qk_dim, x.shape[1], c.rope)
+ return self.encoder(x, quant=quant, mask=mask, valid=seen[..., None], rope=rope), lengths
+
+ def decode(self, tokens, enc, lengths, quant=False):
+ c = self.cfg
+ pe = nn.sigmoid(self.pe_gate).astype(enc.dtype) * sinusoids(c, enc.shape[1]).astype(enc.dtype)
+ mem, mem_mask = _aq(enc + pe, quant), jnp.arange(enc.shape[1]) < lengths[:, None]
+ mask = make_causal_mask(tokens.shape[1])
+ rope = precompute_rope_freqs(c.qk_dim, tokens.shape[1], c.rope)
+ x = self.embedding(tokens) * math.sqrt(c.dim)
+ orders, heads, _ = engram_geometry(c.needle())
+ indices = engram_indices(tokens, orders, heads, c.slots, 0)
+ ngram_ok = jnp.stack([_mask_diag(mask, o - 1) for o in orders for _ in range(heads)], axis=-1)
+ tap_ok = jnp.stack([_mask_diag(mask, j * max(orders)) for j in range(ENGRAM_CONV_TAPS)])
+ pairs = [e(indices, ngram_ok, tap_ok, quant=quant) for e in self.engrams]
+ kv = (jnp.stack([k for k, _ in pairs]), jnp.stack([v for _, v in pairs])) if pairs else None
+ return self.stack(x, quant=quant, mask=mask, rope=rope, engram_kv=kv, mem=mem, mem_mask=mem_mask)
+
+ def logits(self, tokens, enc, frames, quant=False):
+ x = self.decode(tokens, enc, frames, quant)
+ return _aq(x, quant).astype(jnp.float32) @ self.embedding.embedding.T.astype(jnp.float32)
+
+ def __call__(self, wav, lengths, tokens, quant=False):
+ enc, frames = self.encode(wav, lengths, quant)
+ return self.logits(tokens, enc, frames, quant)
+
+
+def load_whistle(path):
+ from .checkpoints import read_checkpoint
+
+ ckpt = read_checkpoint(path)
+ return ckpt["params"], WhistleConfig.from_saved(ckpt["config"])
diff --git a/needle/model/whistle_finetune.py b/needle/model/whistle_finetune.py
new file mode 100644
index 0000000..e154110
--- /dev/null
+++ b/needle/model/whistle_finetune.py
@@ -0,0 +1,448 @@
+import json
+import os
+import re
+import unicodedata
+from dataclasses import replace
+
+import numpy as np
+
+from .checkpoints import read_adapter, write_adapter
+from .finetune import _training_rng, init_lora, lora_target_paths, merge_lora
+from .whistle import LANGUAGES, Whistle, load_whistle
+
+DEFAULT_BASE = "checkpoints/whistle.safetensors"
+SAMPLE_RATE = 16000
+SECONDS = 30
+SPACE = "▁"
+ROWS_LR = 10.0
+
+
+class WhistleTokenizer:
+ def __init__(self, pieces=(), languages=LANGUAGES):
+ import sentencepiece as spm
+ from sentencepiece import sentencepiece_model_pb2
+ from .tokenizer import get_tokenizer
+
+ self.base = get_tokenizer().sp
+ proto = sentencepiece_model_pb2.ModelProto()
+ proto.ParseFromString(self.base.serialized_model_proto())
+ low = min(p.score for p in proto.pieces)
+ for k, piece in enumerate(pieces):
+ proto.pieces.add(piece=piece, score=low - 1 - k)
+ self.proto, self.pieces, self.languages = proto, tuple(pieces), tuple(languages)
+ self.sp = spm.SentencePieceProcessor(model_proto=proto.SerializeToString())
+
+ @property
+ def text(self):
+ return len(self.proto.pieces)
+
+ def encode(self, text):
+ return self.sp.encode(text)
+
+ def blob(self):
+ from .export import whistle_tokenizer_blob
+ return whistle_tokenizer_blob(self.proto, self.languages)
+
+
+def _symbols(sp, text):
+ out, raw = [], bytearray()
+ for piece in sp.encode(text, out_type=str) + [""]:
+ if piece.startswith("<0x") and piece.endswith(">") and len(piece) == 6:
+ raw.append(int(piece[3:5], 16))
+ continue
+ if raw:
+ out += list(raw.decode("utf-8", "ignore"))
+ raw.clear()
+ if piece:
+ out.append(piece)
+ return out
+
+
+def read_tokens(path):
+ if not os.path.isfile(path):
+ raise SystemExit(f"--tokens takes a file with one token per line; {path} is not a file")
+ with open(path, encoding="utf-8-sig") as handle:
+ return [line.strip() for line in handle if line.strip()]
+
+
+def token_pieces(sp, tokens):
+ known = {sp.id_to_piece(i) for i in range(sp.get_piece_size())}
+ pieces = []
+ for token in tokens:
+ for text in (" " + token, token):
+ symbols = _symbols(sp, text)
+ for end in range(1, len(symbols) + 1):
+ piece = "".join(symbols[:end])
+ if piece not in known:
+ known.add(piece)
+ pieces.append(piece)
+ return pieces
+
+
+def read_audio(path):
+ import wave
+ try:
+ with wave.open(path, "rb") as source:
+ channels, width, rate = source.getnchannels(), source.getsampwidth(), source.getframerate()
+ raw = source.readframes(source.getnframes())
+ if width != 2:
+ from ..agent.whistle import _read_wav
+ return np.asarray(_read_wav(path), np.float32)
+ samples = np.frombuffer(raw, "<i2").reshape(-1, channels).mean(1).astype(np.float32) / 32768.0
+ except (wave.Error, EOFError):
+ try:
+ import soundfile
+ except ImportError:
+ raise RuntimeError(f"{path} is not a PCM WAV file; reading it needs soundfile: "
+ "pip install soundfile") from None
+ data, rate = soundfile.read(path, dtype="float32", always_2d=True)
+ samples = data.mean(1)
+ if rate != SAMPLE_RATE:
+ try:
+ import soxr
+ except ImportError:
+ raise RuntimeError(f"{path} is {rate} Hz; resampling it to {SAMPLE_RATE} Hz needs soxr: "
+ "pip install cactus-needle[mic]") from None
+ samples = soxr.resample(samples, rate, SAMPLE_RATE, quality="HQ")
+ return samples.astype(np.float32)
+
+
+def read_clips(path, language=None):
+ root = os.path.dirname(os.path.abspath(path))
+ clips, skipped = [], 0
+ with open(path, encoding="utf-8") as handle:
+ for number, line in enumerate(handle, 1):
+ if not line.strip():
+ continue
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError as error:
+ raise SystemExit(f"{path}:{number}: not JSON ({error})") from None
+ lang = row.get("language") or language
+ if not lang or not re.fullmatch(r"[A-Za-z0-9_-]+", lang):
+ raise SystemExit(f"{path}:{number}: give the line a \"language\" code such as \"pt\", "
+ "or pass --language")
+ if not row.get("audio"):
+ raise SystemExit(f"{path}:{number}: no \"audio\" path")
+ audio = os.path.join(root, row["audio"])
+ if not os.path.isfile(audio):
+ raise SystemExit(f"{path}:{number}: {audio} does not exist")
+ samples, text = read_audio(audio), (row.get("text") or "").strip()
+ if not text or not len(samples) or len(samples) > SECONDS * SAMPLE_RATE:
+ skipped += 1
+ continue
+ clips.append((samples, text, lang))
+ return clips, skipped
+
+
+def load_base(path):
+ if not os.path.exists(path):
+ from ..agent import fetch
+ print(f" {'fetch':<9} {path} downloading from Hugging Face", flush=True)
+ path = fetch.fetch_checkpoint(os.path.basename(path), os.path.dirname(path) or fetch.CHECKPOINT_PREFIX,
+ generation=fetch.WHISTLE)
+ params, cfg = load_whistle(path)
+ if "encoder" not in params or "stem" not in params:
+ raise SystemExit(f"{path} is not a Whistle checkpoint")
+ return params, cfg
+
+
+def initial_rows(embedding, cfg, tokenizer):
+ embedding = np.asarray(embedding)
+ rows = [embedding[tokenizer.base.encode(piece.replace(SPACE, " "))].mean(0) for piece in tokenizer.pieces]
+ tag = embedding[cfg.text:].mean(0)
+ rows += [tag] * (len(tokenizer.languages) - len(cfg.languages))
+ return np.asarray(rows, np.float32).reshape(-1, embedding.shape[1])
+
+
+def tuned_params(params, cfg, tokenizer, lora, scale, rows):
+ import jax.numpy as jnp
+ params = merge_lora(params, lora, scale) if lora else dict(params)
+ embedding = jnp.asarray(params["embedding"]["embedding"])
+ if rows is not None and len(rows):
+ rows, pieces = jnp.asarray(rows), len(tokenizer.pieces)
+ embedding = jnp.concatenate([embedding[:cfg.text], rows[:pieces], embedding[cfg.text:], rows[pieces:]])
+ return {**params, "embedding": {"embedding": embedding}}
+
+
+def _arrays(items, idx, cfg):
+ wav = np.zeros((len(idx), SECONDS * SAMPLE_RATE), np.float32)
+ lengths = np.zeros(len(idx), np.int32)
+ tokens = np.full((len(idx), cfg.max_tokens + 1), cfg.pad, np.int32)
+ for row, i in enumerate(idx):
+ samples, ids = items[i][:2]
+ wav[row, :len(samples)], lengths[row], tokens[row, :len(ids)] = samples, len(samples), ids
+ return wav, lengths, tokens
+
+
+def _normalizer(language):
+ from whisper_normalizer.basic import BasicTextNormalizer
+ from whisper_normalizer.english import EnglishTextNormalizer
+ return EnglishTextNormalizer() if language == "en" else BasicTextNormalizer()
+
+
+def by_character(text):
+ return any(unicodedata.name(c, "").startswith(("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "THAI")) for c in text)
+
+
+def errors(reference, hypothesis, language):
+ normalize = _normalizer(language)
+ ref, hyp = normalize(reference), normalize(hypothesis)
+ if by_character(reference):
+ ref, hyp = list(ref.replace(" ", "")), list(hyp.replace(" ", ""))
+ else:
+ ref, hyp = ref.split(), hyp.split()
+ row = list(range(len(hyp) + 1))
+ for i, r in enumerate(ref, 1):
+ prev, row[0] = row[0], i
+ for j, h in enumerate(hyp, 1):
+ prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (r != h))
+ return row[-1], len(ref)
+
+
+def _score_quantised(model, params, cfg, tokenizer, lora, scale, rows, held, batch):
+ import jax
+ from .quantize import cq_ste_params, WEIGHT_BITS
+
+ merged = cq_ste_params(tuned_params(params, cfg, tokenizer, lora, scale, rows), WEIGHT_BITS)
+ tuned = model.cfg
+ encode = jax.jit(lambda p, wav, lengths: model.apply({"params": p}, wav, lengths, True, method=Whistle.encode))
+ step = jax.jit(lambda p, tokens, enc, frames: model.apply({"params": p}, tokens, enc, frames, True,
+ method=Whistle.logits))
+ tags = np.arange(tuned.vocab) >= tuned.text
+ edits = units = 0
+ for start in range(0, len(held), batch):
+ part = held[start:start + batch]
+ enc, frames = encode(merged, *_arrays(part, range(len(part)), tuned)[:2])
+ tokens = np.full((len(part), tuned.max_tokens), tuned.pad, np.int32)
+ tokens[:, 0] = tuned.bos
+ done = np.zeros(len(part), bool)
+ for pos in range(tuned.max_tokens - 1):
+ size = min(tuned.max_tokens, (pos // 64 + 1) * 64)
+ logits = np.array(step(merged, tokens[:, :size], enc, frames)[:, pos])
+ logits[:, tags != (pos == 0)] = -np.inf
+ pick = logits.argmax(-1)
+ tokens[:, pos + 1] = np.where(done, tuned.pad, pick)
+ done |= pick == tuned.eos
+ if done.all():
+ break
+ for (_, wanted, text), ids in zip(part, tokens.tolist()):
+ ids = ids[2:ids.index(tuned.eos)] if tuned.eos in ids else ids[2:]
+ language = tuned.languages[wanted[1] - tuned.text]
+ e, n = errors(text, tokenizer.sp.decode([i for i in ids if i != tuned.pad]), language)
+ edits, units = edits + e, units + n
+ return edits, units
+
+
+def rung(params, cfg, layers):
+ from .architecture import ladder_config, ladder_slice
+
+ if not layers or layers == cfg.dec_layers:
+ return params, cfg
+ if not 2 <= layers < cfg.dec_layers:
+ raise ValueError(f"--layers must be in [2, {cfg.dec_layers}], got {layers}")
+ config = cfg.needle()
+ sites = ladder_config(config, layers).engram_layers
+ return ladder_slice(params, config, layers), replace(cfg, dec_layers=layers, sites=sites)
+
+
+def build(params, cfg, tokenizer, out):
+ from .export import write_export
+ from .quantize import WEIGHT_BITS
+ cfg = replace(cfg, text=tokenizer.text, languages=tokenizer.languages)
+ return write_export(params, cfg.needle(), out, bits=WEIGHT_BITS, tokenizer=tokenizer.blob(), audio=cfg)
+
+
+def finetune_main(args):
+ import jax
+ import jax.numpy as jnp
+ import optax
+ from .quantize import WEIGHT_BITS, configure_deploy, cq_ste_params
+
+ if args.epochs < 1 or args.batch_size < 1:
+ raise SystemExit("--epochs and --batch-size must be at least 1")
+ base_path = args.checkpoint or DEFAULT_BASE
+ params, cfg = load_base(base_path)
+ params = jax.tree.map(lambda a: np.asarray(a).astype(np.float32), params)
+ backend = jax.default_backend().lower()
+ if backend == "metal":
+ cfg = replace(cfg, flash=False, remat=False, scan_unroll=max(cfg.enc_layers, cfg.dec_layers))
+ params = jax.device_put(params)
+ print(f" {'backend':<9} {backend} float32")
+
+ clips, skipped = read_clips(args.jsonl_path, args.language)
+ if not clips:
+ raise SystemExit("no usable clips in " + args.jsonl_path)
+ languages = tuple(cfg.languages) + tuple(sorted({lang for *_, lang in clips} - set(cfg.languages)))
+ tokens = read_tokens(args.tokens) if args.tokens else []
+ pieces = token_pieces(WhistleTokenizer().sp, tokens)
+ tokenizer = WhistleTokenizer(pieces, languages)
+ tuned = replace(cfg, text=tokenizer.text, languages=languages)
+ note = f" {skipped} skipped (empty, or over {SECONDS} s)" if skipped else ""
+ print(f" {'data':<9} {len(clips)} clips {sum(len(c[0]) for c in clips) / SAMPLE_RATE / 60:.1f} min{note}")
+ added = [lang for lang in languages if lang not in cfg.languages]
+ print(f" {'vocab':<9} {len(tokens)} tokens {len(pieces)} new pieces"
+ + (f" new languages {', '.join(added)}" if added else ""))
+ split = [t for t in tokens if any(len(tokenizer.encode(x)) != 1 for x in (t, " " + t))]
+ if split:
+ print(f" {'warning':<9} not one piece: {', '.join(split)}")
+
+ items = [(samples, [cfg.bos, tuned.tag(lang), *tokenizer.encode(text), cfg.eos], text)
+ for samples, text, lang in clips]
+ long = sum(len(ids) > cfg.max_tokens + 1 for _, ids, _ in items)
+ items = [item for item in items if len(item[1]) <= cfg.max_tokens + 1]
+ if long:
+ print(f" {'skipped':<9} {long} clips with transcripts over {cfg.max_tokens} tokens")
+ if not items:
+ raise SystemExit("no clips left to train on")
+ if tokens:
+ found = sum(any(token in text for *_, text in items) for token in tokens)
+ print(f" {'tokens':<9} {found}/{len(tokens)} appear in the transcripts")
+
+ configure_deploy(act_bits=8, kv_bits=8)
+ model = Whistle(tuned)
+ print(f" {'numerics':<9} CQ W{WEIGHT_BITS} STE + A8 (matches export)")
+ paths = lora_target_paths(params)
+ scale = args.lora_alpha / args.lora_rank
+ seed = int(getattr(args, "seed", 0))
+ rng = _training_rng(seed)
+ lora = init_lora(params, paths, args.lora_rank, jax.random.PRNGKey(seed))
+ train = {"lora": lora, "rows": jnp.asarray(initial_rows(params["embedding"]["embedding"], cfg, tokenizer))}
+ print(f" {'lora':<9} rank {args.lora_rank} alpha {args.lora_alpha:g} {len(paths)} weight groups"
+ f" +{len(train['rows'])} vocabulary rows")
+
+ n_val = min(int(len(items) * getattr(args, "val_split", 0.1)), len(items) - 1)
+ held = []
+ if n_val > 0:
+ order = rng.permutation(len(items))
+ held, items = [items[i] for i in order[:n_val]], [items[i] for i in order[n_val:]]
+ print(f" {'holdout':<9} {n_val} clips for validation")
+
+ def loss_fn(train, wav, lengths, tokens):
+ merged = cq_ste_params(tuned_params(params, cfg, tokenizer, train["lora"], scale, train["rows"]), WEIGHT_BITS)
+ logits = model.apply({"params": merged}, wav, lengths, tokens[:, :-1], quant=True)
+ targets = tokens[:, 1:]
+ mask = targets != cfg.pad
+ ce = optax.softmax_cross_entropy_with_integer_labels(logits, targets)
+ return (ce * mask).sum() / jnp.maximum(mask.sum(), 1.0)
+
+ batch, count = args.batch_size, len(items)
+ steps_per_epoch = -(-count // batch)
+ total_steps = args.epochs * steps_per_epoch
+ warmup = min(max(1, total_steps // 20), total_steps - 1)
+
+ def adamw(lr):
+ return optax.adamw(optax.warmup_cosine_decay_schedule(
+ init_value=0.0, peak_value=lr, warmup_steps=warmup, decay_steps=total_steps))
+
+ labels = {"lora": jax.tree.map(lambda _: "lora", train["lora"]), "rows": "rows"}
+ optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.multi_transform(
+ {"lora": adamw(args.lr), "rows": adamw(args.lr * ROWS_LR)}, labels))
+ opt_state = optimizer.init(train)
+ print(f" {'schedule':<9} {total_steps} steps warmup {warmup} cosine decay clip 1.0 (compiling...)")
+
+ @jax.jit
+ def train_step(train, opt_state, wav, lengths, tokens):
+ loss, grads = jax.value_and_grad(loss_fn)(train, wav, lengths, tokens)
+ updates, opt_state = optimizer.update(grads, opt_state, train)
+ return optax.apply_updates(train, updates), opt_state, loss
+
+ eval_step = jax.jit(loss_fn)
+
+ every = max(1, total_steps // 50)
+ step_i = 0
+ for epoch in range(args.epochs):
+ order = rng.permutation(count)
+ last = 0.0
+ for start in range(0, count, batch):
+ train, opt_state, loss = train_step(train, opt_state, *_arrays(items, order[start:start + batch], cfg))
+ last = float(loss)
+ step_i += 1
+ if step_i % every == 0:
+ print(f" {'step':<9} {step_i}/{total_steps} loss {last:.4f}", flush=True)
+ if held:
+ val = np.mean([float(eval_step(train, *_arrays(held, range(i, min(i + batch, n_val)), cfg)))
+ for i in range(0, n_val, batch)])
+ print(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f} val {val:.4f}", flush=True)
+ else:
+ print(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f}", flush=True)
+
+ if held and getattr(args, "score", True):
+ edits, units = _score_quantised(model, params, cfg, tokenizer, train["lora"], scale, train["rows"],
+ held, batch)
+ kind = "cer" if all(by_character(text) for *_, text in held) else "wer"
+ if units:
+ print(f" {kind:<9} {edits / units:.1%} on {len(held)} held-out clips, scored on the "
+ f"W{WEIGHT_BITS} weights the archive ships")
+
+ os.makedirs(args.checkpoint_dir, exist_ok=True)
+ out = args.out or os.path.join(args.checkpoint_dir, "whistle_lora.safetensors")
+ write_adapter(out, {
+ "lora": {"/".join(p): {"A": np.asarray(v["A"]), "B": np.asarray(v["B"])} for p, v in train["lora"].items()},
+ "rows": np.asarray(train["rows"]),
+ "scale": float(scale),
+ "base": base_path,
+ "rank": args.lora_rank,
+ "seed": seed,
+ "pieces": list(pieces),
+ "languages": list(languages),
+ })
+ print(f" {'adapter':<9} {out}")
+ print(f" {'next':<9} needle whistle build --lora {out}")
+
+
+def build_main(args):
+ import shutil
+ import jax.numpy as jnp
+ from ..agent import fetch
+ from .export import read_layers
+ from .finetune import upload_archive
+
+ base_archive = fetch.fetch_weights(fetch.WHISTLE, force=True)
+ print(f" {'base':<9} {base_archive} {os.path.getsize(base_archive) / 1e6:.2f} MB")
+ platform = getattr(args, "platform", None)
+ layers = getattr(args, "layers", None)
+ folder = None
+ if platform:
+ folder = os.path.abspath(args.out or platform)
+ for path in fetch.download_platform(platform, os.path.dirname(folder), generation=3, dest=folder):
+ print(f" {'engine':<9} {path} {os.path.getsize(path) / 1e6:.2f} MB")
+ out = os.path.join(folder, fetch.base_weights(fetch.WHISTLE))
+ else:
+ out = args.out
+ if not args.lora and (not layers or layers == read_layers(base_archive)):
+ if not out:
+ raise SystemExit("pass --out <archive.cact>, or --platform to build a runnable folder")
+ shutil.copyfile(base_archive, out)
+ print(f" {'wrote':<9} {out} {os.path.getsize(out) / 1e6:.2f} MB the published base archive")
+ else:
+ adapter = read_adapter(args.lora) if args.lora else None
+ if adapter and adapter.get("languages") is None:
+ raise SystemExit(f"{args.lora} is a Needle adapter; build it with needle build")
+ checkpoint = args.checkpoint
+ if not checkpoint and adapter and adapter.get("base") and os.path.exists(adapter["base"]):
+ checkpoint = adapter["base"]
+ checkpoint = checkpoint or DEFAULT_BASE
+ params, cfg = load_base(checkpoint)
+ tokenizer = WhistleTokenizer(languages=cfg.languages)
+ if args.lora:
+ tokenizer = WhistleTokenizer(adapter.get("pieces") or (), adapter.get("languages") or cfg.languages)
+ lora = {tuple(key.split("/")): {"A": jnp.asarray(v["A"]), "B": jnp.asarray(v["B"])}
+ for key, v in adapter["lora"].items()}
+ params = tuned_params(params, cfg, tokenizer, lora, adapter["scale"], adapter.get("rows"))
+ print(f" {'merged':<9} {len(lora)} weight groups {len(tokenizer.pieces)} new pieces {args.lora}")
+ params, cfg = rung(params, cfg, layers)
+ print(f" {'depth':<9} {cfg.dec_layers} decoder layers")
+ out = out or (os.path.splitext(os.path.basename(checkpoint))[0] + ".cact")
+ info = build(params, cfg, tokenizer, out)
+ print(f" {'wrote':<9} {info['path']} {info['bytes'] / 1e6:.2f} MB {info['tensors']} tensors W4A8"
+ f" languages {', '.join(tokenizer.languages)}")
+ if folder:
+ runner = next((n for n in ("needle", "needle.exe") if os.path.exists(os.path.join(folder, n))), None)
+ if runner:
+ print(f" {'next':<9} {os.path.join(folder, runner)} --model {os.path.basename(out)} --audio clip.wav")
+ else:
+ print(f" {'next':<9} needle.transcribe('clip.wav', weights={out!r})")
+
+ if args.upload:
+ upload_archive(out)
diff --git a/pyproject.toml b/pyproject.toml
index b62705c..ea259e7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,8 @@ train = [
"flax>=0.10.2",
"optax",
"safetensors",
- "sentencepiece"
+ "sentencepiece",
+ "whisper-normalizer"
]
gpu = ["jax[cuda12]"]
metal = [
diff --git a/requirements-train.txt b/requirements-train.txt
index 5035c8a..cb2e295 100644
--- a/requirements-train.txt
+++ b/requirements-train.txt
@@ -8,3 +8,4 @@ flax>=0.10.2
optax
safetensors
sentencepiece
+whisper-normalizer
diff --git a/tests/test_whistle_finetune.py b/tests/test_whistle_finetune.py
new file mode 100644
index 0000000..36656ea
--- /dev/null
+++ b/tests/test_whistle_finetune.py
@@ -0,0 +1,244 @@
+import json
+import types
+import wave
+
+import pytest
+
+TINY = dict(dim=64, kv_heads=2, qk_dim=16, v_dim=16, enc_layers=2, dec_layers=3, conv_kernel=3, lanes=2,
+ enc_lanes=2, sites=(2,), slots=64, hashes=1, stem=8, stem_stages=2, max_tokens=64)
+
+
[email protected](scope="module")
+def tiny_whistle(tmp_path_factory):
+ import numpy as np
+ import jax
+ from needle.model.checkpoints import write_checkpoint
+ from needle.model.whistle import Whistle, WhistleConfig
+
+ cfg = WhistleConfig(**TINY, flash=False)
+ params = jax.tree.map(np.asarray, Whistle(cfg).init(
+ jax.random.PRNGKey(0), np.zeros((1, 3200), np.float32), np.array([3200]), np.zeros((1, 4), np.int32))["params"])
+ path = tmp_path_factory.mktemp("whistle") / "tiny.safetensors"
+ write_checkpoint(str(path), {"params": params, "config": cfg.__dict__})
+ return str(path)
+
+
+def published_whistle(tiny_whistle, tmp_path, monkeypatch):
+ from needle.agent import fetch
+ from needle.model.whistle_finetune import WhistleTokenizer, build, load_base
+
+ archive = str(tmp_path / "whistle.cact")
+ params, cfg = load_base(tiny_whistle)
+ build(params, cfg, WhistleTokenizer(languages=cfg.languages), archive)
+ calls = []
+ monkeypatch.setattr(fetch, "fetch_weights", lambda generation=2, dest_dir=None, force=False:
+ calls.append((generation, force)) or archive)
+ return archive, calls
+
+
+def _build_args(**kw):
+ return types.SimpleNamespace(**{"checkpoint": None, "lora": None, "out": None, "upload": False, "layers": None,
+ "platform": None, **kw})
+
+
+def _clips(folder, lines):
+ import numpy as np
+ rng = np.random.default_rng(0)
+ with open(folder / "clips.jsonl", "w", encoding="utf-8") as out:
+ for i, text in enumerate(lines):
+ with wave.open(str(folder / f"{i}.wav"), "wb") as w:
+ w.setnchannels(1), w.setsampwidth(2), w.setframerate(16000)
+ w.writeframes((rng.standard_normal(16000) * 3000).astype("<i2").tobytes())
+ out.write(json.dumps({"audio": f"{i}.wav", "text": text}, ensure_ascii=False) + "\n")
+ return str(folder / "clips.jsonl")
+
+
+def test_declared_tokens_become_one_piece_anywhere_and_match_the_engine_encoder(tmp_path):
+ from needle.model.export import RefTokenizer, parse_tokenizer_blob
+ from needle.model.whistle_finetune import WhistleTokenizer, read_tokens, token_pieces
+
+ listed = tmp_path / "tokens.txt"
+ listed.write_text("ação\nSão Paulo, SP\n🙂\n\nПривет\nASP.NET\n你好\n", encoding="utf-8")
+ tokens = read_tokens(str(listed))
+ assert tokens == ["ação", "São Paulo, SP", "🙂", "Привет", "ASP.NET", "你好"]
+ base = WhistleTokenizer()
+ tokenizer = WhistleTokenizer(token_pieces(base.sp, tokens))
+ meta = parse_tokenizer_blob(tokenizer.blob())
+ assert set(meta["types"][base.text:tokenizer.text]) == {0}
+ engine = RefTokenizer(meta)
+ for token in tokens:
+ for text in (token, " " + token, f"disse {token} hoje"):
+ ids = tokenizer.encode(text)
+ assert any(tokenizer.sp.id_to_piece(i).replace("\u2581", " ").strip() == token for i in ids)
+ assert engine.encode(text) == ids and tokenizer.sp.decode(ids) == text
+ assert token_pieces(base.sp, ["the"]) == []
+
+
+def test_extended_tokenizer_appends_pieces_and_tags():
+ from needle.model.export import parse_tokenizer_blob
+ from needle.model.whistle_finetune import WhistleTokenizer
+
+ plain = WhistleTokenizer((), ("en",))
+ tokenizer = WhistleTokenizer(("ã", "ão"), ("en", "pt"))
+ assert tokenizer.text == plain.text + 2
+ assert tokenizer.encode("não") == [plain.sp.piece_to_id("n"), plain.text + 1]
+ meta = parse_tokenizer_blob(tokenizer.blob())
+ assert meta["pieces"][plain.text:] == ["ã", "ão", "<|en|>", "<|pt|>"]
+ assert meta["types"][-2:] == [3, 3] and meta["byte_fallback"]
+
+
+def test_finetune_adds_a_language_and_builds_an_archive_with_it(tiny_whistle, published_whistle, tmp_path, capsys):
+ from needle.model.checkpoints import read_adapter
+ from needle.model.export import parse_tokenizer_blob, read_export
+ from needle.model.whistle_finetune import WhistleTokenizer, build_main, finetune_main, token_pieces
+
+ data = _clips(tmp_path, ["não é coração", "ação não", "não não", "coração"] * 2)
+ tokens = tmp_path / "tokens.txt"
+ tokens.write_text("ação\ncoração\n", encoding="utf-8")
+ adapter = str(tmp_path / "adapter.safetensors")
+ finetune_main(types.SimpleNamespace(
+ jsonl_path=data, language="pt", tokens=str(tokens), checkpoint=tiny_whistle, epochs=1, batch_size=2, lr=1e-3,
+ lora_rank=4, lora_alpha=8.0, val_split=0.25, seed=0, checkpoint_dir=str(tmp_path), out=adapter))
+ printed = capsys.readouterr().out
+ assert "CQ W4 STE + A8 (matches export)" in printed and "held-out clips, scored on the W4 weights" in printed
+ saved = read_adapter(adapter)
+ pieces = token_pieces(WhistleTokenizer().sp, ["ação", "coração"])
+ assert saved["languages"][-1] == "pt" and saved["pieces"] == pieces
+ assert saved["rows"].shape == (len(pieces) + 1, TINY["dim"]) and len(saved["lora"]) == 15
+
+ out = str(tmp_path / "tuned.cact")
+ build_main(_build_args(lora=adapter, out=out))
+ header, tensors = read_export(out)
+ tokenizer = parse_tokenizer_blob(tensors[-1])
+ text = 8192 + len(pieces)
+ assert header["vocab_size"] == text + 8 and tensors[0].shape == (text + 8, TINY["dim"])
+ assert tokenizer["pieces"][-1] == "<|pt|>" and tokenizer["pieces"][8192:text] == saved["pieces"]
+ manifest = next(t for t in tensors if getattr(t, "shape", None) == (13,) and t[0] == 3246)
+ assert manifest[2] == text
+ from needle.agent.whistle import LANGUAGES, languages
+ assert languages(out) == LANGUAGES + ("pt",)
+
+
+def test_build_without_an_adapter_copies_the_published_archive(published_whistle, tmp_path):
+ from needle.model.whistle_finetune import build_main
+
+ archive, calls = published_whistle
+ out = tmp_path / "copy.cact"
+ build_main(_build_args(out=str(out)))
+ assert out.read_bytes() == open(archive, "rb").read() and calls == [("whistle", True)]
+ with pytest.raises(SystemExit, match="--out"):
+ build_main(_build_args())
+
+
+def test_build_layers_exports_a_decoder_rung(tiny_whistle, published_whistle, tmp_path):
+ from needle.model.export import read_export
+ from needle.model.whistle_finetune import build_main
+
+ out = str(tmp_path / "rung.cact")
+ build_main(_build_args(checkpoint=tiny_whistle, layers=2, out=out))
+ header, _ = read_export(out)
+ assert header["num_layers"] == 2 and header["engram_layers"] == (1,)
+ with pytest.raises(ValueError, match="--layers"):
+ build_main(_build_args(checkpoint=tiny_whistle, layers=5, out=out))
+
+
+def test_lines_need_a_language(tmp_path):
+ from needle.model.whistle_finetune import read_clips
+
+ data = _clips(tmp_path, ["hello"])
+ with pytest.raises(SystemExit, match="language"):
+ read_clips(data)
+ clips, skipped = read_clips(data, "en")
+ assert len(clips) == 1 and clips[0][1:] == ("hello", "en") and not skipped
+
+
+def test_errors_use_the_whisper_normalizers_and_characters_for_unspaced_scripts():
+ from needle.model.whistle_finetune import errors
+
+ assert errors("Olá, mundo!", "olá mundo", "pt") == (0, 2)
+ assert errors("a b c", "a x c d", "pt") == (2, 3)
+ assert errors("Mr. Smith paid ten dollars.", "mister smith paid $10", "en") == (0, 4)
+ assert errors("今天天气很好。", "今天天汽很好", "zh") == (1, 6)
+ assert errors("こんにちは 世界", "こんにちわ世界", "ja") == (1, 7)
+ assert errors("Привет, мир", "привет мир", "be") == (0, 2)
+
+
+def test_bad_lines_name_the_line(tmp_path):
+ from needle.model.whistle_finetune import read_clips
+
+ data = _clips(tmp_path, ["olá"])
+ for line, message in ([("not json", "2: not JSON"), ('{"text": "a"}', "2: no \"audio\""),
+ ('{"audio": "gone.wav", "text": "a"}', "gone.wav does not exist"),
+ ('{"audio": "0.wav", "text": "a", "language": "p t"}', "2: give the line")]):
+ with open(data, "a", encoding="utf-8") as out:
+ out.write(line + "\n")
+ with pytest.raises(SystemExit, match=message):
+ read_clips(data, "pt")
+ lines = open(data, encoding="utf-8").read().splitlines()[:-1]
+ open(data, "w", encoding="utf-8").write("\n".join(lines) + "\n")
+
+
+def test_tokens_come_from_a_file(tmp_path):
+ from needle.model.whistle_finetune import read_tokens
+
+ with pytest.raises(SystemExit, match="one token per line"):
+ read_tokens("ação,coração")
+ listed = tmp_path / "tokens.txt"
+ listed.write_bytes("ação \r\n\n coração\n".encode("utf-8"))
+ assert read_tokens(str(listed)) == ["ação", "coração"]
+
+
+def test_wrong_kinds_of_files_are_refused(tiny_whistle, tiny_checkpoint, published_whistle, tmp_path):
+ from needle.model.checkpoints import write_adapter
+ from needle.model.finetune import build_main as needle_build
+ from needle.model.whistle_finetune import build_main, load_base
+
+ with pytest.raises(SystemExit, match="not a Whistle checkpoint"):
+ load_base(tiny_checkpoint)
+ needle_adapter = str(tmp_path / "needle.safetensors")
+ write_adapter(needle_adapter, {"lora": {}, "scale": 2.0, "base": tiny_checkpoint, "rank": 4, "seed": 0})
+ with pytest.raises(SystemExit, match="Needle adapter"):
+ build_main(_build_args(lora=needle_adapter, out=str(tmp_path / "x.cact")))
+ whistle_adapter = str(tmp_path / "whistle.safetensors")
+ write_adapter(whistle_adapter, {"lora": {}, "scale": 2.0, "base": tiny_whistle, "rank": 4, "seed": 0,
+ "pieces": [], "languages": ["en"]})
+ with pytest.raises(SystemExit, match="Whistle adapter"):
+ needle_build(types.SimpleNamespace(checkpoint=tiny_checkpoint, lora=whistle_adapter,
+ out=str(tmp_path / "y.cact"), upload=False))
+
+
+def test_finetune_needs_an_epoch_and_a_batch():
+ from needle.model.whistle_finetune import finetune_main
+
+ for epochs, batch in ((0, 8), (1, 0)):
+ with pytest.raises(SystemExit, match="at least 1"):
+ finetune_main(types.SimpleNamespace(epochs=epochs, batch_size=batch))
+
+
+def test_cli_routes_finetune_and_build_and_takes_a_tuned_language(monkeypatch):
+ import sys
+ import needle._telemetry
+ import needle.agent.whistle
+ import needle.cli
+ import needle.model.whistle_finetune as tune
+
+ seen = []
+ monkeypatch.setattr(needle._telemetry, "track", lambda *a, **k: None)
+ monkeypatch.setattr(tune, "finetune_main", lambda args: seen.append(("finetune", args)))
+ monkeypatch.setattr(tune, "build_main", lambda args: seen.append(("build", args)))
+ monkeypatch.setattr(needle.agent.whistle, "playground", lambda args: seen.append(("playground", args)))
+ for argv in (["finetune", "clips.jsonl", "--language", "pt", "--tokens", "tokens.txt"],
+ ["build", "--lora", "a.safetensors", "--layers", "4", "--out", "t.cact"],
+ ["playground", "--language", "pt", "--weights", "t.cact"]):
+ monkeypatch.setattr(sys, "argv", ["needle", "whistle", *argv])
+ needle.cli.main()
+ (_, tuned), (_, built), (_, played) = seen
+ assert (tuned.jsonl_path, tuned.language, tuned.tokens, tuned.batch_size, tuned.lr) == (
+ "clips.jsonl", "pt", "tokens.txt", 16, 1e-4)
+ assert (built.lora, built.layers, built.out, built.platform, built.upload) == ("a.safetensors", 4, "t.cact", None, False)
+ assert played.language == "pt"