Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Add Whistle fine-tuning functionality and related tests
jakmro482361d8d9f9
diff --git a/needle/agent/whistle.py b/needle/agent/whistle.pyindex bd189f9..ea23173 100644--- a/needle/agent/whistle.py+++ b/needle/agent/whistle.py@@ -6,6 +6,7 @@ import jsonimport osimport queueimport shutil+import structimport sysimport threadingimport time@@ -74,6 +75,31 @@ def _read_wav(path):return soxr.resample(numpy.asarray(mono, numpy.float32), rate, SAMPLE_RATE, quality="HQ")+def languages(weights):+ try:+ with open(weights, "rb") as handle:+ raw = handle.read()+ tag, count, codebook = struct.unpack_from("<III", raw)+ except (OSError, struct.error):+ return LANGUAGES+ if tag != 0x05E12A84:+ return LANGUAGES+ record = 196 + codebook * 4+ for _ in range(count):+ if raw[record] == 4:+ offset, size = struct.unpack_from("<QQ", raw, record + 20)+ blob, found = raw[offset:offset + size], []+ at = 24+ for _ in range(struct.unpack_from("<I", blob)[0]):+ length = struct.unpack_from("<H", blob, at + 5)[0]+ piece = blob[at + 7:at + 7 + length].decode("utf-8", "replace")+ at += 7 + length+ found = found + [piece[2:-2]] if piece.startswith("<|") and piece.endswith("|>") else []+ return tuple(found) or LANGUAGES+ record += 44+ return LANGUAGES++def _options(language, keywords):if keywords is not None and not isinstance(keywords, str):keywords = "\n".join(keywords)@@ -357,6 +383,9 @@ def playground(args):"stream": False}print("whistle playground: downloading and initializing the model...", flush=True)whistle = Whistle(weights=args.weights)+ known = languages(whistle.weights)+ if args.language and args.language not in known:+ raise SystemExit("--language is one of " + " ".join(known))if args.audio:return show_transcript(whistle, args.audio, state)print(PLAYGROUND_HELP)@@ -367,8 +396,8 @@ def playground(args):returnif command == "/language":language = rest.strip()- if language and language not in LANGUAGES:- print(" language is one of", " ".join(LANGUAGES))+ if language and language not in known:+ print(" language is one of", " ".join(known))continuestate["language"] = language or Noneprint(" language", state["language"] or "detected")diff --git a/needle/cli.py b/needle/cli.pyindex 71521c0..e9f5c64 100644--- a/needle/cli.py+++ b/needle/cli.py@@ -14,7 +14,7 @@ HELP = """usage: needle <command> [options]download needle3 | needle3.safetensors | <platform> | model-<id> | <org>/<repo>[/<file>.cact]fetch fetch the engine library for this platformplayground serve the browser playground- whistle speech to text: playground | compare+ whistle speech to text: playground | compare | finetune | buildneedle <command> --help for the options of one command.Check the readme for the rest."""@@ -271,8 +271,9 @@ def main():v = verbs.add_parser("playground")v.add_argument("audio", type=str, nargs="?", default=None,help="A WAV file to transcribe once (default: press Enter to speak)")- v.add_argument("--language", type=str, default=None, choices=LANGUAGES,- help="Force the language (default: detect it)")+ v.add_argument("--language", type=str, default=None,+ help="Force the language, one the weights know: " + " ".join(LANGUAGES) + " for the published "+ "weights (default: detect it)")v.add_argument("--keywords", type=str, default="",help="Words and phrases to favour, comma separated")v.add_argument("--word-timestamps", action="store_true",@@ -284,6 +285,39 @@ def main():help="A WAV file to run through every model once (default: press Enter to speak)")v.add_argument("--weights", type=str, default=None,help="whistle.cact to load (default: the published weights)")+ v = verbs.add_parser("finetune")+ v.add_argument("jsonl_path", type=str,+ help='JSONL of {"audio": WAV path, "text": transcript, "language": code}, one clip per line')+ v.add_argument("--language", type=str, default=None,+ help="Language of lines that do not name one; a new code adds that language")+ v.add_argument("--tokens", type=str, default=None,+ help="File of tokens to add to the vocabulary, one per line")+ v.add_argument("--checkpoint", type=str, default=None,+ help="Base checkpoint (default: checkpoints/whistle.safetensors, downloaded if missing)")+ v.add_argument("--epochs", type=int, default=3)+ v.add_argument("--batch-size", type=int, default=16)+ v.add_argument("--lr", type=float, default=1e-4)+ v.add_argument("--lora-rank", type=int, default=16, help="LoRA adapter rank (default: 16)")+ v.add_argument("--lora-alpha", type=float, default=32.0, help="LoRA scaling alpha (default: 32)")+ v.add_argument("--val-split", type=float, default=0.1,+ help="Fraction of clips held out for validation (0 disables)")+ v.add_argument("--seed", type=int, default=0,+ help="Random seed for LoRA init, validation split, and epoch shuffling")+ v.add_argument("--checkpoint-dir", type=str, default="checkpoints")+ v.add_argument("--out", type=str, default=None,+ help="Output adapter path (.safetensors)")+ v = verbs.add_parser("build")+ v.add_argument("checkpoint", type=str, nargs="?", default=None,+ help="Base checkpoint (.safetensors); defaults to the adapter's "+ "base, else the Whistle base (auto-downloads)")+ v.add_argument("--lora", type=str, default=None, help="LoRA adapter to merge before export")+ v.add_argument("--out", type=str, default=None, help="Output .cact path")+ v.add_argument("--upload", action="store_true", help="Push the .cact to $NEEDLE_HF_REPO")+ v.add_argument("--layers", type=int, default=None,+ help="Export the N-layer decoder rung of the base (2..8); default the full 8")+ v.add_argument("--platform", type=str, default=None, choices=fetch.PLATFORMS,+ help="Also download that platform's engine and header, and place the "+ "archive beside them as whistle.cact (--out is then a directory)")args = parser.parse_args()@@ -381,6 +415,10 @@ def main():from .agent.whistle import playground as whistle_mainelif args.verb == "compare":from .agent.whistle import compare as whistle_main+ elif args.verb == "finetune":+ from .model.whistle_finetune import finetune_main as whistle_main+ elif args.verb == "build":+ from .model.whistle_finetune import build_main as whistle_mainelse:- raise SystemExit("needle whistle playground | compare")+ raise SystemExit("needle whistle playground | compare | finetune | build")whistle_main(args)diff --git a/needle/model/checkpoints.py b/needle/model/checkpoints.pyindex 7de5f47..681e994 100644--- a/needle/model/checkpoints.py+++ b/needle/model/checkpoints.py@@ -88,7 +88,7 @@ def write_checkpoint(path, checkpoint):library.save_file(flatten(checkpoint["params"]), os.fspath(path), metadata=metadata)-_ADAPTER_FIELDS = ("scale", "base", "rank", "seed")+_ADAPTER_FIELDS = ("scale", "base", "rank", "seed", "pieces", "languages")def write_adapter(path, adapter):@@ -99,6 +99,8 @@ def write_adapter(path, adapter):for name, value in adapter["lora"].items():tensors[f"lora/{name}/A"] = _contiguous(value["A"])tensors[f"lora/{name}/B"] = _contiguous(value["B"])+ if adapter.get("rows") is not None:+ tensors["rows"] = _contiguous(adapter["rows"])metadata = {key: json.dumps(adapter.get(key), default=str) for key in _ADAPTER_FIELDS}library.save_file(tensors, os.fspath(path), metadata=metadata)@@ -111,11 +113,14 @@ def read_adapter(path):with safe_open(os.fspath(path), framework="np") as handle:metadata = handle.metadata() or {}- lora = {}+ lora, rows = {}, Nonefor name, value in library.load_file(os.fspath(path)).items():+ if name == "rows":+ rows = value+ continue_, key, matrix = name.rsplit("/", 2) if name.count("/") == 2 else (None, *name[5:].rsplit("/", 1))lora.setdefault(key, {})[matrix] = value- adapter = {"lora": lora}+ adapter = {"lora": lora, "rows": rows}for key in _ADAPTER_FIELDS:adapter[key] = _json_or_default(metadata.get(key), None)return adapterdiff --git a/needle/model/export.py b/needle/model/export.pyindex faa2f8f..506a62e 100644--- a/needle/model/export.py+++ b/needle/model/export.py@@ -233,7 +233,7 @@ def _q(name, mat, bits, group):return _Tensor(name, CQ, mat.shape, packed.tobytes() + norms.tobytes(), group, bits)-def _tensors(params, config, bits, group):+def _tensors(params, config, bits, group, audio=None):_, _, orders, heads, sub_dim, sites = _geometry(config)num_tables = len(orders) * headsembedding = np.asarray(_get(params, ("embedding", "embedding")))@@ -308,6 +308,71 @@ def _tensors(params, config, bits, group):ts.append(_fp16("final_norm", params["stack"]["final_norm"]["scale"]))ts += _head_tensors(params, group)+ if audio is not None:+ ts += _audio_tensors(params, audio, bits, group)+ return ts+++AUDIO_MANIFEST_MAGIC = 3246.0+HADAMARD = ("d1", "d2", "b2", "d3", "d4", "w1a", "w1b", "w2a", "w2b", "w3a", "w3b", "cond_v", "cond_u")+PROJ = ("q_proj", "k_proj", "v_proj", "gate_proj", "out_proj")+++def _audio_tensors(params, cfg, bits, group):+ from .whistle import mel_matrix+ ts = [_Tensor("audio.manifest", FP32, (13,), np.asarray([+ AUDIO_MANIFEST_MAGIC, 1, cfg.text, cfg.enc_layers, cfg.enc_lanes, cfg.conv_kernel, cfg.stem,+ cfg.stem_stages, cfg.mels, cfg.rate, cfg.fft, cfg.win, cfg.hop], np.float32).tobytes())]++ def proj(prefix, p, i):+ return [_q(f"{prefix}.{n}", np.asarray(p[n]["kernel"][i]).T, bits, group) for n in PROJ]++ b = params["stack"]["layers"]["block"]+ ca = b["cross_attn"]+ for i in range(cfg.dec_layers):+ ts.append(_fp16(f"layer{i:02d}.cross_norm", b["cross_norm"]["scale"][i]))+ ts += proj(f"layer{i:02d}.cross", ca, i)+ ts += [+ _fp16(f"layer{i:02d}.cross_q_norm", ca["q_norm"]["scale"][i]),+ _fp16(f"layer{i:02d}.cross_k_norm", ca["k_norm"]["scale"][i]),+ _fp16(f"layer{i:02d}.post_cross_norm", b["post_cross_norm"]["scale"][i]),+ _fp16(f"layer{i:02d}.cross_gate", np.asarray(b["cross_gate"][i]).reshape(1)),+ ]+ ts.append(_fp16("pe_gate", np.asarray(params["pe_gate"]).reshape(1)))++ e = params["encoder"]["layers"]["block"]+ for i in range(cfg.enc_layers):+ ts += [_fp16(f"encoder{i:02d}.{n}", e[n]["scale"][i]) for n in (+ "pre_hada_norm_0", "ZCRMSNorm_0", "post_attn_norm", "conv_norm", "conv_out_norm", "pre_hada_norm")]+ ts += proj(f"encoder{i:02d}", e["self_attn"], i)+ ts += [+ _fp16(f"encoder{i:02d}.q_norm", e["self_attn"]["q_norm"]["scale"][i]),+ _fp16(f"encoder{i:02d}.k_norm", e["self_attn"]["k_norm"]["scale"][i]),+ _fp16(f"encoder{i:02d}.attn_gate", np.asarray(e["attn_gate"][i]).reshape(1)),+ _q(f"encoder{i:02d}.pw1", np.asarray(e["pw1"]["kernel"][i]).T, bits, group),+ _q(f"encoder{i:02d}.pw2", np.asarray(e["pw2"]["kernel"][i]).T, bits, group),+ _fp16(f"encoder{i:02d}.dw", np.asarray(e["dw"][i]).reshape(cfg.conv_kernel, -1)),+ ]+ ts += [_fp16(f"encoder{i:02d}.{mlp}.{k}", e[mlp][k][i])+ for mlp in ("hadamard_mlp_0", "hadamard_mlp") for k in HADAMARD]++ enc = params["encoder"]+ for name in ("mhc_a_pre", "mhc_a_post", "mhc_a_res", "mhc_b_pre", "mhc_b_post", "mhc_b_res"):+ ts.append(_fp16(f"encoder.{name}", np.asarray(enc[name])))+ for name in ("mhc_phi_pre", "mhc_phi_post", "mhc_phi_res"):+ phi = np.asarray(enc[name])+ L, nC, lanes = phi.shape+ ts.append(_q(f"encoder.{name}", phi.transpose(0, 2, 1).reshape(L * lanes, nC), bits, group))+ ts.append(_fp16("encoder.final_norm", enc["final_norm"]["scale"]))++ stem = params["stem"]+ ts.append(_fp16("stem.w", np.asarray(stem["w"]).reshape(9, -1)))+ for i in range(1, cfg.stem_stages):+ ts += [_fp16(f"stem.dw_{i}", np.asarray(stem[f"dw_{i}"]).reshape(9, -1)),+ _q(f"stem.pw_{i}", np.asarray(stem[f"pw_{i}"]["kernel"]).T, bits, group)]+ ts.append(_q("stem.out", np.asarray(stem["out"]["kernel"]).T, bits, group))+ mel = mel_matrix(cfg)+ ts.append(_Tensor("mel", FP32, mel.shape, mel.tobytes()))return ts@@ -362,6 +427,22 @@ def _tokenizer_blob(tok):return bytes(out)+_PROTO_TYPES = {1: TK_NORMAL, 2: TK_UNKNOWN, 3: TK_CONTROL, 4: TK_USER_DEFINED, 6: TK_BYTE}+++def whistle_tokenizer_blob(proto, languages):+ spec = proto.trainer_spec+ out = bytearray(struct.pack(_TK_HDR, len(proto.pieces) + len(languages), spec.pad_id, spec.eos_id, spec.bos_id,+ spec.unk_id, int(proto.normalizer_spec.add_dummy_prefix), int(spec.byte_fallback), 0))+ for piece in proto.pieces:+ b = piece.piece.encode("utf-8")+ out += struct.pack(_TK_REC, piece.score, _PROTO_TYPES[piece.type], len(b)) + b+ for language in languages:+ b = f"<|{language}|>".encode("utf-8")+ out += struct.pack(_TK_REC, 0.0, TK_USER_DEFINED, len(b)) + b+ return bytes(out)++def _tokenizer_pieces(blob):return struct.unpack_from(_TK_HDR, blob, 0)[0]@@ -389,9 +470,9 @@ def read_tokenizer_blob(path):raise ValueError(f"{path} carries no tokenizer")-def _pack_cact(params, config, bits, group, tokenizer, kv_window=0):+def _pack_cact(params, config, bits, group, tokenizer, kv_window=0, audio=None):params = {k: v for k, v in params.items()}- ts = _tensors(params, config, bits, group)+ ts = _tensors(params, config, bits, group, audio)vocab = int(config.vocab_size)out_vocab = int(getattr(config, "out_vocab", 0) or 0)if out_vocab > vocab:@@ -447,8 +528,8 @@ def _pack_cact(params, config, bits, group, tokenizer, kv_window=0):def write_export(params, config, path, bits=WEIGHT_BITS, group=128, tokenizer=None,- kv_window=0):- buf, n = _pack_cact(params, config, bits, group, tokenizer, kv_window)+ kv_window=0, audio=None):+ buf, n = _pack_cact(params, config, bits, group, tokenizer, kv_window, audio)with open(path, "wb") as f:f.write(buf)return {"path": path, "bytes": len(buf), "tensors": n}diff --git a/needle/model/finetune.py b/needle/model/finetune.pyindex 7063fcb..a5d49f5 100644--- a/needle/model/finetune.py+++ b/needle/model/finetune.py@@ -309,7 +309,6 @@ def lora_target_paths(params):flat = flatten_dict(params)paths = [path for path in flatif path[-1] == "kernel"- and "stack" in pathand "layers" in pathand any(t in path for t in LORA_TARGETS)]paths = [p for p in paths if jnp.max(jnp.abs(flat[p])) > 1e-6]@@ -538,6 +537,8 @@ def build_main(args):print(f" {'wrote':<9} {out} {os.path.getsize(out) / 1e6:.2f} MB the published base archive")else:adapter = read_adapter(args.lora) if args.lora else None+ if adapter and adapter.get("languages") is not None:+ raise SystemExit(f"{args.lora} is a Whistle adapter; build it with needle whistle build")checkpoint = args.checkpointif not checkpoint and adapter and adapter.get("base") and os.path.exists(adapter["base"]):checkpoint = adapter["base"]@@ -566,12 +567,16 @@ def build_main(args):print(f" {'next':<9} needle.Needle(weights={out!r}, tools=[...])")if args.upload:- repo = os.environ.get("NEEDLE_HF_REPO")- if not repo:- raise SystemExit("set NEEDLE_HF_REPO=<you>/<model> to upload")- from huggingface_hub import HfApi- api = HfApi()- api.create_repo(repo, repo_type="model", exist_ok=True)- api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),- repo_id=repo, repo_type="model")- print(f" {'uploaded':<9} {os.path.basename(out)} {repo}")+ upload_archive(out)+++def upload_archive(out):+ repo = os.environ.get("NEEDLE_HF_REPO")+ if not repo:+ raise SystemExit("set NEEDLE_HF_REPO=<you>/<model> to upload")+ from huggingface_hub import HfApi+ api = HfApi()+ api.create_repo(repo, repo_type="model", exist_ok=True)+ api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),+ repo_id=repo, repo_type="model")+ print(f" {'uploaded':<9} {os.path.basename(out)} {repo}")diff --git a/needle/model/whistle.py b/needle/model/whistle.pynew file mode 100644index 0000000..94a0aa1--- /dev/null+++ b/needle/model/whistle.py@@ -0,0 +1,375 @@+import math+from dataclasses import dataclass, fields+from functools import partial++import numpy as np+import jax+import jax.numpy as jnp+import jax.nn.initializers as jinit+import flax.linen as nn++from . import quantize as _quantize+from .architecture import (+ DTYPE_MAP, ENGRAM_CONV_TAPS, Engram, HadamardMLP, MultiHeadAttention, TransformerConfig, ZCRMSNorm,+ _aq, _mask_diag, _res_identity_init, _rms_unit, _sinkhorn, default_init, engram_geometry,+ engram_indices, make_causal_mask, precompute_rope_freqs, residual_init,+)++LANGUAGES = ("en", "de", "fr", "es", "it", "nl", "pl")+++@dataclass(frozen=True)+class WhistleConfig:+ text: int = 8192+ languages: tuple = LANGUAGES+ dim: int = 512+ kv_heads: int = 2+ qk_dim: int = 48+ v_dim: int = 64+ enc_layers: int = 8+ dec_layers: int = 8+ conv_kernel: int = 9+ taps: int = 3+ lanes: int = 4+ enc_lanes: int = 4+ sites: tuple = (3, 7)+ orders: tuple = (2, 3)+ slots: int = 18432+ hashes: int = 2+ stem: int = 128+ stem_stages: int = 3+ rope: float = 100000.0+ mels: int = 80+ rate: int = 16000+ fft: int = 512+ win: int = 400+ hop: int = 160+ max_tokens: int = 320+ pad: int = 0+ eos: int = 1+ bos: int = 2+ dtype: str = "float32"+ flash: bool = True+ remat: bool = True+ scan_unroll: int = 1++ @classmethod+ def from_saved(cls, saved):+ known = {f.name for f in fields(cls)} - {"dtype", "flash", "remat", "scan_unroll"}+ return cls(**{k: tuple(v) if isinstance(v, list) else v for k, v in saved.items() if k in known})++ @property+ def vocab(self):+ return self.text + len(self.languages)++ @property+ def heads(self):+ return self.dim // self.v_dim++ @property+ def jax_dtype(self):+ return DTYPE_MAP[self.dtype]++ def tag(self, language):+ return self.text + self.languages.index(language)++ def needle(self):+ return TransformerConfig(+ vocab_size=self.vocab, d_model=self.dim, num_heads=self.heads, num_kv_heads=self.kv_heads,+ num_layers=self.dec_layers, qk_head_dim=self.qk_dim, v_head_dim=self.v_dim, rope_theta=self.rope,+ max_seq_len=self.max_tokens, engram_orders=self.orders, engram_heads=self.hashes,+ engram_slots=self.slots, engram_layers=self.sites, global_layers=(), sliding_window=0,+ mhc_lanes=self.lanes, qkv_conv_taps=self.taps)+++def mel_matrix(cfg):+ def to_mel(f):+ return np.where(f < 1000.0, f * 3.0 / 200.0, 15.0 + np.log(np.maximum(f, 1e-9) / 1000.0) * 27.0 / np.log(6.4))++ def to_hz(m):+ return np.where(m < 15.0, m * 200.0 / 3.0, 1000.0 * np.exp((m - 15.0) * np.log(6.4) / 27.0))++ points = to_hz(np.linspace(0.0, to_mel(np.array(cfg.rate / 2.0)), cfg.mels + 2))+ freqs = np.linspace(0.0, cfg.rate / 2, cfg.fft // 2 + 1)[:, None]+ lo, mid, hi = points[:-2], points[1:-1], points[2:]+ bank = np.maximum(0.0, np.minimum((freqs - lo) / (mid - lo), (hi - freqs) / (hi - mid)))+ return (bank * 2.0 / (hi - lo)).astype(np.float32)+++def features(cfg, wav, lengths):+ wav = wav.astype(jnp.float32) * (jnp.arange(wav.shape[1]) < lengths[:, None])+ window = np.hanning(cfg.win).astype(np.float32)+ _, _, z = jax.scipy.signal.stft(+ wav, window=window, nperseg=cfg.win, noverlap=cfg.win - cfg.hop, nfft=cfg.fft, boundary="zeros", padded=False)+ power = jnp.swapaxes(jnp.abs(z * window.sum()) ** 2, 1, 2)+ x = jnp.log(power @ mel_matrix(cfg) + 2.0 ** -24)+ lengths = lengths // cfg.hop+ mask = (jnp.arange(x.shape[1]) < lengths[:, None])[..., None]+ n = jnp.maximum(lengths, 1)[:, None, None]+ mean = (x * mask).sum(1, keepdims=True) / n+ std = jnp.sqrt((((x - mean) * mask) ** 2).sum(1, keepdims=True) / jnp.maximum(n - 1, 1))+ return (x - mean) / (std + 1e-5) * mask, lengths+++def sinusoids(cfg, frames):+ ang = jnp.arange(frames)[:, None] * jnp.exp(-np.log(10000.0) / (cfg.dim // 2 - 1) * jnp.arange(cfg.dim // 2))+ return jnp.concatenate([jnp.sin(ang), jnp.cos(ang)], -1)+++def _cross_attend(q, k, v, mem_mask):+ s = jnp.einsum("bshd,bthd->bhst", q.astype(jnp.float32), k.astype(jnp.float32)) / math.sqrt(q.shape[-1])+ s = jnp.where(mem_mask[:, None, None, :], s, jnp.finfo(jnp.float32).min)+ return jnp.einsum("bhst,bthd->bshd", jax.nn.softmax(s, -1), v.astype(jnp.float32)).astype(v.dtype)+++class Stem(nn.Module):+ cfg: WhistleConfig+ dtype: jnp.dtype = jnp.float32++ @nn.compact+ def __call__(self, x, lengths, quant):+ c = self.cfg++ def down(x, w, lengths):+ x = jax.lax.conv_general_dilated(+ x, w.astype(x.dtype), (2, 2), ((1, 1), (1, 1)),+ dimension_numbers=("NHWC", "HWIO", "NHWC"), feature_group_count=x.shape[-1])+ lengths = (lengths + 1) // 2+ return x * (jnp.arange(x.shape[1]) < lengths[:, None])[..., None, None], lengths++ x, lengths = down(x[..., None], self.param("w", default_init(), (3, 3, 1, c.stem)), lengths)+ for i in range(1, c.stem_stages):+ x, lengths = down(jax.nn.silu(x), self.param(f"dw_{i}", default_init(), (3, 3, 1, c.stem)), lengths)+ x = nn.Dense(c.stem, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name=f"pw_{i}")(x)+ x = jnp.swapaxes(jax.nn.silu(x), 2, 3)+ x = _aq(x.reshape(*x.shape[:2], -1), quant)+ return nn.Dense(c.dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="out")(x), lengths+++class EncoderBlock(nn.Module):+ cfg: WhistleConfig+ dtype: jnp.dtype = jnp.float32++ @nn.compact+ def __call__(self, x, mask, valid, rope, quant):+ c, dt = self.cfg, self.dtype+ x = x + 0.5 * HadamardMLP(c.dim, dt, name="hadamard_mlp_0")(ZCRMSNorm(dtype=dt, name="pre_hada_norm_0")(x))++ skip = x+ h = ZCRMSNorm(dtype=dt, name="ZCRMSNorm_0")(x)+ h = MultiHeadAttention(c.heads, c.kv_heads, c.dim, c.enc_layers, dt, c.flash, c.qk_dim, c.v_dim, 0,+ name="self_attn")(h, mask=mask, rope=rope, quant=quant)+ h = ZCRMSNorm(dtype=dt, name="post_attn_norm")(h)+ x = skip + nn.sigmoid(self.param("attn_gate", jinit.zeros, ())).astype(dt) * h++ h = ZCRMSNorm(dtype=dt, name="conv_norm")(x)+ h = nn.Dense(2 * c.dim, dtype=dt, use_bias=False, kernel_init=default_init(), name="pw1")(_aq(h, quant))+ h = jax.nn.glu(h) * valid+ dw = self.param("dw", default_init(), (c.conv_kernel, 1, c.dim)).astype(h.dtype)+ h = jax.lax.conv_general_dilated(h, dw, (1,), "SAME", dimension_numbers=("NWC", "WIO", "NWC"),+ feature_group_count=c.dim)+ h = jax.nn.silu(ZCRMSNorm(dtype=dt, name="conv_out_norm")(h))+ pw2 = nn.Dense(c.dim, dtype=dt, use_bias=False, kernel_init=residual_init(c.enc_layers), name="pw2")+ x = x + pw2(_aq(h, quant))++ return x + 0.5 * HadamardMLP(c.dim, dt, name="hadamard_mlp")(ZCRMSNorm(dtype=dt, name="pre_hada_norm")(x))+++class CrossAttention(nn.Module):+ cfg: WhistleConfig+ dtype: jnp.dtype = jnp.float32++ @nn.compact+ def __call__(self, x, mem, mem_mask, quant=False):+ c = self.cfg+ heads, qk_hd, v_hd = c.heads, c.qk_dim, c.v_dim+ B = x.shape[0]++ x = _aq(x, quant)+ dense = partial(nn.Dense, dtype=self.dtype, use_bias=False, kernel_init=default_init())+ q = dense(heads * qk_hd, name="q_proj")(x).reshape(B, -1, heads, qk_hd)+ k = dense(heads * qk_hd, name="k_proj")(mem).reshape(B, -1, heads, qk_hd)+ v = dense(heads * v_hd, name="v_proj")(mem).reshape(B, -1, heads, v_hd)+ q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q.transpose(0, 2, 1, 3))+ k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k.transpose(0, 2, 1, 3))+ v = v.transpose(0, 2, 1, 3)++ q = _quantize.maybe_quant_query(q, quant)+ k = _quantize.maybe_quant_kv(k, quant)+ v = _quantize.maybe_quant_kv(v, quant)++ if c.flash:+ fused = max(qk_hd, v_hd)+ qf = q * jnp.asarray(math.sqrt(fused / qk_hd), q.dtype)+ if fused > qk_hd:+ qf = jnp.pad(qf, ((0, 0), (0, 0), (0, 0), (0, fused - qk_hd)))+ k = jnp.pad(k, ((0, 0), (0, 0), (0, 0), (0, fused - qk_hd)))+ vf = jnp.pad(v, ((0, 0), (0, 0), (0, 0), (0, fused - v_hd))) if fused > v_hd else v+ mask = jnp.broadcast_to(mem_mask[:, None, None, :], (B, 1, qf.shape[2], k.shape[2]))+ out = jax.nn.dot_product_attention(qf.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3),+ vf.transpose(0, 2, 1, 3), mask=mask)+ out = out[..., :v_hd].reshape(B, -1, heads * v_hd)+ else:+ out = _cross_attend(q.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3), v.transpose(0, 2, 1, 3), mem_mask)+ out = out.reshape(B, -1, heads * v_hd)++ out = out * nn.sigmoid(dense(heads * v_hd, name="gate_proj")(x))+ out = _aq(out, quant)+ return nn.Dense(c.dim, dtype=self.dtype, use_bias=False, kernel_init=residual_init(c.dec_layers),+ name="out_proj")(out)+++class DecoderBlock(nn.Module):+ cfg: WhistleConfig+ dtype: jnp.dtype = jnp.float32++ def _gate(self, name):+ return nn.sigmoid(self.param(name, jinit.zeros, ())).astype(self.dtype)++ @nn.compact+ def __call__(self, x, mask=None, rope=None, quant=False, engram_kv=None, site_flags=None, mem=None, mem_mask=None):+ c, dt = self.cfg, self.dtype+ if engram_kv is not None:+ ek, ev = engram_kv+ alpha = nn.sigmoid(jnp.einsum("btd,sbtd->sbt", _rms_unit(x), _rms_unit(ek)) / math.sqrt(c.dim))+ x = x + jnp.einsum("s,sbt,sbtd->btd", site_flags.astype(jnp.float32), alpha,+ ev.astype(jnp.float32)).astype(x.dtype)++ skip = x+ x = ZCRMSNorm(dtype=dt)(x)+ x = MultiHeadAttention(c.heads, c.kv_heads, c.dim, c.dec_layers, dt, c.flash, qk_head_dim=c.qk_dim,+ v_head_dim=c.v_dim, qkv_conv_taps=c.taps,+ name="self_attn")(x, mask=mask, rope=rope, quant=quant)+ x = ZCRMSNorm(dtype=dt, name="post_attn_norm")(x)+ x = skip + self._gate("attn_gate") * x++ skip = x+ x = ZCRMSNorm(dtype=dt, name="cross_norm")(x)+ x = CrossAttention(c, dt, name="cross_attn")(x, mem, mem_mask, quant)+ x = ZCRMSNorm(dtype=dt, name="post_cross_norm")(x)+ x = skip + self._gate("cross_gate") * x++ skip = x+ x = ZCRMSNorm(dtype=dt, name="pre_hada_norm")(x)+ return skip + HadamardMLP(c.dim, dt, name="hadamard_mlp")(x)+++class _LaneBody(nn.Module):+ make: object+ dtype: jnp.dtype = jnp.float32++ @nn.compact+ def __call__(self, x, xs, quant, block_kwargs):+ site_flags, hc = xs+ block = self.make(name="block")+ extra = {} if site_flags is None else {"site_flags": site_flags}+ B, T, n, C = x.shape+ xf = x.astype(jnp.float32)+ nx = _aq(_rms_unit(x.reshape(B, T, n * C)), quant)+ hpre = nn.sigmoid(hc["a_pre"] * (nx @ hc["phi_pre"].astype(jnp.float32)) + hc["b_pre"] + hc["pre_off"])+ u = jnp.einsum("btn,btnc->btc", hpre, xf).astype(self.dtype)+ y = block(u, quant=quant, **block_kwargs, **extra) - u+ hpost = 2 * nn.sigmoid(hc["a_post"] * (nx @ hc["phi_post"].astype(jnp.float32)) + hc["b_post"] + hc["post_off"])+ hres = _sinkhorn(hc["a_res"] * (nx @ hc["phi_res"].astype(jnp.float32)).reshape(B, T, n, n) + hc["b_res"])+ out = jnp.einsum("btij,btjc->btic", hres, xf) + hpost[..., None] * y.astype(jnp.float32)[:, :, None, :]+ return out.astype(self.dtype), None+++class LaneStack(nn.Module):+ num_layers: int+ lanes: int+ d_model: int+ make: object+ engram_layers: tuple = ()+ dtype: jnp.dtype = jnp.float32+ remat: bool = True+ unroll: int = 1++ @nn.compact+ def __call__(self, x, quant=False, **block_kwargs):+ x = x.astype(self.dtype)+ site_flags = None+ if block_kwargs.get("engram_kv") is not None:+ flags = np.zeros((self.num_layers, len(self.engram_layers)), np.float32)+ for s, layer in enumerate(self.engram_layers):+ flags[layer, s] = 1.0+ site_flags = jnp.asarray(flags)++ n, L, nC = self.lanes, self.num_layers, self.lanes * self.d_model+ lane = np.eye(n, dtype=np.float32)[np.arange(L) % n]+ hc = {+ "phi_pre": self.param("mhc_phi_pre", default_init(), (L, nC, n)),+ "phi_post": self.param("mhc_phi_post", default_init(), (L, nC, n)),+ "phi_res": self.param("mhc_phi_res", default_init(), (L, nC, n * n)),+ "b_pre": self.param("mhc_b_pre", jinit.zeros, (L, n)),+ "b_post": self.param("mhc_b_post", jinit.zeros, (L, n)),+ "b_res": self.param("mhc_b_res", _res_identity_init, (L, n, n)),+ "a_pre": self.param("mhc_a_pre", jinit.constant(0.01), (L,)),+ "a_post": self.param("mhc_a_post", jinit.constant(0.01), (L,)),+ "a_res": self.param("mhc_a_res", jinit.constant(0.01), (L,)),+ "pre_off": jnp.asarray(8 * lane - 4),+ "post_off": jnp.asarray(-4 * (1 - lane)),+ }+ x = jnp.broadcast_to(x[:, :, None, :], (*x.shape[:2], n, x.shape[-1]))+ scan = nn.scan(nn.remat(_LaneBody) if self.remat else _LaneBody, variable_axes={"params": 0},+ split_rngs={"params": True}, length=L, in_axes=(0, nn.broadcast, nn.broadcast),+ unroll=self.unroll)+ x, _ = scan(self.make, self.dtype, name="layers")(x, (site_flags, hc), quant, block_kwargs)+ return ZCRMSNorm(dtype=self.dtype, name="final_norm")(jnp.mean(x, axis=2))+++class Whistle(nn.Module):+ cfg: WhistleConfig++ def setup(self):+ c, dt = self.cfg, self.cfg.jax_dtype+ self.stem = Stem(c, dt)+ self.encoder = LaneStack(c.enc_layers, c.enc_lanes, c.dim, partial(EncoderBlock, c, dt), dtype=dt,+ remat=c.remat, unroll=c.scan_unroll)+ self.embedding = nn.Embed(c.vocab, c.dim, embedding_init=jinit.normal(stddev=0.02))+ self.stack = LaneStack(c.dec_layers, c.lanes, c.dim, partial(DecoderBlock, c, dt), c.sites, dtype=dt,+ remat=c.remat, unroll=c.scan_unroll)+ orders, heads, sub_dim = engram_geometry(c.needle())+ self.engrams = [Engram(c.dim, len(orders) * heads, c.slots, sub_dim, c.dec_layers, max(orders), dt)+ for _ in c.sites]+ self.pe_gate = self.param("pe_gate", jinit.zeros, ())++ def encode(self, wav, lengths, quant=False):+ c = self.cfg+ x, lengths = features(c, wav, lengths)+ x, lengths = self.stem(x.astype(c.jax_dtype), lengths, quant)+ x = jnp.pad(x, ((0, 0), (0, x.shape[1] % 2), (0, 0)))+ seen = jnp.arange(x.shape[1]) < lengths[:, None]+ mask = jnp.broadcast_to(seen[:, None, None, :], (len(seen), 1, seen.shape[1], seen.shape[1]))+ rope = precompute_rope_freqs(c.qk_dim, x.shape[1], c.rope)+ return self.encoder(x, quant=quant, mask=mask, valid=seen[..., None], rope=rope), lengths++ def decode(self, tokens, enc, lengths, quant=False):+ c = self.cfg+ pe = nn.sigmoid(self.pe_gate).astype(enc.dtype) * sinusoids(c, enc.shape[1]).astype(enc.dtype)+ mem, mem_mask = _aq(enc + pe, quant), jnp.arange(enc.shape[1]) < lengths[:, None]+ mask = make_causal_mask(tokens.shape[1])+ rope = precompute_rope_freqs(c.qk_dim, tokens.shape[1], c.rope)+ x = self.embedding(tokens) * math.sqrt(c.dim)+ orders, heads, _ = engram_geometry(c.needle())+ indices = engram_indices(tokens, orders, heads, c.slots, 0)+ ngram_ok = jnp.stack([_mask_diag(mask, o - 1) for o in orders for _ in range(heads)], axis=-1)+ tap_ok = jnp.stack([_mask_diag(mask, j * max(orders)) for j in range(ENGRAM_CONV_TAPS)])+ pairs = [e(indices, ngram_ok, tap_ok, quant=quant) for e in self.engrams]+ kv = (jnp.stack([k for k, _ in pairs]), jnp.stack([v for _, v in pairs])) if pairs else None+ return self.stack(x, quant=quant, mask=mask, rope=rope, engram_kv=kv, mem=mem, mem_mask=mem_mask)++ def logits(self, tokens, enc, frames, quant=False):+ x = self.decode(tokens, enc, frames, quant)+ return _aq(x, quant).astype(jnp.float32) @ self.embedding.embedding.T.astype(jnp.float32)++ def __call__(self, wav, lengths, tokens, quant=False):+ enc, frames = self.encode(wav, lengths, quant)+ return self.logits(tokens, enc, frames, quant)+++def load_whistle(path):+ from .checkpoints import read_checkpoint++ ckpt = read_checkpoint(path)+ return ckpt["params"], WhistleConfig.from_saved(ckpt["config"])diff --git a/needle/model/whistle_finetune.py b/needle/model/whistle_finetune.pynew file mode 100644index 0000000..e154110--- /dev/null+++ b/needle/model/whistle_finetune.py@@ -0,0 +1,448 @@+import json+import os+import re+import unicodedata+from dataclasses import replace++import numpy as np++from .checkpoints import read_adapter, write_adapter+from .finetune import _training_rng, init_lora, lora_target_paths, merge_lora+from .whistle import LANGUAGES, Whistle, load_whistle++DEFAULT_BASE = "checkpoints/whistle.safetensors"+SAMPLE_RATE = 16000+SECONDS = 30+SPACE = "▁"+ROWS_LR = 10.0+++class WhistleTokenizer:+ def __init__(self, pieces=(), languages=LANGUAGES):+ import sentencepiece as spm+ from sentencepiece import sentencepiece_model_pb2+ from .tokenizer import get_tokenizer++ self.base = get_tokenizer().sp+ proto = sentencepiece_model_pb2.ModelProto()+ proto.ParseFromString(self.base.serialized_model_proto())+ low = min(p.score for p in proto.pieces)+ for k, piece in enumerate(pieces):+ proto.pieces.add(piece=piece, score=low - 1 - k)+ self.proto, self.pieces, self.languages = proto, tuple(pieces), tuple(languages)+ self.sp = spm.SentencePieceProcessor(model_proto=proto.SerializeToString())++ @property+ def text(self):+ return len(self.proto.pieces)++ def encode(self, text):+ return self.sp.encode(text)++ def blob(self):+ from .export import whistle_tokenizer_blob+ return whistle_tokenizer_blob(self.proto, self.languages)+++def _symbols(sp, text):+ out, raw = [], bytearray()+ for piece in sp.encode(text, out_type=str) + [""]:+ if piece.startswith("<0x") and piece.endswith(">") and len(piece) == 6:+ raw.append(int(piece[3:5], 16))+ continue+ if raw:+ out += list(raw.decode("utf-8", "ignore"))+ raw.clear()+ if piece:+ out.append(piece)+ return out+++def read_tokens(path):+ if not os.path.isfile(path):+ raise SystemExit(f"--tokens takes a file with one token per line; {path} is not a file")+ with open(path, encoding="utf-8-sig") as handle:+ return [line.strip() for line in handle if line.strip()]+++def token_pieces(sp, tokens):+ known = {sp.id_to_piece(i) for i in range(sp.get_piece_size())}+ pieces = []+ for token in tokens:+ for text in (" " + token, token):+ symbols = _symbols(sp, text)+ for end in range(1, len(symbols) + 1):+ piece = "".join(symbols[:end])+ if piece not in known:+ known.add(piece)+ pieces.append(piece)+ return pieces+++def read_audio(path):+ import wave+ try:+ with wave.open(path, "rb") as source:+ channels, width, rate = source.getnchannels(), source.getsampwidth(), source.getframerate()+ raw = source.readframes(source.getnframes())+ if width != 2:+ from ..agent.whistle import _read_wav+ return np.asarray(_read_wav(path), np.float32)+ samples = np.frombuffer(raw, "<i2").reshape(-1, channels).mean(1).astype(np.float32) / 32768.0+ except (wave.Error, EOFError):+ try:+ import soundfile+ except ImportError:+ raise RuntimeError(f"{path} is not a PCM WAV file; reading it needs soundfile: "+ "pip install soundfile") from None+ data, rate = soundfile.read(path, dtype="float32", always_2d=True)+ samples = data.mean(1)+ if rate != SAMPLE_RATE:+ try:+ import soxr+ except ImportError:+ raise RuntimeError(f"{path} is {rate} Hz; resampling it to {SAMPLE_RATE} Hz needs soxr: "+ "pip install cactus-needle[mic]") from None+ samples = soxr.resample(samples, rate, SAMPLE_RATE, quality="HQ")+ return samples.astype(np.float32)+++def read_clips(path, language=None):+ root = os.path.dirname(os.path.abspath(path))+ clips, skipped = [], 0+ with open(path, encoding="utf-8") as handle:+ for number, line in enumerate(handle, 1):+ if not line.strip():+ continue+ try:+ row = json.loads(line)+ except json.JSONDecodeError as error:+ raise SystemExit(f"{path}:{number}: not JSON ({error})") from None+ lang = row.get("language") or language+ if not lang or not re.fullmatch(r"[A-Za-z0-9_-]+", lang):+ raise SystemExit(f"{path}:{number}: give the line a \"language\" code such as \"pt\", "+ "or pass --language")+ if not row.get("audio"):+ raise SystemExit(f"{path}:{number}: no \"audio\" path")+ audio = os.path.join(root, row["audio"])+ if not os.path.isfile(audio):+ raise SystemExit(f"{path}:{number}: {audio} does not exist")+ samples, text = read_audio(audio), (row.get("text") or "").strip()+ if not text or not len(samples) or len(samples) > SECONDS * SAMPLE_RATE:+ skipped += 1+ continue+ clips.append((samples, text, lang))+ return clips, skipped+++def load_base(path):+ if not os.path.exists(path):+ from ..agent import fetch+ print(f" {'fetch':<9} {path} downloading from Hugging Face", flush=True)+ path = fetch.fetch_checkpoint(os.path.basename(path), os.path.dirname(path) or fetch.CHECKPOINT_PREFIX,+ generation=fetch.WHISTLE)+ params, cfg = load_whistle(path)+ if "encoder" not in params or "stem" not in params:+ raise SystemExit(f"{path} is not a Whistle checkpoint")+ return params, cfg+++def initial_rows(embedding, cfg, tokenizer):+ embedding = np.asarray(embedding)+ rows = [embedding[tokenizer.base.encode(piece.replace(SPACE, " "))].mean(0) for piece in tokenizer.pieces]+ tag = embedding[cfg.text:].mean(0)+ rows += [tag] * (len(tokenizer.languages) - len(cfg.languages))+ return np.asarray(rows, np.float32).reshape(-1, embedding.shape[1])+++def tuned_params(params, cfg, tokenizer, lora, scale, rows):+ import jax.numpy as jnp+ params = merge_lora(params, lora, scale) if lora else dict(params)+ embedding = jnp.asarray(params["embedding"]["embedding"])+ if rows is not None and len(rows):+ rows, pieces = jnp.asarray(rows), len(tokenizer.pieces)+ embedding = jnp.concatenate([embedding[:cfg.text], rows[:pieces], embedding[cfg.text:], rows[pieces:]])+ return {**params, "embedding": {"embedding": embedding}}+++def _arrays(items, idx, cfg):+ wav = np.zeros((len(idx), SECONDS * SAMPLE_RATE), np.float32)+ lengths = np.zeros(len(idx), np.int32)+ tokens = np.full((len(idx), cfg.max_tokens + 1), cfg.pad, np.int32)+ for row, i in enumerate(idx):+ samples, ids = items[i][:2]+ wav[row, :len(samples)], lengths[row], tokens[row, :len(ids)] = samples, len(samples), ids+ return wav, lengths, tokens+++def _normalizer(language):+ from whisper_normalizer.basic import BasicTextNormalizer+ from whisper_normalizer.english import EnglishTextNormalizer+ return EnglishTextNormalizer() if language == "en" else BasicTextNormalizer()+++def by_character(text):+ return any(unicodedata.name(c, "").startswith(("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "THAI")) for c in text)+++def errors(reference, hypothesis, language):+ normalize = _normalizer(language)+ ref, hyp = normalize(reference), normalize(hypothesis)+ if by_character(reference):+ ref, hyp = list(ref.replace(" ", "")), list(hyp.replace(" ", ""))+ else:+ ref, hyp = ref.split(), hyp.split()+ row = list(range(len(hyp) + 1))+ for i, r in enumerate(ref, 1):+ prev, row[0] = row[0], i+ for j, h in enumerate(hyp, 1):+ prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (r != h))+ return row[-1], len(ref)+++def _score_quantised(model, params, cfg, tokenizer, lora, scale, rows, held, batch):+ import jax+ from .quantize import cq_ste_params, WEIGHT_BITS++ merged = cq_ste_params(tuned_params(params, cfg, tokenizer, lora, scale, rows), WEIGHT_BITS)+ tuned = model.cfg+ encode = jax.jit(lambda p, wav, lengths: model.apply({"params": p}, wav, lengths, True, method=Whistle.encode))+ step = jax.jit(lambda p, tokens, enc, frames: model.apply({"params": p}, tokens, enc, frames, True,+ method=Whistle.logits))+ tags = np.arange(tuned.vocab) >= tuned.text+ edits = units = 0+ for start in range(0, len(held), batch):+ part = held[start:start + batch]+ enc, frames = encode(merged, *_arrays(part, range(len(part)), tuned)[:2])+ tokens = np.full((len(part), tuned.max_tokens), tuned.pad, np.int32)+ tokens[:, 0] = tuned.bos+ done = np.zeros(len(part), bool)+ for pos in range(tuned.max_tokens - 1):+ size = min(tuned.max_tokens, (pos // 64 + 1) * 64)+ logits = np.array(step(merged, tokens[:, :size], enc, frames)[:, pos])+ logits[:, tags != (pos == 0)] = -np.inf+ pick = logits.argmax(-1)+ tokens[:, pos + 1] = np.where(done, tuned.pad, pick)+ done |= pick == tuned.eos+ if done.all():+ break+ for (_, wanted, text), ids in zip(part, tokens.tolist()):+ ids = ids[2:ids.index(tuned.eos)] if tuned.eos in ids else ids[2:]+ language = tuned.languages[wanted[1] - tuned.text]+ e, n = errors(text, tokenizer.sp.decode([i for i in ids if i != tuned.pad]), language)+ edits, units = edits + e, units + n+ return edits, units+++def rung(params, cfg, layers):+ from .architecture import ladder_config, ladder_slice++ if not layers or layers == cfg.dec_layers:+ return params, cfg+ if not 2 <= layers < cfg.dec_layers:+ raise ValueError(f"--layers must be in [2, {cfg.dec_layers}], got {layers}")+ config = cfg.needle()+ sites = ladder_config(config, layers).engram_layers+ return ladder_slice(params, config, layers), replace(cfg, dec_layers=layers, sites=sites)+++def build(params, cfg, tokenizer, out):+ from .export import write_export+ from .quantize import WEIGHT_BITS+ cfg = replace(cfg, text=tokenizer.text, languages=tokenizer.languages)+ return write_export(params, cfg.needle(), out, bits=WEIGHT_BITS, tokenizer=tokenizer.blob(), audio=cfg)+++def finetune_main(args):+ import jax+ import jax.numpy as jnp+ import optax+ from .quantize import WEIGHT_BITS, configure_deploy, cq_ste_params++ if args.epochs < 1 or args.batch_size < 1:+ raise SystemExit("--epochs and --batch-size must be at least 1")+ base_path = args.checkpoint or DEFAULT_BASE+ params, cfg = load_base(base_path)+ params = jax.tree.map(lambda a: np.asarray(a).astype(np.float32), params)+ backend = jax.default_backend().lower()+ if backend == "metal":+ cfg = replace(cfg, flash=False, remat=False, scan_unroll=max(cfg.enc_layers, cfg.dec_layers))+ params = jax.device_put(params)+ print(f" {'backend':<9} {backend} float32")++ clips, skipped = read_clips(args.jsonl_path, args.language)+ if not clips:+ raise SystemExit("no usable clips in " + args.jsonl_path)+ languages = tuple(cfg.languages) + tuple(sorted({lang for *_, lang in clips} - set(cfg.languages)))+ tokens = read_tokens(args.tokens) if args.tokens else []+ pieces = token_pieces(WhistleTokenizer().sp, tokens)+ tokenizer = WhistleTokenizer(pieces, languages)+ tuned = replace(cfg, text=tokenizer.text, languages=languages)+ note = f" {skipped} skipped (empty, or over {SECONDS} s)" if skipped else ""+ print(f" {'data':<9} {len(clips)} clips {sum(len(c[0]) for c in clips) / SAMPLE_RATE / 60:.1f} min{note}")+ added = [lang for lang in languages if lang not in cfg.languages]+ print(f" {'vocab':<9} {len(tokens)} tokens {len(pieces)} new pieces"+ + (f" new languages {', '.join(added)}" if added else ""))+ split = [t for t in tokens if any(len(tokenizer.encode(x)) != 1 for x in (t, " " + t))]+ if split:+ print(f" {'warning':<9} not one piece: {', '.join(split)}")++ items = [(samples, [cfg.bos, tuned.tag(lang), *tokenizer.encode(text), cfg.eos], text)+ for samples, text, lang in clips]+ long = sum(len(ids) > cfg.max_tokens + 1 for _, ids, _ in items)+ items = [item for item in items if len(item[1]) <= cfg.max_tokens + 1]+ if long:+ print(f" {'skipped':<9} {long} clips with transcripts over {cfg.max_tokens} tokens")+ if not items:+ raise SystemExit("no clips left to train on")+ if tokens:+ found = sum(any(token in text for *_, text in items) for token in tokens)+ print(f" {'tokens':<9} {found}/{len(tokens)} appear in the transcripts")++ configure_deploy(act_bits=8, kv_bits=8)+ model = Whistle(tuned)+ print(f" {'numerics':<9} CQ W{WEIGHT_BITS} STE + A8 (matches export)")+ paths = lora_target_paths(params)+ scale = args.lora_alpha / args.lora_rank+ seed = int(getattr(args, "seed", 0))+ rng = _training_rng(seed)+ lora = init_lora(params, paths, args.lora_rank, jax.random.PRNGKey(seed))+ train = {"lora": lora, "rows": jnp.asarray(initial_rows(params["embedding"]["embedding"], cfg, tokenizer))}+ print(f" {'lora':<9} rank {args.lora_rank} alpha {args.lora_alpha:g} {len(paths)} weight groups"+ f" +{len(train['rows'])} vocabulary rows")++ n_val = min(int(len(items) * getattr(args, "val_split", 0.1)), len(items) - 1)+ held = []+ if n_val > 0:+ order = rng.permutation(len(items))+ held, items = [items[i] for i in order[:n_val]], [items[i] for i in order[n_val:]]+ print(f" {'holdout':<9} {n_val} clips for validation")++ def loss_fn(train, wav, lengths, tokens):+ merged = cq_ste_params(tuned_params(params, cfg, tokenizer, train["lora"], scale, train["rows"]), WEIGHT_BITS)+ logits = model.apply({"params": merged}, wav, lengths, tokens[:, :-1], quant=True)+ targets = tokens[:, 1:]+ mask = targets != cfg.pad+ ce = optax.softmax_cross_entropy_with_integer_labels(logits, targets)+ return (ce * mask).sum() / jnp.maximum(mask.sum(), 1.0)++ batch, count = args.batch_size, len(items)+ steps_per_epoch = -(-count // batch)+ total_steps = args.epochs * steps_per_epoch+ warmup = min(max(1, total_steps // 20), total_steps - 1)++ def adamw(lr):+ return optax.adamw(optax.warmup_cosine_decay_schedule(+ init_value=0.0, peak_value=lr, warmup_steps=warmup, decay_steps=total_steps))++ labels = {"lora": jax.tree.map(lambda _: "lora", train["lora"]), "rows": "rows"}+ optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.multi_transform(+ {"lora": adamw(args.lr), "rows": adamw(args.lr * ROWS_LR)}, labels))+ opt_state = optimizer.init(train)+ print(f" {'schedule':<9} {total_steps} steps warmup {warmup} cosine decay clip 1.0 (compiling...)")++ @jax.jit+ def train_step(train, opt_state, wav, lengths, tokens):+ loss, grads = jax.value_and_grad(loss_fn)(train, wav, lengths, tokens)+ updates, opt_state = optimizer.update(grads, opt_state, train)+ return optax.apply_updates(train, updates), opt_state, loss++ eval_step = jax.jit(loss_fn)++ every = max(1, total_steps // 50)+ step_i = 0+ for epoch in range(args.epochs):+ order = rng.permutation(count)+ last = 0.0+ for start in range(0, count, batch):+ train, opt_state, loss = train_step(train, opt_state, *_arrays(items, order[start:start + batch], cfg))+ last = float(loss)+ step_i += 1+ if step_i % every == 0:+ print(f" {'step':<9} {step_i}/{total_steps} loss {last:.4f}", flush=True)+ if held:+ val = np.mean([float(eval_step(train, *_arrays(held, range(i, min(i + batch, n_val)), cfg)))+ for i in range(0, n_val, batch)])+ print(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f} val {val:.4f}", flush=True)+ else:+ print(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f}", flush=True)++ if held and getattr(args, "score", True):+ edits, units = _score_quantised(model, params, cfg, tokenizer, train["lora"], scale, train["rows"],+ held, batch)+ kind = "cer" if all(by_character(text) for *_, text in held) else "wer"+ if units:+ print(f" {kind:<9} {edits / units:.1%} on {len(held)} held-out clips, scored on the "+ f"W{WEIGHT_BITS} weights the archive ships")++ os.makedirs(args.checkpoint_dir, exist_ok=True)+ out = args.out or os.path.join(args.checkpoint_dir, "whistle_lora.safetensors")+ write_adapter(out, {+ "lora": {"/".join(p): {"A": np.asarray(v["A"]), "B": np.asarray(v["B"])} for p, v in train["lora"].items()},+ "rows": np.asarray(train["rows"]),+ "scale": float(scale),+ "base": base_path,+ "rank": args.lora_rank,+ "seed": seed,+ "pieces": list(pieces),+ "languages": list(languages),+ })+ print(f" {'adapter':<9} {out}")+ print(f" {'next':<9} needle whistle build --lora {out}")+++def build_main(args):+ import shutil+ import jax.numpy as jnp+ from ..agent import fetch+ from .export import read_layers+ from .finetune import upload_archive++ base_archive = fetch.fetch_weights(fetch.WHISTLE, force=True)+ print(f" {'base':<9} {base_archive} {os.path.getsize(base_archive) / 1e6:.2f} MB")+ platform = getattr(args, "platform", None)+ layers = getattr(args, "layers", None)+ folder = None+ if platform:+ folder = os.path.abspath(args.out or platform)+ for path in fetch.download_platform(platform, os.path.dirname(folder), generation=3, dest=folder):+ print(f" {'engine':<9} {path} {os.path.getsize(path) / 1e6:.2f} MB")+ out = os.path.join(folder, fetch.base_weights(fetch.WHISTLE))+ else:+ out = args.out+ if not args.lora and (not layers or layers == read_layers(base_archive)):+ if not out:+ raise SystemExit("pass --out <archive.cact>, or --platform to build a runnable folder")+ shutil.copyfile(base_archive, out)+ print(f" {'wrote':<9} {out} {os.path.getsize(out) / 1e6:.2f} MB the published base archive")+ else:+ adapter = read_adapter(args.lora) if args.lora else None+ if adapter and adapter.get("languages") is None:+ raise SystemExit(f"{args.lora} is a Needle adapter; build it with needle build")+ checkpoint = args.checkpoint+ if not checkpoint and adapter and adapter.get("base") and os.path.exists(adapter["base"]):+ checkpoint = adapter["base"]+ checkpoint = checkpoint or DEFAULT_BASE+ params, cfg = load_base(checkpoint)+ tokenizer = WhistleTokenizer(languages=cfg.languages)+ if args.lora:+ tokenizer = WhistleTokenizer(adapter.get("pieces") or (), adapter.get("languages") or cfg.languages)+ lora = {tuple(key.split("/")): {"A": jnp.asarray(v["A"]), "B": jnp.asarray(v["B"])}+ for key, v in adapter["lora"].items()}+ params = tuned_params(params, cfg, tokenizer, lora, adapter["scale"], adapter.get("rows"))+ print(f" {'merged':<9} {len(lora)} weight groups {len(tokenizer.pieces)} new pieces {args.lora}")+ params, cfg = rung(params, cfg, layers)+ print(f" {'depth':<9} {cfg.dec_layers} decoder layers")+ out = out or (os.path.splitext(os.path.basename(checkpoint))[0] + ".cact")+ info = build(params, cfg, tokenizer, out)+ print(f" {'wrote':<9} {info['path']} {info['bytes'] / 1e6:.2f} MB {info['tensors']} tensors W4A8"+ f" languages {', '.join(tokenizer.languages)}")+ if folder:+ runner = next((n for n in ("needle", "needle.exe") if os.path.exists(os.path.join(folder, n))), None)+ if runner:+ print(f" {'next':<9} {os.path.join(folder, runner)} --model {os.path.basename(out)} --audio clip.wav")+ else:+ print(f" {'next':<9} needle.transcribe('clip.wav', weights={out!r})")++ if args.upload:+ upload_archive(out)diff --git a/pyproject.toml b/pyproject.tomlindex b62705c..ea259e7 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -17,7 +17,8 @@ train = ["flax>=0.10.2","optax","safetensors",- "sentencepiece"+ "sentencepiece",+ "whisper-normalizer"]gpu = ["jax[cuda12]"]metal = [diff --git a/requirements-train.txt b/requirements-train.txtindex 5035c8a..cb2e295 100644--- a/requirements-train.txt+++ b/requirements-train.txt@@ -8,3 +8,4 @@ flax>=0.10.2optaxsafetensorssentencepiece+whisper-normalizerdiff --git a/tests/test_whistle_finetune.py b/tests/test_whistle_finetune.pynew file mode 100644index 0000000..36656ea--- /dev/null+++ b/tests/test_whistle_finetune.py@@ -0,0 +1,244 @@+import json+import types+import wave++import pytest++TINY = dict(dim=64, kv_heads=2, qk_dim=16, v_dim=16, enc_layers=2, dec_layers=3, conv_kernel=3, lanes=2,+ enc_lanes=2, sites=(2,), slots=64, hashes=1, stem=8, stem_stages=2, max_tokens=64)++[email protected](scope="module")+def tiny_whistle(tmp_path_factory):+ import numpy as np+ import jax+ from needle.model.checkpoints import write_checkpoint+ from needle.model.whistle import Whistle, WhistleConfig++ cfg = WhistleConfig(**TINY, flash=False)+ params = jax.tree.map(np.asarray, Whistle(cfg).init(+ jax.random.PRNGKey(0), np.zeros((1, 3200), np.float32), np.array([3200]), np.zeros((1, 4), np.int32))["params"])+ path = tmp_path_factory.mktemp("whistle") / "tiny.safetensors"+ write_checkpoint(str(path), {"params": params, "config": cfg.__dict__})+ return str(path)+++def published_whistle(tiny_whistle, tmp_path, monkeypatch):+ from needle.agent import fetch+ from needle.model.whistle_finetune import WhistleTokenizer, build, load_base++ archive = str(tmp_path / "whistle.cact")+ params, cfg = load_base(tiny_whistle)+ build(params, cfg, WhistleTokenizer(languages=cfg.languages), archive)+ calls = []+ monkeypatch.setattr(fetch, "fetch_weights", lambda generation=2, dest_dir=None, force=False:+ calls.append((generation, force)) or archive)+ return archive, calls+++def _build_args(**kw):+ return types.SimpleNamespace(**{"checkpoint": None, "lora": None, "out": None, "upload": False, "layers": None,+ "platform": None, **kw})+++def _clips(folder, lines):+ import numpy as np+ rng = np.random.default_rng(0)+ with open(folder / "clips.jsonl", "w", encoding="utf-8") as out:+ for i, text in enumerate(lines):+ with wave.open(str(folder / f"{i}.wav"), "wb") as w:+ w.setnchannels(1), w.setsampwidth(2), w.setframerate(16000)+ w.writeframes((rng.standard_normal(16000) * 3000).astype("<i2").tobytes())+ out.write(json.dumps({"audio": f"{i}.wav", "text": text}, ensure_ascii=False) + "\n")+ return str(folder / "clips.jsonl")+++def test_declared_tokens_become_one_piece_anywhere_and_match_the_engine_encoder(tmp_path):+ from needle.model.export import RefTokenizer, parse_tokenizer_blob+ from needle.model.whistle_finetune import WhistleTokenizer, read_tokens, token_pieces++ listed = tmp_path / "tokens.txt"+ listed.write_text("ação\nSão Paulo, SP\n🙂\n\nПривет\nASP.NET\n你好\n", encoding="utf-8")+ tokens = read_tokens(str(listed))+ assert tokens == ["ação", "São Paulo, SP", "🙂", "Привет", "ASP.NET", "你好"]+ base = WhistleTokenizer()+ tokenizer = WhistleTokenizer(token_pieces(base.sp, tokens))+ meta = parse_tokenizer_blob(tokenizer.blob())+ assert set(meta["types"][base.text:tokenizer.text]) == {0}+ engine = RefTokenizer(meta)+ for token in tokens:+ for text in (token, " " + token, f"disse {token} hoje"):+ ids = tokenizer.encode(text)+ assert any(tokenizer.sp.id_to_piece(i).replace("\u2581", " ").strip() == token for i in ids)+ assert engine.encode(text) == ids and tokenizer.sp.decode(ids) == text+ assert token_pieces(base.sp, ["the"]) == []+++def test_extended_tokenizer_appends_pieces_and_tags():+ from needle.model.export import parse_tokenizer_blob+ from needle.model.whistle_finetune import WhistleTokenizer++ plain = WhistleTokenizer((), ("en",))+ tokenizer = WhistleTokenizer(("ã", "ão"), ("en", "pt"))+ assert tokenizer.text == plain.text + 2+ assert tokenizer.encode("não") == [plain.sp.piece_to_id("n"), plain.text + 1]+ meta = parse_tokenizer_blob(tokenizer.blob())+ assert meta["pieces"][plain.text:] == ["ã", "ão", "<|en|>", "<|pt|>"]+ assert meta["types"][-2:] == [3, 3] and meta["byte_fallback"]+++def test_finetune_adds_a_language_and_builds_an_archive_with_it(tiny_whistle, published_whistle, tmp_path, capsys):+ from needle.model.checkpoints import read_adapter+ from needle.model.export import parse_tokenizer_blob, read_export+ from needle.model.whistle_finetune import WhistleTokenizer, build_main, finetune_main, token_pieces++ data = _clips(tmp_path, ["não é coração", "ação não", "não não", "coração"] * 2)+ tokens = tmp_path / "tokens.txt"+ tokens.write_text("ação\ncoração\n", encoding="utf-8")+ adapter = str(tmp_path / "adapter.safetensors")+ finetune_main(types.SimpleNamespace(+ jsonl_path=data, language="pt", tokens=str(tokens), checkpoint=tiny_whistle, epochs=1, batch_size=2, lr=1e-3,+ lora_rank=4, lora_alpha=8.0, val_split=0.25, seed=0, checkpoint_dir=str(tmp_path), out=adapter))+ printed = capsys.readouterr().out+ assert "CQ W4 STE + A8 (matches export)" in printed and "held-out clips, scored on the W4 weights" in printed+ saved = read_adapter(adapter)+ pieces = token_pieces(WhistleTokenizer().sp, ["ação", "coração"])+ assert saved["languages"][-1] == "pt" and saved["pieces"] == pieces+ assert saved["rows"].shape == (len(pieces) + 1, TINY["dim"]) and len(saved["lora"]) == 15++ out = str(tmp_path / "tuned.cact")+ build_main(_build_args(lora=adapter, out=out))+ header, tensors = read_export(out)+ tokenizer = parse_tokenizer_blob(tensors[-1])+ text = 8192 + len(pieces)+ assert header["vocab_size"] == text + 8 and tensors[0].shape == (text + 8, TINY["dim"])+ assert tokenizer["pieces"][-1] == "<|pt|>" and tokenizer["pieces"][8192:text] == saved["pieces"]+ manifest = next(t for t in tensors if getattr(t, "shape", None) == (13,) and t[0] == 3246)+ assert manifest[2] == text+ from needle.agent.whistle import LANGUAGES, languages+ assert languages(out) == LANGUAGES + ("pt",)+++def test_build_without_an_adapter_copies_the_published_archive(published_whistle, tmp_path):+ from needle.model.whistle_finetune import build_main++ archive, calls = published_whistle+ out = tmp_path / "copy.cact"+ build_main(_build_args(out=str(out)))+ assert out.read_bytes() == open(archive, "rb").read() and calls == [("whistle", True)]+ with pytest.raises(SystemExit, match="--out"):+ build_main(_build_args())+++def test_build_layers_exports_a_decoder_rung(tiny_whistle, published_whistle, tmp_path):+ from needle.model.export import read_export+ from needle.model.whistle_finetune import build_main++ out = str(tmp_path / "rung.cact")+ build_main(_build_args(checkpoint=tiny_whistle, layers=2, out=out))+ header, _ = read_export(out)+ assert header["num_layers"] == 2 and header["engram_layers"] == (1,)+ with pytest.raises(ValueError, match="--layers"):+ build_main(_build_args(checkpoint=tiny_whistle, layers=5, out=out))+++def test_lines_need_a_language(tmp_path):+ from needle.model.whistle_finetune import read_clips++ data = _clips(tmp_path, ["hello"])+ with pytest.raises(SystemExit, match="language"):+ read_clips(data)+ clips, skipped = read_clips(data, "en")+ assert len(clips) == 1 and clips[0][1:] == ("hello", "en") and not skipped+++def test_errors_use_the_whisper_normalizers_and_characters_for_unspaced_scripts():+ from needle.model.whistle_finetune import errors++ assert errors("Olá, mundo!", "olá mundo", "pt") == (0, 2)+ assert errors("a b c", "a x c d", "pt") == (2, 3)+ assert errors("Mr. Smith paid ten dollars.", "mister smith paid $10", "en") == (0, 4)+ assert errors("今天天气很好。", "今天天汽很好", "zh") == (1, 6)+ assert errors("こんにちは 世界", "こんにちわ世界", "ja") == (1, 7)+ assert errors("Привет, мир", "привет мир", "be") == (0, 2)+++def test_bad_lines_name_the_line(tmp_path):+ from needle.model.whistle_finetune import read_clips++ data = _clips(tmp_path, ["olá"])+ for line, message in ([("not json", "2: not JSON"), ('{"text": "a"}', "2: no \"audio\""),+ ('{"audio": "gone.wav", "text": "a"}', "gone.wav does not exist"),+ ('{"audio": "0.wav", "text": "a", "language": "p t"}', "2: give the line")]):+ with open(data, "a", encoding="utf-8") as out:+ out.write(line + "\n")+ with pytest.raises(SystemExit, match=message):+ read_clips(data, "pt")+ lines = open(data, encoding="utf-8").read().splitlines()[:-1]+ open(data, "w", encoding="utf-8").write("\n".join(lines) + "\n")+++def test_tokens_come_from_a_file(tmp_path):+ from needle.model.whistle_finetune import read_tokens++ with pytest.raises(SystemExit, match="one token per line"):+ read_tokens("ação,coração")+ listed = tmp_path / "tokens.txt"+ listed.write_bytes("ação \r\n\n coração\n".encode("utf-8"))+ assert read_tokens(str(listed)) == ["ação", "coração"]+++def test_wrong_kinds_of_files_are_refused(tiny_whistle, tiny_checkpoint, published_whistle, tmp_path):+ from needle.model.checkpoints import write_adapter+ from needle.model.finetune import build_main as needle_build+ from needle.model.whistle_finetune import build_main, load_base++ with pytest.raises(SystemExit, match="not a Whistle checkpoint"):+ load_base(tiny_checkpoint)+ needle_adapter = str(tmp_path / "needle.safetensors")+ write_adapter(needle_adapter, {"lora": {}, "scale": 2.0, "base": tiny_checkpoint, "rank": 4, "seed": 0})+ with pytest.raises(SystemExit, match="Needle adapter"):+ build_main(_build_args(lora=needle_adapter, out=str(tmp_path / "x.cact")))+ whistle_adapter = str(tmp_path / "whistle.safetensors")+ write_adapter(whistle_adapter, {"lora": {}, "scale": 2.0, "base": tiny_whistle, "rank": 4, "seed": 0,+ "pieces": [], "languages": ["en"]})+ with pytest.raises(SystemExit, match="Whistle adapter"):+ needle_build(types.SimpleNamespace(checkpoint=tiny_checkpoint, lora=whistle_adapter,+ out=str(tmp_path / "y.cact"), upload=False))+++def test_finetune_needs_an_epoch_and_a_batch():+ from needle.model.whistle_finetune import finetune_main++ for epochs, batch in ((0, 8), (1, 0)):+ with pytest.raises(SystemExit, match="at least 1"):+ finetune_main(types.SimpleNamespace(epochs=epochs, batch_size=batch))+++def test_cli_routes_finetune_and_build_and_takes_a_tuned_language(monkeypatch):+ import sys+ import needle._telemetry+ import needle.agent.whistle+ import needle.cli+ import needle.model.whistle_finetune as tune++ seen = []+ monkeypatch.setattr(needle._telemetry, "track", lambda *a, **k: None)+ monkeypatch.setattr(tune, "finetune_main", lambda args: seen.append(("finetune", args)))+ monkeypatch.setattr(tune, "build_main", lambda args: seen.append(("build", args)))+ monkeypatch.setattr(needle.agent.whistle, "playground", lambda args: seen.append(("playground", args)))+ for argv in (["finetune", "clips.jsonl", "--language", "pt", "--tokens", "tokens.txt"],+ ["build", "--lora", "a.safetensors", "--layers", "4", "--out", "t.cact"],+ ["playground", "--language", "pt", "--weights", "t.cact"]):+ monkeypatch.setattr(sys, "argv", ["needle", "whistle", *argv])+ needle.cli.main()+ (_, tuned), (_, built), (_, played) = seen+ assert (tuned.jsonl_path, tuned.language, tuned.tokens, tuned.batch_size, tuned.lr) == (+ "clips.jsonl", "pt", "tokens.txt", 16, 1e-4)+ assert (built.lora, built.layers, built.out, built.platform, built.upload) == ("a.safetensors", 4, "t.cact", None, False)+ assert played.language == "pt"