Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
remove legacy evaluate
HenryNdubuakua65b02d349b3
diff --git a/src/evaluate.py b/src/evaluate.pydeleted file mode 100644index 911b853..0000000--- a/src/evaluate.py+++ /dev/null@@ -1,379 +0,0 @@-import argparse-import math--import jax-import jax.numpy as jnp-import numpy as np-from datasets import load_dataset-from flax import jax_utils--from .data import get_tokenizer-from .model import (- EncoderDecoderTransformer,- TransformerConfig,- make_causal_mask,- make_padding_mask,-)-from .run import load_checkpoint---def _make_p_encode(model):- """Create a pmap'd encode function."""- def _encode(params, src, src_mask):- return model.apply(- {"params": params}, src, src_mask=src_mask, method="encode",- )- return jax.pmap(_encode, axis_name="batch")---def _make_p_decode(model):- """Create a pmap'd decode function."""- def _decode(params, dec_input, encoder_out, tgt_mask, _unused_cross_mask):- return model.apply(- {"params": params}, dec_input, encoder_out,- self_mask=tgt_mask, method="decode",- )- return jax.pmap(_decode, axis_name="batch")---def _shard_single(x, num_devices):- """Replicate a single-sample batch across all devices for pmap."""- return jnp.broadcast_to(x, (num_devices, *x.shape[1:]))---def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, sos_id=None,- p_encode=None, p_decode=None, num_devices=1):- """Compute average negative log-likelihood of dec_tokens given enc_tokens."""- sos = sos_id if sos_id is not None else pad_id- enc_input = jnp.array([enc_tokens])- src_mask = make_padding_mask(enc_input, pad_id)-- if p_encode is not None and num_devices > 1:- enc_s = _shard_single(enc_input, num_devices)- src_mask_s = _shard_single(src_mask, num_devices)- encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]- else:- p = params if num_devices <= 1 else jax_utils.unreplicate(params)- encoder_out = model.apply(- {"params": p}, enc_input, src_mask=src_mask, method="encode",- )-- dec_in = [sos] + list(dec_tokens[:-1])- dec_input = jnp.array([dec_in])- tgt_mask = make_causal_mask(len(dec_in))-- if p_decode is not None and num_devices > 1:- dec_s = _shard_single(dec_input, num_devices)- tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]- else:- p = params if num_devices <= 1 else jax_utils.unreplicate(params)- logits = model.apply(- {"params": p}, dec_input, encoder_out,- self_mask=tgt_mask, method="decode",- )[0]-- log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])- target_ids = jnp.array(dec_tokens)- token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]- return float(jnp.mean(token_lls))---def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,- num_devices=1, p_encode=None, p_decode=None):- """Perplexity on WikiText-2 test split."""- ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")-- pad_id = tokenizer.pad_token_id- sos_id = tokenizer.eos_token_id- total_nll = 0.0- total_tokens = 0- evaluated = 0-- single_params = jax_utils.unreplicate(params) if num_devices > 1 else params-- for example in ds:- text = example["text"].strip()- if len(text) < 20:- continue-- tokens = tokenizer.encode(text)- if len(tokens) < 6:- continue-- tokens = tokens[:max_len]- split = max(2, len(tokens) // 3)- enc_tokens = tokens[:split]- dec_tokens = tokens[split:]-- enc_input = jnp.array([enc_tokens])- src_mask = make_padding_mask(enc_input, pad_id)-- if p_encode is not None and num_devices > 1:- enc_s = _shard_single(enc_input, num_devices)- src_mask_s = _shard_single(src_mask, num_devices)- encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]- else:- encoder_out = model.apply(- {"params": single_params}, enc_input, src_mask=src_mask, method="encode",- )-- dec_in = [sos_id] + list(dec_tokens[:-1])- dec_input = jnp.array([dec_in])- tgt_mask = make_causal_mask(len(dec_in))-- if p_decode is not None and num_devices > 1:- dec_s = _shard_single(dec_input, num_devices)- tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]- else:- logits = model.apply(- {"params": single_params}, dec_input, encoder_out,- self_mask=tgt_mask, method="decode",- )-- log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])- target_ids = jnp.array(dec_tokens)- token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]-- total_nll += -float(jnp.sum(token_lls))- total_tokens += len(dec_tokens)- evaluated += 1-- if evaluated >= max_samples:- break-- ppl = math.exp(total_nll / max(total_tokens, 1))- return {"perplexity": ppl, "samples": evaluated, "tokens": total_tokens}---def eval_lambada(model, params, tokenizer, max_samples=500,- num_devices=1, p_encode=None, p_decode=None):- """Accuracy of predicting the final word on LAMBADA."""- ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")-- pad_id = tokenizer.pad_token_id- sos_id = tokenizer.eos_token_id- correct = 0- total = 0-- single_params = jax_utils.unreplicate(params) if num_devices > 1 else params-- for example in ds:- text = example["text"].strip()- words = text.rsplit(" ", 1)- if len(words) < 2:- continue-- context, last_word = words- context_tokens = tokenizer.encode(context)- target_tokens = tokenizer.encode(" " + last_word)-- if not context_tokens or not target_tokens:- continue-- enc_input = jnp.array([context_tokens])- src_mask = make_padding_mask(enc_input, pad_id)-- if p_encode is not None and num_devices > 1:- enc_s = _shard_single(enc_input, num_devices)- src_mask_s = _shard_single(src_mask, num_devices)- encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]- else:- encoder_out = model.apply(- {"params": single_params}, enc_input, src_mask=src_mask, method="encode",- )-- dec_in = jnp.array([[sos_id]])- tgt_mask = make_causal_mask(1)-- if p_decode is not None and num_devices > 1:- dec_s = _shard_single(dec_in, num_devices)- tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]- else:- logits = model.apply(- {"params": single_params}, dec_in, encoder_out,- self_mask=tgt_mask, method="decode",- )-- predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))- if predicted == target_tokens[0]:- correct += 1- total += 1-- if total >= max_samples:- break-- acc = correct / max(total, 1)- return {"accuracy": acc, "correct": correct, "total": total}---def eval_hellaswag(model, params, tokenizer, max_samples=500,- num_devices=1, p_encode=None, p_decode=None):- """Accuracy on HellaSwag by scoring each candidate ending."""- ds = load_dataset("Rowan/hellaswag", split="validation")-- pad_id = tokenizer.pad_token_id- sos_id = tokenizer.eos_token_id- correct = 0- total = 0-- for example in ds:- ctx = example["ctx"].strip()- endings = example["endings"]- label = int(example["label"])-- enc_tokens = tokenizer.encode(ctx)- if not enc_tokens:- continue-- enc_tokens = enc_tokens[:192]-- scores = []- for ending in endings:- dec_tokens = tokenizer.encode(" " + ending.strip())- if not dec_tokens:- scores.append(float("-inf"))- continue- dec_tokens = dec_tokens[:64]- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,- sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,- num_devices=num_devices)- scores.append(score)-- predicted = int(np.argmax(scores))- if predicted == label:- correct += 1- total += 1-- if total >= max_samples:- break-- acc = correct / max(total, 1)- return {"accuracy": acc, "correct": correct, "total": total}---def eval_arc_easy(model, params, tokenizer, max_samples=500,- num_devices=1, p_encode=None, p_decode=None):- """Accuracy on ARC-Easy by scoring each answer choice."""- ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")-- pad_id = tokenizer.pad_token_id- sos_id = tokenizer.eos_token_id- correct = 0- total = 0-- for example in ds:- question = example["question"].strip()- choices = example["choices"]- answer_key = example["answerKey"]-- labels = choices["label"]- texts = choices["text"]-- enc_tokens = tokenizer.encode(question)[:128]- if not enc_tokens:- continue-- scores = []- for choice_text in texts:- dec_tokens = tokenizer.encode(" " + choice_text.strip())- if not dec_tokens:- scores.append(float("-inf"))- continue- dec_tokens = dec_tokens[:64]- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,- sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,- num_devices=num_devices)- scores.append(score)-- predicted_idx = int(np.argmax(scores))- predicted_label = labels[predicted_idx]- if predicted_label == answer_key:- correct += 1- total += 1-- if total >= max_samples:- break-- acc = correct / max(total, 1)- return {"accuracy": acc, "correct": correct, "total": total}---BENCHMARKS = {- "wikitext2": eval_wikitext2,- "lambada": eval_lambada,- "hellaswag": eval_hellaswag,- "arc_easy": eval_arc_easy,-}---def main(args):- num_devices = jax.local_device_count()- print(f"Detected {num_devices} device(s) for data-parallel evaluation")-- print(f"Loading checkpoint: {args.checkpoint}")- params, config = load_checkpoint(args.checkpoint)- model = EncoderDecoderTransformer(config)- tokenizer = get_tokenizer()-- param_count = sum(x.size for x in jax.tree.leaves(params))- print(f"Model parameters: {param_count:,}")-- # Replicate params across devices for pmap- if num_devices > 1:- params = jax_utils.replicate(params)- p_encode = _make_p_encode(model)- p_decode = _make_p_decode(model)- print(f"Params replicated across {num_devices} devices")- else:- p_encode = None- p_decode = None-- benchmarks = args.benchmarks or list(BENCHMARKS.keys())-- results = {}- for name in benchmarks:- if name not in BENCHMARKS:- print(f"Unknown benchmark: {name}, skipping")- continue-- print(f"\n{'=' * 50}")- print(f"Benchmark: {name}")- print("=" * 50)-- result = BENCHMARKS[name](- model, params, tokenizer, max_samples=args.max_samples,- num_devices=num_devices, p_encode=p_encode, p_decode=p_decode,- )- results[name] = result-- for k, v in result.items():- if isinstance(v, float):- print(f" {k}: {v:.4f}")- else:- print(f" {k}: {v}")-- print(f"\n{'=' * 50}")- print("Summary")- print("=" * 50)- for name, result in results.items():- metric = "perplexity" if "perplexity" in result else "accuracy"- print(f" {name:>12s}: {result[metric]:.4f} ({metric})")---def parse_args():- parser = argparse.ArgumentParser(description="Evaluate on established NLP benchmarks")- parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")- parser.add_argument("--benchmarks", type=str, nargs="*", choices=list(BENCHMARKS.keys()),- help="Benchmarks to run (default: all)")- parser.add_argument("--max-samples", type=int, default=500)- return parser.parse_args()---if __name__ == "__main__":- main(parse_args())