Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
simplify validation
HenryNdubuaku402783d23614
diff --git a/src/eval.py b/src/eval.pyindex 9d6d5e1..db85dce 100644--- a/src/eval.py+++ b/src/eval.py@@ -345,7 +345,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=}-def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10)):+def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10), ds=None):"""Benchmark contrastive retrieval: Recall@k and MRR over validation set.For each query, ranks all tools from that example by cosine similarity@@ -354,7 +354,8 @@ def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256,import jsonfrom .run import encode_for_retrieval- ds = load_tool_calls("validation", max_samples=num_samples)+ if ds is None:+ ds = load_tool_calls("validation", max_samples=num_samples)queries = []all_tool_strs = []diff --git a/src/train.py b/src/train.pyindex 9762ab1..8dc806a 100644--- a/src/train.py+++ b/src/train.py@@ -1008,6 +1008,7 @@ def train(args):retrieval_metrics = benchmark_retrieval(eval_model, eval_params, tokenizer,num_samples=min(500, getattr(args, "max_eval_samples", 500)),+ ds=val_ds,)del eval_params