Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

simplify validation

HenryNdubuaku402783d23614
diff --git a/src/eval.py b/src/eval.py
index 9d6d5e1..db85dce 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -345,7 +345,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
}
-def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10)):
+def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10), ds=None):
"""Benchmark contrastive retrieval: Recall@k and MRR over validation set.
For each query, ranks all tools from that example by cosine similarity
@@ -354,7 +354,8 @@ def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256,
import json
from .run import encode_for_retrieval
- ds = load_tool_calls("validation", max_samples=num_samples)
+ if ds is None:
+ ds = load_tool_calls("validation", max_samples=num_samples)
queries = []
all_tool_strs = []
diff --git a/src/train.py b/src/train.py
index 9762ab1..8dc806a 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1008,6 +1008,7 @@ def train(args):
retrieval_metrics = benchmark_retrieval(
eval_model, eval_params, tokenizer,
num_samples=min(500, getattr(args, "max_eval_samples", 500)),
+ ds=val_ds,
)
del eval_params