Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku5a6bde355880
diff --git a/src/cli.py b/src/cli.pyindex 1ada00e..64b73db 100644--- a/src/cli.py+++ b/src/cli.py@@ -17,7 +17,7 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--d-model", type=int, default=768)p.add_argument("--num-heads", type=int, default=16)p.add_argument("--num-kv-heads", type=int, default=8)p.add_argument("--num-layers", type=int, default=4)@@ -46,7 +46,7 @@ def main():help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")p.add_argument("--mat-shared-input", action="store_true",help="Each unique input is repeated across all mat widths (default: unique input per width)")- p.add_argument("--dropout", type=float, default=0.05,+ p.add_argument("--dropout", type=float, default=0.1,help="Dropout rate for residual connections (default: 0.1)")p = sub.add_parser("tokenize", add_help=False)diff --git a/src/eval.py b/src/eval.pyindex a21ea5a..822cbcc 100644--- a/src/eval.py+++ b/src/eval.py@@ -192,6 +192,14 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=json_parse_errors = 0empty_ref = 0empty_pred = 0+ args_correct = 0+ args_total = 0+ halluc_params = 0+ total_pred_params = 0+ missing_params = 0+ total_ref_params = 0+ correct_values = 0+ matched_params = 0samples = []def call_key(c):@@ -240,6 +248,43 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=fp_calls += len(pred_keys - ref_keys)fn_calls += len(ref_keys - pred_keys)+ ref_by_name = {}+ for c in ref_calls:+ if isinstance(c, dict) and "name" in c:+ ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))+ for c in pred_calls:+ if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:+ args_total += 1+ pa = json.dumps(c.get("arguments", {}), sort_keys=True)+ if any(pa == json.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):+ args_correct += 1++ try:+ tool_defs = json.loads(ex["tools"])+ tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}+ except (json.JSONDecodeError, TypeError):+ tool_param_map = {}+ for c in pred_calls:+ if not isinstance(c, dict) or "name" not in c:+ continue+ cname = c["name"]+ if cname not in tool_param_map:+ continue+ schema_keys = tool_param_map[cname]+ p_keys = set((c.get("arguments") or {}).keys())+ total_pred_params += len(p_keys)+ halluc_params += len(p_keys - schema_keys)+ if cname in ref_by_name:+ ref_args = ref_by_name[cname][0]+ r_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())+ total_ref_params += len(r_keys)+ missing_params += len(r_keys - p_keys)+ m_keys = p_keys & r_keys+ matched_params += len(m_keys)+ for k in m_keys:+ if json.dumps(c.get("arguments", {})[k], sort_keys=True) == json.dumps(ref_args[k], sort_keys=True):+ correct_values += 1+name_precision = tp_names / max(tp_names + fp_names, 1)name_recall = tp_names / max(tp_names + fn_names, 1)name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)@@ -255,6 +300,10 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len="name_precision": name_precision,"name_recall": name_recall,"name_f1": name_f1,+ "args_acc": args_correct / max(args_total, 1),+ "param_haluc": halluc_params / max(total_pred_params, 1),+ "param_miss": missing_params / max(total_ref_params, 1),+ "value_acc": correct_values / max(matched_params, 1),"call_precision": call_precision,"call_recall": call_recall,"call_f1": call_f1,@@ -298,6 +347,11 @@ def main(args):print(f" Precision {tc['name_precision']:>10.3f}")print(f" Recall {tc['name_recall']:>10.3f}")print(f" F1 {tc['name_f1']:>10.3f}")+ print(f" ─── Parameter-Level ────────────────")+ print(f" Param haluc {tc['param_haluc']:>10.1%}")+ print(f" Param miss {tc['param_miss']:>10.1%}")+ print(f" Value acc {tc['value_acc']:>10.1%}")+ print(f" Args acc {tc['args_acc']:>10.1%}")print(f" ─── Full Call (name+args) ───────────")print(f" Precision {tc['call_precision']:>10.3f}")print(f" Recall {tc['call_recall']:>10.3f}")diff --git a/src/train.py b/src/train.pyindex 797c607..c9e34a8 100644--- a/src/train.py+++ b/src/train.py@@ -748,7 +748,6 @@ def train(args):max_enc_len=args.max_enc_len,)- # Split results backdisplay_preds = all_preds[:len(display_pairs)]tc_preds = all_preds[len(display_pairs):]@@ -764,6 +763,9 @@ def train(args):tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0tc_args_correct, tc_args_total = 0, 0+ tc_halluc_params, tc_total_pred_params = 0, 0+ tc_missing_params, tc_total_ref_params = 0, 0+ tc_correct_values, tc_matched_params = 0, 0def _call_key(c):if not isinstance(c, dict): return None@@ -809,6 +811,34 @@ def train(args):if any(pred_args == _json_mod.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):tc_args_correct += 1+ # Parameter-level metrics+ try:+ tool_defs = _json_mod.loads(ex["tools"])+ tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}+ except (ValueError, TypeError):+ tool_param_map = {}+ for c in pred_calls:+ if not isinstance(c, dict) or "name" not in c:+ continue+ cname = c["name"]+ if cname not in tool_param_map:+ continue+ schema_keys = tool_param_map[cname]+ pred_keys = set((c.get("arguments") or {}).keys())+ tc_total_pred_params += len(pred_keys)+ tc_halluc_params += len(pred_keys - schema_keys)+ # Find matching reference call for missing/value metrics+ if cname in ref_by_name:+ ref_args = ref_by_name[cname][0]+ ref_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())+ tc_total_ref_params += len(ref_keys)+ tc_missing_params += len(ref_keys - pred_keys)+ matched_keys = pred_keys & ref_keys+ tc_matched_params += len(matched_keys)+ for k in matched_keys:+ if _json_mod.dumps(c.get("arguments", {})[k], sort_keys=True) == _json_mod.dumps(ref_args[k], sort_keys=True):+ tc_correct_values += 1+tc_metrics = {}if tc_n > 0:tc_metrics["parse_rate"] = 1.0 - tc_parse_err / tc_n@@ -820,6 +850,9 @@ def train(args):cr_ = tc_call_tp + tc_call_fntc_metrics["call_f1"] = 2 * tc_call_tp / max(cp_ + cr_, 1)tc_metrics["args_acc"] = tc_args_correct / max(tc_args_total, 1)+ tc_metrics["param_haluc"] = tc_halluc_params / max(tc_total_pred_params, 1)+ tc_metrics["param_miss"] = tc_missing_params / max(tc_total_ref_params, 1)+ tc_metrics["value_acc"] = tc_correct_values / max(tc_matched_params, 1)del eval_params@@ -842,6 +875,9 @@ def train(args):print(f" ─── Tool-Call Accuracy ({tc_n} samples) ──")print(f" JSON parse {tc_metrics['parse_rate']:>10.1%}")print(f" Name F1 {tc_metrics['name_f1']:>10.1%}")+ print(f" Param haluc {tc_metrics['param_haluc']:>10.1%}")+ print(f" Param miss {tc_metrics['param_miss']:>10.1%}")+ print(f" Value acc {tc_metrics['value_acc']:>10.1%}")print(f" Args acc {tc_metrics['args_acc']:>10.1%}")print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")print(f" Exact match {tc_metrics['exact_match']:>10.1%}")@@ -879,6 +915,9 @@ def train(args):log_dict["epoch/tc_parse_rate"] = tc_metrics["parse_rate"]log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]+ log_dict["epoch/tc_param_haluc"] = tc_metrics["param_haluc"]+ log_dict["epoch/tc_param_miss"] = tc_metrics["param_miss"]+ log_dict["epoch/tc_value_acc"] = tc_metrics["value_acc"]log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]wandb.log(log_dict)