Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku5a6bde355880
diff --git a/src/cli.py b/src/cli.py
index 1ada00e..64b73db 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -17,7 +17,7 @@ def main():
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=512)
+ p.add_argument("--d-model", type=int, default=768)
p.add_argument("--num-heads", type=int, default=16)
p.add_argument("--num-kv-heads", type=int, default=8)
p.add_argument("--num-layers", type=int, default=4)
@@ -46,7 +46,7 @@ def main():
help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")
p.add_argument("--mat-shared-input", action="store_true",
help="Each unique input is repeated across all mat widths (default: unique input per width)")
- p.add_argument("--dropout", type=float, default=0.05,
+ p.add_argument("--dropout", type=float, default=0.1,
help="Dropout rate for residual connections (default: 0.1)")
p = sub.add_parser("tokenize", add_help=False)
diff --git a/src/eval.py b/src/eval.py
index a21ea5a..822cbcc 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -192,6 +192,14 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
json_parse_errors = 0
empty_ref = 0
empty_pred = 0
+ args_correct = 0
+ args_total = 0
+ halluc_params = 0
+ total_pred_params = 0
+ missing_params = 0
+ total_ref_params = 0
+ correct_values = 0
+ matched_params = 0
samples = []
def call_key(c):
@@ -240,6 +248,43 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
fp_calls += len(pred_keys - ref_keys)
fn_calls += len(ref_keys - pred_keys)
+ ref_by_name = {}
+ for c in ref_calls:
+ if isinstance(c, dict) and "name" in c:
+ ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
+ for c in pred_calls:
+ if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
+ args_total += 1
+ pa = json.dumps(c.get("arguments", {}), sort_keys=True)
+ if any(pa == json.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
+ args_correct += 1
+
+ try:
+ tool_defs = json.loads(ex["tools"])
+ tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}
+ except (json.JSONDecodeError, TypeError):
+ tool_param_map = {}
+ for c in pred_calls:
+ if not isinstance(c, dict) or "name" not in c:
+ continue
+ cname = c["name"]
+ if cname not in tool_param_map:
+ continue
+ schema_keys = tool_param_map[cname]
+ p_keys = set((c.get("arguments") or {}).keys())
+ total_pred_params += len(p_keys)
+ halluc_params += len(p_keys - schema_keys)
+ if cname in ref_by_name:
+ ref_args = ref_by_name[cname][0]
+ r_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())
+ total_ref_params += len(r_keys)
+ missing_params += len(r_keys - p_keys)
+ m_keys = p_keys & r_keys
+ matched_params += len(m_keys)
+ for k in m_keys:
+ if json.dumps(c.get("arguments", {})[k], sort_keys=True) == json.dumps(ref_args[k], sort_keys=True):
+ correct_values += 1
+
name_precision = tp_names / max(tp_names + fp_names, 1)
name_recall = tp_names / max(tp_names + fn_names, 1)
name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
@@ -255,6 +300,10 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
"name_precision": name_precision,
"name_recall": name_recall,
"name_f1": name_f1,
+ "args_acc": args_correct / max(args_total, 1),
+ "param_haluc": halluc_params / max(total_pred_params, 1),
+ "param_miss": missing_params / max(total_ref_params, 1),
+ "value_acc": correct_values / max(matched_params, 1),
"call_precision": call_precision,
"call_recall": call_recall,
"call_f1": call_f1,
@@ -298,6 +347,11 @@ def main(args):
print(f" Precision {tc['name_precision']:>10.3f}")
print(f" Recall {tc['name_recall']:>10.3f}")
print(f" F1 {tc['name_f1']:>10.3f}")
+ print(f" ─── Parameter-Level ────────────────")
+ print(f" Param haluc {tc['param_haluc']:>10.1%}")
+ print(f" Param miss {tc['param_miss']:>10.1%}")
+ print(f" Value acc {tc['value_acc']:>10.1%}")
+ print(f" Args acc {tc['args_acc']:>10.1%}")
print(f" ─── Full Call (name+args) ───────────")
print(f" Precision {tc['call_precision']:>10.3f}")
print(f" Recall {tc['call_recall']:>10.3f}")
diff --git a/src/train.py b/src/train.py
index 797c607..c9e34a8 100644
--- a/src/train.py
+++ b/src/train.py
@@ -748,7 +748,6 @@ def train(args):
max_enc_len=args.max_enc_len,
)
- # Split results back
display_preds = all_preds[:len(display_pairs)]
tc_preds = all_preds[len(display_pairs):]
@@ -764,6 +763,9 @@ def train(args):
tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0
tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0
tc_args_correct, tc_args_total = 0, 0
+ tc_halluc_params, tc_total_pred_params = 0, 0
+ tc_missing_params, tc_total_ref_params = 0, 0
+ tc_correct_values, tc_matched_params = 0, 0
def _call_key(c):
if not isinstance(c, dict): return None
@@ -809,6 +811,34 @@ def train(args):
if any(pred_args == _json_mod.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
tc_args_correct += 1
+ # Parameter-level metrics
+ try:
+ tool_defs = _json_mod.loads(ex["tools"])
+ tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}
+ except (ValueError, TypeError):
+ tool_param_map = {}
+ for c in pred_calls:
+ if not isinstance(c, dict) or "name" not in c:
+ continue
+ cname = c["name"]
+ if cname not in tool_param_map:
+ continue
+ schema_keys = tool_param_map[cname]
+ pred_keys = set((c.get("arguments") or {}).keys())
+ tc_total_pred_params += len(pred_keys)
+ tc_halluc_params += len(pred_keys - schema_keys)
+ # Find matching reference call for missing/value metrics
+ if cname in ref_by_name:
+ ref_args = ref_by_name[cname][0]
+ ref_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())
+ tc_total_ref_params += len(ref_keys)
+ tc_missing_params += len(ref_keys - pred_keys)
+ matched_keys = pred_keys & ref_keys
+ tc_matched_params += len(matched_keys)
+ for k in matched_keys:
+ if _json_mod.dumps(c.get("arguments", {})[k], sort_keys=True) == _json_mod.dumps(ref_args[k], sort_keys=True):
+ tc_correct_values += 1
+
tc_metrics = {}
if tc_n > 0:
tc_metrics["parse_rate"] = 1.0 - tc_parse_err / tc_n
@@ -820,6 +850,9 @@ def train(args):
cr_ = tc_call_tp + tc_call_fn
tc_metrics["call_f1"] = 2 * tc_call_tp / max(cp_ + cr_, 1)
tc_metrics["args_acc"] = tc_args_correct / max(tc_args_total, 1)
+ tc_metrics["param_haluc"] = tc_halluc_params / max(tc_total_pred_params, 1)
+ tc_metrics["param_miss"] = tc_missing_params / max(tc_total_ref_params, 1)
+ tc_metrics["value_acc"] = tc_correct_values / max(tc_matched_params, 1)
del eval_params
@@ -842,6 +875,9 @@ def train(args):
print(f" ─── Tool-Call Accuracy ({tc_n} samples) ──")
print(f" JSON parse {tc_metrics['parse_rate']:>10.1%}")
print(f" Name F1 {tc_metrics['name_f1']:>10.1%}")
+ print(f" Param haluc {tc_metrics['param_haluc']:>10.1%}")
+ print(f" Param miss {tc_metrics['param_miss']:>10.1%}")
+ print(f" Value acc {tc_metrics['value_acc']:>10.1%}")
print(f" Args acc {tc_metrics['args_acc']:>10.1%}")
print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")
print(f" Exact match {tc_metrics['exact_match']:>10.1%}")
@@ -879,6 +915,9 @@ def train(args):
log_dict["epoch/tc_parse_rate"] = tc_metrics["parse_rate"]
log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]
log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]
+ log_dict["epoch/tc_param_haluc"] = tc_metrics["param_haluc"]
+ log_dict["epoch/tc_param_miss"] = tc_metrics["param_miss"]
+ log_dict["epoch/tc_value_acc"] = tc_metrics["value_acc"]
log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]
log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
wandb.log(log_dict)