Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku07a01ab8cec8
diff --git a/src/train.py b/src/train.pyindex dcd861d..e321bb7 100644--- a/src/train.py+++ b/src/train.py@@ -872,55 +872,21 @@ def train(args):display_pairs = display_with + display_withoutimport json as _json_mod- _STRUCTURED_TYPES = {"integer", "number", "boolean", "float", "int", "bool"}- tc_per_bucket = 30+ tc_per_bucket = 50tc_rng = np.random.RandomState(epoch + 42)def _classify_sample(ex):- """Classify a sample as (arg_type, call_type) for stratified eval.-- arg_type: 'structured', 'semantic', or 'empty'- call_type: 'single', 'multi', or 'empty'- """+ """Classify a sample as single or multi call."""try:- tools = _json_mod.loads(ex["tools"])answers = _json_mod.loads(ex["answers"])except (ValueError, TypeError):- return "empty", "empty"+ return "empty"if not answers or not isinstance(answers, list):- return "empty", "empty"-- call_type = "single" if len(answers) == 1 else "multi"+ return "empty"+ return "single" if len(answers) == 1 else "multi"- type_map = {}- for t in tools:- if not isinstance(t, dict) or "name" not in t:- continue- for k, v in (t.get("parameters") or {}).items():- ptype = v.get("type", "string") if isinstance(v, dict) else "string"- if not isinstance(ptype, str):- ptype = "string"- type_map[(t["name"], k)] = ptype-- n_struct, n_sem = 0, 0- for call in answers:- if not isinstance(call, dict):- continue- cname = call.get("name")- if not isinstance(cname, str):- continue- for k in (call.get("arguments") or {}).keys():- if type_map.get((cname, k), "string") in _STRUCTURED_TYPES:- n_struct += 1- else:- n_sem += 1- if n_struct + n_sem == 0:- return "empty", call_type- arg_type = "struct" if n_struct >= n_sem else "semantic"- return arg_type, call_type-- # 4 pools: (structured|semantic) × (single|multi), each balanced by tool count- _pool_names = ["struct_single", "struct_multi", "semantic_single", "semantic_multi"]+ # 2 pools: single/multi, each balanced by tool count+ _pool_names = ["single", "multi"]_pool_buckets = {name: {t: [] for t in range(11)} for name in _pool_names}for k in range(len(val_kept)):@@ -929,12 +895,10 @@ def train(args):nc = min(len(_json_mod.loads(ex["tools"])), 10)except (ValueError, TypeError):nc = 0- arg_type, call_type = _classify_sample(ex)- if arg_type == "empty":+ call_type = _classify_sample(ex)+ if call_type == "empty":continue- pool_key = f"{arg_type}_{call_type}"- if pool_key in _pool_buckets:- _pool_buckets[pool_key][nc].append(k)+ _pool_buckets[call_type][nc].append(k)def _balanced_sample(buckets, per_bucket, rng):pool = []@@ -1137,13 +1101,13 @@ def train(args):for name in _pool_names:pool_metrics[name], pool_pc[name] = _eval_pool(eval_pools[name], pool_preds[name])- _best_metric = pool_metrics.get("struct_single", {}).get("call_f1", 0)+ _best_metric = pool_metrics.get("single", {}).get("call_f1", 0)if _best_metric > best_call_f1:best_call_f1 = _best_metricbest_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_{config.d_ff}_best.pkl")import shutil as _shutil_shutil.copy2(ckpt_path, best_ckpt_path)- print(f" ** New best struct call_f1={best_call_f1:.1%} → {best_ckpt_path}")+ print(f" ** New best single call_f1={best_call_f1:.1%} → {best_ckpt_path}")_upload_checkpoint(best_ckpt_path)if _MAT_FACTORS:from .export import export_submodel@@ -1213,10 +1177,8 @@ def train(args):print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {d['name_tp']:>4} {d['name_fp']:>4} {d['name_fn']:>4} {cf1:>7.1%} {d['call_tp']:>4} {d['call_fp']:>4} {d['call_fn']:>4} {ex_:>5.1%} {pr_:>5.1%}")_label_map = {- "struct_single": "Structured / Single-Call",- "struct_multi": "Structured / Multi-Call",- "semantic_single": "Semantic / Single-Call",- "semantic_multi": "Semantic / Multi-Call",+ "single": "Single-Call",+ "multi": "Multi-Call",}for name in _pool_names:_print_tc_metrics(_label_map[name], pool_metrics[name], pool_pc[name])