Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku62b60b80daf3
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex e4fb8f9..7c6db82 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -21,6 +21,8 @@ import timefrom concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Path+import itertools+import requestslogging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")@@ -35,9 +37,7 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"MODEL_POOL = ["google/gemini-3.1-flash-lite-preview",- # "anthropic/claude-opus-4.6",- # "x-ai/grok-4.1-fast",- # "openai/gpt-5.3-chat",+ "google/gemini-3-flash-preview",]DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"@@ -309,15 +309,24 @@ def load_queries_from_gcs(max_samples=None):return samples-def _pick_model(models, rng):- """Randomly select a model from the pool."""- return rng.choice(models)+class _KeyModelRotator:+ """Thread-safe round-robin across (api_key, model) pairs."""++ def __init__(self, api_keys, models):+ pairs = [(k, m) for k in api_keys for m in models]+ self._cycle = itertools.cycle(pairs)+ import threading+ self._lock = threading.Lock()+ def next(self):+ with self._lock:+ return next(self._cycle)-def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):++def process_sample(sample, rotator, source_tag="synth-pleias"):"""Process a single sample: call LLM, validate, return row or None."""synth_id, language, query = sample- model = _pick_model(models, rng)+ api_key, model = rotator.next()result = call_openrouter(query, api_key, model=model)if result is None:return None@@ -328,7 +337,7 @@ def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):return row-def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, source_tag,+def _process_batch(samples, label, batch_idx, rotator, args, tqdm, source_tag,since_last_upload):"""Process a batch of samples, append to checkpoint. Returns (generated, failed)."""logger.info(f"\n{'=' * 60}")@@ -345,14 +354,13 @@ def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, sourcgenerated = 0failed = 0out_file = open(_CHECKPOINT_PATH, "a")- rng = random.Random(42 + batch_idx)try:with ThreadPoolExecutor(max_workers=args.workers) as executor:futures = {}for sample in pending:fut = executor.submit(- process_sample, sample, api_key, models, rng, source_tag,+ process_sample, sample, rotator, source_tag,)futures[fut] = sample[0]@@ -419,8 +427,8 @@ def main():help=f"OpenRouter models to rotate (default: built-in pool)")parser.add_argument("--model", type=str, default=None,help="Use a single model instead of the pool")- parser.add_argument("--workers", type=int, default=32,- help="Concurrent API requests (default: 32)")+ parser.add_argument("--workers", type=int, default=128,+ help="Concurrent API requests (default: 128)")parser.add_argument("--max-shards", type=int, default=None,help="Max parquet shards to process (default: all)")parser.add_argument("--resume", action="store_true",@@ -431,10 +439,12 @@ def main():help="Upload to GCS every N successful generations (default: 1000)")args = parser.parse_args()- api_key = os.environ.get("OPENROUTER_API_KEY")- if not api_key:- print("Error: set OPENROUTER_API_KEY environment variable")+ raw_keys = os.environ.get("OPENROUTER_API_KEY", "")+ api_keys = [k.strip() for k in raw_keys.split(",") if k.strip()]+ if not api_keys:+ print("Error: set OPENROUTER_API_KEY environment variable (comma-separated for multiple)")sys.exit(1)+ logger.info(f"API keys: {len(api_keys)}")if args.model:models = [args.model]@@ -443,6 +453,9 @@ def main():else:models = list(MODEL_POOL)logger.info(f"Model pool ({len(models)}): {', '.join(models)}")+ logger.info(f"Key x Model pairs: {len(api_keys) * len(models)} (each gets its own rate limit)")++ rotator = _KeyModelRotator(api_keys, models)_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)@@ -461,11 +474,10 @@ def main():batch_idx = 0since_last_upload = [0]- # ── GCS raw_data queries ──gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)if gcs_samples:gen, fail = _process_batch(- gcs_samples, "GCS raw_data", batch_idx, api_key, models,+ gcs_samples, "GCS raw_data", batch_idx, rotator,args, tqdm, source_tag="synth-gcs",since_last_upload=since_last_upload,)@@ -473,7 +485,6 @@ def main():total_failed += failbatch_idx += 1- # ── PleIAs/SYNTH parquet shards ──hf_token = os.environ.get("HF_TOKEN")shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)@@ -481,7 +492,7 @@ def main():parquet_path = download_shard(shard_url, shard_idx, hf_token)samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)gen, fail = _process_batch(- samples, f"PleIAs shard {shard_idx}", batch_idx, api_key, models,+ samples, f"PleIAs shard {shard_idx}", batch_idx, rotator,args, tqdm, source_tag="synth-pleias",since_last_upload=since_last_upload,)diff --git a/src/data.py b/src/data.pyindex 8076f2e..402cc97 100644--- a/src/data.py+++ b/src/data.py@@ -98,7 +98,6 @@ BOS_ID = 2UNK_ID = 3TOOL_CALL_ID = 4TOOLS_ID = 5-DEFER_ID = 6DEFAULT_MAX_ENC_LEN = 1024DEFAULT_MAX_DEC_LEN = 512@@ -288,10 +287,6 @@ class NeedleTokenizer:def tools_token_id(self):return TOOLS_ID- @property- def defer_token_id(self):- return DEFER_ID-@propertydef vocab_size(self):return self.sp.GetPieceSize()@@ -418,6 +413,11 @@ def load_tool_calls(split="train", max_samples=None, return_global_indices=Falseds = ds.select(global_indices.tolist())+ # Normalize <defer> / empty answers to []+ ds = ds.map(lambda ex: {+ "answers": "[]" if ex["answers"].strip() in ("", "<defer>") else ex["answers"]+ })+if max_samples:limit = min(max_samples, len(ds))ds = ds.select(range(limit))@@ -507,8 +507,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_tool_call_id = tokenizer.tool_call_token_idtools_sep_id = tokenizer.tools_token_id- defer_id = tokenizer.defer_token_id-enc_texts = [ex["query"] for ex in ds]tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]@@ -520,16 +518,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_return sans_texts = [_compact(a) for a in ans_texts]+ # Normalize any remaining <defer> or empty answers to []+ ans_texts = ["[]" if a.strip() in ("", "<defer>") else a for a in ans_texts]tools_texts = [_compact(t) for t in tools_texts]- empty_tools_mask = []- for t in tools_texts:- try:- parsed = _json.loads(t)- empty_tools_mask.append(isinstance(parsed, list) and len(parsed) == 0)- except (ValueError, TypeError):- empty_tools_mask.append(False)-tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)if shuffle_tools:@@ -562,21 +554,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_n = len(ds)- # Track which samples have empty answers (no tool calls to make)- empty_ans_mask = []- for a in ans_texts:- stripped = a.strip()- empty_ans_mask.append(stripped in ("", "[]"))-- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools, is_empty_ans,+ def _fill_sample(j, e_tok, t_tok, a_tok, ans_text,enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):"""Fill arrays at position j for one sample. Returns False if skipped."""# Encoder: [query..., <tools>, tools...] truncated to max_enc_len- # For empty tools, use <defer> token instead of tokenized "[]"- if is_empty_tools:- t_tok = [defer_id]-- # Reserve 1 slot for <tools> separator + at least 1 token for toolsmax_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools tokenif len(e_tok) > max_query:e_tok = e_tok[:max_query]@@ -587,37 +568,29 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_el = len(enc_seq)enc_arr[j, :el] = enc_seq- if is_empty_ans:- # Empty answer: decoder input [EOS, <defer>], target [<defer>, EOS]- dec_in_arr[j, 0] = eos_id- dec_in_arr[j, 1] = defer_id- dec_tgt_arr[j, 0] = defer_id- dec_tgt_arr[j, 1] = eos_id- lm_arr[j, 0] = 3.0 # High weight on <defer> prediction- lm_arr[j, 1] = 1.0 # EOS- else:- al = len(a_tok)- # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1- if 2 + al + 1 > max_dec_len:- return False-- # Decoder input: [EOS, <tool_call>, answer...]- dec_in_arr[j, 0] = eos_id- dec_in_arr[j, 1] = tool_call_id- if al > 0:- dec_in_arr[j, 2:2 + al] = a_tok-- # Decoder target: [<tool_call>, answer..., EOS]- dec_tgt_arr[j, 0] = tool_call_id- if al > 0:- dec_tgt_arr[j, 1:1 + al] = a_tok- dec_tgt_arr[j, 1 + al] = eos_id-- # Token-level loss weighting- token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,- w_name=w_name, w_value=w_value, w_key=w_key)- lm_arr[j, 1:1 + len(token_weights)] = token_weights- lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight+ al = len(a_tok)+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1+ if 2 + al + 1 > max_dec_len:+ return False++ # Decoder input: [EOS, <tool_call>, answer...]+ dec_in_arr[j, 0] = eos_id+ dec_in_arr[j, 1] = tool_call_id+ if al > 0:+ dec_in_arr[j, 2:2 + al] = a_tok++ # Decoder target: [<tool_call>, answer..., EOS]+ dec_tgt_arr[j, 0] = tool_call_id+ if al > 0:+ dec_tgt_arr[j, 1:1 + al] = a_tok+ dec_tgt_arr[j, 1 + al] = eos_id++ # Token-level loss weighting+ lm_arr[j, 0] = 1.0 # <tool_call> prediction+ token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,+ w_name=w_name, w_value=w_value, w_key=w_key)+ lm_arr[j, 1:1 + len(token_weights)] = token_weights+ lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weightreturn Trueos.makedirs(CACHE_DIR, exist_ok=True)@@ -630,8 +603,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_skipped = 0for i in tqdm(range(n), desc="Building pairs"):if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],- all_ans_tokens[i], ans_texts[i], empty_tools_mask[i],- empty_ans_mask[i],+ all_ans_tokens[i], ans_texts[i],enc_inputs, dec_inputs, dec_targets, loss_mask):skipped += 1diff --git a/src/eval.py b/src/eval.pyindex 8cf17f6..9d6d5e1 100644--- a/src/eval.py+++ b/src/eval.py@@ -256,16 +256,16 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=if i < 10:samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))- ref_is_defer = ref_text.strip() in ("", "[]", "<defer>")- pred_is_defer = pred_text.strip() in ("", "[]", "<defer>")- if ref_is_defer:+ ref_is_empty = ref_text.strip() in ("", "[]")+ pred_is_empty = pred_text.strip() in ("", "[]")+ if ref_is_empty:empty_ref += 1- if pred_is_defer:+ if pred_is_empty:empty_pred += 1- if ref_is_defer and pred_is_defer:+ if ref_is_empty and pred_is_empty:exact_match += 1- elif not ref_is_defer and not pred_is_defer and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):+ elif not ref_is_empty and not pred_is_empty and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):exact_match += 1ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}diff --git a/src/run.py b/src/run.pyindex ce86bd0..5b3d798 100644--- a/src/run.py+++ b/src/run.py@@ -91,25 +91,10 @@ def load_checkpoint(path):def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):- """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len.-- Empty tools ([]) are replaced with the <defer> token.- """+ """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""tools_sep_id = tokenizer.tools_token_id- defer_id = tokenizer.defer_token_idq_toks = tokenizer.encode(query)-- # Use <defer> token for empty tools- try:- parsed = _json.loads(tools)- is_empty = isinstance(parsed, list) and len(parsed) == 0- except (ValueError, TypeError):- is_empty = False-- if is_empty:- t_toks = [defer_id]- else:- t_toks = tokenizer.encode(tools)+ t_toks = tokenizer.encode(tools)max_query = max_enc_len - 2if len(q_toks) > max_query:@@ -123,8 +108,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA"""Generate tool-call output.Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.- Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.- If <defer> is predicted, returns "<defer>" (task routed to cloud model).+ Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens."""name_map = {}if normalize:@@ -135,7 +119,6 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MApad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id- defer_id = tokenizer.defer_token_idsrc_mask = make_padding_mask(enc_input, pad_id)encoder_out, enc_mask = model.apply(@@ -162,12 +145,6 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAif next_token == eos_id:break- # If model predicts <defer> as first token, route to cloud- if next_token == defer_id and i == 0:- if stream:- sys.stdout.write("<defer>\n")- return "<defer>"-generated_tokens.append(next_token)dec_buffer = dec_buffer.at[0, i + 1].set(next_token)@@ -193,7 +170,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE"""Batch-generate tool-call outputs for multiple examples at once.Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.- Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.+ Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens.Returns a list of decoded strings, one per example."""@@ -209,7 +186,6 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEB = len(queries)pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id- defer_id = tokenizer.defer_token_id# --- Encode: [query..., <tools>, tools...] per example, truncated ---enc_token_lists = []@@ -226,16 +202,13 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE{"params": params}, enc_input, src_mask=src_mask, method="encode")- # --- Decoder prefix: [EOS] only, model chooses <tool_call> or <defer> ---dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)dec_buffer[:, 0] = eos_iddec_buffer = jnp.array(dec_buffer)decode_fn = _get_decode_fn(model, max_gen_len)- # --- Autoregressive decoding ---finished = [False] * B- deferred = [False] * Bgen_tokens = [[] for _ in range(B)]logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)@@ -248,11 +221,6 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEif next_token == eos_id:finished[i] = Truecontinue- # <defer> as first token means no tool call- if next_token == defer_id and pos == 0:- finished[i] = True- deferred[i] = True- continuegen_tokens[i].append(next_token)dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)@@ -263,13 +231,10 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEresults = []for i in range(B):- if deferred[i]:- results.append("<defer>")- else:- text = tokenizer.decode(gen_tokens[i])- if text.startswith("<tool_call>"):- text = text[len("<tool_call>"):]- results.append(text)+ text = tokenizer.decode(gen_tokens[i])+ if text.startswith("<tool_call>"):+ text = text[len("<tool_call>"):]+ results.append(text)if normalize and name_maps:results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]return resultsdiff --git a/src/train.py b/src/train.pyindex bc98a48..078f4e1 100644--- a/src/train.py+++ b/src/train.py@@ -787,10 +787,10 @@ def train(args):breaklocal_idx = int(val_kept[k])ex = val_ds[local_idx]- is_defer = ex["answers"].strip() in ("", "[]", "<defer>")- if not is_defer and len(display_with) < 4:+ is_empty = ex["answers"].strip() in ("", "[]")+ if not is_empty and len(display_with) < 4:display_with.append(ex)- elif is_defer and len(display_without) < 1:+ elif is_empty and len(display_without) < 1:display_without.append(ex)display_pairs = display_with + display_without@@ -807,10 +807,10 @@ def train(args):local_idx = int(val_kept[k])ex = val_ds[local_idx]ref_text = ex["answers"].strip()- is_defer = ref_text in ("", "[]", "<defer>")- if not is_defer and len(tc_with) < tc_with_n:+ is_empty = ref_text in ("", "[]")+ if not is_empty and len(tc_with) < tc_with_n:tc_with.append(ex)- elif is_defer and len(tc_without) < tc_without_n:+ elif is_empty and len(tc_without) < tc_without_n:tc_without.append(ex)tc_eval_pairs = tc_with + tc_without@@ -831,8 +831,8 @@ def train(args):unified_samples = []for ex, pred in zip(display_pairs, display_preds):ref = ex["answers"].strip()- if ref in ("", "[]"):- ref = "<defer>"+ if ref == "":+ ref = "[]"unified_samples.append({"query": ex["query"],"tools": ex["tools"],@@ -854,23 +854,23 @@ def train(args):for ex, pred_text in zip(tc_eval_pairs, tc_preds):ref_text = ex["answers"].strip()pred_text = pred_text.strip()- ref_is_defer = ref_text in ("", "[]")- pred_is_defer = pred_text in ("<defer>", "[]", "")+ ref_is_empty = ref_text in ("", "[]")+ pred_is_empty = pred_text in ("[]", "")try:- ref_calls = _json_mod.loads(ref_text) if not ref_is_defer else []+ ref_calls = _json_mod.loads(ref_text) if not ref_is_empty else []except (ValueError, TypeError):ref_calls = []try:- pred_calls = _json_mod.loads(pred_text) if not pred_is_defer else []+ pred_calls = _json_mod.loads(pred_text) if not pred_is_empty else []if not isinstance(pred_calls, list):pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []except (ValueError, TypeError):tc_parse_err += 1pred_calls = []tc_n += 1- if ref_is_defer and pred_is_defer:+ if ref_is_empty and pred_is_empty:tc_exact += 1- elif not ref_is_defer and not pred_is_defer and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):+ elif not ref_is_empty and not pred_is_empty and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):tc_exact += 1ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}@@ -1001,7 +1001,7 @@ def train(args):if len(s["tools"]) > 120:tools_short += "..."print(f" Tools: {tools_short}")- print(f" Ref: {s['ref'][:200] or '<defer>'}")+ print(f" Ref: {s['ref'][:200]}")print(f" Text: {s['text'][:200] or '(empty)'}")if j < len(unified_samples) - 1:print()