Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku1a6b1a703877
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex 3b77207..e4fb8f9 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -276,6 +276,39 @@ def _validate_and_format(synth_id, language, query, tools, answer, model, source}+def load_queries_from_gcs(max_samples=None):+ """Pull raw_data from GCS bucket and extract queries for synthesis."""+ sys.path.insert(0, str(_PROJECT_ROOT))+ from src.gcs import download_directory++ local_dir = _PARQUET_CACHE_DIR / "raw_data"+ if not (local_dir / "dataset_info.json").exists():+ ok = download_directory("raw_data", str(local_dir))+ if not ok:+ logger.warning("No raw_data found in GCS, skipping")+ return []++ from datasets import load_from_disk+ ds = load_from_disk(str(local_dir))+ logger.info(f"Loaded {len(ds):,} raw_data queries from GCS")++ samples = []+ for i, ex in enumerate(ds):+ synth_id = f"gcs-{i}"+ query = ex.get("query", "")+ if not query or len(query.strip()) < 10:+ continue+ samples.append((synth_id, "en", query))++ rng = random.Random(43)+ rng.shuffle(samples)+ if max_samples:+ samples = samples[:max_samples]++ logger.info(f" Selected {len(samples):,} GCS queries")+ return samples++def _pick_model(models, rng):"""Randomly select a model from the pool."""return rng.choice(models)@@ -428,6 +461,19 @@ def main():batch_idx = 0since_last_upload = [0]+ # ── GCS raw_data queries ──+ gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)+ if gcs_samples:+ gen, fail = _process_batch(+ gcs_samples, "GCS raw_data", batch_idx, api_key, models,+ args, tqdm, source_tag="synth-gcs",+ since_last_upload=since_last_upload,+ )+ total_generated += gen+ total_failed += fail+ batch_idx += 1++ # ── PleIAs/SYNTH parquet shards ──hf_token = os.environ.get("HF_TOKEN")shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)diff --git a/src/data.py b/src/data.pyindex 9c40fd2..6ab4c9d 100644--- a/src/data.py+++ b/src/data.py@@ -554,14 +554,15 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_n = len(ds)- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools,+ # Track which samples have empty answers (no tool calls to make)+ empty_ans_mask = []+ for a in ans_texts:+ stripped = a.strip()+ empty_ans_mask.append(stripped in ("", "[]"))++ def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools, is_empty_ans,enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):"""Fill arrays at position j for one sample. Returns False if skipped."""- al = len(a_tok)- # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1- if 2 + al + 1 > max_dec_len:- return False-# Encoder: [query..., <tools>, tools...] truncated to max_enc_len# For empty tools, use <defer> token instead of tokenized "[]"if is_empty_tools:@@ -578,23 +579,37 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_el = len(enc_seq)enc_arr[j, :el] = enc_seq- # Decoder input: [EOS, <tool_call>, answer...]- dec_in_arr[j, 0] = eos_id- dec_in_arr[j, 1] = tool_call_id- if al > 0:- dec_in_arr[j, 2:2 + al] = a_tok-- # Decoder target: [<tool_call>, answer..., EOS]- dec_tgt_arr[j, 0] = tool_call_id- if al > 0:- dec_tgt_arr[j, 1:1 + al] = a_tok- dec_tgt_arr[j, 1 + al] = eos_id-- # Token-level loss weighting- token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,- w_name=w_name, w_value=w_value, w_key=w_key)- lm_arr[j, 1:1 + len(token_weights)] = token_weights- lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight+ if is_empty_ans:+ # Empty answer: decoder input [EOS, <defer>], target [<defer>, EOS]+ dec_in_arr[j, 0] = eos_id+ dec_in_arr[j, 1] = defer_id+ dec_tgt_arr[j, 0] = defer_id+ dec_tgt_arr[j, 1] = eos_id+ lm_arr[j, 0] = 3.0 # High weight on <defer> prediction+ lm_arr[j, 1] = 1.0 # EOS+ else:+ al = len(a_tok)+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1+ if 2 + al + 1 > max_dec_len:+ return False++ # Decoder input: [EOS, <tool_call>, answer...]+ dec_in_arr[j, 0] = eos_id+ dec_in_arr[j, 1] = tool_call_id+ if al > 0:+ dec_in_arr[j, 2:2 + al] = a_tok++ # Decoder target: [<tool_call>, answer..., EOS]+ dec_tgt_arr[j, 0] = tool_call_id+ if al > 0:+ dec_tgt_arr[j, 1:1 + al] = a_tok+ dec_tgt_arr[j, 1 + al] = eos_id++ # Token-level loss weighting+ token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,+ w_name=w_name, w_value=w_value, w_key=w_key)+ lm_arr[j, 1:1 + len(token_weights)] = token_weights+ lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weightreturn Trueos.makedirs(CACHE_DIR, exist_ok=True)@@ -608,6 +623,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_for i in tqdm(range(n), desc="Building pairs"):if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],all_ans_tokens[i], ans_texts[i], empty_tools_mask[i],+ empty_ans_mask[i],enc_inputs, dec_inputs, dec_targets, loss_mask):skipped += 1diff --git a/src/eval.py b/src/eval.pyindex 7b30dfd..8cf17f6 100644--- a/src/eval.py+++ b/src/eval.py@@ -256,12 +256,16 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=if i < 10:samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))- if len(ref_calls) == 0:+ ref_is_defer = ref_text.strip() in ("", "[]", "<defer>")+ pred_is_defer = pred_text.strip() in ("", "[]", "<defer>")+ if ref_is_defer:empty_ref += 1- if len(pred_calls) == 0:+ if pred_is_defer:empty_pred += 1- if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):+ if ref_is_defer and pred_is_defer:+ exact_match += 1+ elif not ref_is_defer and not pred_is_defer and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):exact_match += 1ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}@@ -469,7 +473,7 @@ def main(args):print(f" Recall {tc['call_recall']:>10.3f}")print(f" F1 {tc['call_f1']:>10.3f}")print(f" ─────────────────────────────────────")- print(f" Empty ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")+ print(f" Defer ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")if tc["samples"]:print(f"\n samples:")for query, ref, pred in tc["samples"][:5]:diff --git a/src/gcs.py b/src/gcs.pyindex 9c635c7..2593121 100644--- a/src/gcs.py+++ b/src/gcs.py@@ -13,6 +13,7 @@ import subprocessBUCKET = "cactus-dataset"+_RAW_DATA_PREFIX = "raw_data"_SYNTH_DATA_PREFIX = "synth_tool_calls"_TOKENIZED_DATA_PREFIX = "tokenized_data"_TOKENIZER_PREFIX = "tokenizer"@@ -53,6 +54,16 @@ def download_directory(gcs_prefix, local_dir):# ── Convenience wrappers ──+def upload_raw_data(local_dir):+ """Upload raw tool-calling dataset to GCS."""+ upload_directory(local_dir, _RAW_DATA_PREFIX)+++def download_raw_data(local_dir):+ """Download raw tool-calling dataset from GCS. Returns True if successful."""+ return download_directory(_RAW_DATA_PREFIX, local_dir)++def download_synth_data(local_dir):"""Download synthesized tool-calling dataset from GCS. Returns True if successful."""return download_directory(_SYNTH_DATA_PREFIX, local_dir)diff --git a/src/run.py b/src/run.pyindex dbdd8f3..ce86bd0 100644--- a/src/run.py+++ b/src/run.py@@ -123,7 +123,8 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA"""Generate tool-call output.Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.- Decoder: prefilled with [EOS, <tool_call>], then greedy decode.+ Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.+ If <defer> is predicted, returns "<defer>" (task routed to cloud model)."""name_map = {}if normalize:@@ -134,19 +135,15 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MApad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id- tool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_id+ defer_id = tokenizer.defer_token_idsrc_mask = make_padding_mask(enc_input, pad_id)encoder_out, enc_mask = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")- prefix = [eos_id, tool_call_id]- prefix_len = 2-dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)- dec_buffer = dec_buffer.at[0, 1].set(tool_call_id)decode_fn = _get_decode_fn(model, max_gen_len)@@ -158,13 +155,19 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAlogits = decode_fn(params, dec_buffer, encoder_out, enc_mask)- for i in range(prefix_len - 1, max_gen_len - 1):+ for i in range(0, max_gen_len - 1):next_logits = logits[0, i]next_token = int(jnp.argmax(next_logits))if next_token == eos_id:break+ # If model predicts <defer> as first token, route to cloud+ if next_token == defer_id and i == 0:+ if stream:+ sys.stdout.write("<defer>\n")+ return "<defer>"+generated_tokens.append(next_token)dec_buffer = dec_buffer.at[0, i + 1].set(next_token)@@ -178,6 +181,9 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAsys.stdout.write("\n")result = tokenizer.decode(generated_tokens)+ # Strip leading <tool_call> token text from output if present+ if result.startswith("<tool_call>"):+ result = result[len("<tool_call>"):]if normalize and name_map:result = restore_tool_names(result, name_map)return result@@ -187,7 +193,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE"""Batch-generate tool-call outputs for multiple examples at once.Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.- Decoder: uniform prefix [EOS, <tool_call>] for all examples.+ Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.Returns a list of decoded strings, one per example."""@@ -203,7 +209,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEB = len(queries)pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id- tool_call_id = tokenizer.tool_call_token_id+ defer_id = tokenizer.defer_token_id# --- Encode: [query..., <tools>, tools...] per example, truncated ---enc_token_lists = []@@ -220,22 +226,21 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE{"params": params}, enc_input, src_mask=src_mask, method="encode")- # --- Decoder prefix: uniform [EOS, <tool_call>] (prefix_len=2) ---- prefix_len = 2+ # --- Decoder prefix: [EOS] only, model chooses <tool_call> or <defer> ---dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)dec_buffer[:, 0] = eos_id- dec_buffer[:, 1] = tool_call_iddec_buffer = jnp.array(dec_buffer)decode_fn = _get_decode_fn(model, max_gen_len)# --- Autoregressive decoding ---finished = [False] * B+ deferred = [False] * Bgen_tokens = [[] for _ in range(B)]logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)- for pos in range(prefix_len - 1, max_gen_len - 1):+ for pos in range(0, max_gen_len - 1):for i in range(B):if finished[i]:continue@@ -243,6 +248,11 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEif next_token == eos_id:finished[i] = Truecontinue+ # <defer> as first token means no tool call+ if next_token == defer_id and pos == 0:+ finished[i] = True+ deferred[i] = True+ continuegen_tokens[i].append(next_token)dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)@@ -251,7 +261,15 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DElogits = decode_fn(params, dec_buffer, encoder_out, enc_mask)- results = [tokenizer.decode(toks) for toks in gen_tokens]+ results = []+ for i in range(B):+ if deferred[i]:+ results.append("<defer>")+ else:+ text = tokenizer.decode(gen_tokens[i])+ if text.startswith("<tool_call>"):+ text = text[len("<tool_call>"):]+ results.append(text)if normalize and name_maps:results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]return resultsdiff --git a/src/train.py b/src/train.pyindex 9eefc6f..25c991e 100644--- a/src/train.py+++ b/src/train.py@@ -787,10 +787,10 @@ def train(args):breaklocal_idx = int(val_kept[k])ex = val_ds[local_idx]- is_empty = ex["answers"].strip() in ("", "[]")- if not is_empty and len(display_with) < 4:+ is_defer = ex["answers"].strip() in ("", "[]", "<defer>")+ if not is_defer and len(display_with) < 4:display_with.append(ex)- elif is_empty and len(display_without) < 1:+ elif is_defer and len(display_without) < 1:display_without.append(ex)display_pairs = display_with + display_without@@ -807,10 +807,10 @@ def train(args):local_idx = int(val_kept[k])ex = val_ds[local_idx]ref_text = ex["answers"].strip()- is_empty = ref_text in ("", "[]")- if not is_empty and len(tc_with) < tc_with_n:+ is_defer = ref_text in ("", "[]", "<defer>")+ if not is_defer and len(tc_with) < tc_with_n:tc_with.append(ex)- elif is_empty and len(tc_without) < tc_without_n:+ elif is_defer and len(tc_without) < tc_without_n:tc_without.append(ex)tc_eval_pairs = tc_with + tc_without@@ -830,10 +830,13 @@ def train(args):unified_samples = []for ex, pred in zip(display_pairs, display_preds):+ ref = ex["answers"].strip()+ if ref in ("", "[]"):+ ref = "<defer>"unified_samples.append({"query": ex["query"],"tools": ex["tools"],- "ref": ex["answers"],+ "ref": ref,"text": pred.strip(),})@@ -851,19 +854,23 @@ def train(args):for ex, pred_text in zip(tc_eval_pairs, tc_preds):ref_text = ex["answers"].strip()pred_text = pred_text.strip()+ ref_is_defer = ref_text in ("", "[]")+ pred_is_defer = pred_text in ("<defer>", "[]", "")try:- ref_calls = _json_mod.loads(ref_text)+ ref_calls = _json_mod.loads(ref_text) if not ref_is_defer else []except (ValueError, TypeError):ref_calls = []try:- pred_calls = _json_mod.loads(pred_text)+ pred_calls = _json_mod.loads(pred_text) if not pred_is_defer else []if not isinstance(pred_calls, list):pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []except (ValueError, TypeError):tc_parse_err += 1pred_calls = []tc_n += 1- if _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):+ if ref_is_defer and pred_is_defer:+ tc_exact += 1+ elif not ref_is_defer and not pred_is_defer and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):tc_exact += 1ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}@@ -994,7 +1001,7 @@ def train(args):if len(s["tools"]) > 120:tools_short += "..."print(f" Tools: {tools_short}")- print(f" Ref: {s['ref'][:200] or '[]'}")+ print(f" Ref: {s['ref'][:200] or '<defer>'}")print(f" Text: {s['text'][:200] or '(empty)'}")if j < len(unified_samples) - 1:print()