Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku62b60b80daf3
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
index e4fb8f9..7c6db82 100644
--- a/scripts/synthesize_tools_data.py
+++ b/scripts/synthesize_tools_data.py
@@ -21,6 +21,8 @@ import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
+import itertools
+
import requests
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
@@ -35,9 +37,7 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
MODEL_POOL = [
"google/gemini-3.1-flash-lite-preview",
- # "anthropic/claude-opus-4.6",
- # "x-ai/grok-4.1-fast",
- # "openai/gpt-5.3-chat",
+ "google/gemini-3-flash-preview",
]
DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"
@@ -309,15 +309,24 @@ def load_queries_from_gcs(max_samples=None):
return samples
-def _pick_model(models, rng):
- """Randomly select a model from the pool."""
- return rng.choice(models)
+class _KeyModelRotator:
+ """Thread-safe round-robin across (api_key, model) pairs."""
+
+ def __init__(self, api_keys, models):
+ pairs = [(k, m) for k in api_keys for m in models]
+ self._cycle = itertools.cycle(pairs)
+ import threading
+ self._lock = threading.Lock()
+ def next(self):
+ with self._lock:
+ return next(self._cycle)
-def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):
+
+def process_sample(sample, rotator, source_tag="synth-pleias"):
"""Process a single sample: call LLM, validate, return row or None."""
synth_id, language, query = sample
- model = _pick_model(models, rng)
+ api_key, model = rotator.next()
result = call_openrouter(query, api_key, model=model)
if result is None:
return None
@@ -328,7 +337,7 @@ def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):
return row
-def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, source_tag,
+def _process_batch(samples, label, batch_idx, rotator, args, tqdm, source_tag,
since_last_upload):
"""Process a batch of samples, append to checkpoint. Returns (generated, failed)."""
logger.info(f"\n{'=' * 60}")
@@ -345,14 +354,13 @@ def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, sourc
generated = 0
failed = 0
out_file = open(_CHECKPOINT_PATH, "a")
- rng = random.Random(42 + batch_idx)
try:
with ThreadPoolExecutor(max_workers=args.workers) as executor:
futures = {}
for sample in pending:
fut = executor.submit(
- process_sample, sample, api_key, models, rng, source_tag,
+ process_sample, sample, rotator, source_tag,
)
futures[fut] = sample[0]
@@ -419,8 +427,8 @@ def main():
help=f"OpenRouter models to rotate (default: built-in pool)")
parser.add_argument("--model", type=str, default=None,
help="Use a single model instead of the pool")
- parser.add_argument("--workers", type=int, default=32,
- help="Concurrent API requests (default: 32)")
+ parser.add_argument("--workers", type=int, default=128,
+ help="Concurrent API requests (default: 128)")
parser.add_argument("--max-shards", type=int, default=None,
help="Max parquet shards to process (default: all)")
parser.add_argument("--resume", action="store_true",
@@ -431,10 +439,12 @@ def main():
help="Upload to GCS every N successful generations (default: 1000)")
args = parser.parse_args()
- api_key = os.environ.get("OPENROUTER_API_KEY")
- if not api_key:
- print("Error: set OPENROUTER_API_KEY environment variable")
+ raw_keys = os.environ.get("OPENROUTER_API_KEY", "")
+ api_keys = [k.strip() for k in raw_keys.split(",") if k.strip()]
+ if not api_keys:
+ print("Error: set OPENROUTER_API_KEY environment variable (comma-separated for multiple)")
sys.exit(1)
+ logger.info(f"API keys: {len(api_keys)}")
if args.model:
models = [args.model]
@@ -443,6 +453,9 @@ def main():
else:
models = list(MODEL_POOL)
logger.info(f"Model pool ({len(models)}): {', '.join(models)}")
+ logger.info(f"Key x Model pairs: {len(api_keys) * len(models)} (each gets its own rate limit)")
+
+ rotator = _KeyModelRotator(api_keys, models)
_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
@@ -461,11 +474,10 @@ def main():
batch_idx = 0
since_last_upload = [0]
- # ── GCS raw_data queries ──
gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)
if gcs_samples:
gen, fail = _process_batch(
- gcs_samples, "GCS raw_data", batch_idx, api_key, models,
+ gcs_samples, "GCS raw_data", batch_idx, rotator,
args, tqdm, source_tag="synth-gcs",
since_last_upload=since_last_upload,
)
@@ -473,7 +485,6 @@ def main():
total_failed += fail
batch_idx += 1
- # ── PleIAs/SYNTH parquet shards ──
hf_token = os.environ.get("HF_TOKEN")
shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)
@@ -481,7 +492,7 @@ def main():
parquet_path = download_shard(shard_url, shard_idx, hf_token)
samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)
gen, fail = _process_batch(
- samples, f"PleIAs shard {shard_idx}", batch_idx, api_key, models,
+ samples, f"PleIAs shard {shard_idx}", batch_idx, rotator,
args, tqdm, source_tag="synth-pleias",
since_last_upload=since_last_upload,
)
diff --git a/src/data.py b/src/data.py
index 8076f2e..402cc97 100644
--- a/src/data.py
+++ b/src/data.py
@@ -98,7 +98,6 @@ BOS_ID = 2
UNK_ID = 3
TOOL_CALL_ID = 4
TOOLS_ID = 5
-DEFER_ID = 6
DEFAULT_MAX_ENC_LEN = 1024
DEFAULT_MAX_DEC_LEN = 512
@@ -288,10 +287,6 @@ class NeedleTokenizer:
def tools_token_id(self):
return TOOLS_ID
- @property
- def defer_token_id(self):
- return DEFER_ID
-
@property
def vocab_size(self):
return self.sp.GetPieceSize()
@@ -418,6 +413,11 @@ def load_tool_calls(split="train", max_samples=None, return_global_indices=False
ds = ds.select(global_indices.tolist())
+ # Normalize <defer> / empty answers to []
+ ds = ds.map(lambda ex: {
+ "answers": "[]" if ex["answers"].strip() in ("", "<defer>") else ex["answers"]
+ })
+
if max_samples:
limit = min(max_samples, len(ds))
ds = ds.select(range(limit))
@@ -507,8 +507,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
tool_call_id = tokenizer.tool_call_token_id
tools_sep_id = tokenizer.tools_token_id
- defer_id = tokenizer.defer_token_id
-
enc_texts = [ex["query"] for ex in ds]
tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
@@ -520,16 +518,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
return s
ans_texts = [_compact(a) for a in ans_texts]
+ # Normalize any remaining <defer> or empty answers to []
+ ans_texts = ["[]" if a.strip() in ("", "<defer>") else a for a in ans_texts]
tools_texts = [_compact(t) for t in tools_texts]
- empty_tools_mask = []
- for t in tools_texts:
- try:
- parsed = _json.loads(t)
- empty_tools_mask.append(isinstance(parsed, list) and len(parsed) == 0)
- except (ValueError, TypeError):
- empty_tools_mask.append(False)
-
tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)
if shuffle_tools:
@@ -562,21 +554,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
n = len(ds)
- # Track which samples have empty answers (no tool calls to make)
- empty_ans_mask = []
- for a in ans_texts:
- stripped = a.strip()
- empty_ans_mask.append(stripped in ("", "[]"))
-
- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools, is_empty_ans,
+ def _fill_sample(j, e_tok, t_tok, a_tok, ans_text,
enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):
"""Fill arrays at position j for one sample. Returns False if skipped."""
# Encoder: [query..., <tools>, tools...] truncated to max_enc_len
- # For empty tools, use <defer> token instead of tokenized "[]"
- if is_empty_tools:
- t_tok = [defer_id]
-
- # Reserve 1 slot for <tools> separator + at least 1 token for tools
max_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools token
if len(e_tok) > max_query:
e_tok = e_tok[:max_query]
@@ -587,37 +568,29 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
el = len(enc_seq)
enc_arr[j, :el] = enc_seq
- if is_empty_ans:
- # Empty answer: decoder input [EOS, <defer>], target [<defer>, EOS]
- dec_in_arr[j, 0] = eos_id
- dec_in_arr[j, 1] = defer_id
- dec_tgt_arr[j, 0] = defer_id
- dec_tgt_arr[j, 1] = eos_id
- lm_arr[j, 0] = 3.0 # High weight on <defer> prediction
- lm_arr[j, 1] = 1.0 # EOS
- else:
- al = len(a_tok)
- # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
- if 2 + al + 1 > max_dec_len:
- return False
-
- # Decoder input: [EOS, <tool_call>, answer...]
- dec_in_arr[j, 0] = eos_id
- dec_in_arr[j, 1] = tool_call_id
- if al > 0:
- dec_in_arr[j, 2:2 + al] = a_tok
-
- # Decoder target: [<tool_call>, answer..., EOS]
- dec_tgt_arr[j, 0] = tool_call_id
- if al > 0:
- dec_tgt_arr[j, 1:1 + al] = a_tok
- dec_tgt_arr[j, 1 + al] = eos_id
-
- # Token-level loss weighting
- token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,
- w_name=w_name, w_value=w_value, w_key=w_key)
- lm_arr[j, 1:1 + len(token_weights)] = token_weights
- lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight
+ al = len(a_tok)
+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
+ if 2 + al + 1 > max_dec_len:
+ return False
+
+ # Decoder input: [EOS, <tool_call>, answer...]
+ dec_in_arr[j, 0] = eos_id
+ dec_in_arr[j, 1] = tool_call_id
+ if al > 0:
+ dec_in_arr[j, 2:2 + al] = a_tok
+
+ # Decoder target: [<tool_call>, answer..., EOS]
+ dec_tgt_arr[j, 0] = tool_call_id
+ if al > 0:
+ dec_tgt_arr[j, 1:1 + al] = a_tok
+ dec_tgt_arr[j, 1 + al] = eos_id
+
+ # Token-level loss weighting
+ lm_arr[j, 0] = 1.0 # <tool_call> prediction
+ token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,
+ w_name=w_name, w_value=w_value, w_key=w_key)
+ lm_arr[j, 1:1 + len(token_weights)] = token_weights
+ lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight
return True
os.makedirs(CACHE_DIR, exist_ok=True)
@@ -630,8 +603,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
skipped = 0
for i in tqdm(range(n), desc="Building pairs"):
if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],
- all_ans_tokens[i], ans_texts[i], empty_tools_mask[i],
- empty_ans_mask[i],
+ all_ans_tokens[i], ans_texts[i],
enc_inputs, dec_inputs, dec_targets, loss_mask):
skipped += 1
diff --git a/src/eval.py b/src/eval.py
index 8cf17f6..9d6d5e1 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -256,16 +256,16 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
if i < 10:
samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
- ref_is_defer = ref_text.strip() in ("", "[]", "<defer>")
- pred_is_defer = pred_text.strip() in ("", "[]", "<defer>")
- if ref_is_defer:
+ ref_is_empty = ref_text.strip() in ("", "[]")
+ pred_is_empty = pred_text.strip() in ("", "[]")
+ if ref_is_empty:
empty_ref += 1
- if pred_is_defer:
+ if pred_is_empty:
empty_pred += 1
- if ref_is_defer and pred_is_defer:
+ if ref_is_empty and pred_is_empty:
exact_match += 1
- elif not ref_is_defer and not pred_is_defer and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
+ elif not ref_is_empty and not pred_is_empty and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
exact_match += 1
ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
diff --git a/src/run.py b/src/run.py
index ce86bd0..5b3d798 100644
--- a/src/run.py
+++ b/src/run.py
@@ -91,25 +91,10 @@ def load_checkpoint(path):
def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):
- """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len.
-
- Empty tools ([]) are replaced with the <defer> token.
- """
+ """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""
tools_sep_id = tokenizer.tools_token_id
- defer_id = tokenizer.defer_token_id
q_toks = tokenizer.encode(query)
-
- # Use <defer> token for empty tools
- try:
- parsed = _json.loads(tools)
- is_empty = isinstance(parsed, list) and len(parsed) == 0
- except (ValueError, TypeError):
- is_empty = False
-
- if is_empty:
- t_toks = [defer_id]
- else:
- t_toks = tokenizer.encode(tools)
+ t_toks = tokenizer.encode(tools)
max_query = max_enc_len - 2
if len(q_toks) > max_query:
@@ -123,8 +108,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
"""Generate tool-call output.
Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
- Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.
- If <defer> is predicted, returns "<defer>" (task routed to cloud model).
+ Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens.
"""
name_map = {}
if normalize:
@@ -135,7 +119,6 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
- defer_id = tokenizer.defer_token_id
src_mask = make_padding_mask(enc_input, pad_id)
encoder_out, enc_mask = model.apply(
@@ -162,12 +145,6 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
if next_token == eos_id:
break
- # If model predicts <defer> as first token, route to cloud
- if next_token == defer_id and i == 0:
- if stream:
- sys.stdout.write("<defer>\n")
- return "<defer>"
-
generated_tokens.append(next_token)
dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
@@ -193,7 +170,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
"""Batch-generate tool-call outputs for multiple examples at once.
Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
- Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.
+ Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens.
Returns a list of decoded strings, one per example.
"""
@@ -209,7 +186,6 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
B = len(queries)
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
- defer_id = tokenizer.defer_token_id
# --- Encode: [query..., <tools>, tools...] per example, truncated ---
enc_token_lists = []
@@ -226,16 +202,13 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- # --- Decoder prefix: [EOS] only, model chooses <tool_call> or <defer> ---
dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)
dec_buffer[:, 0] = eos_id
dec_buffer = jnp.array(dec_buffer)
decode_fn = _get_decode_fn(model, max_gen_len)
- # --- Autoregressive decoding ---
finished = [False] * B
- deferred = [False] * B
gen_tokens = [[] for _ in range(B)]
logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
@@ -248,11 +221,6 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
if next_token == eos_id:
finished[i] = True
continue
- # <defer> as first token means no tool call
- if next_token == defer_id and pos == 0:
- finished[i] = True
- deferred[i] = True
- continue
gen_tokens[i].append(next_token)
dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)
@@ -263,13 +231,10 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
results = []
for i in range(B):
- if deferred[i]:
- results.append("<defer>")
- else:
- text = tokenizer.decode(gen_tokens[i])
- if text.startswith("<tool_call>"):
- text = text[len("<tool_call>"):]
- results.append(text)
+ text = tokenizer.decode(gen_tokens[i])
+ if text.startswith("<tool_call>"):
+ text = text[len("<tool_call>"):]
+ results.append(text)
if normalize and name_maps:
results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]
return results
diff --git a/src/train.py b/src/train.py
index bc98a48..078f4e1 100644
--- a/src/train.py
+++ b/src/train.py
@@ -787,10 +787,10 @@ def train(args):
break
local_idx = int(val_kept[k])
ex = val_ds[local_idx]
- is_defer = ex["answers"].strip() in ("", "[]", "<defer>")
- if not is_defer and len(display_with) < 4:
+ is_empty = ex["answers"].strip() in ("", "[]")
+ if not is_empty and len(display_with) < 4:
display_with.append(ex)
- elif is_defer and len(display_without) < 1:
+ elif is_empty and len(display_without) < 1:
display_without.append(ex)
display_pairs = display_with + display_without
@@ -807,10 +807,10 @@ def train(args):
local_idx = int(val_kept[k])
ex = val_ds[local_idx]
ref_text = ex["answers"].strip()
- is_defer = ref_text in ("", "[]", "<defer>")
- if not is_defer and len(tc_with) < tc_with_n:
+ is_empty = ref_text in ("", "[]")
+ if not is_empty and len(tc_with) < tc_with_n:
tc_with.append(ex)
- elif is_defer and len(tc_without) < tc_without_n:
+ elif is_empty and len(tc_without) < tc_without_n:
tc_without.append(ex)
tc_eval_pairs = tc_with + tc_without
@@ -831,8 +831,8 @@ def train(args):
unified_samples = []
for ex, pred in zip(display_pairs, display_preds):
ref = ex["answers"].strip()
- if ref in ("", "[]"):
- ref = "<defer>"
+ if ref == "":
+ ref = "[]"
unified_samples.append({
"query": ex["query"],
"tools": ex["tools"],
@@ -854,23 +854,23 @@ def train(args):
for ex, pred_text in zip(tc_eval_pairs, tc_preds):
ref_text = ex["answers"].strip()
pred_text = pred_text.strip()
- ref_is_defer = ref_text in ("", "[]")
- pred_is_defer = pred_text in ("<defer>", "[]", "")
+ ref_is_empty = ref_text in ("", "[]")
+ pred_is_empty = pred_text in ("[]", "")
try:
- ref_calls = _json_mod.loads(ref_text) if not ref_is_defer else []
+ ref_calls = _json_mod.loads(ref_text) if not ref_is_empty else []
except (ValueError, TypeError):
ref_calls = []
try:
- pred_calls = _json_mod.loads(pred_text) if not pred_is_defer else []
+ pred_calls = _json_mod.loads(pred_text) if not pred_is_empty else []
if not isinstance(pred_calls, list):
pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
except (ValueError, TypeError):
tc_parse_err += 1
pred_calls = []
tc_n += 1
- if ref_is_defer and pred_is_defer:
+ if ref_is_empty and pred_is_empty:
tc_exact += 1
- elif not ref_is_defer and not pred_is_defer and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):
+ elif not ref_is_empty and not pred_is_empty and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):
tc_exact += 1
ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
@@ -1001,7 +1001,7 @@ def train(args):
if len(s["tools"]) > 120:
tools_short += "..."
print(f" Tools: {tools_short}")
- print(f" Ref: {s['ref'][:200] or '<defer>'}")
+ print(f" Ref: {s['ref'][:200]}")
print(f" Text: {s['text'][:200] or '(empty)'}")
if j < len(unified_samples) - 1:
print()