Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku1a6b1a703877
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
index 3b77207..e4fb8f9 100644
--- a/scripts/synthesize_tools_data.py
+++ b/scripts/synthesize_tools_data.py
@@ -276,6 +276,39 @@ def _validate_and_format(synth_id, language, query, tools, answer, model, source
}
+def load_queries_from_gcs(max_samples=None):
+ """Pull raw_data from GCS bucket and extract queries for synthesis."""
+ sys.path.insert(0, str(_PROJECT_ROOT))
+ from src.gcs import download_directory
+
+ local_dir = _PARQUET_CACHE_DIR / "raw_data"
+ if not (local_dir / "dataset_info.json").exists():
+ ok = download_directory("raw_data", str(local_dir))
+ if not ok:
+ logger.warning("No raw_data found in GCS, skipping")
+ return []
+
+ from datasets import load_from_disk
+ ds = load_from_disk(str(local_dir))
+ logger.info(f"Loaded {len(ds):,} raw_data queries from GCS")
+
+ samples = []
+ for i, ex in enumerate(ds):
+ synth_id = f"gcs-{i}"
+ query = ex.get("query", "")
+ if not query or len(query.strip()) < 10:
+ continue
+ samples.append((synth_id, "en", query))
+
+ rng = random.Random(43)
+ rng.shuffle(samples)
+ if max_samples:
+ samples = samples[:max_samples]
+
+ logger.info(f" Selected {len(samples):,} GCS queries")
+ return samples
+
+
def _pick_model(models, rng):
"""Randomly select a model from the pool."""
return rng.choice(models)
@@ -428,6 +461,19 @@ def main():
batch_idx = 0
since_last_upload = [0]
+ # ── GCS raw_data queries ──
+ gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)
+ if gcs_samples:
+ gen, fail = _process_batch(
+ gcs_samples, "GCS raw_data", batch_idx, api_key, models,
+ args, tqdm, source_tag="synth-gcs",
+ since_last_upload=since_last_upload,
+ )
+ total_generated += gen
+ total_failed += fail
+ batch_idx += 1
+
+ # ── PleIAs/SYNTH parquet shards ──
hf_token = os.environ.get("HF_TOKEN")
shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)
diff --git a/src/data.py b/src/data.py
index 9c40fd2..6ab4c9d 100644
--- a/src/data.py
+++ b/src/data.py
@@ -554,14 +554,15 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
n = len(ds)
- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools,
+ # Track which samples have empty answers (no tool calls to make)
+ empty_ans_mask = []
+ for a in ans_texts:
+ stripped = a.strip()
+ empty_ans_mask.append(stripped in ("", "[]"))
+
+ def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools, is_empty_ans,
enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):
"""Fill arrays at position j for one sample. Returns False if skipped."""
- al = len(a_tok)
- # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
- if 2 + al + 1 > max_dec_len:
- return False
-
# Encoder: [query..., <tools>, tools...] truncated to max_enc_len
# For empty tools, use <defer> token instead of tokenized "[]"
if is_empty_tools:
@@ -578,23 +579,37 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
el = len(enc_seq)
enc_arr[j, :el] = enc_seq
- # Decoder input: [EOS, <tool_call>, answer...]
- dec_in_arr[j, 0] = eos_id
- dec_in_arr[j, 1] = tool_call_id
- if al > 0:
- dec_in_arr[j, 2:2 + al] = a_tok
-
- # Decoder target: [<tool_call>, answer..., EOS]
- dec_tgt_arr[j, 0] = tool_call_id
- if al > 0:
- dec_tgt_arr[j, 1:1 + al] = a_tok
- dec_tgt_arr[j, 1 + al] = eos_id
-
- # Token-level loss weighting
- token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,
- w_name=w_name, w_value=w_value, w_key=w_key)
- lm_arr[j, 1:1 + len(token_weights)] = token_weights
- lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight
+ if is_empty_ans:
+ # Empty answer: decoder input [EOS, <defer>], target [<defer>, EOS]
+ dec_in_arr[j, 0] = eos_id
+ dec_in_arr[j, 1] = defer_id
+ dec_tgt_arr[j, 0] = defer_id
+ dec_tgt_arr[j, 1] = eos_id
+ lm_arr[j, 0] = 3.0 # High weight on <defer> prediction
+ lm_arr[j, 1] = 1.0 # EOS
+ else:
+ al = len(a_tok)
+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
+ if 2 + al + 1 > max_dec_len:
+ return False
+
+ # Decoder input: [EOS, <tool_call>, answer...]
+ dec_in_arr[j, 0] = eos_id
+ dec_in_arr[j, 1] = tool_call_id
+ if al > 0:
+ dec_in_arr[j, 2:2 + al] = a_tok
+
+ # Decoder target: [<tool_call>, answer..., EOS]
+ dec_tgt_arr[j, 0] = tool_call_id
+ if al > 0:
+ dec_tgt_arr[j, 1:1 + al] = a_tok
+ dec_tgt_arr[j, 1 + al] = eos_id
+
+ # Token-level loss weighting
+ token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,
+ w_name=w_name, w_value=w_value, w_key=w_key)
+ lm_arr[j, 1:1 + len(token_weights)] = token_weights
+ lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight
return True
os.makedirs(CACHE_DIR, exist_ok=True)
@@ -608,6 +623,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
for i in tqdm(range(n), desc="Building pairs"):
if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],
all_ans_tokens[i], ans_texts[i], empty_tools_mask[i],
+ empty_ans_mask[i],
enc_inputs, dec_inputs, dec_targets, loss_mask):
skipped += 1
diff --git a/src/eval.py b/src/eval.py
index 7b30dfd..8cf17f6 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -256,12 +256,16 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
if i < 10:
samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
- if len(ref_calls) == 0:
+ ref_is_defer = ref_text.strip() in ("", "[]", "<defer>")
+ pred_is_defer = pred_text.strip() in ("", "[]", "<defer>")
+ if ref_is_defer:
empty_ref += 1
- if len(pred_calls) == 0:
+ if pred_is_defer:
empty_pred += 1
- if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
+ if ref_is_defer and pred_is_defer:
+ exact_match += 1
+ elif not ref_is_defer and not pred_is_defer and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
exact_match += 1
ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
@@ -469,7 +473,7 @@ def main(args):
print(f" Recall {tc['call_recall']:>10.3f}")
print(f" F1 {tc['call_f1']:>10.3f}")
print(f" ─────────────────────────────────────")
- print(f" Empty ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")
+ print(f" Defer ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")
if tc["samples"]:
print(f"\n samples:")
for query, ref, pred in tc["samples"][:5]:
diff --git a/src/gcs.py b/src/gcs.py
index 9c635c7..2593121 100644
--- a/src/gcs.py
+++ b/src/gcs.py
@@ -13,6 +13,7 @@ import subprocess
BUCKET = "cactus-dataset"
+_RAW_DATA_PREFIX = "raw_data"
_SYNTH_DATA_PREFIX = "synth_tool_calls"
_TOKENIZED_DATA_PREFIX = "tokenized_data"
_TOKENIZER_PREFIX = "tokenizer"
@@ -53,6 +54,16 @@ def download_directory(gcs_prefix, local_dir):
# ── Convenience wrappers ──
+def upload_raw_data(local_dir):
+ """Upload raw tool-calling dataset to GCS."""
+ upload_directory(local_dir, _RAW_DATA_PREFIX)
+
+
+def download_raw_data(local_dir):
+ """Download raw tool-calling dataset from GCS. Returns True if successful."""
+ return download_directory(_RAW_DATA_PREFIX, local_dir)
+
+
def download_synth_data(local_dir):
"""Download synthesized tool-calling dataset from GCS. Returns True if successful."""
return download_directory(_SYNTH_DATA_PREFIX, local_dir)
diff --git a/src/run.py b/src/run.py
index dbdd8f3..ce86bd0 100644
--- a/src/run.py
+++ b/src/run.py
@@ -123,7 +123,8 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
"""Generate tool-call output.
Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
- Decoder: prefilled with [EOS, <tool_call>], then greedy decode.
+ Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.
+ If <defer> is predicted, returns "<defer>" (task routed to cloud model).
"""
name_map = {}
if normalize:
@@ -134,19 +135,15 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
- tool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_id
+ defer_id = tokenizer.defer_token_id
src_mask = make_padding_mask(enc_input, pad_id)
encoder_out, enc_mask = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- prefix = [eos_id, tool_call_id]
- prefix_len = 2
-
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
- dec_buffer = dec_buffer.at[0, 1].set(tool_call_id)
decode_fn = _get_decode_fn(model, max_gen_len)
@@ -158,13 +155,19 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
- for i in range(prefix_len - 1, max_gen_len - 1):
+ for i in range(0, max_gen_len - 1):
next_logits = logits[0, i]
next_token = int(jnp.argmax(next_logits))
if next_token == eos_id:
break
+ # If model predicts <defer> as first token, route to cloud
+ if next_token == defer_id and i == 0:
+ if stream:
+ sys.stdout.write("<defer>\n")
+ return "<defer>"
+
generated_tokens.append(next_token)
dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
@@ -178,6 +181,9 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
sys.stdout.write("\n")
result = tokenizer.decode(generated_tokens)
+ # Strip leading <tool_call> token text from output if present
+ if result.startswith("<tool_call>"):
+ result = result[len("<tool_call>"):]
if normalize and name_map:
result = restore_tool_names(result, name_map)
return result
@@ -187,7 +193,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
"""Batch-generate tool-call outputs for multiple examples at once.
Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
- Decoder: uniform prefix [EOS, <tool_call>] for all examples.
+ Decoder: prefilled with [EOS], model predicts <tool_call> or <defer> first.
Returns a list of decoded strings, one per example.
"""
@@ -203,7 +209,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
B = len(queries)
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
- tool_call_id = tokenizer.tool_call_token_id
+ defer_id = tokenizer.defer_token_id
# --- Encode: [query..., <tools>, tools...] per example, truncated ---
enc_token_lists = []
@@ -220,22 +226,21 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- # --- Decoder prefix: uniform [EOS, <tool_call>] (prefix_len=2) ---
- prefix_len = 2
+ # --- Decoder prefix: [EOS] only, model chooses <tool_call> or <defer> ---
dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)
dec_buffer[:, 0] = eos_id
- dec_buffer[:, 1] = tool_call_id
dec_buffer = jnp.array(dec_buffer)
decode_fn = _get_decode_fn(model, max_gen_len)
# --- Autoregressive decoding ---
finished = [False] * B
+ deferred = [False] * B
gen_tokens = [[] for _ in range(B)]
logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
- for pos in range(prefix_len - 1, max_gen_len - 1):
+ for pos in range(0, max_gen_len - 1):
for i in range(B):
if finished[i]:
continue
@@ -243,6 +248,11 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
if next_token == eos_id:
finished[i] = True
continue
+ # <defer> as first token means no tool call
+ if next_token == defer_id and pos == 0:
+ finished[i] = True
+ deferred[i] = True
+ continue
gen_tokens[i].append(next_token)
dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)
@@ -251,7 +261,15 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
- results = [tokenizer.decode(toks) for toks in gen_tokens]
+ results = []
+ for i in range(B):
+ if deferred[i]:
+ results.append("<defer>")
+ else:
+ text = tokenizer.decode(gen_tokens[i])
+ if text.startswith("<tool_call>"):
+ text = text[len("<tool_call>"):]
+ results.append(text)
if normalize and name_maps:
results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]
return results
diff --git a/src/train.py b/src/train.py
index 9eefc6f..25c991e 100644
--- a/src/train.py
+++ b/src/train.py
@@ -787,10 +787,10 @@ def train(args):
break
local_idx = int(val_kept[k])
ex = val_ds[local_idx]
- is_empty = ex["answers"].strip() in ("", "[]")
- if not is_empty and len(display_with) < 4:
+ is_defer = ex["answers"].strip() in ("", "[]", "<defer>")
+ if not is_defer and len(display_with) < 4:
display_with.append(ex)
- elif is_empty and len(display_without) < 1:
+ elif is_defer and len(display_without) < 1:
display_without.append(ex)
display_pairs = display_with + display_without
@@ -807,10 +807,10 @@ def train(args):
local_idx = int(val_kept[k])
ex = val_ds[local_idx]
ref_text = ex["answers"].strip()
- is_empty = ref_text in ("", "[]")
- if not is_empty and len(tc_with) < tc_with_n:
+ is_defer = ref_text in ("", "[]", "<defer>")
+ if not is_defer and len(tc_with) < tc_with_n:
tc_with.append(ex)
- elif is_empty and len(tc_without) < tc_without_n:
+ elif is_defer and len(tc_without) < tc_without_n:
tc_without.append(ex)
tc_eval_pairs = tc_with + tc_without
@@ -830,10 +830,13 @@ def train(args):
unified_samples = []
for ex, pred in zip(display_pairs, display_preds):
+ ref = ex["answers"].strip()
+ if ref in ("", "[]"):
+ ref = "<defer>"
unified_samples.append({
"query": ex["query"],
"tools": ex["tools"],
- "ref": ex["answers"],
+ "ref": ref,
"text": pred.strip(),
})
@@ -851,19 +854,23 @@ def train(args):
for ex, pred_text in zip(tc_eval_pairs, tc_preds):
ref_text = ex["answers"].strip()
pred_text = pred_text.strip()
+ ref_is_defer = ref_text in ("", "[]")
+ pred_is_defer = pred_text in ("<defer>", "[]", "")
try:
- ref_calls = _json_mod.loads(ref_text)
+ ref_calls = _json_mod.loads(ref_text) if not ref_is_defer else []
except (ValueError, TypeError):
ref_calls = []
try:
- pred_calls = _json_mod.loads(pred_text)
+ pred_calls = _json_mod.loads(pred_text) if not pred_is_defer else []
if not isinstance(pred_calls, list):
pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
except (ValueError, TypeError):
tc_parse_err += 1
pred_calls = []
tc_n += 1
- if _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):
+ if ref_is_defer and pred_is_defer:
+ tc_exact += 1
+ elif not ref_is_defer and not pred_is_defer and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True):
tc_exact += 1
ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
@@ -994,7 +1001,7 @@ def train(args):
if len(s["tools"]) > 120:
tools_short += "..."
print(f" Tools: {tools_short}")
- print(f" Ref: {s['ref'][:200] or '[]'}")
+ print(f" Ref: {s['ref'][:200] or '<defer>'}")
print(f" Text: {s['text'][:200] or '(empty)'}")
if j < len(unified_samples) - 1:
print()