Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakue129015be121
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
index 96d986d..0b2a291 100644
--- a/scripts/build_dataset.py
+++ b/scripts/build_dataset.py
@@ -72,6 +72,13 @@ def main():
combined.save_to_disk(args.output)
logger.info(f"Saved {len(combined)} rows to {args.output}/")
+ # Persist to GCS
+ try:
+ from src.gcs import upload_raw_data
+ upload_raw_data(args.output)
+ except Exception as e:
+ logger.warning(f"GCS upload failed (non-fatal): {e}")
+
print_summary(combined)
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
new file mode 100644
index 0000000..6fcf697
--- /dev/null
+++ b/scripts/synthesize_tools_data.py
@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""Synthesize tool-calling data from PleIAs/SYNTH queries via OpenRouter.
+
+Downloads the first parquet shard of PleIAs/SYNTH (~1M diverse multilingual
+queries), then prompts an LLM to generate (tools, answer) pairs for each query.
+
+Usage:
+ python scripts/synthesize_tools_data.py
+ python scripts/synthesize_tools_data.py --max-samples 10000 --model google/gemini-2.5-flash
+ python scripts/synthesize_tools_data.py --resume # continue from last checkpoint
+"""
+
+import argparse
+import json
+import logging
+import os
+import random
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import requests
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+_OUTPUT_DIR = _PROJECT_ROOT / "data" / "synth_tool_calls"
+_CHECKPOINT_PATH = _OUTPUT_DIR / "checkpoint.jsonl"
+_PARQUET_CACHE = _PROJECT_ROOT / ".data_cache" / "synth_shard0.parquet"
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+DEFAULT_MODEL = "google/gemini-2.5-flash"
+
+SYSTEM_PROMPT = """\
+You are a synthetic data generator for training a tool-calling AI model.
+
+Given a user query, you must invent a realistic set of tools (APIs/functions) that \
+could help answer it, then produce the correct tool call(s).
+
+Rules:
+1. Invent 3-8 tools as JSON. Each tool has: name, description, parameters (dict of param_name -> {type, description}). \
+Only ONE or TWO tools should be relevant; the rest are distractors.
+2. Tool names should be snake_case, realistic API-style (e.g. search_web, get_weather, translate_text).
+3. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].
+4. If the query genuinely needs no tool, return an empty list [].
+5. Argument values MUST be grounded in the query — do not hallucinate values.
+6. Respond in the SAME LANGUAGE as the query.
+7. Return ONLY valid JSON, no markdown fences, no commentary.
+
+Output format (strict JSON object):
+{"tools": [...], "answer": [...]}"""
+
+USER_TEMPLATE = "Query: {query}"
+
+
+def download_first_shard(token):
+ """Download the first parquet shard of PleIAs/SYNTH."""
+ if _PARQUET_CACHE.exists():
+ logger.info(f"Using cached shard: {_PARQUET_CACHE}")
+ return _PARQUET_CACHE
+
+ _PARQUET_CACHE.parent.mkdir(parents=True, exist_ok=True)
+
+ url = (
+ "https://huggingface.co/datasets/PleIAs/SYNTH"
+ "/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet"
+ )
+ headers = {}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+
+ logger.info(f"Downloading first shard from PleIAs/SYNTH...")
+ resp = requests.get(url, headers=headers, stream=True, timeout=120)
+ resp.raise_for_status()
+
+ total = int(resp.headers.get("content-length", 0))
+ downloaded = 0
+ with open(_PARQUET_CACHE, "wb") as f:
+ for chunk in resp.iter_content(chunk_size=8 * 1024 * 1024):
+ f.write(chunk)
+ downloaded += len(chunk)
+ if total:
+ pct = downloaded / total * 100
+ print(f"\r {downloaded / 1e6:.0f}/{total / 1e6:.0f} MB ({pct:.0f}%)", end="", flush=True)
+ print()
+ logger.info(f"Saved to {_PARQUET_CACHE}")
+ return _PARQUET_CACHE
+
+
+def load_queries(path, max_samples=None):
+ """Load queries from parquet, return list of (synth_id, language, query)."""
+ import pyarrow.parquet as pq
+
+ logger.info(f"Reading parquet: {path}")
+ table = pq.read_table(path, columns=["synth_id", "language", "query"])
+ n = table.num_rows
+ logger.info(f" {n:,} rows in shard")
+
+ ids = table.column("synth_id").to_pylist()
+ langs = table.column("language").to_pylist()
+ queries = table.column("query").to_pylist()
+
+ samples = list(zip(ids, langs, queries))
+
+ # Shuffle for diversity (don't process in order)
+ rng = random.Random(42)
+ rng.shuffle(samples)
+
+ if max_samples:
+ samples = samples[:max_samples]
+
+ logger.info(f" Selected {len(samples):,} samples")
+ return samples
+
+
+def load_checkpoint():
+ """Load completed synth_ids from checkpoint file."""
+ done = set()
+ if _CHECKPOINT_PATH.exists():
+ with open(_CHECKPOINT_PATH) as f:
+ for line in f:
+ try:
+ row = json.loads(line)
+ done.add(row["synth_id"])
+ except (json.JSONDecodeError, KeyError):
+ continue
+ logger.info(f"Resuming: {len(done):,} already completed")
+ return done
+
+
+def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):
+ """Call OpenRouter API to generate tools + answer for a query."""
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json",
+ }
+ payload = {
+ "model": model,
+ "messages": [
+ {"role": "system", "content": SYSTEM_PROMPT},
+ {"role": "user", "content": USER_TEMPLATE.format(query=query)},
+ ],
+ "temperature": 0.7,
+ "max_tokens": 2048,
+ }
+
+ for attempt in range(max_retries):
+ try:
+ resp = requests.post(
+ OPENROUTER_URL, headers=headers, json=payload, timeout=60
+ )
+ if resp.status_code == 429:
+ wait = min(2 ** attempt * 2, 30)
+ time.sleep(wait)
+ continue
+ resp.raise_for_status()
+ content = resp.json()["choices"][0]["message"]["content"]
+ return _parse_response(content)
+ except (requests.RequestException, KeyError, IndexError) as e:
+ if attempt == max_retries - 1:
+ return None
+ time.sleep(2 ** attempt)
+ return None
+
+
+def _parse_response(content):
+ """Parse LLM response into (tools, answer) or None."""
+ content = content.strip()
+ # Strip markdown fences if present
+ if content.startswith("```"):
+ content = content.split("\n", 1)[-1]
+ if content.endswith("```"):
+ content = content[:-3]
+ content = content.strip()
+
+ try:
+ obj = json.loads(content)
+ except json.JSONDecodeError:
+ return None
+
+ if not isinstance(obj, dict):
+ return None
+
+ tools = obj.get("tools")
+ answer = obj.get("answer")
+
+ if not isinstance(tools, list):
+ return None
+ if not isinstance(answer, list):
+ return None
+
+ # Validate tool structure
+ for t in tools:
+ if not isinstance(t, dict) or "name" not in t:
+ return None
+
+ # Validate answer structure
+ for a in answer:
+ if not isinstance(a, dict) or "name" not in a:
+ return None
+ if "arguments" not in a:
+ a["arguments"] = {}
+
+ return tools, answer
+
+
+def _validate_and_format(synth_id, language, query, tools, answer):
+ """Validate the generated data and return a formatted row or None."""
+ # Check that every called tool exists in the tool list
+ tool_names = {t["name"] for t in tools}
+ for a in answer:
+ if a["name"] not in tool_names:
+ return None
+
+ # Check argument keys are valid
+ tool_params = {}
+ for t in tools:
+ params = t.get("parameters") or {}
+ tool_params[t["name"]] = set(params.keys())
+
+ for a in answer:
+ args = a.get("arguments", {})
+ if not isinstance(args, dict):
+ return None
+ schema_params = tool_params.get(a["name"], set())
+ if args and not set(args.keys()).issubset(schema_params):
+ return None
+
+ return {
+ "synth_id": synth_id,
+ "language": language,
+ "query": query,
+ "answers": json.dumps(answer),
+ "tools": json.dumps(tools),
+ "source": "synth-pleias",
+ }
+
+
+def process_sample(sample, api_key, model):
+ """Process a single sample: call LLM, validate, return row or None."""
+ synth_id, language, query = sample
+ result = call_openrouter(query, api_key, model=model)
+ if result is None:
+ return None
+ tools, answer = result
+ return _validate_and_format(synth_id, language, query, tools, answer)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")
+ parser.add_argument("--max-samples", type=int, default=None,
+ help="Max queries to process (default: all in shard)")
+ parser.add_argument("--model", type=str, default=DEFAULT_MODEL,
+ help=f"OpenRouter model (default: {DEFAULT_MODEL})")
+ parser.add_argument("--workers", type=int, default=32,
+ help="Concurrent API requests (default: 32)")
+ parser.add_argument("--resume", action="store_true",
+ help="Resume from checkpoint")
+ parser.add_argument("--batch-size", type=int, default=500,
+ help="Samples between progress logs (default: 500)")
+ args = parser.parse_args()
+
+ api_key = os.environ.get("OPENROUTER_API_KEY")
+ if not api_key:
+ print("Error: set OPENROUTER_API_KEY environment variable")
+ sys.exit(1)
+
+ hf_token = os.environ.get("HF_TOKEN")
+ parquet_path = download_first_shard(hf_token)
+ samples = load_queries(parquet_path, max_samples=args.max_samples)
+
+ _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+ done_ids = load_checkpoint() if args.resume else set()
+ if not args.resume and _CHECKPOINT_PATH.exists():
+ _CHECKPOINT_PATH.unlink()
+
+ pending = [s for s in samples if s[0] not in done_ids]
+ logger.info(f"Processing {len(pending):,} samples ({len(done_ids):,} already done)")
+
+ generated = 0
+ failed = 0
+ out_file = open(_CHECKPOINT_PATH, "a")
+
+ try:
+ with ThreadPoolExecutor(max_workers=args.workers) as executor:
+ futures = {}
+ for sample in pending:
+ fut = executor.submit(process_sample, sample, api_key, args.model)
+ futures[fut] = sample[0] # synth_id
+
+ for i, fut in enumerate(as_completed(futures), 1):
+ try:
+ row = fut.result()
+ except Exception:
+ failed += 1
+ row = None
+
+ if row is not None:
+ out_file.write(json.dumps(row) + "\n")
+ generated += 1
+ else:
+ failed += 1
+
+ if i % args.batch_size == 0:
+ out_file.flush()
+ total = generated + failed
+ rate = generated / max(total, 1) * 100
+ logger.info(
+ f" [{total:,}/{len(pending):,}] "
+ f"generated={generated:,} failed={failed:,} "
+ f"({rate:.0f}% success)"
+ )
+ except KeyboardInterrupt:
+ logger.info("\nInterrupted — progress saved to checkpoint")
+ finally:
+ out_file.flush()
+ out_file.close()
+
+ logger.info(f"\nDone: {generated:,} generated, {failed:,} failed")
+ logger.info(f"Output: {_CHECKPOINT_PATH}")
+
+ # Convert checkpoint to HF dataset format
+ if generated > 0:
+ _export_dataset()
+
+
+def _export_dataset():
+ """Convert checkpoint JSONL to the unified format expected by build_dataset.py."""
+ from datasets import Dataset
+
+ rows = []
+ with open(_CHECKPOINT_PATH) as f:
+ for line in f:
+ try:
+ row = json.loads(line)
+ rows.append({
+ "query": row["query"],
+ "answers": row["answers"],
+ "tools": row["tools"],
+ "source": row.get("source", "synth-pleias"),
+ })
+ except (json.JSONDecodeError, KeyError):
+ continue
+
+ if not rows:
+ return
+
+ ds = Dataset.from_list(rows)
+ out_path = _OUTPUT_DIR / "dataset"
+ ds.save_to_disk(str(out_path))
+ logger.info(f"Exported {len(rows):,} examples to {out_path}/")
+ logger.info(
+ "To include in training, add to tools_data.py or merge with unified dataset."
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/data.py b/src/data.py
index 40bba44..a25eca0 100644
--- a/src/data.py
+++ b/src/data.py
@@ -183,7 +183,6 @@ def _token_weights_for_answer(answer_str, token_ids, sp_model,
v_str = _json.dumps(v) if not isinstance(v, str) else v
_mark_json_value(answer_str, char_w, k, v_str, w_value)
- # Map char weights to token weights via SentencePiece pieces
pieces = sp_model.Encode(answer_str, out_type=str)
pos = 0
for i, piece in enumerate(pieces):
@@ -212,7 +211,6 @@ def _load_unified_dataset():
if _unified_dataset_cache is not None:
return _unified_dataset_cache
- # Try each candidate path: shm first, then disk
for path in [_SHM_UNIFIED_DIR, _DISK_UNIFIED_DIR]:
if os.path.exists(path) and any(
f.endswith(".arrow") for f in os.listdir(path)
@@ -226,6 +224,17 @@ def _load_unified_dataset():
print(f"Warning: failed to load from {path}: {e}")
continue
+ try:
+ from .gcs import download_raw_data
+ target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR
+ if download_raw_data(target):
+ ds = load_from_disk(target)
+ print(f"Loaded unified dataset from GCS -> {target} ({len(ds)} rows)")
+ _unified_dataset_cache = _set_audio_backend(ds)
+ return _unified_dataset_cache
+ except Exception as e:
+ print(f"Warning: GCS download failed: {e}")
+
raise FileNotFoundError(
f"Unified dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "
f"Run 'python scripts/build_dataset.py' first."
@@ -359,6 +368,12 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
+ if not os.path.exists(model_path):
+ try:
+ from .gcs import download_tokenizer
+ download_tokenizer(TOKENIZER_DIR)
+ except Exception:
+ pass
if not os.path.exists(model_path):
train_tokenizer(max_samples=max_samples)
return NeedleTokenizer(model_path)
@@ -773,6 +788,13 @@ def load_prepared_data(split, mmap=False):
kept_indices.
"""
meta = _load_cache_metadata(split)
+ if meta is None:
+ try:
+ from .gcs import download_tokenized_data
+ download_tokenized_data(CACHE_DIR)
+ meta = _load_cache_metadata(split)
+ except Exception:
+ pass
if meta is None:
raise FileNotFoundError(
f"No prepared data found for split '{split}'. Run 'needle tokenize' first."
@@ -781,7 +803,6 @@ def load_prepared_data(split, mmap=False):
text_cache_id = meta["text_cache_id"]
cache_path = os.path.join(CACHE_DIR, text_cache_id)
- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
if not os.path.exists(cache_path + "_enc.npy"):
raise FileNotFoundError(
f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
diff --git a/src/gcs.py b/src/gcs.py
new file mode 100644
index 0000000..82164ff
--- /dev/null
+++ b/src/gcs.py
@@ -0,0 +1,100 @@
+"""GCS sync utilities for persisting datasets to gs://cactus-dataset.
+
+Uploads and downloads three artifact types:
+ - raw_data/ unified HuggingFace dataset (.arrow files)
+ - tokenized_data/ pre-tokenized .npy arrays + metadata JSONs
+ - tokenizer/ SentencePiece model + vocab
+"""
+
+import os
+
+import gcsfs
+
+BUCKET = "cactus-dataset"
+
+_RAW_DATA_PREFIX = "raw_data"
+_TOKENIZED_DATA_PREFIX = "tokenized_data"
+_TOKENIZER_PREFIX = "tokenizer"
+
+
+def _get_fs():
+ return gcsfs.GCSFileSystem()
+
+
+def upload_directory(local_dir, gcs_prefix):
+ """Upload all files in local_dir to gs://BUCKET/gcs_prefix/."""
+ if not os.path.isdir(local_dir):
+ print(f"[GCS] Skipping upload: {local_dir} does not exist")
+ return
+ fs = _get_fs()
+ dest = f"{BUCKET}/{gcs_prefix}"
+ count = 0
+ for root, _, files in os.walk(local_dir):
+ for fname in files:
+ local_path = os.path.join(root, fname)
+ rel = os.path.relpath(local_path, local_dir)
+ remote_path = f"{dest}/{rel}"
+ fs.put(local_path, remote_path)
+ count += 1
+ print(f"[GCS] Uploaded {count} files to gs://{dest}/")
+
+
+def download_directory(gcs_prefix, local_dir):
+ """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if files were downloaded."""
+ fs = _get_fs()
+ src = f"{BUCKET}/{gcs_prefix}"
+ try:
+ remote_files = fs.ls(src, detail=False)
+ except FileNotFoundError:
+ return False
+ if not remote_files:
+ return False
+ os.makedirs(local_dir, exist_ok=True)
+ count = 0
+ for remote_path in fs.find(src):
+ rel = os.path.relpath(remote_path, src)
+ local_path = os.path.join(local_dir, rel)
+ os.makedirs(os.path.dirname(local_path), exist_ok=True)
+ fs.get(remote_path, local_path)
+ count += 1
+ print(f"[GCS] Downloaded {count} files from gs://{src}/ to {local_dir}/")
+ return count > 0
+
+
+# ── Convenience wrappers ──
+
+
+def upload_raw_data(local_dir):
+ """Upload unified dataset to GCS."""
+ print("[GCS] Uploading raw dataset...")
+ upload_directory(local_dir, _RAW_DATA_PREFIX)
+
+
+def download_raw_data(local_dir):
+ """Download unified dataset from GCS. Returns True if successful."""
+ print("[GCS] Downloading raw dataset...")
+ return download_directory(_RAW_DATA_PREFIX, local_dir)
+
+
+def upload_tokenized_data(cache_dir):
+ """Upload tokenized .npy files and metadata to GCS."""
+ print("[GCS] Uploading tokenized data...")
+ upload_directory(cache_dir, _TOKENIZED_DATA_PREFIX)
+
+
+def download_tokenized_data(cache_dir):
+ """Download tokenized data from GCS. Returns True if successful."""
+ print("[GCS] Downloading tokenized data...")
+ return download_directory(_TOKENIZED_DATA_PREFIX, cache_dir)
+
+
+def upload_tokenizer(tokenizer_dir):
+ """Upload tokenizer model and vocab to GCS."""
+ print("[GCS] Uploading tokenizer...")
+ upload_directory(tokenizer_dir, _TOKENIZER_PREFIX)
+
+
+def download_tokenizer(tokenizer_dir):
+ """Download tokenizer from GCS. Returns True if successful."""
+ print("[GCS] Downloading tokenizer...")
+ return download_directory(_TOKENIZER_PREFIX, tokenizer_dir)
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index a1d17cf..cf66bc0 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -23,6 +23,7 @@ from .data import (
prepare_tool_call_pairs,
train_tokenizer,
)
+from .gcs import upload_tokenized_data, upload_tokenizer
def _clear_local_caches():
@@ -67,4 +68,10 @@ def tokenize(args):
_save_cache_metadata(split, text_cache_id, len(kept_indices),
max_enc_len, max_dec_len)
+ try:
+ upload_tokenizer(TOKENIZER_DIR)
+ upload_tokenized_data(CACHE_DIR)
+ except Exception as e:
+ print(f"[GCS] Upload failed (non-fatal): {e}")
+
print("\n=== Tokenization pipeline complete ===")