Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
refactors
HenryNdubuakuafddf8b42386
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex 4b8bae3..3b77207 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -1,18 +1,13 @@#!/usr/bin/env python3-"""Synthesize tool-calling data from multiple query sources via OpenRouter.--Query sources:- 1. PleIAs/SYNTH parquet shards (~154k queries per shard, 7 shards)- 2. GCS raw_data — queries extracted from the existing unified tool-calling dataset+"""Synthesize tool-calling data from PleIAs/SYNTH via OpenRouter.Each query is sent to a randomly-selected frontier model to generate(tools, answer) pairs, then validated and checkpointed as JSONL.Usage:- python scripts/synthesize_tools_data.py # all sources+ python scripts/synthesize_tools_data.pypython scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-previewpython scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100- python scripts/synthesize_tools_data.py --no-gcs # skip GCS queriespython scripts/synthesize_tools_data.py --resume # continue from checkpoint"""@@ -152,48 +147,6 @@ def load_queries_from_parquet(path, max_samples=None):return samples-def load_queries_from_gcs(max_samples=None):- """Download raw_data from GCS and extract queries for re-synthesis.-- Returns list of (sample_id, language, query) tuples, where sample_id- is prefixed with 'gcs-' to avoid collisions with PleIAs IDs.- """- sys.path.insert(0, str(_PROJECT_ROOT))- from src.gcs import download_raw_data-- local_dir = _PARQUET_CACHE_DIR / "gcs_raw_data"- if not local_dir.exists():- logger.info("Downloading raw_data from GCS...")- ok = download_raw_data(str(local_dir))- if not ok:- logger.warning("No raw_data found in GCS, skipping")- return []- else:- logger.info(f"Using cached GCS raw_data: {local_dir}")-- from datasets import load_from_disk- ds = load_from_disk(str(local_dir))- logger.info(f"Loaded {len(ds):,} rows from GCS raw_data")-- samples = []- for i in range(len(ds)):- ex = ds[i]- query = ex.get("query", "").strip()- if not query:- continue- sample_id = f"gcs-{i}"- samples.append((sample_id, "en", query))-- rng = random.Random(42)- rng.shuffle(samples)-- if max_samples:- samples = samples[:max_samples]-- logger.info(f" Selected {len(samples):,} GCS queries")- return samples--def load_checkpoint():"""Load completed synth_ids from checkpoint file."""done = set()@@ -437,8 +390,6 @@ def main():help="Concurrent API requests (default: 32)")parser.add_argument("--max-shards", type=int, default=None,help="Max parquet shards to process (default: all)")- parser.add_argument("--no-gcs", action="store_true",- help="Skip GCS raw_data queries")parser.add_argument("--resume", action="store_true",help="Resume from checkpoint")parser.add_argument("--batch-size", type=int, default=500,@@ -475,19 +426,7 @@ def main():total_generated = 0total_failed = 0batch_idx = 0- since_last_upload = [0]-- if not args.no_gcs:- gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)- if gcs_samples:- gen, fail = _process_batch(- gcs_samples, "GCS raw_data", batch_idx, api_key, models,- args, tqdm, source_tag="synth-gcs",- since_last_upload=since_last_upload,- )- total_generated += gen- total_failed += fail- batch_idx += 1+ since_last_upload = [0]hf_token = os.environ.get("HF_TOKEN")shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)diff --git a/src/data.py b/src/data.pyindex 8934139..aaf5b5e 100644--- a/src/data.py+++ b/src/data.py@@ -39,7 +39,7 @@ _DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")-_MIN_SHM_BYTES = 200 * 1024**3+_MIN_SHM_BYTES = 20 * 1024**3def _shm_available():@@ -237,19 +237,19 @@ def _load_unified_dataset():continuetry:- from .gcs import download_raw_data+ from .gcs import download_synth_datatarget = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR- if download_raw_data(target):+ if download_synth_data(target):ds = load_from_disk(target)- print(f"Loaded unified dataset from GCS -> {target} ({len(ds)} rows)")+ print(f"Loaded synth dataset from GCS -> {target} ({len(ds)} rows)")_unified_dataset_cache = _set_audio_backend(ds)return _unified_dataset_cacheexcept Exception as e:print(f"Warning: GCS download failed: {e}")raise FileNotFoundError(- f"Unified dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "- f"Run 'python scripts/build_dataset.py' first."+ f"Dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "+ f"Run 'needle tokenize' or 'python scripts/synthesize_tools_data.py' first.")diff --git a/src/gcs.py b/src/gcs.pyindex 21d503c..9c635c7 100644--- a/src/gcs.py+++ b/src/gcs.py@@ -1,7 +1,7 @@"""GCS sync utilities for persisting datasets to gs://cactus-dataset.Uploads and downloads three artifact types:- - raw_data/ unified HuggingFace dataset (.arrow files)+ - synth_tool_calls/ synthesized tool-calling dataset (.arrow files)- tokenized_data/ pre-tokenized .npy arrays + metadata JSONs- tokenizer/ SentencePiece model + vocab@@ -13,7 +13,7 @@ import subprocessBUCKET = "cactus-dataset"-_RAW_DATA_PREFIX = "raw_data"+_SYNTH_DATA_PREFIX = "synth_tool_calls"_TOKENIZED_DATA_PREFIX = "tokenized_data"_TOKENIZER_PREFIX = "tokenizer"@@ -53,14 +53,9 @@ def download_directory(gcs_prefix, local_dir):# ── Convenience wrappers ──-def upload_raw_data(local_dir):- """Upload unified dataset to GCS."""- upload_directory(local_dir, _RAW_DATA_PREFIX)---def download_raw_data(local_dir):- """Download unified dataset from GCS. Returns True if successful."""- return download_directory(_RAW_DATA_PREFIX, local_dir)+def download_synth_data(local_dir):+ """Download synthesized tool-calling dataset from GCS. Returns True if successful."""+ return download_directory(_SYNTH_DATA_PREFIX, local_dir)def upload_tokenized_data(cache_dir):diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 791a76b..9735a6c 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -1,7 +1,8 @@"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.-Trains the SentencePiece tokenizer and tokenizes train + val splits,-caching everything locally.+Downloads the synthesized tool-calling dataset from GCS+(gs://cactus-dataset/synth_tool_calls/), trains the SentencePiece tokenizer,+and tokenizes train + val splits, caching everything locally.Usage:needle tokenize # full run@@ -18,6 +19,9 @@ from .data import (TOKENIZER_DIR,_cache_key,_save_cache_metadata,+ _DISK_UNIFIED_DIR,+ _SHM_UNIFIED_DIR,+ _shm_available,get_tokenizer,load_tool_calls,prepare_tool_call_pairs,@@ -26,17 +30,33 @@ from .data import (def _clear_local_caches():- """Remove local .data_cache/ and tokenizer/ directories."""- for d in [CACHE_DIR, TOKENIZER_DIR]:+ """Remove local .data_cache/, tokenizer/, and stale unified dataset directories."""+ for d in [CACHE_DIR, TOKENIZER_DIR, _DISK_UNIFIED_DIR, _SHM_UNIFIED_DIR]:if os.path.exists(d):print(f"Removing {d}/ ...")shutil.rmtree(d)+def _download_synth_dataset():+ """Download synthesized tool-calling dataset from GCS."""+ from .gcs import download_synth_data++ target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR+ print(f"Downloading synth dataset to {target} ...")+ if not download_synth_data(target):+ raise FileNotFoundError(+ "Synth dataset not found at gs://cactus-dataset/synth_tool_calls/. "+ "Run 'python scripts/synthesize_tools_data.py' first."+ )++def tokenize(args):print("=== Clearing existing caches ===")_clear_local_caches()+ print("\n=== Downloading synth dataset from GCS ===")+ _download_synth_dataset()+print("\n=== Training tokenizer ===")train_tokenizer(max_samples=args.max_samples, force=True)