Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
streamline dataset synthesis
HenryNdubuaku75082a016f3a
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex 78caaaa..6478efa 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -37,7 +37,6 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"MODEL_POOL = ["google/gemini-3.1-flash-lite-preview",- "google/gemini-2.5-flash-lite",]DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"@@ -276,38 +275,6 @@ def _validate_and_format(synth_id, language, query, tools, answer, model, source}-def load_queries_from_gcs(max_samples=None):- """Pull raw_data from GCS bucket and extract queries for synthesis."""- sys.path.insert(0, str(_PROJECT_ROOT))- from src.gcs import download_directory-- local_dir = _PARQUET_CACHE_DIR / "raw_data"- if not (local_dir / "dataset_info.json").exists():- ok = download_directory("raw_data", str(local_dir))- if not ok:- logger.warning("No raw_data found in GCS, skipping")- return []-- from datasets import load_from_disk- ds = load_from_disk(str(local_dir))- logger.info(f"Loaded {len(ds):,} raw_data queries from GCS")-- samples = []- for i, ex in enumerate(ds):- synth_id = f"gcs-{i}"- query = ex.get("query", "")- if not query or len(query.strip()) < 10:- continue- samples.append((synth_id, "en", query))-- rng = random.Random(43)- rng.shuffle(samples)- if max_samples:- samples = samples[:max_samples]-- logger.info(f" Selected {len(samples):,} GCS queries")- return samples-class _KeyModelRotator:"""Thread-safe round-robin across (api_key, model) pairs."""@@ -474,17 +441,6 @@ def main():batch_idx = 0since_last_upload = [0]- gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)- if gcs_samples:- gen, fail = _process_batch(- gcs_samples, "GCS raw_data", batch_idx, rotator,- args, tqdm, source_tag="synth-gcs",- since_last_upload=since_last_upload,- )- total_generated += gen- total_failed += fail- batch_idx += 1-hf_token = os.environ.get("HF_TOKEN")shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)