Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

refactors

HenryNdubuakuafddf8b42386
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
index 4b8bae3..3b77207 100644
--- a/scripts/synthesize_tools_data.py
+++ b/scripts/synthesize_tools_data.py
@@ -1,18 +1,13 @@
#!/usr/bin/env python3
-"""Synthesize tool-calling data from multiple query sources via OpenRouter.
-
-Query sources:
- 1. PleIAs/SYNTH parquet shards (~154k queries per shard, 7 shards)
- 2. GCS raw_data — queries extracted from the existing unified tool-calling dataset
+"""Synthesize tool-calling data from PleIAs/SYNTH via OpenRouter.
Each query is sent to a randomly-selected frontier model to generate
(tools, answer) pairs, then validated and checkpointed as JSONL.
Usage:
- python scripts/synthesize_tools_data.py # all sources
+ python scripts/synthesize_tools_data.py
python scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-preview
python scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100
- python scripts/synthesize_tools_data.py --no-gcs # skip GCS queries
python scripts/synthesize_tools_data.py --resume # continue from checkpoint
"""
@@ -152,48 +147,6 @@ def load_queries_from_parquet(path, max_samples=None):
return samples
-def load_queries_from_gcs(max_samples=None):
- """Download raw_data from GCS and extract queries for re-synthesis.
-
- Returns list of (sample_id, language, query) tuples, where sample_id
- is prefixed with 'gcs-' to avoid collisions with PleIAs IDs.
- """
- sys.path.insert(0, str(_PROJECT_ROOT))
- from src.gcs import download_raw_data
-
- local_dir = _PARQUET_CACHE_DIR / "gcs_raw_data"
- if not local_dir.exists():
- logger.info("Downloading raw_data from GCS...")
- ok = download_raw_data(str(local_dir))
- if not ok:
- logger.warning("No raw_data found in GCS, skipping")
- return []
- else:
- logger.info(f"Using cached GCS raw_data: {local_dir}")
-
- from datasets import load_from_disk
- ds = load_from_disk(str(local_dir))
- logger.info(f"Loaded {len(ds):,} rows from GCS raw_data")
-
- samples = []
- for i in range(len(ds)):
- ex = ds[i]
- query = ex.get("query", "").strip()
- if not query:
- continue
- sample_id = f"gcs-{i}"
- samples.append((sample_id, "en", query))
-
- rng = random.Random(42)
- rng.shuffle(samples)
-
- if max_samples:
- samples = samples[:max_samples]
-
- logger.info(f" Selected {len(samples):,} GCS queries")
- return samples
-
-
def load_checkpoint():
"""Load completed synth_ids from checkpoint file."""
done = set()
@@ -437,8 +390,6 @@ def main():
help="Concurrent API requests (default: 32)")
parser.add_argument("--max-shards", type=int, default=None,
help="Max parquet shards to process (default: all)")
- parser.add_argument("--no-gcs", action="store_true",
- help="Skip GCS raw_data queries")
parser.add_argument("--resume", action="store_true",
help="Resume from checkpoint")
parser.add_argument("--batch-size", type=int, default=500,
@@ -475,19 +426,7 @@ def main():
total_generated = 0
total_failed = 0
batch_idx = 0
- since_last_upload = [0]
-
- if not args.no_gcs:
- gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)
- if gcs_samples:
- gen, fail = _process_batch(
- gcs_samples, "GCS raw_data", batch_idx, api_key, models,
- args, tqdm, source_tag="synth-gcs",
- since_last_upload=since_last_upload,
- )
- total_generated += gen
- total_failed += fail
- batch_idx += 1
+ since_last_upload = [0]
hf_token = os.environ.get("HF_TOKEN")
shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)
diff --git a/src/data.py b/src/data.py
index 8934139..aaf5b5e 100644
--- a/src/data.py
+++ b/src/data.py
@@ -39,7 +39,7 @@ _DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")
-_MIN_SHM_BYTES = 200 * 1024**3
+_MIN_SHM_BYTES = 20 * 1024**3
def _shm_available():
@@ -237,19 +237,19 @@ def _load_unified_dataset():
continue
try:
- from .gcs import download_raw_data
+ from .gcs import download_synth_data
target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR
- if download_raw_data(target):
+ if download_synth_data(target):
ds = load_from_disk(target)
- print(f"Loaded unified dataset from GCS -> {target} ({len(ds)} rows)")
+ print(f"Loaded synth dataset from GCS -> {target} ({len(ds)} rows)")
_unified_dataset_cache = _set_audio_backend(ds)
return _unified_dataset_cache
except Exception as e:
print(f"Warning: GCS download failed: {e}")
raise FileNotFoundError(
- f"Unified dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "
- f"Run 'python scripts/build_dataset.py' first."
+ f"Dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "
+ f"Run 'needle tokenize' or 'python scripts/synthesize_tools_data.py' first."
)
diff --git a/src/gcs.py b/src/gcs.py
index 21d503c..9c635c7 100644
--- a/src/gcs.py
+++ b/src/gcs.py
@@ -1,7 +1,7 @@
"""GCS sync utilities for persisting datasets to gs://cactus-dataset.
Uploads and downloads three artifact types:
- - raw_data/ unified HuggingFace dataset (.arrow files)
+ - synth_tool_calls/ synthesized tool-calling dataset (.arrow files)
- tokenized_data/ pre-tokenized .npy arrays + metadata JSONs
- tokenizer/ SentencePiece model + vocab
@@ -13,7 +13,7 @@ import subprocess
BUCKET = "cactus-dataset"
-_RAW_DATA_PREFIX = "raw_data"
+_SYNTH_DATA_PREFIX = "synth_tool_calls"
_TOKENIZED_DATA_PREFIX = "tokenized_data"
_TOKENIZER_PREFIX = "tokenizer"
@@ -53,14 +53,9 @@ def download_directory(gcs_prefix, local_dir):
# ── Convenience wrappers ──
-def upload_raw_data(local_dir):
- """Upload unified dataset to GCS."""
- upload_directory(local_dir, _RAW_DATA_PREFIX)
-
-
-def download_raw_data(local_dir):
- """Download unified dataset from GCS. Returns True if successful."""
- return download_directory(_RAW_DATA_PREFIX, local_dir)
+def download_synth_data(local_dir):
+ """Download synthesized tool-calling dataset from GCS. Returns True if successful."""
+ return download_directory(_SYNTH_DATA_PREFIX, local_dir)
def upload_tokenized_data(cache_dir):
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 791a76b..9735a6c 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -1,7 +1,8 @@
"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.
-Trains the SentencePiece tokenizer and tokenizes train + val splits,
-caching everything locally.
+Downloads the synthesized tool-calling dataset from GCS
+(gs://cactus-dataset/synth_tool_calls/), trains the SentencePiece tokenizer,
+and tokenizes train + val splits, caching everything locally.
Usage:
needle tokenize # full run
@@ -18,6 +19,9 @@ from .data import (
TOKENIZER_DIR,
_cache_key,
_save_cache_metadata,
+ _DISK_UNIFIED_DIR,
+ _SHM_UNIFIED_DIR,
+ _shm_available,
get_tokenizer,
load_tool_calls,
prepare_tool_call_pairs,
@@ -26,17 +30,33 @@ from .data import (
def _clear_local_caches():
- """Remove local .data_cache/ and tokenizer/ directories."""
- for d in [CACHE_DIR, TOKENIZER_DIR]:
+ """Remove local .data_cache/, tokenizer/, and stale unified dataset directories."""
+ for d in [CACHE_DIR, TOKENIZER_DIR, _DISK_UNIFIED_DIR, _SHM_UNIFIED_DIR]:
if os.path.exists(d):
print(f"Removing {d}/ ...")
shutil.rmtree(d)
+def _download_synth_dataset():
+ """Download synthesized tool-calling dataset from GCS."""
+ from .gcs import download_synth_data
+
+ target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR
+ print(f"Downloading synth dataset to {target} ...")
+ if not download_synth_data(target):
+ raise FileNotFoundError(
+ "Synth dataset not found at gs://cactus-dataset/synth_tool_calls/. "
+ "Run 'python scripts/synthesize_tools_data.py' first."
+ )
+
+
def tokenize(args):
print("=== Clearing existing caches ===")
_clear_local_caches()
+ print("\n=== Downloading synth dataset from GCS ===")
+ _download_synth_dataset()
+
print("\n=== Training tokenizer ===")
train_tokenizer(max_samples=args.max_samples, force=True)