Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
bug fixes
HenryNdubuakufe0214186bcb
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.pyindex 0b2a291..6e52425 100644--- a/scripts/build_dataset.py+++ b/scripts/build_dataset.py@@ -73,11 +73,8 @@ def main():logger.info(f"Saved {len(combined)} rows to {args.output}/")# Persist to GCS- try:- from src.gcs import upload_raw_data- upload_raw_data(args.output)- except Exception as e:- logger.warning(f"GCS upload failed (non-fatal): {e}")+ from src.gcs import upload_raw_data+ upload_raw_data(args.output)print_summary(combined)diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex f8d95e6..124c5a5 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -39,7 +39,7 @@ from collections import Counterfrom datasets import load_dataset, Dataset, concatenate_datasets-_NUM_WORKERS = max(1, (os.cpu_count() or 1) // 2)+_NUM_WORKERS = min(20, max(1, (os.cpu_count() or 1) // 4))DATASETS = {diff --git a/src/data.py b/src/data.pyindex a25eca0..0a895b6 100644--- a/src/data.py+++ b/src/data.py@@ -356,7 +356,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):user_defined_symbols=["<tool_call>", "<transcribe>", "<tools>"],byte_fallback=True,normalization_rule_name="identity",- num_threads=os.cpu_count(),+ num_threads=min(20, max(1, (os.cpu_count() or 1) // 4)),train_extremely_large_corpus=False,minloglevel=2,)@@ -495,7 +495,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_if shuffle_tools:tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]- num_workers = max(1, (os.cpu_count() or 1) // 2)+ num_workers = min(20, max(1, (os.cpu_count() or 1) // 4))model_path = TOKENIZER_PREFIX + ".model"chunk_size = max(1, len(enc_texts) // (num_workers * 4))diff --git a/src/gcs.py b/src/gcs.pyindex 82164ff..21d503c 100644--- a/src/gcs.py+++ b/src/gcs.py@@ -4,11 +4,12 @@ Uploads and downloads three artifact types:- raw_data/ unified HuggingFace dataset (.arrow files)- tokenized_data/ pre-tokenized .npy arrays + metadata JSONs- tokenizer/ SentencePiece model + vocab++Uses `gcloud storage` CLI (authenticated via `gcloud auth login`)."""import os--import gcsfs+import subprocessBUCKET = "cactus-dataset"@@ -17,48 +18,36 @@ _TOKENIZED_DATA_PREFIX = "tokenized_data"_TOKENIZER_PREFIX = "tokenizer"-def _get_fs():- return gcsfs.GCSFileSystem()--def upload_directory(local_dir, gcs_prefix):"""Upload all files in local_dir to gs://BUCKET/gcs_prefix/."""if not os.path.isdir(local_dir):print(f"[GCS] Skipping upload: {local_dir} does not exist")return- fs = _get_fs()- dest = f"{BUCKET}/{gcs_prefix}"- count = 0- for root, _, files in os.walk(local_dir):- for fname in files:- local_path = os.path.join(root, fname)- rel = os.path.relpath(local_path, local_dir)- remote_path = f"{dest}/{rel}"- fs.put(local_path, remote_path)- count += 1- print(f"[GCS] Uploaded {count} files to gs://{dest}/")+ dest = f"gs://{BUCKET}/{gcs_prefix}/"+ print(f"[GCS] Uploading {local_dir} -> {dest}")+ subprocess.run(+ ["gcloud", "storage", "cp", "-r", f"{local_dir}/*", dest],+ check=True,+ )+ print(f"[GCS] Upload complete: {dest}")def download_directory(gcs_prefix, local_dir):- """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if files were downloaded."""- fs = _get_fs()- src = f"{BUCKET}/{gcs_prefix}"- try:- remote_files = fs.ls(src, detail=False)- except FileNotFoundError:- return False- if not remote_files:- return False+ """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if successful."""+ src = f"gs://{BUCKET}/{gcs_prefix}/"os.makedirs(local_dir, exist_ok=True)- count = 0- for remote_path in fs.find(src):- rel = os.path.relpath(remote_path, src)- local_path = os.path.join(local_dir, rel)- os.makedirs(os.path.dirname(local_path), exist_ok=True)- fs.get(remote_path, local_path)- count += 1- print(f"[GCS] Downloaded {count} files from gs://{src}/ to {local_dir}/")- return count > 0+ print(f"[GCS] Downloading {src} -> {local_dir}")+ result = subprocess.run(+ ["gcloud", "storage", "cp", "-r", f"{src}*", local_dir],+ capture_output=True, text=True,+ )+ if result.returncode != 0:+ if "not found" in result.stderr.lower() or "CommandException" in result.stderr:+ print(f"[GCS] Nothing found at {src}")+ return False+ raise RuntimeError(f"gcloud storage download failed: {result.stderr}")+ print(f"[GCS] Download complete: {local_dir}")+ return True# ── Convenience wrappers ──@@ -66,35 +55,29 @@ def download_directory(gcs_prefix, local_dir):def upload_raw_data(local_dir):"""Upload unified dataset to GCS."""- print("[GCS] Uploading raw dataset...")upload_directory(local_dir, _RAW_DATA_PREFIX)def download_raw_data(local_dir):"""Download unified dataset from GCS. Returns True if successful."""- print("[GCS] Downloading raw dataset...")return download_directory(_RAW_DATA_PREFIX, local_dir)def upload_tokenized_data(cache_dir):"""Upload tokenized .npy files and metadata to GCS."""- print("[GCS] Uploading tokenized data...")upload_directory(cache_dir, _TOKENIZED_DATA_PREFIX)def download_tokenized_data(cache_dir):"""Download tokenized data from GCS. Returns True if successful."""- print("[GCS] Downloading tokenized data...")return download_directory(_TOKENIZED_DATA_PREFIX, cache_dir)def upload_tokenizer(tokenizer_dir):"""Upload tokenizer model and vocab to GCS."""- print("[GCS] Uploading tokenizer...")upload_directory(tokenizer_dir, _TOKENIZER_PREFIX)def download_tokenizer(tokenizer_dir):"""Download tokenizer from GCS. Returns True if successful."""- print("[GCS] Downloading tokenizer...")return download_directory(_TOKENIZER_PREFIX, tokenizer_dir)diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 8c2ce15..791a76b 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -67,11 +67,8 @@ def tokenize(args):_save_cache_metadata(split, text_cache_id, len(kept_indices),max_enc_len, max_dec_len)- try:- from .gcs import upload_tokenized_data, upload_tokenizer- upload_tokenizer(TOKENIZER_DIR)- upload_tokenized_data(CACHE_DIR)- except Exception as e:- print(f"[GCS] Upload failed (non-fatal): {e}")+ from .gcs import upload_tokenized_data, upload_tokenizer+ upload_tokenizer(TOKENIZER_DIR)+ upload_tokenized_data(CACHE_DIR)print("\n=== Tokenization pipeline complete ===")