Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

bug fixes

HenryNdubuakufe0214186bcb
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
index 0b2a291..6e52425 100644
--- a/scripts/build_dataset.py
+++ b/scripts/build_dataset.py
@@ -73,11 +73,8 @@ def main():
logger.info(f"Saved {len(combined)} rows to {args.output}/")
# Persist to GCS
- try:
- from src.gcs import upload_raw_data
- upload_raw_data(args.output)
- except Exception as e:
- logger.warning(f"GCS upload failed (non-fatal): {e}")
+ from src.gcs import upload_raw_data
+ upload_raw_data(args.output)
print_summary(combined)
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index f8d95e6..124c5a5 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -39,7 +39,7 @@ from collections import Counter
from datasets import load_dataset, Dataset, concatenate_datasets
-_NUM_WORKERS = max(1, (os.cpu_count() or 1) // 2)
+_NUM_WORKERS = min(20, max(1, (os.cpu_count() or 1) // 4))
DATASETS = {
diff --git a/src/data.py b/src/data.py
index a25eca0..0a895b6 100644
--- a/src/data.py
+++ b/src/data.py
@@ -356,7 +356,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
user_defined_symbols=["<tool_call>", "<transcribe>", "<tools>"],
byte_fallback=True,
normalization_rule_name="identity",
- num_threads=os.cpu_count(),
+ num_threads=min(20, max(1, (os.cpu_count() or 1) // 4)),
train_extremely_large_corpus=False,
minloglevel=2,
)
@@ -495,7 +495,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
if shuffle_tools:
tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]
- num_workers = max(1, (os.cpu_count() or 1) // 2)
+ num_workers = min(20, max(1, (os.cpu_count() or 1) // 4))
model_path = TOKENIZER_PREFIX + ".model"
chunk_size = max(1, len(enc_texts) // (num_workers * 4))
diff --git a/src/gcs.py b/src/gcs.py
index 82164ff..21d503c 100644
--- a/src/gcs.py
+++ b/src/gcs.py
@@ -4,11 +4,12 @@ Uploads and downloads three artifact types:
- raw_data/ unified HuggingFace dataset (.arrow files)
- tokenized_data/ pre-tokenized .npy arrays + metadata JSONs
- tokenizer/ SentencePiece model + vocab
+
+Uses `gcloud storage` CLI (authenticated via `gcloud auth login`).
"""
import os
-
-import gcsfs
+import subprocess
BUCKET = "cactus-dataset"
@@ -17,48 +18,36 @@ _TOKENIZED_DATA_PREFIX = "tokenized_data"
_TOKENIZER_PREFIX = "tokenizer"
-def _get_fs():
- return gcsfs.GCSFileSystem()
-
-
def upload_directory(local_dir, gcs_prefix):
"""Upload all files in local_dir to gs://BUCKET/gcs_prefix/."""
if not os.path.isdir(local_dir):
print(f"[GCS] Skipping upload: {local_dir} does not exist")
return
- fs = _get_fs()
- dest = f"{BUCKET}/{gcs_prefix}"
- count = 0
- for root, _, files in os.walk(local_dir):
- for fname in files:
- local_path = os.path.join(root, fname)
- rel = os.path.relpath(local_path, local_dir)
- remote_path = f"{dest}/{rel}"
- fs.put(local_path, remote_path)
- count += 1
- print(f"[GCS] Uploaded {count} files to gs://{dest}/")
+ dest = f"gs://{BUCKET}/{gcs_prefix}/"
+ print(f"[GCS] Uploading {local_dir} -> {dest}")
+ subprocess.run(
+ ["gcloud", "storage", "cp", "-r", f"{local_dir}/*", dest],
+ check=True,
+ )
+ print(f"[GCS] Upload complete: {dest}")
def download_directory(gcs_prefix, local_dir):
- """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if files were downloaded."""
- fs = _get_fs()
- src = f"{BUCKET}/{gcs_prefix}"
- try:
- remote_files = fs.ls(src, detail=False)
- except FileNotFoundError:
- return False
- if not remote_files:
- return False
+ """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if successful."""
+ src = f"gs://{BUCKET}/{gcs_prefix}/"
os.makedirs(local_dir, exist_ok=True)
- count = 0
- for remote_path in fs.find(src):
- rel = os.path.relpath(remote_path, src)
- local_path = os.path.join(local_dir, rel)
- os.makedirs(os.path.dirname(local_path), exist_ok=True)
- fs.get(remote_path, local_path)
- count += 1
- print(f"[GCS] Downloaded {count} files from gs://{src}/ to {local_dir}/")
- return count > 0
+ print(f"[GCS] Downloading {src} -> {local_dir}")
+ result = subprocess.run(
+ ["gcloud", "storage", "cp", "-r", f"{src}*", local_dir],
+ capture_output=True, text=True,
+ )
+ if result.returncode != 0:
+ if "not found" in result.stderr.lower() or "CommandException" in result.stderr:
+ print(f"[GCS] Nothing found at {src}")
+ return False
+ raise RuntimeError(f"gcloud storage download failed: {result.stderr}")
+ print(f"[GCS] Download complete: {local_dir}")
+ return True
# ── Convenience wrappers ──
@@ -66,35 +55,29 @@ def download_directory(gcs_prefix, local_dir):
def upload_raw_data(local_dir):
"""Upload unified dataset to GCS."""
- print("[GCS] Uploading raw dataset...")
upload_directory(local_dir, _RAW_DATA_PREFIX)
def download_raw_data(local_dir):
"""Download unified dataset from GCS. Returns True if successful."""
- print("[GCS] Downloading raw dataset...")
return download_directory(_RAW_DATA_PREFIX, local_dir)
def upload_tokenized_data(cache_dir):
"""Upload tokenized .npy files and metadata to GCS."""
- print("[GCS] Uploading tokenized data...")
upload_directory(cache_dir, _TOKENIZED_DATA_PREFIX)
def download_tokenized_data(cache_dir):
"""Download tokenized data from GCS. Returns True if successful."""
- print("[GCS] Downloading tokenized data...")
return download_directory(_TOKENIZED_DATA_PREFIX, cache_dir)
def upload_tokenizer(tokenizer_dir):
"""Upload tokenizer model and vocab to GCS."""
- print("[GCS] Uploading tokenizer...")
upload_directory(tokenizer_dir, _TOKENIZER_PREFIX)
def download_tokenizer(tokenizer_dir):
"""Download tokenizer from GCS. Returns True if successful."""
- print("[GCS] Downloading tokenizer...")
return download_directory(_TOKENIZER_PREFIX, tokenizer_dir)
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 8c2ce15..791a76b 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -67,11 +67,8 @@ def tokenize(args):
_save_cache_metadata(split, text_cache_id, len(kept_indices),
max_enc_len, max_dec_len)
- try:
- from .gcs import upload_tokenized_data, upload_tokenizer
- upload_tokenizer(TOKENIZER_DIR)
- upload_tokenized_data(CACHE_DIR)
- except Exception as e:
- print(f"[GCS] Upload failed (non-fatal): {e}")
+ from .gcs import upload_tokenized_data, upload_tokenizer
+ upload_tokenizer(TOKENIZER_DIR)
+ upload_tokenized_data(CACHE_DIR)
print("\n=== Tokenization pipeline complete ===")