Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
merged updates
Karen Mosoyan8bf2c109d597
diff --cc src/data.pyindex fcfd7da,45a313a..5d40484--- a/src/data.py+++ b/src/data.py@@@ -256,12 -259,29 +259,33 @@@ def _gcs_cache_upload(cache_id, suffixe)+ def _gcs_download_shards(cache_id, n_shards, shard_suffixes):+ """Download sharded .npy files from GCS. Returns True if all found."""+ import subprocess+ os.makedirs(CACHE_DIR, exist_ok=True)+ srcs = []+ for suffix in shard_suffixes:+ for i in range(n_shards):+ srcs.append(f"{GCS_CACHE_PATH}/{cache_id}{suffix}_{i:05d}.npy")+ if not srcs:+ return True+ subprocess.run(+ ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],+ capture_output=True, text=True,+ )+ for suffix in shard_suffixes:+ for i in range(n_shards):+ if not os.path.exists(os.path.join(CACHE_DIR, f"{cache_id}{suffix}_{i:05d}.npy")):+ return False+ return True++-def load_tool_calls(split="train", max_samples=None):- """Load tool-calling dataset, splitting 90/10 for train/val."""+def load_tool_calls(split="train", max_samples=None, return_global_indices=False):+ """Load tool-calling dataset, splitting 90/10 for train/val.++ If return_global_indices is True, also return a numpy array mapping each+ split-local row position back to its row id in the full unified dataset.+ """ds = _load_unified_dataset()n = len(ds)if split in ("validation", "val", "test"):diff --cc src/tokenize_data.pyindex 800ba08,9a1740e..04a6b5b--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@@ -71,23 -68,18 +68,22 @@@ def tokenize(args)for split in ("train", "val"):print(f"\n--- {split} split ---")- ds = load_tool_calls(split=split, max_samples=args.max_samples)+ ds, global_indices = load_tool_calls(+ split=split,+ max_samples=args.max_samples,+ return_global_indices=True,+ )_, _, _, _, kept_indices = prepare_tool_call_pairs(- ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len+ ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,+ batch_size=batch_size,)text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)- # Precompute mel spectrogramsmel_cache_id = precompute_mels(- kept_indices, n_mels=n_mels, max_mel_len=max_mel_len,+ global_indices[kept_indices], n_mels=n_mels, max_mel_len=max_mel_len,- cache_id_prefix=split,+ cache_id_prefix=split, batch_size=batch_size,)- # Save metadata for train.py to find caches_save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),max_enc_len, max_dec_len, n_mels, max_mel_len)