Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

merged updates

Karen Mosoyan8bf2c109d597
diff --cc src/data.py
index fcfd7da,45a313a..5d40484
--- a/src/data.py
+++ b/src/data.py
@@@ -256,12 -259,29 +259,33 @@@ def _gcs_cache_upload(cache_id, suffixe
)
+ def _gcs_download_shards(cache_id, n_shards, shard_suffixes):
+ """Download sharded .npy files from GCS. Returns True if all found."""
+ import subprocess
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ srcs = []
+ for suffix in shard_suffixes:
+ for i in range(n_shards):
+ srcs.append(f"{GCS_CACHE_PATH}/{cache_id}{suffix}_{i:05d}.npy")
+ if not srcs:
+ return True
+ subprocess.run(
+ ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],
+ capture_output=True, text=True,
+ )
+ for suffix in shard_suffixes:
+ for i in range(n_shards):
+ if not os.path.exists(os.path.join(CACHE_DIR, f"{cache_id}{suffix}_{i:05d}.npy")):
+ return False
+ return True
+
+
-def load_tool_calls(split="train", max_samples=None):
- """Load tool-calling dataset, splitting 90/10 for train/val."""
+def load_tool_calls(split="train", max_samples=None, return_global_indices=False):
+ """Load tool-calling dataset, splitting 90/10 for train/val.
+
+ If return_global_indices is True, also return a numpy array mapping each
+ split-local row position back to its row id in the full unified dataset.
+ """
ds = _load_unified_dataset()
n = len(ds)
if split in ("validation", "val", "test"):
diff --cc src/tokenize_data.py
index 800ba08,9a1740e..04a6b5b
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@@ -71,23 -68,18 +68,22 @@@ def tokenize(args)
for split in ("train", "val"):
print(f"\n--- {split} split ---")
- ds = load_tool_calls(split=split, max_samples=args.max_samples)
+ ds, global_indices = load_tool_calls(
+ split=split,
+ max_samples=args.max_samples,
+ return_global_indices=True,
+ )
_, _, _, _, kept_indices = prepare_tool_call_pairs(
- ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len
+ ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
+ batch_size=batch_size,
)
text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
- # Precompute mel spectrograms
mel_cache_id = precompute_mels(
- kept_indices, n_mels=n_mels, max_mel_len=max_mel_len,
+ global_indices[kept_indices], n_mels=n_mels, max_mel_len=max_mel_len,
- cache_id_prefix=split,
+ cache_id_prefix=split, batch_size=batch_size,
)
- # Save metadata for train.py to find caches
_save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),
max_enc_len, max_dec_len, n_mels, max_mel_len)