Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku82fb1e43e0e5
diff --git a/scripts/generate_data.py b/scripts/generate_data.pyindex b4856b8..daf58c3 100644--- a/scripts/generate_data.py+++ b/scripts/generate_data.py@@ -765,6 +765,15 @@ def _load_existing():else:print(f"Downloading existing dataset from {HF_DATASET_REPO}...")ds = load_dataset(HF_DATASET_REPO, split="train", token=True)+ # Save locally so subsequent chunks don't re-download+ os.makedirs(local, exist_ok=True)+ ds.save_to_disk(local)+ # Clear HF download cache to avoid double storage+ hf_cache = os.path.expanduser("~/.cache/huggingface/datasets")+ if os.path.exists(hf_cache):+ import shutil+ shutil.rmtree(hf_cache)+ print(f"Cleared HF dataset cache ({hf_cache})")print(f"Downloaded: {len(ds)} rows")return ds@@ -829,7 +838,7 @@ def main():existing = None if args.dry_run else _load_existing()seen_queries = set()if existing and not args.dry_run:- seen_queries = set(existing["query"][:50000])+ seen_queries = set(existing["query"])while remaining > 0:chunk_size = min(remaining, args.upload_every)diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex b05ccf4..ff1235e 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -110,18 +110,17 @@ def tokenize(args):max_samples=args.max_samples,return_global_indices=True,)- w_name = getattr(args, "w_name", 3.0)- w_value = getattr(args, "w_value", 2.0)- w_key = getattr(args, "w_key", 1.5)shuffle_tools = getattr(args, "shuffle_tools", True)max_tool_len = getattr(args, "max_tool_len", 256)prepare_tool_call_pairs._max_tool_len = max_tool_len_, _, _, _, kept_indices, _ = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,- w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,+ shuffle_tools=shuffle_tools,)+ # Cache key must match the one used inside prepare_tool_call_pairs+ # (weights are no longer part of the cache key — class labels stored instead)text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,- w_name, w_value, w_key, shuffle_tools)+ 1.0, 1.0, 1.0, shuffle_tools)cache_path = os.path.join(CACHE_DIR, text_cache_id)enc_vl = VarLenArray.load(cache_path + "_enc", max_enc_len)@@ -130,6 +129,13 @@ def tokenize(args):loss_vl = VarLenArray.load(cache_path + "_loss", max_dec_len)pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl)+ # Clean up unpacked VarLenArrays — only packed arrays are needed for training+ for suffix in ("_enc", "_dec_in", "_dec_tgt", "_loss"):+ for ext in ("_data.npy", "_offsets.npy"):+ path = cache_path + suffix + ext+ if os.path.exists(path):+ os.remove(path)+_save_cache_metadata(split, text_cache_id, len(kept_indices),max_enc_len, max_dec_len, max_tool_len)