Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix tokenize bugs
HenryNdubuaku0d2abe0cd3b5
diff --git a/src/data.py b/src/data.pyindex 77fcef0..6069cfd 100644--- a/src/data.py+++ b/src/data.py@@ -6,7 +6,6 @@ import queueimport threadingos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"-os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"import logginglogging.getLogger("transformers").setLevel(logging.ERROR)diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 015287a..985107b 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -124,4 +124,8 @@ def tokenize(args):_push_to_hf(CACHE_DIR, TOKENIZER_DIR)+ if os.path.exists(LOCAL_UNIFIED_DIR):+ print(f"\n=== Removing raw dataset ({LOCAL_UNIFIED_DIR}) ===")+ shutil.rmtree(LOCAL_UNIFIED_DIR)+print("\n=== Tokenization pipeline complete ===")