Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

fix tokenize bugs

HenryNdubuaku0d2abe0cd3b5
diff --git a/src/data.py b/src/data.py
index 77fcef0..6069cfd 100644
--- a/src/data.py
+++ b/src/data.py
@@ -6,7 +6,6 @@ import queue
import threading
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
-os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
import logging
logging.getLogger("transformers").setLevel(logging.ERROR)
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 015287a..985107b 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -124,4 +124,8 @@ def tokenize(args):
_push_to_hf(CACHE_DIR, TOKENIZER_DIR)
+ if os.path.exists(LOCAL_UNIFIED_DIR):
+ print(f"\n=== Removing raw dataset ({LOCAL_UNIFIED_DIR}) ===")
+ shutil.rmtree(LOCAL_UNIFIED_DIR)
+
print("\n=== Tokenization pipeline complete ===")