Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix auth
HenryNdubuaku8b643c253efd
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.pyindex 5b8b0b7..96d986d 100644--- a/scripts/build_dataset.py+++ b/scripts/build_dataset.py@@ -13,7 +13,6 @@ import loggingimport osimport sys-# Redirect HF cache to /dev/shm when RAM is plentiful (before importing datasets)_shm = "/dev/shm"if os.path.isdir(_shm):try:@@ -21,11 +20,17 @@ if os.path.isdir(_shm):if _st.f_bavail * _st.f_frsize >= 200 * 1024**3:_hf_cache = os.path.join(_shm, "needle_hf_cache")os.makedirs(_hf_cache, exist_ok=True)- os.environ.setdefault("HF_HOME", _hf_cache)os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(_hf_cache, "datasets"))except OSError:pass+_token_path = os.path.expanduser("~/.cache/huggingface/token")+if os.path.isfile(_token_path) and "HF_TOKEN" not in os.environ:+ with open(_token_path) as _f:+ _tok = _f.read().strip()+ if _tok:+ os.environ["HF_TOKEN"] = _tok+import numpy as npsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))