Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Fix tokenizer bug
HenryNdubuaku375a4a0be288
diff --git a/needle/dataset/dataset.py b/needle/dataset/dataset.pyindex fb185b8..3d4a1ea 100644--- a/needle/dataset/dataset.py+++ b/needle/dataset/dataset.py@@ -21,7 +21,6 @@ from .tokenizer import (DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN,to_snake_case, NeedleTokenizer, train_tokenizer, get_tokenizer,_download_tokenizer_from_hf, TOKENIZER_DIR, TOKENIZER_PREFIX,- _HF_TOKENIZER_REPO,)import re as _rediff --git a/needle/dataset/tokenize.py b/needle/dataset/tokenize.pyindex 793ed6d..cfc446c 100644--- a/needle/dataset/tokenize.py+++ b/needle/dataset/tokenize.py@@ -28,24 +28,25 @@ from .dataset import ()-_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"+_HF_MODEL_REPO = "Cactus-Compute/needle"_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"def _push_to_hf(cache_dir, tokenizer_dir):- """Upload tokenizer and tokenized data to their respective HuggingFace repos."""+ """Upload tokenizer and tokenized data to HuggingFace."""from huggingface_hub import HfApiapi = HfApi()- api.create_repo(_HF_TOKENIZER_REPO, repo_type="dataset", private=True, exist_ok=True)- print(f"Uploading tokenizer to {_HF_TOKENIZER_REPO} ...")- api.upload_folder(- folder_path=tokenizer_dir,- repo_id=_HF_TOKENIZER_REPO,- repo_type="dataset",- allow_patterns=["*.model", "*.vocab"],- )+ print(f"Uploading tokenizer to {_HF_MODEL_REPO} ...")+ for fname in os.listdir(tokenizer_dir):+ if fname.endswith((".model", ".vocab")):+ api.upload_file(+ path_or_fileobj=os.path.join(tokenizer_dir, fname),+ path_in_repo=f"tokenizer/{fname}",+ repo_id=_HF_MODEL_REPO,+ repo_type="model",+ )api.create_repo(_HF_TOKENIZED_REPO, repo_type="dataset", private=True, exist_ok=True)print(f"Uploading tokenized data to {_HF_TOKENIZED_REPO} ...")@@ -54,7 +55,7 @@ def _push_to_hf(cache_dir, tokenizer_dir):repo_id=_HF_TOKENIZED_REPO,repo_type="dataset",allow_patterns=["*.npy", "*.json"],- delete_patterns=["*.npy", "*.json"],+ delete_patterns=["*.npy", "*.json"],)print("HuggingFace upload complete.")diff --git a/needle/dataset/tokenizer.py b/needle/dataset/tokenizer.pyindex a29c8ed..efa2cc8 100644--- a/needle/dataset/tokenizer.py+++ b/needle/dataset/tokenizer.py@@ -19,7 +19,7 @@ DEFAULT_MAX_ENC_LEN = 1024DEFAULT_MAX_DEC_LEN = 512DEFAULT_MAX_GEN_LEN = 512-_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"+_HF_MODEL_REPO = "Cactus-Compute/needle"def to_snake_case(name):@@ -86,19 +86,22 @@ class NeedleTokenizer:def _download_tokenizer_from_hf():"""Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""- from huggingface_hub import snapshot_download+ from huggingface_hub import hf_hub_download- local = snapshot_download(- _HF_TOKENIZER_REPO,- repo_type="dataset",- )os.makedirs(TOKENIZER_DIR, exist_ok=True)- for fname in os.listdir(local):- if fname.startswith("."):- continue+ for fname in ["needle.model", "needle.vocab"]:+ hf_hub_download(+ repo_id=_HF_MODEL_REPO,+ filename=f"tokenizer/{fname}",+ repo_type="model",+ local_dir=TOKENIZER_DIR,+ force_download=True,+ )+ # hf_hub_download puts it in tokenizer/tokenizer/needle.model, move up+ nested = os.path.join(TOKENIZER_DIR, "tokenizer", fname)dst = os.path.join(TOKENIZER_DIR, fname)- if not os.path.exists(dst):- os.symlink(os.path.join(local, fname), dst)+ if os.path.exists(nested) and not os.path.exists(dst):+ os.rename(nested, dst)def get_tokenizer(max_samples=None):