Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Fix tokenizer bug

HenryNdubuaku375a4a0be288
diff --git a/needle/dataset/dataset.py b/needle/dataset/dataset.py
index fb185b8..3d4a1ea 100644
--- a/needle/dataset/dataset.py
+++ b/needle/dataset/dataset.py
@@ -21,7 +21,6 @@ from .tokenizer import (
DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN,
to_snake_case, NeedleTokenizer, train_tokenizer, get_tokenizer,
_download_tokenizer_from_hf, TOKENIZER_DIR, TOKENIZER_PREFIX,
- _HF_TOKENIZER_REPO,
)
import re as _re
diff --git a/needle/dataset/tokenize.py b/needle/dataset/tokenize.py
index 793ed6d..cfc446c 100644
--- a/needle/dataset/tokenize.py
+++ b/needle/dataset/tokenize.py
@@ -28,24 +28,25 @@ from .dataset import (
)
-_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"
+_HF_MODEL_REPO = "Cactus-Compute/needle"
_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"
def _push_to_hf(cache_dir, tokenizer_dir):
- """Upload tokenizer and tokenized data to their respective HuggingFace repos."""
+ """Upload tokenizer and tokenized data to HuggingFace."""
from huggingface_hub import HfApi
api = HfApi()
- api.create_repo(_HF_TOKENIZER_REPO, repo_type="dataset", private=True, exist_ok=True)
- print(f"Uploading tokenizer to {_HF_TOKENIZER_REPO} ...")
- api.upload_folder(
- folder_path=tokenizer_dir,
- repo_id=_HF_TOKENIZER_REPO,
- repo_type="dataset",
- allow_patterns=["*.model", "*.vocab"],
- )
+ print(f"Uploading tokenizer to {_HF_MODEL_REPO} ...")
+ for fname in os.listdir(tokenizer_dir):
+ if fname.endswith((".model", ".vocab")):
+ api.upload_file(
+ path_or_fileobj=os.path.join(tokenizer_dir, fname),
+ path_in_repo=f"tokenizer/{fname}",
+ repo_id=_HF_MODEL_REPO,
+ repo_type="model",
+ )
api.create_repo(_HF_TOKENIZED_REPO, repo_type="dataset", private=True, exist_ok=True)
print(f"Uploading tokenized data to {_HF_TOKENIZED_REPO} ...")
@@ -54,7 +55,7 @@ def _push_to_hf(cache_dir, tokenizer_dir):
repo_id=_HF_TOKENIZED_REPO,
repo_type="dataset",
allow_patterns=["*.npy", "*.json"],
- delete_patterns=["*.npy", "*.json"],
+ delete_patterns=["*.npy", "*.json"],
)
print("HuggingFace upload complete.")
diff --git a/needle/dataset/tokenizer.py b/needle/dataset/tokenizer.py
index a29c8ed..efa2cc8 100644
--- a/needle/dataset/tokenizer.py
+++ b/needle/dataset/tokenizer.py
@@ -19,7 +19,7 @@ DEFAULT_MAX_ENC_LEN = 1024
DEFAULT_MAX_DEC_LEN = 512
DEFAULT_MAX_GEN_LEN = 512
-_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"
+_HF_MODEL_REPO = "Cactus-Compute/needle"
def to_snake_case(name):
@@ -86,19 +86,22 @@ class NeedleTokenizer:
def _download_tokenizer_from_hf():
"""Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""
- from huggingface_hub import snapshot_download
+ from huggingface_hub import hf_hub_download
- local = snapshot_download(
- _HF_TOKENIZER_REPO,
- repo_type="dataset",
- )
os.makedirs(TOKENIZER_DIR, exist_ok=True)
- for fname in os.listdir(local):
- if fname.startswith("."):
- continue
+ for fname in ["needle.model", "needle.vocab"]:
+ hf_hub_download(
+ repo_id=_HF_MODEL_REPO,
+ filename=f"tokenizer/{fname}",
+ repo_type="model",
+ local_dir=TOKENIZER_DIR,
+ force_download=True,
+ )
+ # hf_hub_download puts it in tokenizer/tokenizer/needle.model, move up
+ nested = os.path.join(TOKENIZER_DIR, "tokenizer", fname)
dst = os.path.join(TOKENIZER_DIR, fname)
- if not os.path.exists(dst):
- os.symlink(os.path.join(local, fname), dst)
+ if os.path.exists(nested) and not os.path.exists(dst):
+ os.rename(nested, dst)
def get_tokenizer(max_samples=None):