Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Saving: drop the training truncation and padding from saved tokenizers (#16102)

Leo Borcherding25262ce38233
diff --git a/tests/saving/test_save_clears_runtime_truncation.py b/tests/saving/test_save_clears_runtime_truncation.py
new file mode 100644
index 00000000000..68bd466d79f
--- /dev/null
+++ b/tests/saving/test_save_clears_runtime_truncation.py
@@ -0,0 +1,396 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+import ast
+import json
+import os
+import types
+from contextlib import contextmanager
+from pathlib import Path
+
+from tokenizers import Tokenizer, models, pre_tokenizers
+from transformers import (
+ PreTrainedTokenizerFast,
+ Qwen2TokenizerFast,
+ Qwen2VLImageProcessor,
+ Qwen3VLProcessor,
+ Qwen3VLVideoProcessor,
+)
+import pytest
+
+
+class _Logger:
+ def warning_once(self, *args, **kwargs):
+ pass
+
+
+def _noop(*args, **kwargs):
+ pass
+
+
+def _load_save_helpers():
+ source = Path(__file__).parents[2] / "unsloth" / "save.py"
+ tree = ast.parse(source.read_text(encoding = "utf-8"))
+ wanted = {
+ "_without_runtime_truncation",
+ "_unsloth_tokenizer_save_pretrained",
+ "_source_tokenizer_class",
+ "_restore_source_tokenizer_class",
+ }
+ nodes = [
+ node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted
+ ]
+ module = ast.Module(body = nodes, type_ignores = [])
+ ast.fix_missing_locations(module)
+ namespace = {
+ "contextmanager": contextmanager,
+ "json": json,
+ "os": os,
+ "logger": _Logger(),
+ "_resolve_hub_repo_cached_file": lambda *args, **kwargs: None,
+ "_tokenizer_cache_dir": lambda tokenizer: None,
+ "_tokenizer_revision": lambda tokenizer: None,
+ "_preserve_sentencepiece_tokenizer_assets": _noop,
+ "_preserve_tokenizer_eos_token": _noop,
+ "_preserve_repaired_tokenizer_class": _noop,
+ }
+ exec(compile(module, str(source), "exec"), namespace)
+ return namespace["_unsloth_tokenizer_save_pretrained"]
+
+
+def _trained_tokenizer():
+ vocab = {"[PAD]": 0, "[UNK]": 1, **{w: i + 2 for i, w in enumerate("abcdefgh")}}
+ backend = Tokenizer(models.WordLevel(vocab, unk_token = "[UNK]"))
+ backend.pre_tokenizer = pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object = backend, pad_token = "[PAD]", unk_token = "[UNK]"
+ )
+ tokenizer(["a b c d e f g h"], truncation = True, max_length = 3, padding = "max_length")
+ assert tokenizer._tokenizer.truncation["max_length"] == 3
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+ return tokenizer
+
+
+def _load_repair_helper():
+ source = Path(__file__).parents[2] / "unsloth" / "tokenizer_utils.py"
+ tree = ast.parse(source.read_text(encoding = "utf-8"))
+ wanted = {
+ "_tokenizer_objects",
+ "_repair_tokenizer_backend_from_json",
+ "_inferred_tokenizer_max_length",
+ "_resolve_tokenizer_json",
+ }
+ module = ast.Module(
+ body = [
+ node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted
+ ],
+ type_ignores = [],
+ )
+ namespace = {"os": os, "json": json}
+ exec(compile(ast.fix_missing_locations(module), str(source), "exec"), namespace)
+ return namespace["_repair_tokenizer_backend_from_json"]
+
+
+def _capture_loaded_limit(tokenizer, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ return _load_repair_helper()(tokenizer)
+
+
+def test_saved_tokenizer_json_has_no_training_truncation(tmp_path):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+
+ save(tokenizer, str(tmp_path))
+
+ saved = json.loads((tmp_path / "tokenizer.json").read_text(encoding = "utf-8"))
+ assert saved["truncation"] is None
+ assert saved["padding"] is None
+ reloaded = PreTrainedTokenizerFast.from_pretrained(str(tmp_path))
+ assert len(reloaded("a b c d e f g h")["input_ids"]) == 8
+
+
+def test_training_truncation_is_restored_after_save(tmp_path):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+
+ save(tokenizer, str(tmp_path))
+
+ assert tokenizer._tokenizer.truncation["max_length"] == 3
+ assert tokenizer._tokenizer.padding["length"] == 3
+
+
+def test_processor_wrapper_clears_inner_tokenizer_truncation(tmp_path):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+ processor = types.SimpleNamespace(
+ tokenizer = tokenizer,
+ original_save_pretrained = lambda path, **kwargs: tokenizer.save_pretrained(path),
+ )
+
+ save(processor, str(tmp_path))
+
+ saved = json.loads((tmp_path / "tokenizer.json").read_text(encoding = "utf-8"))
+ assert saved["truncation"] is None
+ assert tokenizer._tokenizer.truncation["max_length"] == 3
+
+
+def test_resaving_a_truncated_checkpoint_drops_leaked_max_length(tmp_path, monkeypatch):
+ save = _load_save_helpers()
+ broken = tmp_path / "broken"
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(str(broken))
+ reloaded = PreTrainedTokenizerFast.from_pretrained(str(broken))
+ _capture_loaded_limit(reloaded, monkeypatch)
+ reloaded.original_save_pretrained = reloaded.save_pretrained
+
+ fixed = tmp_path / "fixed"
+ save(reloaded, str(fixed))
+
+ config = json.loads((fixed / "tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config.get("max_length") is None
+ assert (
+ len(PreTrainedTokenizerFast.from_pretrained(str(fixed))("a b c d e f g h")["input_ids"])
+ == 8
+ )
+
+
+class _PaddingClearFails:
+ def __init__(self, tokenizer):
+ self._inner = tokenizer._tokenizer
+
+ def __getattr__(self, name):
+ return getattr(self._inner, name)
+
+ def no_padding(self):
+ raise RuntimeError("no_padding failed")
+
+
+def test_partial_clear_failure_still_restores_truncation(tmp_path):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+ inner = tokenizer._tokenizer
+ processor = types.SimpleNamespace(
+ tokenizer = types.SimpleNamespace(_tokenizer = _PaddingClearFails(tokenizer)),
+ original_save_pretrained = lambda path, **kwargs: None,
+ )
+
+ save(processor, str(tmp_path))
+
+ assert inner.truncation["max_length"] == 3
+ assert inner.padding["length"] == 3
+
+
[email protected]("padding_only", [False, True])
+def test_nested_save_cleans_limit_before_upload(tmp_path, monkeypatch, padding_only):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+ if padding_only:
+ tokenizer._tokenizer.no_truncation()
+ tokenizer.save_pretrained(tmp_path / "broken")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "broken")
+ _capture_loaded_limit(tokenizer, monkeypatch)
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+ tokenizer.save_pretrained = types.MethodType(save, tokenizer)
+ before = dict(tokenizer.init_kwargs)
+
+ def inspect_upload(repo_id, **kwargs):
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert "max_length" not in config
+
+ tokenizer.push_to_hub = inspect_upload
+ processor = types.SimpleNamespace(
+ tokenizer = tokenizer,
+ original_save_pretrained = lambda path, **kwargs: tokenizer.save_pretrained(
+ path, push_to_hub = True
+ ),
+ )
+ save(processor, tmp_path / "fixed")
+ assert tokenizer.init_kwargs == before
+ assert tokenizer._tokenizer.padding["length"] == 3
+
+
+def test_failed_save_restores_serialization_limit(tmp_path, monkeypatch):
+ save = _load_save_helpers()
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "broken")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "broken")
+ _capture_loaded_limit(tokenizer, monkeypatch)
+
+ def fail(*args, **kwargs):
+ assert "max_length" not in tokenizer.init_kwargs
+ raise ValueError("save failed")
+
+ tokenizer.original_save_pretrained = fail
+ with pytest.raises(ValueError, match = "save failed"):
+ save(tokenizer, tmp_path)
+ assert tokenizer.init_kwargs["max_length"] == 3
+ assert tokenizer._tokenizer.truncation["max_length"] == 3
+
+
[email protected]("retrain", [False, True])
+def test_loaded_limit_survives_tokenizer_calls(tmp_path, monkeypatch, retrain):
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ repair = _load_repair_helper()
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "broken")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "broken")
+ processor = types.SimpleNamespace(tokenizer = tokenizer)
+ assert repair(processor) is processor
+
+ if retrain:
+ tokenizer("a b c d e f g h", truncation = True, max_length = 7, padding = "max_length")
+ else:
+ tokenizer("a")
+ repair(processor)
+ before = dict(tokenizer.init_kwargs)
+ truncation, padding = tokenizer._tokenizer.truncation, tokenizer._tokenizer.padding
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert "max_length" not in config
+ assert tokenizer._unsloth_loaded_runtime_max_length == 3
+ assert tokenizer.init_kwargs == before
+ assert tokenizer._tokenizer.truncation == truncation
+ assert tokenizer._tokenizer.padding == padding
+
+
+def test_loaded_limit_preserves_explicit_override_and_restores_on_error(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "broken")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "broken")
+ _load_repair_helper()(tokenizer)
+ tokenizer("a")
+ before = dict(tokenizer.init_kwargs)
+
+ def fail(*args, **kwargs):
+ assert "max_length" not in tokenizer.init_kwargs
+ raise ValueError("save failed")
+
+ tokenizer.original_save_pretrained = fail
+ with pytest.raises(ValueError, match = "save failed"):
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+ assert tokenizer.init_kwargs == before
+ assert tokenizer._tokenizer.truncation is None
+ assert tokenizer._tokenizer.padding is None
+
+ tokenizer.init_kwargs["max_length"] = 11
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config["max_length"] == 11
+ assert tokenizer.init_kwargs["max_length"] == 11
+
+
+def test_limit_without_loaded_provenance_is_not_recaptured(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ repair = _load_repair_helper()
+ tokenizer = _trained_tokenizer()
+ tokenizer.init_kwargs["max_length"] = 11
+ repair(tokenizer)
+ tokenizer("a b", truncation = True, max_length = 11)
+ repair(tokenizer)
+ tokenizer("a")
+ _load_save_helpers()(tokenizer, tmp_path)
+ config = json.loads((tmp_path / "tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config["max_length"] == 11
+
+
[email protected]("reload_source", [False, True])
+def test_matching_explicit_limit_keeps_processor_default(tmp_path, monkeypatch, reload_source):
+ tokenizer = Qwen2TokenizerFast(
+ tokenizer_object = _trained_tokenizer()._tokenizer,
+ pad_token = "[PAD]",
+ unk_token = "[UNK]",
+ max_length = 3,
+ )
+ if reload_source:
+ tokenizer.save_pretrained(tmp_path / "source")
+ tokenizer = Qwen2TokenizerFast.from_pretrained(tmp_path / "source")
+ _capture_loaded_limit(tokenizer, monkeypatch)
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+ before = dict(tokenizer.init_kwargs)
+
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config["max_length"] == 3
+ assert tokenizer.init_kwargs == before
+ processor = Qwen3VLProcessor(
+ image_processor = Qwen2VLImageProcessor(),
+ tokenizer = Qwen2TokenizerFast.from_pretrained(tmp_path / "fixed"),
+ video_processor = Qwen3VLVideoProcessor(),
+ )
+ assert len(processor(text = "a b c d e f g h")["input_ids"][0]) == 3
+
+
[email protected]("config_contents", [None, "not json", "[]"])
+def test_unknown_source_preserves_init_limit(tmp_path, monkeypatch, config_contents):
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "source")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "source")
+ config_path = tmp_path / "source/tokenizer_config.json"
+ if config_contents is None:
+ config_path.unlink()
+ else:
+ config_path.write_text(config_contents, encoding = "utf-8")
+ _capture_loaded_limit(tokenizer, monkeypatch)
+ tokenizer("a")
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config["max_length"] == 3
+
+
+def test_inferred_limit_uses_supplied_hub_cache_and_revision(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "source")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "source")
+ tokenizer.name_or_path = "test/runtime-limit"
+ revision = "a" * 40
+ cached = tmp_path / "hub/models--test--runtime-limit/snapshots" / revision
+ cached.mkdir(parents = True)
+ (cached / "tokenizer_config.json").write_text("{}", encoding = "utf-8")
+ repair = _load_repair_helper()
+ repair(tokenizer, cache_dir = str(tmp_path / "hub"), revision = revision)
+ tokenizer("a")
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert "max_length" not in config
+
+
+def test_missing_custom_cache_config_does_not_use_default_cache(tmp_path, monkeypatch):
+ from huggingface_hub import constants, try_to_load_from_cache
+
+ monkeypatch.setenv("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "1")
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "default_hub"))
+ tokenizer = _trained_tokenizer()
+ tokenizer.save_pretrained(tmp_path / "source")
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(tmp_path / "source")
+ tokenizer.name_or_path = "test/runtime-limit"
+ revision = "b" * 40
+ cached = tmp_path / "default_hub/models--test--runtime-limit/snapshots" / revision
+ cached.mkdir(parents = True)
+ cached_config = cached / "tokenizer_config.json"
+ cached_config.write_text("{}", encoding = "utf-8")
+ custom_cache = tmp_path / "custom_hub"
+ custom_cache.mkdir()
+ repair = _load_repair_helper()
+ assert try_to_load_from_cache(
+ tokenizer.name_or_path, "tokenizer_config.json", revision = revision
+ ) == str(cached_config)
+
+ repair(tokenizer, cache_dir = str(custom_cache), revision = revision)
+ tokenizer("a")
+ tokenizer.original_save_pretrained = tokenizer.save_pretrained
+ _load_save_helpers()(tokenizer, tmp_path / "fixed")
+
+ config = json.loads((tmp_path / "fixed/tokenizer_config.json").read_text(encoding = "utf-8"))
+ assert config["max_length"] == 3
+ assert tokenizer._unsloth_loaded_runtime_max_length is None
diff --git a/unsloth/save.py b/unsloth/save.py
index 6c049403775..16e52267af2 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -986,6 +986,48 @@ def _preserve_tokenizer_eos_token(
)
+@contextmanager
+def _without_runtime_truncation(
+ tokenizer,
+ save_directory = None,
+ filename_prefix = None,
+):
+ source_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ backend = getattr(source_tokenizer, "_tokenizer", None)
+ truncation = getattr(backend, "truncation", None)
+ padding = getattr(backend, "padding", None)
+ init_kwargs = getattr(source_tokenizer, "init_kwargs", {})
+ loaded_limit = getattr(source_tokenizer, "_unsloth_loaded_runtime_max_length", None)
+ max_length = init_kwargs.get("max_length")
+ remove_max_length = loaded_limit is not None and max_length == loaded_limit
+ if truncation is None and padding is None and not remove_max_length:
+ yield
+ return
+ if remove_max_length:
+ init_kwargs.pop("max_length")
+ try:
+ try:
+ backend.no_truncation()
+ backend.no_padding()
+ except Exception as error:
+ logger.warning_once(
+ f"Unsloth: Could not clear tokenizer truncation before saving: {error}"
+ )
+ yield
+ finally:
+ if remove_max_length:
+ init_kwargs["max_length"] = max_length
+ try:
+ if truncation is not None:
+ backend.enable_truncation(**truncation)
+ if padding is not None:
+ backend.enable_padding(**padding)
+ except Exception as error:
+ logger.warning_once(
+ f"Unsloth: Could not restore tokenizer truncation after saving: {error}"
+ )
+
+
def _preserve_repaired_tokenizer_class(
tokenizer,
save_directory,
@@ -1652,7 +1694,12 @@ def unsloth_save_model(
old_padding_side = _tokenizer.padding_side
_tokenizer.padding_side = "left"
- tokenizer.save_pretrained(**tokenizer_save_settings)
+ with _without_runtime_truncation(
+ tokenizer,
+ tokenizer_save_settings["save_directory"],
+ tokenizer_save_settings.get("filename_prefix"),
+ ):
+ tokenizer.save_pretrained(**tokenizer_save_settings)
_tokenizer.padding_side = old_padding_side
@@ -1921,7 +1968,12 @@ def unsloth_save_model(
old_padding_side = _tokenizer.padding_side
_tokenizer.padding_side = "left"
- tokenizer.save_pretrained(**tokenizer_save_settings)
+ with _without_runtime_truncation(
+ tokenizer,
+ tokenizer_save_settings["save_directory"],
+ tokenizer_save_settings.get("filename_prefix"),
+ ):
+ tokenizer.save_pretrained(**tokenizer_save_settings)
_preserve_tokenizer_eos_token(
tokenizer,
tokenizer_save_settings["save_directory"],
@@ -8826,11 +8878,12 @@ def _unsloth_tokenizer_save_pretrained(
save_kwargs["legacy_format"] = legacy_format
if filename_prefix is not None:
save_kwargs["filename_prefix"] = filename_prefix
- result = self.original_save_pretrained(
- save_directory,
- push_to_hub = False,
- **save_kwargs,
- )
+ with _without_runtime_truncation(self, save_directory, filename_prefix):
+ result = self.original_save_pretrained(
+ save_directory,
+ push_to_hub = False,
+ **save_kwargs,
+ )
_preserve_sentencepiece_tokenizer_assets(
self,
save_directory,
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 7e1839d0cbf..3fe73d47242 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -14,6 +14,7 @@ from transformers.convert_slow_tokenizer import convert_slow_tokenizer
from transformers import PreTrainedTokenizerFast
import re
import os
+import json
import shutil
import tempfile
import weakref
@@ -301,22 +302,23 @@ def _resolve_tokenizer_json(
tokenizer,
cache_dir = None,
revision = None,
+ filename = "tokenizer.json",
+ allow_default_cache = True,
):
name = getattr(tokenizer, "name_or_path", None)
if not isinstance(name, str) or not name:
return None
if os.path.isdir(name):
- path = os.path.join(name, "tokenizer.json")
+ path = os.path.join(name, filename)
return path if os.path.isfile(path) else None
try:
from huggingface_hub import try_to_load_from_cache
except Exception:
return None
- for _cache_dir in dict.fromkeys((cache_dir, None)):
+ cache_dirs = (cache_dir, None) if allow_default_cache else (cache_dir,)
+ for _cache_dir in dict.fromkeys(cache_dirs):
try:
- path = try_to_load_from_cache(
- name, "tokenizer.json", cache_dir = _cache_dir, revision = revision
- )
+ path = try_to_load_from_cache(name, filename, cache_dir = _cache_dir, revision = revision)
except Exception:
path = None
if isinstance(path, str) and os.path.isfile(path):
@@ -324,6 +326,34 @@ def _resolve_tokenizer_json(
return None
+def _inferred_tokenizer_max_length(
+ tokenizer,
+ cache_dir = None,
+ revision = None,
+):
+ backend = getattr(tokenizer, "_tokenizer", None)
+ truncation = getattr(backend, "truncation", None)
+ padding = getattr(backend, "padding", None)
+ limit = truncation["max_length"] if truncation else padding.get("length") if padding else None
+ if limit is None or getattr(tokenizer, "init_kwargs", {}).get("max_length") != limit:
+ return None
+ path = _resolve_tokenizer_json(
+ tokenizer,
+ cache_dir = cache_dir,
+ revision = revision,
+ filename = "tokenizer_config.json",
+ allow_default_cache = False,
+ )
+ if path is None:
+ return None
+ try:
+ with open(path, "r", encoding = "utf-8") as file:
+ config = json.load(file)
+ return limit if isinstance(config, dict) and "max_length" not in config else None
+ except Exception:
+ return None
+
+
def _repair_one_tokenizer_backend(
tokenizer,
cache_dir = None,
@@ -381,6 +411,13 @@ def _repair_tokenizer_backend_from_json(
cache_dir = None,
revision = None,
):
+ for obj in _tokenizer_objects(tokenizer):
+ backend = getattr(obj, "_tokenizer", None)
+ if backend is None or hasattr(obj, "_unsloth_loaded_runtime_max_length"):
+ continue
+ obj._unsloth_loaded_runtime_max_length = _inferred_tokenizer_max_length(
+ obj, cache_dir = cache_dir, revision = revision
+ )
if tokenizer is None or os.environ.get("UNSLOTH_DISABLE_TOKENIZER_JSON_REPAIR", "0") == "1":
return tokenizer
for obj in _tokenizer_objects(tokenizer):