Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FastSentenceTransformer: reuse the loaded model for Transformer subclass modules (#13591)

Daniel Han2e556ba2ecdd
diff --git a/tests/test_st_transformer_subclass_reuse.py b/tests/test_st_transformer_subclass_reuse.py
new file mode 100644
index 00000000000..d1de103155e
--- /dev/null
+++ b/tests/test_st_transformer_subclass_reuse.py
@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""A modules.json Transformer subclass (tencent/WeMM-Embedding) must wrap the model Unsloth loaded, not load a second copy (#13566)."""
+
+import json
+import sys
+
+import pytest
+
+pytest.importorskip("sentence_transformers")
+
+from unsloth import FastSentenceTransformer # noqa: E402
+
+PKG = "unsloth_st_subclass_pkg"
+
+
+def subclass_model(tmp_path, monkeypatch):
+ from transformers import BertConfig, BertModel, BertTokenizerFast
+
+ vocab = tmp_path / "vocab.txt"
+ vocab.write_text(
+ "\n".join(["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "a", "red", "square"]),
+ encoding = "utf-8",
+ )
+ tokenizer = BertTokenizerFast(vocab_file = str(vocab))
+ config = BertConfig(
+ vocab_size = 8,
+ hidden_size = 16,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 32,
+ max_position_embeddings = 64,
+ )
+ model = BertModel(config).eval()
+
+ model_dir = tmp_path / "model"
+ model.save_pretrained(model_dir)
+ tokenizer.save_pretrained(model_dir)
+
+ pkg = tmp_path / "site" / PKG
+ pkg.mkdir(parents = True)
+ (pkg / "__init__.py").write_text(
+ "from sentence_transformers.models import Transformer\n"
+ "class RepoTransformer(Transformer):\n"
+ " def repo_method(self):\n"
+ " return 'repo'\n"
+ "class OwnLoadTransformer(Transformer):\n"
+ " @classmethod\n"
+ " def load(cls, input_path):\n"
+ " obj = cls.__new__(cls)\n"
+ " obj.loaded_by_own_load = input_path\n"
+ " return obj\n"
+ "class NotATransformer:\n"
+ " @classmethod\n"
+ " def load(cls, path, **kwargs):\n"
+ " return cls()\n",
+ encoding = "utf-8",
+ )
+ monkeypatch.syspath_prepend(str(tmp_path / "site"))
+
+ def write_modules(type_ref):
+ (model_dir / "modules.json").write_text(
+ json.dumps([{"idx": 0, "name": "0", "path": "", "type": type_ref}]),
+ encoding = "utf-8",
+ )
+
+ yield model_dir, model, tokenizer, write_modules
+
+ for name in [m for m in sys.modules if m.split(".")[0] == PKG]:
+ del sys.modules[name]
+
+
+def _load(model_dir, model, tokenizer):
+ modules, no_modules_json = FastSentenceTransformer._load_modules(
+ str(model_dir),
+ None,
+ model,
+ tokenizer,
+ 32,
+ "mean",
+ trust_remote_code = True,
+ )
+ assert no_modules_json is False
+ return modules["0"]
+
+
+def test_a_transformer_subclass_wraps_the_loaded_model(subclass_model):
+ model_dir, model, tokenizer, write_modules = subclass_model
+ write_modules(f"{PKG}.RepoTransformer")
+
+ module = _load(model_dir, model, tokenizer)
+
+ assert type(module).__name__ == "RepoTransformer"
+ assert module.repo_method() == "repo"
+ assert module.auto_model is model
+ assert module.max_seq_length == 32
+
+
+def test_the_stock_transformer_still_wraps_the_loaded_model(subclass_model):
+ from sentence_transformers.models import Transformer
+
+ model_dir, model, tokenizer, write_modules = subclass_model
+ write_modules("sentence_transformers.models.Transformer")
+
+ module = _load(model_dir, model, tokenizer)
+
+ assert type(module) is Transformer
+ assert module.auto_model is model
+
+
+def test_a_custom_module_that_is_not_a_transformer_keeps_its_own_load(subclass_model):
+ model_dir, model, tokenizer, write_modules = subclass_model
+ write_modules(f"{PKG}.NotATransformer")
+
+ module = _load(model_dir, model, tokenizer)
+
+ assert type(module).__name__ == "NotATransformer"
+
+
+def test_a_transformer_subclass_with_its_own_load_keeps_it(subclass_model):
+ model_dir, model, tokenizer, write_modules = subclass_model
+ write_modules(f"{PKG}.OwnLoadTransformer")
+
+ module = _load(model_dir, model, tokenizer)
+
+ assert type(module).__name__ == "OwnLoadTransformer"
+ assert module.loaded_by_own_load
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 66667906698..efbfb6c5bec 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -1106,10 +1106,15 @@ class FastSentenceTransformer(FastModel):
cache_dir = None,
revision = None,
module_subfolder = "",
+ transformer_class = None,
):
"""Helper to create and configure a Transformer module."""
from sentence_transformers.models import Transformer
+ # A repo's Transformer subclass, built around our model.
+ if transformer_class is not None:
+ Transformer = transformer_class
+
# Prevents loading the model a second time and redirects AutoProcessor/AutoTokenizer so Transformer.__init__ picks up our pre-fixed tokenizer.
from transformers import AutoProcessor, AutoTokenizer
@@ -1362,7 +1367,7 @@ class FastSentenceTransformer(FastModel):
@staticmethod
def _is_transformer_module_ref(class_ref):
"""Name ST's Transformer without importing anything, since importing to compare is the same
- execution the gate refuses. Any other spelling is caught by `is Transformer` in
+ execution the gate refuses. Any other spelling (or a subclass) is caught by `issubclass` in
_load_modules, after gating; refusals all live in _resolve_module_class."""
if not isinstance(class_ref, str):
return False
@@ -2063,7 +2068,13 @@ class FastSentenceTransformer(FastModel):
cache_dir = cache_dir,
revision = revision,
)
- is_transformer_module = module_class is Transformer
+ # Stock load() would re-read the weights unpatched; a subclass's own load() may restore its state, so keep it.
+ is_transformer_module = (
+ isinstance(module_class, type)
+ and issubclass(module_class, Transformer)
+ and inspect.getattr_static(module_class, "load", None)
+ is inspect.getattr_static(Transformer, "load", None)
+ )
if is_transformer_module:
transformer_module = FastSentenceTransformer._create_transformer_module(
@@ -2076,6 +2087,7 @@ class FastSentenceTransformer(FastModel):
cache_dir,
revision,
module_subfolder = module_config.get("path") or "",
+ transformer_class = module_class,
)
modules[name] = transformer_module
else: