Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 SentenceTransformer save: honour create_model_card=False and keep training mode (#15431)

Daniel Han4fbf9edef7f5
diff --git a/tests/python/_st_fsdp2_save_worker.py b/tests/python/_st_fsdp2_save_worker.py
index b2c0dfbce15..50d31d1c935 100644
--- a/tests/python/_st_fsdp2_save_worker.py
+++ b/tests/python/_st_fsdp2_save_worker.py
@@ -128,3 +128,92 @@ def run_rank(rank, port, checkpoint, out_dir, queue):
raise
if result.get("error") is None:
dist.destroy_process_group()
+
+
+def run_rank_save_options(rank, port, checkpoint, out_dir, queue):
+ """create_model_card=False must skip the usage snippet; no save may leave eval mode."""
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ LOCAL_WORLD_SIZE = "2",
+ ACCELERATE_USE_CPU = "true",
+ UNSLOTH_FSDP2 = "1",
+ )
+ runpy.run_path(
+ os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "conftest.py")
+ )
+ import torch
+ import torch.distributed as dist
+
+ torch.set_num_threads(1)
+ from sentence_transformers import SentenceTransformer
+ from sentence_transformers.models import Pooling, Transformer
+
+ import unsloth # noqa: F401
+ from unsloth import FastSentenceTransformer
+ from unsloth.models.fsdp2_native import FSDP2Config, shard_model
+
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ result = {}
+ try:
+ torch.manual_seed(0)
+ transformer = Transformer(checkpoint)
+ model = SentenceTransformer(
+ modules = [transformer, Pooling(transformer.get_word_embedding_dimension())],
+ device = "cpu",
+ )
+ model._unsloth_fast_encoder = True
+ model._compile_mode = None
+ model = FastSentenceTransformer.get_peft_model(
+ model,
+ r = 4,
+ lora_alpha = 8,
+ target_modules = ["query", "key", "value", "dense"],
+ use_gradient_checkpointing = False,
+ )
+ shard_model(model, FSDP2Config(device = "cpu", reshard_after_forward = True))
+ n_sharded = sum(type(p).__name__ == "DTensor" for p in model.parameters())
+ card_cls = type(model.model_card_data)
+ original_run = card_cls.run_usage_snippet
+ runs = []
+
+ def counted(self, *args, **kwargs):
+ runs.append(1)
+ return original_run(self, *args, **kwargs)
+
+ card_cls.run_usage_snippet = counted
+ try:
+ model.train()
+ model.model_card_data.similarities = None
+ no_card = os.path.join(out_dir, "no_card")
+ model.save_pretrained(no_card, create_model_card = False)
+ result["no_card_runs"] = len(runs)
+ result["no_card_eval_modules"] = sum(not m.training for m in model.modules())
+ dist.barrier()
+ if rank == 0:
+ readme = os.path.join(no_card, "README.md")
+ result["no_card_has_st_card"] = os.path.exists(readme) and (
+ "library_name: sentence-transformers" in open(readme, encoding = "utf-8").read()
+ )
+ model.train()
+ model.model_card_data.similarities = None
+ model.save_pretrained(os.path.join(out_dir, "card"))
+ result["card_runs"] = len(runs) - result["no_card_runs"]
+ result["card_eval_modules"] = sum(not m.training for m in model.modules())
+ finally:
+ card_cls.run_usage_snippet = original_run
+ result.update(
+ error = None,
+ n_sharded = n_sharded,
+ still_sharded = sum(type(p).__name__ == "DTensor" for p in model.parameters()),
+ )
+ dist.barrier()
+ queue.put((rank, result))
+ except BaseException:
+ import traceback
+ queue.put((rank, {"error": traceback.format_exc()}))
+ raise
+ dist.destroy_process_group()
diff --git a/tests/python/test_sentence_transformer_fsdp2_save.py b/tests/python/test_sentence_transformer_fsdp2_save.py
index bf6b5fac2c0..99b0df4741d 100644
--- a/tests/python/test_sentence_transformer_fsdp2_save.py
+++ b/tests/python/test_sentence_transformer_fsdp2_save.py
@@ -66,7 +66,7 @@ def _tiny_bert(path):
return str(path)
-def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):
+def _run_two_ranks(target, tmp_path):
import torch.multiprocessing as mp
checkpoint = _tiny_bert(tmp_path / "bert")
@@ -74,7 +74,7 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):
queue = ctx.Queue()
port = _free_port()
procs = [
- ctx.Process(target = worker.run_rank, args = (r, port, checkpoint, str(tmp_path), queue))
+ ctx.Process(target = target, args = (r, port, checkpoint, str(tmp_path), queue))
for r in range(2)
]
for p in procs:
@@ -96,6 +96,11 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):
for rank, out in sorted(results.items()):
assert out["error"] is None, (rank, out["error"])
assert sorted(results) == [0, 1]
+ return results
+
+
+def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):
+ results = _run_two_ranks(worker.run_rank, tmp_path)
for rank in (0, 1):
assert results[rank]["n_sharded"] > 0
# The gather put the shards back afterwards.
@@ -108,3 +113,16 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):
assert fsdp2[name] == reference[name], name
# Trainer.save_model (checkpoints, SentenceTransformerTrainer._save -> save_pretrained) writes the same.
assert results[0]["trainer"] == fsdp2
+
+
+def test_fsdp2_sentence_transformer_save_honours_card_option_and_training_mode(tmp_path):
+ results = _run_two_ranks(worker.run_rank_save_options, tmp_path)
+ for rank in (0, 1):
+ out = results[rank]
+ assert out["n_sharded"] > 0
+ assert out["still_sharded"] == out["n_sharded"]
+ assert out["no_card_runs"] == 0, (rank, out)
+ assert out["no_card_eval_modules"] == 0, (rank, out)
+ assert out["card_runs"] == 1, (rank, out)
+ assert out["card_eval_modules"] == 0, (rank, out)
+ assert results[0]["no_card_has_st_card"] is False
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 670d31d48e0..55be99db598 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -93,6 +93,8 @@ def fsdp2_st_usage_snippet(model):
from .fsdp2_native import native_fsdp2_unsharded
model.__dict__["to"] = lambda *args, **kwargs: model
+ # encode() switches the model to eval mode; a save must not change training mode.
+ modes = [(module, module.training) for module in model.modules()]
try:
# Whole weights on every rank for the snippet: older sentence-transformers' encode calls
# self.forward, which skips the root unit's FSDP2 hooks.
@@ -102,6 +104,8 @@ def fsdp2_st_usage_snippet(model):
print(f"Unsloth: FSDP2 save: model card usage snippet not computed ({exc}).")
finally:
model.__dict__.pop("to", None)
+ for module, training in modes:
+ module.training = training
card.__dict__["run_usage_snippet"] = lambda *args, **kwargs: None
try:
yield
@@ -109,6 +113,14 @@ def fsdp2_st_usage_snippet(model):
card.__dict__.pop("run_usage_snippet", None)
+def _save_creates_model_card(save, model, args, kwargs):
+ try:
+ bound = inspect.signature(save).bind_partial(model, *args, **kwargs)
+ except (TypeError, ValueError):
+ return kwargs.get("create_model_card", True) is not False
+ return bound.arguments.get("create_model_card", True) is not False
+
+
def _install_fsdp2_collective_st_save(SentenceTransformer):
"""Under native FSDP2 (UNSLOTH_FSDP2=1) the encoder's weights are DTensor shards, and ST's `save`
(behind `save_pretrained` and the Trainer's `_save`) reads them directly, on every rank, into the
@@ -145,9 +157,15 @@ def _install_fsdp2_collective_st_save(SentenceTransformer):
# The writer inside a gather already made (Trainer.save_model under native FSDP2, which
# also ran fsdp2_st_usage_snippet on every rank).
return plainly_named(self, args, kwargs)
+ # No card: skip the snippet's unshard + forward.
+ snippet = (
+ fsdp2_st_usage_snippet(self)
+ if _save_creates_model_card(original, self, args, kwargs)
+ else contextlib.nullcontext()
+ )
# Every parameter, not just the trainable ones: ST writes its other modules (Dense, ...) too,
# and embedding models are small.
- with fsdp2_st_usage_snippet(self), fsdp2_gathered(self) as writer:
+ with snippet, fsdp2_gathered(self) as writer:
if writer is False:
return None
return plainly_named(self, args, kwargs)