Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 SentenceTransformer save: honour create_model_card=False and keep training mode (#15431)
Daniel Han4fbf9edef7f5
diff --git a/tests/python/_st_fsdp2_save_worker.py b/tests/python/_st_fsdp2_save_worker.pyindex b2c0dfbce15..50d31d1c935 100644--- a/tests/python/_st_fsdp2_save_worker.py+++ b/tests/python/_st_fsdp2_save_worker.py@@ -128,3 +128,92 @@ def run_rank(rank, port, checkpoint, out_dir, queue):raiseif result.get("error") is None:dist.destroy_process_group()+++def run_rank_save_options(rank, port, checkpoint, out_dir, queue):+ """create_model_card=False must skip the usage snippet; no save may leave eval mode."""+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ LOCAL_WORLD_SIZE = "2",+ ACCELERATE_USE_CPU = "true",+ UNSLOTH_FSDP2 = "1",+ )+ runpy.run_path(+ os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "conftest.py")+ )+ import torch+ import torch.distributed as dist++ torch.set_num_threads(1)+ from sentence_transformers import SentenceTransformer+ from sentence_transformers.models import Pooling, Transformer++ import unsloth # noqa: F401+ from unsloth import FastSentenceTransformer+ from unsloth.models.fsdp2_native import FSDP2Config, shard_model++ dist.init_process_group("gloo", rank = rank, world_size = 2)+ result = {}+ try:+ torch.manual_seed(0)+ transformer = Transformer(checkpoint)+ model = SentenceTransformer(+ modules = [transformer, Pooling(transformer.get_word_embedding_dimension())],+ device = "cpu",+ )+ model._unsloth_fast_encoder = True+ model._compile_mode = None+ model = FastSentenceTransformer.get_peft_model(+ model,+ r = 4,+ lora_alpha = 8,+ target_modules = ["query", "key", "value", "dense"],+ use_gradient_checkpointing = False,+ )+ shard_model(model, FSDP2Config(device = "cpu", reshard_after_forward = True))+ n_sharded = sum(type(p).__name__ == "DTensor" for p in model.parameters())+ card_cls = type(model.model_card_data)+ original_run = card_cls.run_usage_snippet+ runs = []++ def counted(self, *args, **kwargs):+ runs.append(1)+ return original_run(self, *args, **kwargs)++ card_cls.run_usage_snippet = counted+ try:+ model.train()+ model.model_card_data.similarities = None+ no_card = os.path.join(out_dir, "no_card")+ model.save_pretrained(no_card, create_model_card = False)+ result["no_card_runs"] = len(runs)+ result["no_card_eval_modules"] = sum(not m.training for m in model.modules())+ dist.barrier()+ if rank == 0:+ readme = os.path.join(no_card, "README.md")+ result["no_card_has_st_card"] = os.path.exists(readme) and (+ "library_name: sentence-transformers" in open(readme, encoding = "utf-8").read()+ )+ model.train()+ model.model_card_data.similarities = None+ model.save_pretrained(os.path.join(out_dir, "card"))+ result["card_runs"] = len(runs) - result["no_card_runs"]+ result["card_eval_modules"] = sum(not m.training for m in model.modules())+ finally:+ card_cls.run_usage_snippet = original_run+ result.update(+ error = None,+ n_sharded = n_sharded,+ still_sharded = sum(type(p).__name__ == "DTensor" for p in model.parameters()),+ )+ dist.barrier()+ queue.put((rank, result))+ except BaseException:+ import traceback+ queue.put((rank, {"error": traceback.format_exc()}))+ raise+ dist.destroy_process_group()diff --git a/tests/python/test_sentence_transformer_fsdp2_save.py b/tests/python/test_sentence_transformer_fsdp2_save.pyindex bf6b5fac2c0..99b0df4741d 100644--- a/tests/python/test_sentence_transformer_fsdp2_save.py+++ b/tests/python/test_sentence_transformer_fsdp2_save.py@@ -66,7 +66,7 @@ def _tiny_bert(path):return str(path)-def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):+def _run_two_ranks(target, tmp_path):import torch.multiprocessing as mpcheckpoint = _tiny_bert(tmp_path / "bert")@@ -74,7 +74,7 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):queue = ctx.Queue()port = _free_port()procs = [- ctx.Process(target = worker.run_rank, args = (r, port, checkpoint, str(tmp_path), queue))+ ctx.Process(target = target, args = (r, port, checkpoint, str(tmp_path), queue))for r in range(2)]for p in procs:@@ -96,6 +96,11 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):for rank, out in sorted(results.items()):assert out["error"] is None, (rank, out["error"])assert sorted(results) == [0, 1]+ return results+++def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):+ results = _run_two_ranks(worker.run_rank, tmp_path)for rank in (0, 1):assert results[rank]["n_sharded"] > 0# The gather put the shards back afterwards.@@ -108,3 +113,16 @@ def test_fsdp2_sentence_transformer_save_equals_one_process(tmp_path):assert fsdp2[name] == reference[name], name# Trainer.save_model (checkpoints, SentenceTransformerTrainer._save -> save_pretrained) writes the same.assert results[0]["trainer"] == fsdp2+++def test_fsdp2_sentence_transformer_save_honours_card_option_and_training_mode(tmp_path):+ results = _run_two_ranks(worker.run_rank_save_options, tmp_path)+ for rank in (0, 1):+ out = results[rank]+ assert out["n_sharded"] > 0+ assert out["still_sharded"] == out["n_sharded"]+ assert out["no_card_runs"] == 0, (rank, out)+ assert out["no_card_eval_modules"] == 0, (rank, out)+ assert out["card_runs"] == 1, (rank, out)+ assert out["card_eval_modules"] == 0, (rank, out)+ assert results[0]["no_card_has_st_card"] is Falsediff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex 670d31d48e0..55be99db598 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -93,6 +93,8 @@ def fsdp2_st_usage_snippet(model):from .fsdp2_native import native_fsdp2_unshardedmodel.__dict__["to"] = lambda *args, **kwargs: model+ # encode() switches the model to eval mode; a save must not change training mode.+ modes = [(module, module.training) for module in model.modules()]try:# Whole weights on every rank for the snippet: older sentence-transformers' encode calls# self.forward, which skips the root unit's FSDP2 hooks.@@ -102,6 +104,8 @@ def fsdp2_st_usage_snippet(model):print(f"Unsloth: FSDP2 save: model card usage snippet not computed ({exc}).")finally:model.__dict__.pop("to", None)+ for module, training in modes:+ module.training = trainingcard.__dict__["run_usage_snippet"] = lambda *args, **kwargs: Nonetry:yield@@ -109,6 +113,14 @@ def fsdp2_st_usage_snippet(model):card.__dict__.pop("run_usage_snippet", None)+def _save_creates_model_card(save, model, args, kwargs):+ try:+ bound = inspect.signature(save).bind_partial(model, *args, **kwargs)+ except (TypeError, ValueError):+ return kwargs.get("create_model_card", True) is not False+ return bound.arguments.get("create_model_card", True) is not False++def _install_fsdp2_collective_st_save(SentenceTransformer):"""Under native FSDP2 (UNSLOTH_FSDP2=1) the encoder's weights are DTensor shards, and ST's `save`(behind `save_pretrained` and the Trainer's `_save`) reads them directly, on every rank, into the@@ -145,9 +157,15 @@ def _install_fsdp2_collective_st_save(SentenceTransformer):# The writer inside a gather already made (Trainer.save_model under native FSDP2, which# also ran fsdp2_st_usage_snippet on every rank).return plainly_named(self, args, kwargs)+ # No card: skip the snippet's unshard + forward.+ snippet = (+ fsdp2_st_usage_snippet(self)+ if _save_creates_model_card(original, self, args, kwargs)+ else contextlib.nullcontext()+ )# Every parameter, not just the trainable ones: ST writes its other modules (Dense, ...) too,# and embedding models are small.- with fsdp2_st_usage_snippet(self), fsdp2_gathered(self) as writer:+ with snippet, fsdp2_gathered(self) as writer:if writer is False:return Nonereturn plainly_named(self, args, kwargs)