Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SentenceTransformer: add opt-in FP32 merged-pair ranking loss (#11867)

Etherl676e09f6e435
diff --git a/tests/python/test_sentence_transformer_merged_pair_loss.py b/tests/python/test_sentence_transformer_merged_pair_loss.py
new file mode 100644
index 00000000000..1d50293229b
--- /dev/null
+++ b/tests/python/test_sentence_transformer_merged_pair_loss.py
@@ -0,0 +1,295 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+import copy
+import importlib.util
+import inspect
+from pathlib import Path
+
+import pytest
+import torch
+from torch import nn
+
+
+pytest.importorskip("sentence_transformers")
+SOURCE = Path(__file__).resolve().parents[2] / "unsloth/models/sentence_transformer_loss.py"
+SPEC = importlib.util.spec_from_file_location("st_merged_loss_under_test", SOURCE)
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+class TinyEncoder(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embedding = nn.Embedding(64, 8)
+ self.projection = nn.Linear(8, 8)
+
+ def forward(self, features):
+ values = self.embedding(features["input_ids"])
+ mask = features["attention_mask"].unsqueeze(-1)
+ pooled = (values * mask).sum(1) / mask.sum(1)
+ return {"sentence_embedding": self.projection(pooled)}
+
+
+class MixedDtypeTinyEncoder(TinyEncoder):
+ def __init__(self):
+ super().__init__()
+ self.projection.to(torch.bfloat16)
+
+ def forward(self, features):
+ values = self.embedding(features["input_ids"])
+ mask = features["attention_mask"].unsqueeze(-1)
+ pooled = (values * mask).sum(1) / mask.sum(1)
+ return {"sentence_embedding": self.projection(pooled.to(torch.bfloat16))}
+
+
+class BatchNormTinyEncoder(TinyEncoder):
+ def __init__(self):
+ super().__init__()
+ self.normalizer = nn.BatchNorm1d(8)
+
+ def forward(self, features):
+ values = self.embedding(features["input_ids"])
+ mask = features["attention_mask"].unsqueeze(-1)
+ pooled = (values * mask).sum(1) / mask.sum(1)
+ return {"sentence_embedding": self.projection(self.normalizer(pooled))}
+
+
+def pair_features(task_mismatch = False):
+ left = {
+ "input_ids": torch.tensor([[1, 2, 3, 0], [4, 5, 6, 7], [8, 9, 0, 0]]),
+ "attention_mask": torch.tensor([[1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 0, 0]]),
+ }
+ right = {
+ "input_ids": torch.tensor([[3, 2, 1, 10, 0, 0], [7, 6, 5, 4, 11, 12], [9, 8, 13, 0, 0, 0]]),
+ "attention_mask": torch.tensor(
+ [[1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0]]
+ ),
+ }
+ if task_mismatch:
+ left["task"] = "query"
+ right["task"] = "document"
+ return left, right
+
+
[email protected]("task_mismatch", [False, True])
+def test_loss_gradients_and_adamw_update(task_mismatch):
+ try:
+ from sentence_transformers.sentence_transformer.losses import MultipleNegativesRankingLoss
+ except ImportError:
+ from sentence_transformers.losses import MultipleNegativesRankingLoss
+
+ torch.manual_seed(4460)
+ reference_model = TinyEncoder()
+ candidate_model = copy.deepcopy(reference_model)
+ features = pair_features(task_mismatch)
+ reference = MultipleNegativesRankingLoss(reference_model)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(candidate_model)
+ ref_loss = reference([dict(x) for x in features], None)
+ got_loss = candidate([dict(x) for x in features], None)
+ torch.testing.assert_close(got_loss, ref_loss, atol = 1e-6, rtol = 1e-6)
+ ref_loss.backward()
+ got_loss.backward()
+ for a, b in zip(reference_model.parameters(), candidate_model.parameters()):
+ torch.testing.assert_close(b.grad, a.grad, atol = 1e-6, rtol = 1e-6)
+ ref_opt = torch.optim.AdamW(reference_model.parameters(), lr = 1e-3)
+ got_opt = torch.optim.AdamW(candidate_model.parameters(), lr = 1e-3)
+ ref_opt.step()
+ got_opt.step()
+ for a, b in zip(reference_model.parameters(), candidate_model.parameters()):
+ torch.testing.assert_close(b, a, atol = 1e-6, rtol = 1e-6)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (
+ (0, 1) if task_mismatch or MODULE.merge_feature_batches is None else (1, 0)
+ )
+
+
+def test_older_sentence_transformers_helper_falls_back(monkeypatch):
+ monkeypatch.setattr(MODULE, "merge_feature_batches", None)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder())
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_wrapper_can_disable_column_merging():
+ if MODULE.merge_feature_batches is None:
+ pytest.skip("current SentenceTransformers merged-feature helper is unavailable")
+ from sentence_transformers.base.losses.merged_forward import column_merging_disabled
+
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder())
+ with column_merging_disabled():
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
[email protected]("case", ["packed", "hard_negative", "prompt_mismatch"])
+def test_ineligible_features_use_reference_loss(case):
+ try:
+ from sentence_transformers.sentence_transformer.losses import MultipleNegativesRankingLoss
+ except ImportError:
+ from sentence_transformers.losses import MultipleNegativesRankingLoss
+
+ torch.manual_seed(4460)
+ reference_model = TinyEncoder()
+ candidate_model = copy.deepcopy(reference_model)
+ features = list(pair_features())
+ if case == "packed":
+ for column in features:
+ column["cu_seq_lens_q"] = torch.tensor([0, 2, 4, 6])
+ elif case == "hard_negative":
+ features.append({key: value.clone() for key, value in features[1].items()})
+ features[-1]["input_ids"] = features[-1]["input_ids"].roll(1, dims = 0)
+ else:
+ features[0]["prompt_length"] = 1
+ features[1]["prompt_length"] = 2
+ reference = MultipleNegativesRankingLoss(reference_model)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(candidate_model)
+ expected = reference([dict(column) for column in features], None)
+ actual = candidate([dict(column) for column in features], None)
+ torch.testing.assert_close(actual, expected, atol = 1e-6, rtol = 1e-6)
+ actual.backward()
+ assert all(parameter.grad is not None for parameter in candidate_model.parameters())
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_distributed_gather_option_keeps_stock_forward():
+ if (
+ "gather_across_devices"
+ not in inspect.signature(MODULE.MultipleNegativesRankingLoss).parameters
+ ):
+ pytest.skip("this SentenceTransformers MNRL has no gather_across_devices option")
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder(), gather_across_devices = True)
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_long_asymmetric_columns_keep_stock_forward():
+ features = list(pair_features())
+ features[0]["input_ids"] = torch.nn.functional.pad(features[0]["input_ids"], (0, 40))
+ features[0]["attention_mask"] = torch.nn.functional.pad(features[0]["attention_mask"], (0, 40))
+ features[1]["input_ids"] = torch.nn.functional.pad(features[1]["input_ids"], (0, 186))
+ features[1]["attention_mask"] = torch.nn.functional.pad(features[1]["attention_mask"], (0, 186))
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder())
+ loss = candidate(features, None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_autocast_keeps_stock_forward():
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder())
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_non_fp32_weights_keep_stock_forward(dtype):
+ candidate = MODULE.FastMultipleNegativesRankingLoss(TinyEncoder().to(dtype))
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_mixed_dtype_weights_keep_stock_forward():
+ model = MixedDtypeTinyEncoder()
+ assert next(model.parameters()).dtype == torch.float32
+ candidate = MODULE.FastMultipleNegativesRankingLoss(model)
+ loss = candidate(list(pair_features()), None)
+ assert torch.isfinite(loss)
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_batch_norm_keeps_stock_running_statistics():
+ try:
+ from sentence_transformers.sentence_transformer.losses import MultipleNegativesRankingLoss
+ except ImportError:
+ from sentence_transformers.losses import MultipleNegativesRankingLoss
+
+ torch.manual_seed(4460)
+ reference_model = BatchNormTinyEncoder()
+ candidate_model = copy.deepcopy(reference_model)
+ reference = MultipleNegativesRankingLoss(reference_model)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(candidate_model)
+ expected = reference(list(pair_features()), None)
+ actual = candidate(list(pair_features()), None)
+ torch.testing.assert_close(actual, expected, atol = 1e-6, rtol = 1e-6)
+ expected.backward()
+ actual.backward()
+ for reference_parameter, candidate_parameter in zip(
+ reference_model.parameters(), candidate_model.parameters()
+ ):
+ torch.testing.assert_close(
+ candidate_parameter.grad, reference_parameter.grad, atol = 1e-6, rtol = 1e-6
+ )
+ reference_optimizer = torch.optim.AdamW(reference_model.parameters(), lr = 1e-3)
+ candidate_optimizer = torch.optim.AdamW(candidate_model.parameters(), lr = 1e-3)
+ reference_optimizer.step()
+ candidate_optimizer.step()
+ for reference_parameter, candidate_parameter in zip(
+ reference_model.parameters(), candidate_model.parameters()
+ ):
+ torch.testing.assert_close(candidate_parameter, reference_parameter, atol = 1e-6, rtol = 1e-6)
+ for name, buffer in reference_model.named_buffers():
+ torch.testing.assert_close(dict(candidate_model.named_buffers())[name], buffer)
+ assert candidate_model.normalizer.num_batches_tracked == 2
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+
+
+def test_batch_norm_added_after_loss_construction_keeps_stock_semantics():
+ torch.manual_seed(4460)
+ reference_model = TinyEncoder()
+ candidate_model = copy.deepcopy(reference_model)
+ reference = MODULE.MultipleNegativesRankingLoss(reference_model)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(candidate_model)
+ reference_model.projection = nn.Sequential(reference_model.projection, nn.BatchNorm1d(8))
+ candidate_model.projection = nn.Sequential(candidate_model.projection, nn.BatchNorm1d(8))
+ expected = reference(list(pair_features()), None)
+ actual = candidate(list(pair_features()), None)
+ torch.testing.assert_close(actual, expected, atol = 1e-6, rtol = 1e-6)
+ expected.backward()
+ actual.backward()
+ for left, right in zip(reference_model.parameters(), candidate_model.parameters()):
+ torch.testing.assert_close(right.grad, left.grad, atol = 1e-6, rtol = 1e-6)
+ for name, buffer in reference_model.named_buffers():
+ torch.testing.assert_close(dict(candidate_model.named_buffers())[name], buffer)
+ assert candidate_model.projection[1].num_batches_tracked == 2
+ assert (candidate.merged_calls, candidate.fallback_calls) == (0, 1)
+ torch.optim.AdamW(reference_model.parameters(), lr = 1e-3).step()
+ torch.optim.AdamW(candidate_model.parameters(), lr = 1e-3).step()
+ for left, right in zip(reference_model.parameters(), candidate_model.parameters()):
+ torch.testing.assert_close(right, left, atol = 1e-6, rtol = 1e-6)
+
+
+ "options",
+ [
+ {"directions": ("query_to_doc", "doc_to_query")},
+ {"directions": ("query_to_doc", "query_to_query", "doc_to_query", "doc_to_doc")},
+ {"hardness_mode": "in_batch_negatives", "hardness_strength": 5.0},
+ ],
+)
+def test_optional_global_negative_modes_preserve_loss_and_gradients(options):
+ if MODULE.merge_feature_batches is None:
+ pytest.skip("current SentenceTransformers merged-feature helper is unavailable")
+ from sentence_transformers.sentence_transformer.losses import MultipleNegativesRankingLoss
+
+ torch.manual_seed(4460)
+ reference_model = TinyEncoder()
+ candidate_model = copy.deepcopy(reference_model)
+ reference = MultipleNegativesRankingLoss(reference_model, **options)
+ candidate = MODULE.FastMultipleNegativesRankingLoss(candidate_model, **options)
+ expected = reference([dict(column) for column in pair_features()], None)
+ actual = candidate([dict(column) for column in pair_features()], None)
+ torch.testing.assert_close(actual, expected, atol = 1e-6, rtol = 1e-6)
+ expected.backward()
+ actual.backward()
+ for reference_parameter, candidate_parameter in zip(
+ reference_model.parameters(), candidate_model.parameters()
+ ):
+ torch.testing.assert_close(
+ candidate_parameter.grad, reference_parameter.grad, atol = 1e-6, rtol = 1e-6
+ )
+ assert (candidate.merged_calls, candidate.fallback_calls) == (1, 0)
diff --git a/unsloth/models/sentence_transformer_loss.py b/unsloth/models/sentence_transformer_loss.py
new file mode 100644
index 00000000000..3243d2758e9
--- /dev/null
+++ b/unsloth/models/sentence_transformer_loss.py
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+import torch
+
+try:
+ from sentence_transformers.sentence_transformer.losses import MultipleNegativesRankingLoss
+except ImportError:
+ from sentence_transformers.losses import MultipleNegativesRankingLoss
+
+try:
+ from sentence_transformers.base.losses.merged_forward import merge_feature_batches
+except ImportError:
+ merge_feature_batches = None
+
+
+class FastMultipleNegativesRankingLoss(MultipleNegativesRankingLoss):
+ """MNRL embedding anchor+positive in one forward; falls back to stock when parity is unproven."""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.merged_calls = 0
+ self.fallback_calls = 0
+
+ def forward(self, sentence_features, labels):
+ features = list(sentence_features)
+ parameter = next(self.model.parameters(), None)
+ if (
+ len(features) != 2
+ or merge_feature_batches is None
+ or not hasattr(self, "compute_loss_from_embeddings")
+ or getattr(self, "gather_across_devices", False)
+ or torch.is_autocast_enabled("cuda")
+ or torch.is_autocast_enabled("cpu")
+ or parameter is None
+ or parameter.device.type not in ("cpu", "cuda")
+ or any(
+ value.dtype != torch.float32 or value.device != parameter.device
+ for value in self.model.parameters()
+ )
+ or any(
+ isinstance(module, torch.nn.modules.batchnorm._BatchNorm)
+ for module in self.model.modules()
+ )
+ ):
+ self.fallback_calls += 1
+ return super().forward(features, labels)
+
+ tensors = [feature.get("input_ids", feature.get("attention_mask")) for feature in features]
+ if any(not isinstance(tensor, torch.Tensor) or tensor.ndim != 2 for tensor in tensors):
+ self.fallback_calls += 1
+ return super().forward(features, labels)
+ widths = [tensor.shape[1] for tensor in tensors]
+ # Padding a short query to a long document width regressed throughput.
+ if max(widths) > 128 and max(widths) >= 2 * min(widths):
+ self.fallback_calls += 1
+ return super().forward(features, labels)
+
+ merged = merge_feature_batches(features)
+ if merged is None:
+ self.fallback_calls += 1
+ return super().forward(features, labels)
+
+ rows = tensors[0].shape[0]
+ embeddings = self.model(merged)["sentence_embedding"]
+ if embeddings.shape[0] != 2 * rows:
+ raise RuntimeError("Merged SentenceTransformer output has the wrong batch size")
+ self.merged_calls += 1
+ return self.compute_loss_from_embeddings(list(embeddings.split(rows, dim = 0)), labels)