Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for #1234: embedding contrastive in-batch negatives require batch_size >= 2."""
from __future__ import annotations
from unittest.mock import patch
import pytest
import yaml
from soup_cli.config.loader import load_config_from_string
from soup_cli.utils.gpu import model_size_from_name
def _tiny_bert_dir(tmp_path):
"""Build a tiny offline BERT model and tokenizer on CPU without downloading."""
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast
model_dir = tmp_path / "tiny_bert"
model_dir.mkdir(parents=True, exist_ok=True)
config = BertConfig(
vocab_size=64,
hidden_size=32,
num_hidden_layers=2,
num_attention_heads=2,
intermediate_size=64,
max_position_embeddings=128,
)
model = BertModel(config).eval()
model.save_pretrained(str(model_dir))
vocab = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "hello": 4, "world": 5, "python": 6}
tok = Tokenizer(models.WordLevel(vocab=vocab, unk_token="[UNK]"))
tok.pre_tokenizer = pre_tokenizers.Whitespace()
fast = PreTrainedTokenizerFast(
tokenizer_object=tok,
unk_token="[UNK]",
pad_token="[PAD]",
cls_token="[CLS]",
sep_token="[SEP]",
)
fast.save_pretrained(str(model_dir))
return str(model_dir)
class TestParseTimeRefusal:
"""Explicit batch_size: 1 with embedding_loss: contrastive must be refused at parse time."""
def test_refuse_explicit_batch_size_1_contrastive(self):
cfg_str = """
base: test-model
task: embedding
data:
train: dummy.jsonl
max_length: 64
training:
batch_size: 1
embedding_loss: contrastive
"""
with pytest.raises(ValueError) as excinfo:
load_config_from_string(cfg_str)
err = str(excinfo.value)
assert "contrastive" in err
assert "batch_size" in err
assert "batch_size >= 2" in err
def test_refuse_explicit_batch_size_1_with_gradient_accumulation(self):
cfg_str = """
base: test-model
task: embedding
data:
train: dummy.jsonl
max_length: 64
training:
batch_size: 1
gradient_accumulation_steps: 4
embedding_loss: contrastive
"""
with pytest.raises(ValueError) as excinfo:
load_config_from_string(cfg_str)
err = str(excinfo.value)
assert "contrastive" in err
assert "batch_size" in err
def test_allow_batch_size_1_cosine(self):
cfg_str = """
base: test-model
task: embedding
data:
train: dummy.jsonl
max_length: 64
training:
batch_size: 1
embedding_loss: cosine
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.batch_size == 1
assert cfg.training.embedding_loss == "cosine"
def test_allow_batch_size_1_triplet(self):
cfg_str = """
base: test-model
task: embedding
data:
train: dummy.jsonl
max_length: 64
training:
batch_size: 1
embedding_loss: triplet
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.batch_size == 1
assert cfg.training.embedding_loss == "triplet"
def test_allow_batch_size_2_contrastive(self):
cfg_str = """
base: test-model
task: embedding
data:
train: dummy.jsonl
max_length: 64
training:
batch_size: 2
embedding_loss: contrastive
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.batch_size == 2
assert cfg.training.embedding_loss == "contrastive"
class TestAutoBatchSizeFlooring:
"""batch_size: auto resolves to at least 2 when effective loss is contrastive."""
@pytest.mark.parametrize("gib", [0, 16, 24, 40])
def test_auto_batch_size_floored_at_2_contrastive(self, tmp_path, monkeypatch, gib):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"batch_size": "auto",
"embedding_loss": "contrastive",
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
mock_gpu = {"memory_total_bytes": int(gib * 1024**3)}
with patch("soup_cli.utils.gpu.get_gpu_info", return_value=mock_gpu):
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
dataset = {
"train": [
{"anchor": "hello", "positive": "world"},
{"anchor": "hi", "positive": "there"},
{"anchor": "good", "positive": "day"},
{"anchor": "great", "positive": "job"},
]
}
wrapper.setup(dataset)
assert wrapper.args.per_device_train_batch_size >= 2
assert wrapper.args.dataloader_drop_last is True
def test_triplet_fallback_to_contrastive_with_batch_size_1_rejected_at_setup(
self, tmp_path, monkeypatch
):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"batch_size": 1,
"embedding_loss": "triplet",
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
# Dataset without 'negative' column triggers fallback to contrastive
dataset = {
"train": [
{"anchor": "hello", "positive": "world"},
{"anchor": "hi", "positive": "there"},
]
}
with pytest.raises(ValueError) as excinfo:
wrapper.setup(dataset)
err = str(excinfo.value)
assert "contrastive in-batch negatives need batch_size >= 2" in err
def test_triplet_with_negatives_at_batch_size_1_allowed(self, tmp_path, monkeypatch):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"batch_size": 1,
"embedding_loss": "triplet",
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
dataset = {
"train": [
{"anchor": "hello", "positive": "world", "negative": "bad"},
{"anchor": "hi", "positive": "there", "negative": "wrong"},
]
}
wrapper.setup(dataset)
assert wrapper.args.per_device_train_batch_size == 1
assert wrapper.args.dataloader_drop_last is False
def test_contrastive_batch_size_capped_at_dataset_size_with_drop_last(
self, tmp_path, monkeypatch
):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"batch_size": 4,
"embedding_loss": "contrastive",
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
dataset = {
"train": [
{"anchor": "hello", "positive": "world"},
{"anchor": "hi", "positive": "there"},
{"anchor": "good", "positive": "day"},
]
}
wrapper.setup(dataset)
assert wrapper.args.per_device_train_batch_size == 3
assert wrapper.args.dataloader_drop_last is True
class TestRealCpuEmbeddingTraining:
"""Verify that template training block on CPU updates weights and has non-zero loss."""
def test_cpu_template_training_trains_and_updates_lora(self, tmp_path, monkeypatch):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
# Match the shipped template configuration
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"epochs": 1,
"lr": 1e-4,
"batch_size": "auto",
"gradient_accumulation_steps": 1,
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
"quantization": "none",
"embedding_loss": "contrastive",
"embedding_margin": 0.5,
"embedding_pooling": "mean",
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
dataset = {
"train": [
{"anchor": "hello world", "positive": "hello world"},
{"anchor": "python programming", "positive": "python language"},
{"anchor": "data science", "positive": "machine learning"},
{"anchor": "good morning", "positive": "good day"},
{"anchor": "computer vision", "positive": "image processing"},
]
}
wrapper.setup(dataset)
assert wrapper.args.per_device_train_batch_size >= 2
assert wrapper.args.dataloader_drop_last is True
res = wrapper.train()
assert res["total_steps"] > 0
assert res["final_loss"] > 0.0
# Verify LoRA B weights moved (not all zeros)
moved = 0
for name, param in wrapper.model.named_parameters():
if "lora_B" in name:
if (param != 0).any():
moved += 1
assert moved > 0, "LoRA B weights should have been updated by training"
class TestRuntimeGuard:
"""Runtime guard in _compute_loss raises if similarity.size(0) < 2."""
def test_compute_loss_raises_on_single_sample_batch(self, tmp_path, monkeypatch):
import torch
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": model_path,
"task": "embedding",
"data": {"train": "train.jsonl", "max_length": 64},
"training": {
"batch_size": 2,
"embedding_loss": "contrastive",
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = EmbeddingTrainerWrapper(cfg, device="cpu")
dataset = {
"train": [
{"anchor": "hello", "positive": "world"},
{"anchor": "hi", "positive": "there"},
]
}
wrapper.setup(dataset)
# Force a single-item input batch to compute_loss
device = next(wrapper.model.parameters()).device
single_inputs = {
"anchor_input_ids": torch.ones((1, 4), dtype=torch.long, device=device),
"anchor_attention_mask": torch.ones((1, 4), dtype=torch.long, device=device),
"positive_input_ids": torch.ones((1, 4), dtype=torch.long, device=device),
"positive_attention_mask": torch.ones((1, 4), dtype=torch.long, device=device),
}
with pytest.raises(ValueError) as excinfo:
wrapper.trainer.compute_loss(wrapper.model, single_inputs)
assert "contrastive in-batch negatives need batch_size >= 2" in str(excinfo.value)
class TestModelSizeFromNameEncoder:
"""Encoder models (BGE, E5, GTE, etc.) resolve to accurate parameter sizes."""
def test_bge_base_size(self):
assert model_size_from_name("BAAI/bge-base-en-v1.5") == 0.11
def test_bge_large_size(self):
assert model_size_from_name("BAAI/bge-large-en-v1.5") == 0.335
def test_bge_small_size(self):
assert model_size_from_name("BAAI/bge-small-en-v1.5") == 0.033
def test_encoder_without_size_defaults_to_large(self):
assert model_size_from_name("BAAI/bge-m3") == 0.335
ROWS = [
{"anchor": "hello", "positive": "world"},
{"anchor": "hi", "positive": "there"},
{"anchor": "good", "positive": "day"},
{"anchor": "great", "positive": "job"},
]
def _wrapper(tmp_path, monkeypatch, **training):
from soup_cli.trainer.embedding import EmbeddingTrainerWrapper
model_path = _tiny_bert_dir(tmp_path)
monkeypatch.chdir(tmp_path)
body = {
"quantization": "none",
"lora": {"r": 8, "alpha": 16, "target_modules": ["query", "value"]},
**training,
}
cfg = load_config_from_string(
yaml.safe_dump({
"base": model_path,
"task": "embedding",
# the shipped template's max_length: the estimator sizes for it
"data": {"train": "train.jsonl", "max_length": 512},
"training": body,
"output": str(tmp_path / "out"),
})
)
return EmbeddingTrainerWrapper(cfg, device="cpu")
class TestTheIssueEstimatesHitTheFloor:
"""The issue's table: bge-base guessed as 7B at max_length 512 estimates 1/1/2/4
on CPU/16/24/40 GiB, i.e. 0/0/0/1 after the embedding // 3. The tiny local BERT
is sized from its own weights (24k params), so without pinning the size the
16/24/40 GiB cases resolve to 5-10 and never reach the floor."""
@pytest.mark.parametrize("gib", [0, 16, 24, 40])
def test_auto_resolves_to_exactly_two(self, tmp_path, monkeypatch, gib):
wrapper = _wrapper(
tmp_path, monkeypatch, batch_size="auto", embedding_loss="contrastive"
)
with patch("soup_cli.trainer.embedding.model_size_from_name", return_value=7.0), patch(
"soup_cli.utils.gpu.get_gpu_info",
return_value={"memory_total_bytes": int(gib * 1024**3)},
):
wrapper.setup({"train": list(ROWS)})
assert wrapper.args.per_device_train_batch_size == 2
assert wrapper.args.dataloader_drop_last is True
def test_triplet_without_negatives_is_floored_too(self, tmp_path, monkeypatch):
"""The triplet -> contrastive fallback is contrastive for the floor as well."""
wrapper = _wrapper(
tmp_path, monkeypatch, batch_size="auto", embedding_loss="triplet"
)
with patch("soup_cli.trainer.embedding.model_size_from_name", return_value=7.0), patch(
"soup_cli.utils.gpu.get_gpu_info", return_value={"memory_total_bytes": 0}
):
wrapper.setup({"train": list(ROWS)})
assert wrapper.args.per_device_train_batch_size == 2
assert wrapper.args.dataloader_drop_last is True
def test_contrastive_refuses_a_one_row_dataset(tmp_path, monkeypatch):
wrapper = _wrapper(tmp_path, monkeypatch, batch_size=2, embedding_loss="contrastive")
with pytest.raises(ValueError, match="at least 2 training samples; got 1"):
wrapper.setup({"train": ROWS[:1]})