Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(unlearn): respect explicit unlearn_alpha: 0.0 for pure forget loss (#2108) (#2114)
Nariman Abdikarim72aab2aab5a2
diff --git a/changelog.d/0.76.0/2114.fixed.md b/changelog.d/0.76.0/2114.fixed.mdnew file mode 100644index 00000000..3beb41a8--- /dev/null+++ b/changelog.d/0.76.0/2114.fixed.md@@ -0,0 +1 @@+- Respect explicit `training.unlearn_alpha: 0.0` for pure forget loss in unlearn training (NPO, SimNPO, and RMU) instead of silently falling back to 1.0 (#2108, #2114 by @Nari-Ab).diff --git a/src/soup_cli/trainer/unlearn.py b/src/soup_cli/trainer/unlearn.pyindex 610b4568..7fbf2919 100644--- a/src/soup_cli/trainer/unlearn.py+++ b/src/soup_cli/trainer/unlearn.py@@ -451,7 +451,11 @@ class UnlearnTrainerWrapper:retain_ce = self._retain_ce(tokenize_pair, dev)beta = _DEFAULT_BETA- alpha = float(self.config.training.unlearn_alpha or 1.0)+ alpha = (+ 1.0+ if self.config.training.unlearn_alpha is None+ else float(self.config.training.unlearn_alpha)+ )out = self.model(input_ids=input_ids)policy_logps = sequence_logprob(out.logits, labels)if method == "npo":@@ -552,7 +556,11 @@ class UnlearnTrainerWrapper:if retain_acts is None or retain_frozen is None:# No retain set: pure forget steering.return torch.mean((forget_acts - control_vec) ** 2)- alpha = float(self.config.training.unlearn_alpha or 1.0)+ alpha = (+ 1.0+ if self.config.training.unlearn_alpha is None+ else float(self.config.training.unlearn_alpha)+ )return rmu_loss(forget_acts, control_vec, retain_acts, retain_frozen, alpha=alpha,)diff --git a/tests/test_unlearn_alpha.py b/tests/test_unlearn_alpha.pynew file mode 100644index 00000000..20542c95--- /dev/null+++ b/tests/test_unlearn_alpha.py@@ -0,0 +1,122 @@+"""Tests for unlearn_alpha weighting in UnlearnTrainerWrapper (#2108)."""++from __future__ import annotations++from unittest.mock import MagicMock, patch++import pytest+import torch++from soup_cli.config.schema import SoupConfig+from soup_cli.trainer.unlearn import UnlearnTrainerWrapper+++def _make_wrapper(method: str, unlearn_alpha: float | None) -> UnlearnTrainerWrapper:+ training_dict: dict = {"unlearn_method": method}+ if unlearn_alpha is not None:+ training_dict["unlearn_alpha"] = unlearn_alpha+ cfg = SoupConfig(+ base="test-model",+ task="unlearn",+ data={"train": "test.jsonl", "forget_set": "f.jsonl", "retain_set": "r.jsonl"},+ training=training_dict,+ )+ wrapper = UnlearnTrainerWrapper(cfg)+ wrapper.tokenizer = MagicMock()+ wrapper._retain = [("r_prompt", "r_target")]+ return wrapper+++def _dummy_tokenize_pair(tok, p, t, max_length=None):+ return (torch.tensor([[1, 2]]), torch.tensor([[1, 2]]))+++class TestUnlearnAlphaPreference:+ """_step_preference (NPO / SimNPO) retain loss weighting (#2108)."""++ @pytest.mark.parametrize("method", ["npo", "simnpo"])+ @pytest.mark.parametrize(+ ("alpha_config", "expected_alpha"),+ [+ (0.0, 0.0),+ (None, 1.0),+ (0.5, 0.5),+ ],+ )+ def test_preference_retain_weight(+ self, method: str, alpha_config: float | None, expected_alpha: float,+ ):+ wrapper = _make_wrapper(method, alpha_config)+ wrapper._retain_ce = lambda tp, dev: torch.tensor(10.0)+ wrapper.model = MagicMock(return_value=MagicMock(logits=torch.zeros((1, 2, 10))))+ wrapper.ref_model = MagicMock(return_value=MagicMock(logits=torch.zeros((1, 2, 10))))++ with (+ patch("soup_cli.utils.unlearn_kernels.npo_loss", return_value=torch.tensor(1.0)),+ patch("soup_cli.utils.unlearn_kernels.simnpo_loss", return_value=torch.tensor(1.0)),+ patch(+ "soup_cli.utils.unlearn_kernels.sequence_logprob",+ return_value=torch.tensor([0.0]),+ ),+ ):+ total_loss = wrapper._step_preference(+ _dummy_tokenize_pair, "prompt", "target", method, "cpu",+ )++ expected_total = 1.0 + expected_alpha * 10.0+ assert total_loss is not None+ msg = f"Expected loss {expected_total} for alpha={alpha_config}, got {total_loss.item()}"+ assert torch.isclose(total_loss, torch.tensor(expected_total)), msg+++class TestUnlearnAlphaRMU:+ """_step_rmu retain loss weighting (#2108)."""++ @pytest.mark.parametrize(+ ("alpha_config", "expected_alpha"),+ [+ (0.0, 0.0),+ (None, 1.0),+ (0.5, 0.5),+ ],+ )+ def test_rmu_retain_weight(self, alpha_config: float | None, expected_alpha: float):+ wrapper = _make_wrapper("rmu", alpha_config)++ class DummyLayer(torch.nn.Module):+ def forward(self, x):+ return x++ dummy_layer = DummyLayer()+ wrapper._resolve_ref_rmu_layer = lambda: dummy_layer++ class DummyModel(torch.nn.Module):+ def forward(self, input_ids):+ acts = torch.ones((1, 2, 4))+ for hook in list(dummy_layer._forward_hooks.values()):+ hook(dummy_layer, None, (acts,))+ return acts++ wrapper.model = DummyModel()+ wrapper.ref_model = DummyModel()++ recorded_alphas = []++ def fake_rmu_loss(forget_acts, control_vec, retain_acts, retain_frozen, *, alpha=1.0):+ recorded_alphas.append(alpha)+ return torch.tensor(0.0)++ with patch("soup_cli.utils.unlearn_kernels.rmu_loss", fake_rmu_loss):+ wrapper._step_rmu(+ _dummy_tokenize_pair,+ "f_prompt",+ "f_target",+ ("r_prompt", "r_target"),+ torch.zeros(4),+ dummy_layer,+ "cpu",+ )++ assert len(recorded_alphas) == 1+ msg = f"Expected alpha {expected_alpha} for alpha={alpha_config}, got {recorded_alphas[0]}"+ assert recorded_alphas[0] == expected_alpha, msg