Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(unlearn): respect explicit unlearn_alpha: 0.0 for pure forget loss (#2108) (#2114)

Nariman Abdikarim72aab2aab5a2
diff --git a/changelog.d/0.76.0/2114.fixed.md b/changelog.d/0.76.0/2114.fixed.md
new file mode 100644
index 00000000..3beb41a8
--- /dev/null
+++ b/changelog.d/0.76.0/2114.fixed.md
@@ -0,0 +1 @@
+- Respect explicit `training.unlearn_alpha: 0.0` for pure forget loss in unlearn training (NPO, SimNPO, and RMU) instead of silently falling back to 1.0 (#2108, #2114 by @Nari-Ab).
diff --git a/src/soup_cli/trainer/unlearn.py b/src/soup_cli/trainer/unlearn.py
index 610b4568..7fbf2919 100644
--- a/src/soup_cli/trainer/unlearn.py
+++ b/src/soup_cli/trainer/unlearn.py
@@ -451,7 +451,11 @@ class UnlearnTrainerWrapper:
retain_ce = self._retain_ce(tokenize_pair, dev)
beta = _DEFAULT_BETA
- alpha = float(self.config.training.unlearn_alpha or 1.0)
+ alpha = (
+ 1.0
+ if self.config.training.unlearn_alpha is None
+ else float(self.config.training.unlearn_alpha)
+ )
out = self.model(input_ids=input_ids)
policy_logps = sequence_logprob(out.logits, labels)
if method == "npo":
@@ -552,7 +556,11 @@ class UnlearnTrainerWrapper:
if retain_acts is None or retain_frozen is None:
# No retain set: pure forget steering.
return torch.mean((forget_acts - control_vec) ** 2)
- alpha = float(self.config.training.unlearn_alpha or 1.0)
+ alpha = (
+ 1.0
+ if self.config.training.unlearn_alpha is None
+ else float(self.config.training.unlearn_alpha)
+ )
return rmu_loss(
forget_acts, control_vec, retain_acts, retain_frozen, alpha=alpha,
)
diff --git a/tests/test_unlearn_alpha.py b/tests/test_unlearn_alpha.py
new file mode 100644
index 00000000..20542c95
--- /dev/null
+++ b/tests/test_unlearn_alpha.py
@@ -0,0 +1,122 @@
+"""Tests for unlearn_alpha weighting in UnlearnTrainerWrapper (#2108)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import torch
+
+from soup_cli.config.schema import SoupConfig
+from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
+
+
+def _make_wrapper(method: str, unlearn_alpha: float | None) -> UnlearnTrainerWrapper:
+ training_dict: dict = {"unlearn_method": method}
+ if unlearn_alpha is not None:
+ training_dict["unlearn_alpha"] = unlearn_alpha
+ cfg = SoupConfig(
+ base="test-model",
+ task="unlearn",
+ data={"train": "test.jsonl", "forget_set": "f.jsonl", "retain_set": "r.jsonl"},
+ training=training_dict,
+ )
+ wrapper = UnlearnTrainerWrapper(cfg)
+ wrapper.tokenizer = MagicMock()
+ wrapper._retain = [("r_prompt", "r_target")]
+ return wrapper
+
+
+def _dummy_tokenize_pair(tok, p, t, max_length=None):
+ return (torch.tensor([[1, 2]]), torch.tensor([[1, 2]]))
+
+
+class TestUnlearnAlphaPreference:
+ """_step_preference (NPO / SimNPO) retain loss weighting (#2108)."""
+
+ @pytest.mark.parametrize("method", ["npo", "simnpo"])
+ @pytest.mark.parametrize(
+ ("alpha_config", "expected_alpha"),
+ [
+ (0.0, 0.0),
+ (None, 1.0),
+ (0.5, 0.5),
+ ],
+ )
+ def test_preference_retain_weight(
+ self, method: str, alpha_config: float | None, expected_alpha: float,
+ ):
+ wrapper = _make_wrapper(method, alpha_config)
+ wrapper._retain_ce = lambda tp, dev: torch.tensor(10.0)
+ wrapper.model = MagicMock(return_value=MagicMock(logits=torch.zeros((1, 2, 10))))
+ wrapper.ref_model = MagicMock(return_value=MagicMock(logits=torch.zeros((1, 2, 10))))
+
+ with (
+ patch("soup_cli.utils.unlearn_kernels.npo_loss", return_value=torch.tensor(1.0)),
+ patch("soup_cli.utils.unlearn_kernels.simnpo_loss", return_value=torch.tensor(1.0)),
+ patch(
+ "soup_cli.utils.unlearn_kernels.sequence_logprob",
+ return_value=torch.tensor([0.0]),
+ ),
+ ):
+ total_loss = wrapper._step_preference(
+ _dummy_tokenize_pair, "prompt", "target", method, "cpu",
+ )
+
+ expected_total = 1.0 + expected_alpha * 10.0
+ assert total_loss is not None
+ msg = f"Expected loss {expected_total} for alpha={alpha_config}, got {total_loss.item()}"
+ assert torch.isclose(total_loss, torch.tensor(expected_total)), msg
+
+
+class TestUnlearnAlphaRMU:
+ """_step_rmu retain loss weighting (#2108)."""
+
+ @pytest.mark.parametrize(
+ ("alpha_config", "expected_alpha"),
+ [
+ (0.0, 0.0),
+ (None, 1.0),
+ (0.5, 0.5),
+ ],
+ )
+ def test_rmu_retain_weight(self, alpha_config: float | None, expected_alpha: float):
+ wrapper = _make_wrapper("rmu", alpha_config)
+
+ class DummyLayer(torch.nn.Module):
+ def forward(self, x):
+ return x
+
+ dummy_layer = DummyLayer()
+ wrapper._resolve_ref_rmu_layer = lambda: dummy_layer
+
+ class DummyModel(torch.nn.Module):
+ def forward(self, input_ids):
+ acts = torch.ones((1, 2, 4))
+ for hook in list(dummy_layer._forward_hooks.values()):
+ hook(dummy_layer, None, (acts,))
+ return acts
+
+ wrapper.model = DummyModel()
+ wrapper.ref_model = DummyModel()
+
+ recorded_alphas = []
+
+ def fake_rmu_loss(forget_acts, control_vec, retain_acts, retain_frozen, *, alpha=1.0):
+ recorded_alphas.append(alpha)
+ return torch.tensor(0.0)
+
+ with patch("soup_cli.utils.unlearn_kernels.rmu_loss", fake_rmu_loss):
+ wrapper._step_rmu(
+ _dummy_tokenize_pair,
+ "f_prompt",
+ "f_target",
+ ("r_prompt", "r_target"),
+ torch.zeros(4),
+ dummy_layer,
+ "cpu",
+ )
+
+ assert len(recorded_alphas) == 1
+ msg = f"Expected alpha {expected_alpha} for alpha={alpha_config}, got {recorded_alphas[0]}"
+ assert recorded_alphas[0] == expected_alpha, msg