Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

30.6 KB
Raw Blame
"""Part E — v0.32.1 stability live (#56, #57, #58, #59) for v0.33.0.

Covers:
  - #56 run_lr_sweep — in-process LR-sweep loop with mocked model + DataLoader.
  - #57 SoupTrainerCallback._write_spike_recovery_hint — writes JSON hint
    when watchdog fires and loss_spike_recovery is enabled.
  - #58 SFTTrainerWrapper._resolve_mixed_precision — wires
    pick_mixed_precision into bf16/fp16 flags; preserves legacy default
    when auto flag is False.
  - #59 SoupTrainerCallback grad-accum advisory — fires once on threshold
    crossing.
"""

from __future__ import annotations

import json
import re
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest

_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")


def _plain(text: str) -> str:
    """ANSI-stripped, whitespace-collapsed CLI output (#1068: Rich colours numbers)."""
    return " ".join(_ANSI_RE.sub("", text).split())

# ---------------------------------------------------------------------------
# #56 — run_lr_sweep
# ---------------------------------------------------------------------------


class TestRunLRSweep:
    def test_empty_schedule_rejected(self):
        from soup_cli.utils.lr_finder import run_lr_sweep

        with pytest.raises(ValueError, match="schedule must be non-empty"):
            run_lr_sweep(
                model=MagicMock(), dataloader=iter([]),
                schedule=[], optimizer_factory=lambda p: MagicMock(),
            )

    def test_loop_records_loss_per_step(self):
        from soup_cli.utils.lr_finder import run_lr_sweep

        # Fake model returning a tensor-like loss
        def _fake_loss_value(value):
            obj = MagicMock()
            obj.detach = MagicMock(return_value=obj)
            obj.item = MagicMock(return_value=value)
            obj.backward = MagicMock(return_value=None)
            return obj

        loss_values = [3.0, 2.0, 1.5, 1.0]

        class FakeModel:
            def __init__(self):
                self._idx = 0

            def parameters(self):
                return []

            def __call__(self, **batch):
                value = loss_values[self._idx]
                self._idx += 1
                return {"loss": _fake_loss_value(value)}

        model = FakeModel()

        # Fake optimizer with mutable param_groups
        class FakeOptim:
            def __init__(self, _params):
                self.param_groups = [{"lr": 0.0}]

            def zero_grad(self, set_to_none: bool = False):  # noqa: ARG002
                pass

            def step(self):
                pass

        dl = iter([{"input_ids": MagicMock()}] * 4)
        schedule = [1e-6, 1e-5, 1e-4, 1e-3]

        losses = run_lr_sweep(
            model=model, dataloader=dl, schedule=schedule,
            optimizer_factory=FakeOptim,
        )
        assert losses == loss_values

    def test_diverged_loss_breaks_loop(self):
        from soup_cli.utils.lr_finder import run_lr_sweep

        loss_values = [3.0, float("inf"), 1.0]

        def _wrap(value):
            obj = MagicMock()
            obj.detach = MagicMock(return_value=obj)
            obj.item = MagicMock(return_value=value)
            obj.backward = MagicMock(return_value=None)
            return obj

        class FakeModel:
            def __init__(self):
                self._idx = 0

            def parameters(self):
                return []

            def __call__(self, **batch):
                value = loss_values[self._idx]
                self._idx += 1
                return {"loss": _wrap(value)}

        class FakeOptim:
            def __init__(self, _params):
                self.param_groups = [{"lr": 0.0}]

            def zero_grad(self, set_to_none: bool = False):  # noqa: ARG002
                pass

            def step(self):
                pass

        dl = iter([{"x": MagicMock()}] * 3)
        losses = run_lr_sweep(
            model=FakeModel(), dataloader=dl,
            schedule=[1e-6, 1e-5, 1e-4],
            optimizer_factory=FakeOptim,
        )
        # Loop terminates after the inf — only the first finite loss kept.
        assert losses == [3.0]


# ---------------------------------------------------------------------------
# #58 — auto mixed-precision push
# ---------------------------------------------------------------------------


class TestResolveMixedPrecision:
    def test_auto_flag_off_asks_the_card(self, monkeypatch):
        """Was ``test_auto_flag_off_preserves_legacy``, asserting bf16 on any
        CUDA device. That "legacy" is what transformers refuses outright on a
        pre-Ampere card — *"Your setup doesn't support bf16/gpu"* — so every
        run on a T4 or P100 died before step 0 (#385). The contract is now: bf16
        when the card has it, fp16 when it does not. Both directions asserted,
        because a resolver hardcoded either way would satisfy only one."""
        import torch

        from soup_cli.trainer.sft import SFTTrainerWrapper

        wrapper = SFTTrainerWrapper.__new__(SFTTrainerWrapper)
        wrapper.device = "cuda"
        tcfg = SimpleNamespace(auto_mixed_precision=False)

        class _Cuda:
            def __init__(self, bf16):
                self._bf16 = bf16

            def is_available(self):
                return True

            def is_bf16_supported(self, including_emulation: bool = True):
                # Mirrors the real signature: the bare call is permissive
                # and says True on a T4 through emulation (#385 follow-up).
                return self._bf16 if not including_emulation else True

            def get_device_capability(self, device=None):
                return (8, 0) if self._bf16 else (7, 5)

        monkeypatch.setattr(torch, "cuda", _Cuda(True))
        assert wrapper._resolve_mixed_precision(tcfg, "any") == (True, False)

        monkeypatch.setattr(torch, "cuda", _Cuda(False))
        assert wrapper._resolve_mixed_precision(tcfg, "any") == (False, True)

    def test_auto_flag_off_cpu(self):
        from soup_cli.trainer.sft import SFTTrainerWrapper

        wrapper = SFTTrainerWrapper.__new__(SFTTrainerWrapper)
        wrapper.device = "cpu"
        tcfg = SimpleNamespace(auto_mixed_precision=False)
        bf16, fp16 = wrapper._resolve_mixed_precision(tcfg, "any")
        assert (bf16, fp16) == (False, False)

    def test_auto_flag_cpu_returns_no(self):
        from soup_cli.trainer.sft import SFTTrainerWrapper

        wrapper = SFTTrainerWrapper.__new__(SFTTrainerWrapper)
        wrapper.device = "cpu"
        tcfg = SimpleNamespace(auto_mixed_precision=True)
        assert wrapper._resolve_mixed_precision(tcfg, "any") == (False, False)

    def test_auto_flag_picks_bf16_on_ampere(self, monkeypatch):
        """Ampere (cc 8.6) + non-quirk model → bf16."""
        import torch

        from soup_cli.trainer.sft import SFTTrainerWrapper

        wrapper = SFTTrainerWrapper.__new__(SFTTrainerWrapper)
        wrapper.device = "cuda"
        monkeypatch.setattr(
            torch.cuda, "get_device_capability",
            lambda *_a, **_k: (8, 6),
            raising=False,
        )
        tcfg = SimpleNamespace(auto_mixed_precision=True)
        bf16, fp16 = wrapper._resolve_mixed_precision(tcfg, "neutral-model")
        assert (bf16, fp16) == (True, False)

    def test_auto_flag_picks_fp16_for_qwen2_on_ampere(self, monkeypatch):
        import torch

        from soup_cli.trainer.sft import SFTTrainerWrapper

        wrapper = SFTTrainerWrapper.__new__(SFTTrainerWrapper)
        wrapper.device = "cuda"
        monkeypatch.setattr(
            torch.cuda, "get_device_capability",
            lambda *_a, **_k: (8, 6),
            raising=False,
        )
        tcfg = SimpleNamespace(auto_mixed_precision=True)
        bf16, fp16 = wrapper._resolve_mixed_precision(
            tcfg, "Qwen/Qwen2-7B-Instruct",
        )
        assert (bf16, fp16) == (False, True)


# ---------------------------------------------------------------------------
# #57 — spike recovery hint
# ---------------------------------------------------------------------------


def _make_callback(tmp_path, **kwargs):
    from soup_cli.monitoring.callback import SoupTrainerCallback

    display = MagicMock()
    return SoupTrainerCallback(
        display=display,
        tracker=None,
        run_id="t",
        output_dir=str(tmp_path),
        **kwargs,
    )


class TestSpikeRecoveryHint:
    def test_writes_hint_file(self, tmp_path, monkeypatch):
        # Containment guard requires output_dir to live under cwd.
        monkeypatch.chdir(tmp_path)
        cb = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        args = SimpleNamespace(
            learning_rate=1e-3, output_dir=str(tmp_path),
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        hint = tmp_path / "spike_recovery.json"
        assert hint.exists()
        data = json.loads(hint.read_text(encoding="utf-8"))
        assert data["previous_lr"] == pytest.approx(1e-3)
        assert data["recommended_lr"] == pytest.approx(5e-4)
        assert data["should_recover"] is True
        assert data["attempts"] == 1

    def test_attempts_counter_increments(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        cb = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=3,
            spike_recovery_lr_decay=0.5,
        )
        args = SimpleNamespace(
            learning_rate=1e-3, output_dir=str(tmp_path),
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        cb._write_spike_recovery_hint(args, loss=10.0)
        data = json.loads((tmp_path / "spike_recovery.json").read_text())
        assert data["attempts"] == 2

    def test_fresh_cap_1_allows_one_recovery_and_refuses_second(self, tmp_path, monkeypatch):
        """Cap 1 permits exactly one recovery (prior attempts=0), refusing the second."""
        monkeypatch.chdir(tmp_path)
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))

        # Launch 1: fresh instance, prior count = 0 -> recovery permitted, attempts incremented to 1
        cb1 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=1,
            spike_recovery_lr_decay=0.5,
        )
        cb1._write_spike_recovery_hint(args, loss=10.0)
        data1 = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data1["attempts"] == 1
        assert data1["should_recover"] is True, "First launch under cap 1 must permit recovery"

        # Launch 2 (re-launch): new instance reads persisted count 1 -> budget exhausted
        cb2 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=1,
            spike_recovery_lr_decay=0.5,
        )
        cb2._write_spike_recovery_hint(args, loss=12.0)
        data2 = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data2["attempts"] == 2
        assert data2["should_recover"] is False, "Second launch under cap 1 must refuse recovery"

    def test_cap_2_two_allowed_attempts_across_new_instances(self, tmp_path, monkeypatch):
        """Under cap 2, both attempt 1 and attempt 2 are permitted; attempt 3 is refused."""
        monkeypatch.chdir(tmp_path)
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))

        # Launch 1: prior count = 0 -> permitted, persisted 1
        cb1 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        cb1._write_spike_recovery_hint(args, loss=10.0)
        data1 = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data1["attempts"] == 1
        assert data1["should_recover"] is True, "First attempt under cap 2 must be permitted"

        # Launch 2: new instance reads persisted count 1 -> second attempt is permitted
        cb2 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        cb2._write_spike_recovery_hint(args, loss=12.0)
        data2 = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data2["attempts"] == 2
        assert data2["should_recover"] is True, "Second attempt under cap 2 must be permitted"

        # Launch 3: new instance reads persisted count 2 -> budget exhausted, recovery refused
        cb3 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        cb3._write_spike_recovery_hint(args, loss=14.0)
        data3 = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data3["attempts"] == 3
        assert data3["should_recover"] is False, "Third attempt under cap 2 must be refused"

    @pytest.mark.parametrize(
        "bad_payload",
        [
            "corrupted non-json",
            '{"attempts": null}',
            '{"attempts": []}',
            '{"attempts": {}}',
            '{"attempts": "not_an_int"}',
            '{"attempts": true}',
            '{"attempts": 1e999}',
            '{"attempts": -1e999}',
            '{"attempts": Infinity}',
            '{"attempts": -Infinity}',
            '{"attempts": NaN}',
        ],
    )
    def test_corrupt_and_malformed_hint_files_fall_back_gracefully(
        self, tmp_path, monkeypatch, bad_payload
    ):
        """Malformed JSON or invalid attempt types fall back safely to 0 without crashing."""
        monkeypatch.chdir(tmp_path)
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))
        (tmp_path / "spike_recovery.json").write_text(bad_payload, encoding="utf-8")

        cb = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        data = json.loads((tmp_path / "spike_recovery.json").read_text(encoding="utf-8"))
        assert data["attempts"] == 1
        assert data["should_recover"] is True

    def test_preserves_valid_positive_persisted_count(self, tmp_path, monkeypatch):
        """Valid persisted count from prior launch is safely parsed and incremented."""
        monkeypatch.chdir(tmp_path)
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))
        hint_path = tmp_path / "spike_recovery.json"
        hint_path.write_text(json.dumps({"attempts": 2, "max_attempts": 3}), encoding="utf-8")

        cb1 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=3,
            spike_recovery_lr_decay=0.5,
        )
        cb1._write_spike_recovery_hint(args, loss=10.0)
        data1 = json.loads(hint_path.read_text(encoding="utf-8"))
        assert data1["attempts"] == 3
        assert data1["should_recover"] is True, "Prior count 2 with cap 3 must allow 3rd attempt"

        # Subsequent re-launch with prior count 3
        cb2 = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=3,
            spike_recovery_lr_decay=0.5,
        )
        cb2._write_spike_recovery_hint(args, loss=12.0)
        data2 = json.loads(hint_path.read_text(encoding="utf-8"))
        assert data2["attempts"] == 4
        assert data2["should_recover"] is False, "Prior count 3 with cap 3 must refuse recovery"

    def test_preserves_large_positive_integer_count_and_exhausts_cap(
        self, tmp_path, monkeypatch
    ):
        """Large integer count (10**309) is preserved without float conversion and exhausts cap."""
        monkeypatch.chdir(tmp_path)
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))
        hint_path = tmp_path / "spike_recovery.json"
        huge_attempts = 10**309
        hint_path.write_text(
            json.dumps({"attempts": huge_attempts, "max_attempts": 2}),
            encoding="utf-8",
        )

        cb = _make_callback(
            tmp_path,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        data = json.loads(hint_path.read_text(encoding="utf-8"))
        assert data["attempts"] == huge_attempts + 1
        assert data["should_recover"] is False, (
            "Prior large count must exhaust cap and refuse recovery"
        )

    @pytest.mark.parametrize(
        "bad_payload",
        [
            '{"attempts": null}',
            '{"attempts": 1e999}',
            '{"attempts": -1e999}',
            '{"attempts": Infinity}',
            '{"attempts": -Infinity}',
            '{"attempts": NaN}',
            '{"attempts": "not_an_int"}',
        ],
    )
    def test_watchdog_path_stops_training_with_bad_hints(
        self, tmp_path, monkeypatch, bad_payload
    ):
        """Watchdog path stops training safely when hint file is malformed or non-finite."""
        monkeypatch.chdir(tmp_path)
        hint_path = tmp_path / "spike_recovery.json"
        hint_path.write_text(bad_payload, encoding="utf-8")

        cb = _make_callback(
            tmp_path,
            loss_watchdog=True,
            loss_watchdog_threshold=5.0,
            loss_watchdog_patience=1,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))
        state = SimpleNamespace(global_step=1, epoch=1.0)
        control = SimpleNamespace(should_training_stop=False)

        cb.on_log(args, state, control, logs={"loss": 10.0})

        assert control.should_training_stop is True, "Watchdog must stop training on loss spike"
        assert cb._watchdog_fired is True
        data = json.loads(hint_path.read_text(encoding="utf-8"))
        assert data["attempts"] == 1
        assert data["should_recover"] is True

    def test_watchdog_path_preserves_large_integer_count_and_exhausts_cap(
        self, tmp_path, monkeypatch
    ):
        """Watchdog path preserves large positive integer and exhausts recovery budget."""
        monkeypatch.chdir(tmp_path)
        hint_path = tmp_path / "spike_recovery.json"
        huge_attempts = 10**309
        hint_path.write_text(
            json.dumps({"attempts": huge_attempts, "max_attempts": 2}),
            encoding="utf-8",
        )

        cb = _make_callback(
            tmp_path,
            loss_watchdog=True,
            loss_watchdog_threshold=5.0,
            loss_watchdog_patience=1,
            spike_recovery=True,
            spike_recovery_max_attempts=2,
            spike_recovery_lr_decay=0.5,
        )
        args = SimpleNamespace(learning_rate=1e-3, output_dir=str(tmp_path))
        state = SimpleNamespace(global_step=1, epoch=1.0)
        control = SimpleNamespace(should_training_stop=False)

        cb.on_log(args, state, control, logs={"loss": 10.0})

        assert control.should_training_stop is True, "Watchdog must stop training on loss spike"
        assert cb._watchdog_fired is True
        data = json.loads(hint_path.read_text(encoding="utf-8"))
        assert data["attempts"] == huge_attempts + 1
        assert data["should_recover"] is False, "Exhausted count must keep recovery disabled"

    def test_disabled_when_strategy_not_set(self, tmp_path):
        cb = _make_callback(tmp_path, spike_recovery=False)
        args = SimpleNamespace(
            learning_rate=1e-3, output_dir=str(tmp_path),
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        # No hint file written.
        assert not (tmp_path / "spike_recovery.json").exists()

    def test_outside_cwd_skipped(self, tmp_path):
        """Containment guard: when output_dir is outside cwd, skip silently."""
        # No monkeypatch.chdir — tmp_path is outside the test's actual cwd.
        cb = _make_callback(
            tmp_path,
            spike_recovery=True,
        )
        args = SimpleNamespace(
            learning_rate=1e-3, output_dir=str(tmp_path),
        )
        cb._write_spike_recovery_hint(args, loss=10.0)
        # No hint written; no exception raised.
        assert not (tmp_path / "spike_recovery.json").exists()


# ---------------------------------------------------------------------------
# #59 — grad-accum advisory
# ---------------------------------------------------------------------------


class TestGradAccumAdvisory:
    def test_advise_fires_once_under_pressure(self, tmp_path, monkeypatch, capsys):
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.5,
            grad_accum_total_vram_gb=10.0,
            grad_accum_current_steps=1,
            grad_accum_current_batch=4,
        )

        # Mock torch presence + memory probe — high pressure (8 GB / 10 GB = 80%).
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.max_memory_allocated.return_value = 8 * (1024**3)
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is True

        # Second call is a no-op (one-shot).
        cb._grad_accum_monitor.observe = MagicMock()
        cb._maybe_advise_grad_accum()
        # Already advised, so monitor.observe shouldn't be called.
        cb._grad_accum_monitor.observe.assert_not_called()

    def test_no_advice_when_under_threshold(self, tmp_path, monkeypatch):
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.9,
            grad_accum_total_vram_gb=10.0,
            grad_accum_current_steps=1,
            grad_accum_current_batch=4,
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.max_memory_allocated.return_value = 5 * (1024**3)
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False

    def test_no_advice_when_disabled(self, tmp_path):
        cb = _make_callback(tmp_path, grad_accum_auto_tune=False)
        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False

    def test_no_advice_when_cuda_unavailable(self, tmp_path, monkeypatch):
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.5,
            grad_accum_total_vram_gb=10.0,
            grad_accum_current_steps=1,
            grad_accum_current_batch=4,
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = False
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False

    # -- #1620: pressure is measured against the card, not a fixed total ----

    def test_advises_on_small_card_using_measured_total(self, tmp_path, monkeypatch, capsys):
        """An 8 GB card at 97.5% pressure with no explicit total advises.

        Against the old fixed 24 GB default this peak could never fire.
        """
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.92,
            grad_accum_current_steps=1,
            grad_accum_current_batch=8,
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.current_device.return_value = 0
        fake_torch.cuda.max_memory_allocated.return_value = int(7.8 * (1024**3))
        fake_torch.cuda.get_device_properties.return_value.total_memory = 8 * (1024**3)
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is True
        assert cb._grad_accum_monitor.total_vram_gb == pytest.approx(8.0)
        # The advisory carries the measured numbers, not just the pair
        # (#1068: strip ANSI before reading numbers out of Rich output).
        assert "peak 7.8 of 8.0 GB (97%)" in _plain(capsys.readouterr().out)

    def test_no_advice_on_large_card_using_measured_total(self, tmp_path, monkeypatch):
        """An 80 GB card at 28.7% pressure with no explicit total stays quiet."""
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.92,
            grad_accum_current_steps=1,
            grad_accum_current_batch=8,
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.current_device.return_value = 0
        fake_torch.cuda.max_memory_allocated.return_value = 23 * (1024**3)
        fake_torch.cuda.get_device_properties.return_value.total_memory = 80 * (1024**3)
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False
        assert cb._grad_accum_monitor.total_vram_gb == pytest.approx(80.0)

    def test_trainer_built_callback_measures_the_card(self, tmp_path, monkeypatch):
        """Built exactly as the trainers build it (#802 kwargs), the advisory
        uses the card's total, not the old 24 GB default."""
        from soup_cli.monitoring.callback import (
            SoupTrainerCallback,
            soup_callback_kwargs,
        )

        tcfg = SimpleNamespace(
            grad_accum_auto_tune=True,
            gradient_accumulation_steps=1,
            batch_size=8,
        )
        cb = SoupTrainerCallback(
            display=MagicMock(), **soup_callback_kwargs(tcfg)
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.current_device.return_value = 0
        fake_torch.cuda.max_memory_allocated.return_value = int(7.8 * (1024**3))
        fake_torch.cuda.get_device_properties.return_value.total_memory = 8 * (1024**3)
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is True
        assert cb._grad_accum_monitor.total_vram_gb == pytest.approx(8.0)

    def test_no_advice_when_device_properties_raise(self, tmp_path, monkeypatch):
        """An unreadable card total means no advice and no exception."""
        cb = _make_callback(
            tmp_path,
            grad_accum_auto_tune=True,
            grad_accum_pressure_threshold=0.92,
            grad_accum_current_steps=1,
            grad_accum_current_batch=8,
        )
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.current_device.return_value = 0
        fake_torch.cuda.max_memory_allocated.return_value = int(7.8 * (1024**3))
        fake_torch.cuda.get_device_properties.side_effect = RuntimeError("no props")
        monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)

        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False
        assert cb._grad_accum_monitor is None

    # -- review follow-ups: the gate, the enabled guard, the device identity ----

    def _fake_cuda(self, current=0, peaks=None, totals=None):
        peaks, totals = peaks or {0: 7.8}, totals or {0: 8}
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.current_device.return_value = current
        fake_torch.cuda.max_memory_allocated.side_effect = (
            lambda device=0: int(peaks.get(device, 0.0) * (1024**3))
        )
        fake_torch.cuda.get_device_properties.side_effect = (
            lambda device: SimpleNamespace(total_memory=int(totals[device] * (1024**3)))
        )
        return fake_torch

    def test_on_log_drives_the_advisory(self, tmp_path, monkeypatch):
        """The advisory must fire through on_log, not only via a direct call:
        the on_log gate is what makes a real run advise at all."""
        cb = _make_callback(tmp_path, grad_accum_auto_tune=True,
                            grad_accum_pressure_threshold=0.92,
                            grad_accum_current_steps=1, grad_accum_current_batch=8)
        monkeypatch.setitem(__import__("sys").modules, "torch", self._fake_cuda())
        state = SimpleNamespace(global_step=1, epoch=0.1, max_steps=10, log_history=[])
        cb.on_log(SimpleNamespace(output_dir=str(tmp_path)), state, SimpleNamespace(),
                  logs={"loss": 1.0, "learning_rate": 1e-4})
        assert cb._grad_accum_advised is True
        assert cb._grad_accum_monitor.total_vram_gb == pytest.approx(8.0)

    def test_disabled_callback_stays_quiet_under_pressure(self, tmp_path, monkeypatch):
        """The enabled guard keeps a callback built with auto_tune off silent
        even when pressure is over the threshold."""
        cb = _make_callback(tmp_path, grad_accum_auto_tune=False,
                            grad_accum_current_batch=8)
        monkeypatch.setitem(__import__("sys").modules, "torch", self._fake_cuda())
        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is False
        assert cb._grad_accum_monitor is None

    def test_peak_and_total_come_from_the_current_device(self, tmp_path, monkeypatch):
        """The peak and the total must both come from the device the run uses:
        device 1 holds the 7.8 GB peak and the 8 GB card, device 0 is idle."""
        cb = _make_callback(tmp_path, grad_accum_auto_tune=True,
                            grad_accum_pressure_threshold=0.92,
                            grad_accum_current_steps=1, grad_accum_current_batch=8)
        monkeypatch.setitem(__import__("sys").modules, "torch", self._fake_cuda(
            current=1, peaks={0: 0.0, 1: 7.8}, totals={0: 80, 1: 8}))
        cb._maybe_advise_grad_accum()
        assert cb._grad_accum_advised is True
        assert cb._grad_accum_monitor.total_vram_gb == pytest.approx(8.0)

    def test_live_panel_reads_the_current_device(self, tmp_path, monkeypatch):
        """The live memory panel reports the device the run uses, not device 0."""
        cb = _make_callback(tmp_path)
        monkeypatch.setitem(__import__("sys").modules, "torch", self._fake_cuda(
            current=1, peaks={0: 0.0, 1: 7.8}, totals={0: 80, 1: 8}))
        state = SimpleNamespace(global_step=1, epoch=0.1, max_steps=10, log_history=[])
        cb.on_log(SimpleNamespace(output_dir=str(tmp_path)), state, SimpleNamespace(),
                  logs={"loss": 1.0, "learning_rate": 1e-4})
        assert cb.display.update.call_args.kwargs["gpu_mem"] == "7.8/8.0 GB"