Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

99.5 KB
Raw Blame
"""v0.71.33 — `soup draft`: train-your-own speculative-decoding draft.

Covers:
* ``utils/adapter_fuse.py`` — the shared LoRA -> dense fuse extracted from
  ``commands/shrink.py`` (v0.71.29) so ``shrink`` and ``draft`` share one path.
* ``utils/draft.py`` — pure acceptance kernel + tokenizer-equality guard +
  local draft registry + torch-lazy measurement.
* ``utils/spec_pairing.py`` — local-registry lookup before the static map.
* ``commands/draft.py`` — ``soup draft distill / measure / list``.
"""

from __future__ import annotations

import ast
import math
import os
import re
from pathlib import Path
from typing import Sequence

import pytest

# Rich styles console output, so a phrase assertion must never read raw
# `result.output`: an escape sequence can land inside the phrase and a wrap can
# split it across a line. Strip ANSI, then collapse whitespace — the repo-wide
# rule enforced by `test_cli_help_assertions_are_ansi_safe.py`.
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")


def _plain(text: str) -> str:
    return re.sub(r"\s+", " ", _ANSI_RE.sub("", text or ""))


# ---------------------------------------------------------------------------
# Task 1 — shared adapter fuse
# ---------------------------------------------------------------------------
class TestAdapterFuse:
    def test_module_exports_fuse_and_release(self):
        from soup_cli.utils.adapter_fuse import fuse_adapter_into, release_cuda

        assert callable(fuse_adapter_into)
        assert callable(release_cuda)

    def test_shrink_reuses_the_shared_implementation(self):
        """shrink must not keep a second copy of the fuse (no behavioural drift)."""
        from soup_cli.commands import shrink
        from soup_cli.utils.adapter_fuse import fuse_adapter_into, release_cuda

        assert shrink._fuse_adapter is fuse_adapter_into
        assert shrink._release_cuda is release_cuda

    def test_no_top_level_torch(self):
        import soup_cli

        path = Path(soup_cli.__file__).parent / "utils" / "adapter_fuse.py"
        tree = ast.parse(path.read_text(encoding="utf-8"))
        for node in tree.body:
            if isinstance(node, ast.Import):
                for alias in node.names:
                    assert not alias.name.startswith(
                        ("torch", "transformers", "peft")
                    ), f"top-level import of {alias.name}"
            elif isinstance(node, ast.ImportFrom) and node.module:
                assert not node.module.startswith(
                    ("torch", "transformers", "peft")
                ), f"top-level import from {node.module}"

    def test_failed_save_does_not_orphan_the_staging_dir(self, tmp_path, monkeypatch):
        """python-review HIGH: a save that raises mid-write used to leave a
        full model's worth of bytes in a .fuse_* dir next to the model."""
        import sys
        import types

        from soup_cli.utils import adapter_fuse

        base = tmp_path / "base"
        base.mkdir()
        (base / "config.json").write_text("{}", encoding="utf-8")
        monkeypatch.chdir(tmp_path)

        class _Merged:
            def save_pretrained(self, path):
                # The staging dir already exists at this point — this is exactly
                # where a disk-full / interrupted write lands.
                Path(path).joinpath("partial.bin").write_bytes(b"xxxx")
                raise RuntimeError("disk full")

        class _Base:
            @staticmethod
            def from_pretrained(*a, **kw):
                return _Base()

        class _Peft:
            @staticmethod
            def from_pretrained(*a, **kw):
                obj = _Peft()
                obj.merge_and_unload = lambda: _Merged()  # noqa: E731
                return obj

        class _Tok:
            @staticmethod
            def from_pretrained(*a, **kw):
                return _Tok()

            def save_pretrained(self, path):
                pass

        fake_tf = types.ModuleType("transformers")
        fake_tf.AutoModelForCausalLM = _Base
        fake_tf.AutoTokenizer = _Tok
        fake_peft = types.ModuleType("peft")
        fake_peft.PeftModel = _Peft
        monkeypatch.setitem(sys.modules, "transformers", fake_tf)
        monkeypatch.setitem(sys.modules, "peft", fake_peft)

        with pytest.raises(RuntimeError, match="disk full"):
            adapter_fuse.fuse_adapter_into(
                base_dir="base", adapter_dir=str(tmp_path / "ad")
            )
        assert list(tmp_path.glob(".fuse_*")) == [], "staging dir was orphaned"
        assert base.exists(), "the base model must survive a failed fuse"

    def test_fuse_adapter_into_is_the_in_place_special_case(self, monkeypatch):
        """shrink's in-place fuse == merge(base=out=base_dir)."""
        from soup_cli.utils import adapter_fuse

        seen: dict = {}
        monkeypatch.setattr(
            adapter_fuse,
            "merge_adapter_to_dense",
            lambda **kw: seen.update(kw),
        )
        adapter_fuse.fuse_adapter_into(base_dir="model", adapter_dir="ad", trc=True)
        assert seen["base_model"] == "model"
        assert seen["out_dir"] == "model"
        assert seen["adapter_dir"] == "ad"
        assert seen["trc"] is True

    def test_merge_rejects_pickle_weights_in_adapter_dir(self, tmp_path, monkeypatch):
        """security CRITICAL: the adapter dir may be attacker-produced; refuse
        pickle weights before PEFT torch.load's them."""
        from soup_cli.utils import adapter_fuse

        monkeypatch.chdir(tmp_path)
        adapter = tmp_path / "out" / "_adapter"
        adapter.mkdir(parents=True)
        (adapter / "adapter_model.bin").write_bytes(b"\x80\x04pickle")  # unsafe ext

        with pytest.raises(ValueError, match="(?i)unsafe|pickle|safetensors"):
            adapter_fuse.merge_adapter_to_dense(
                base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
            )

    def test_trainer_checkpoint_pickles_are_not_mistaken_for_an_attack(
        self, tmp_path, monkeypatch
    ):
        """Live-smoke regression: the HF Trainer writes checkpoint-N/optimizer.pt
        (a pickle IT wrote). A recursive scan refused Soup's own distill output
        and made `soup draft distill` impossible. Only TOP-LEVEL weights — the
        ones PEFT actually loads — are the threat surface."""
        from soup_cli.utils.strict_safetensors import (
            assert_safe_top_level_weights,
            check_strict_safetensors,
        )

        monkeypatch.chdir(tmp_path)  # check_strict_safetensors is cwd-contained
        adapter = tmp_path / "_adapter"
        (adapter / "checkpoint-60").mkdir(parents=True)
        (adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
        # Both pickles below are written by the HF Trainer itself, and NEITHER is
        # ever deserialized by from_pretrained. Both broke the live smoke.
        (adapter / "training_args.bin").write_bytes(b"\x80\x04pickle")
        (adapter / "checkpoint-60" / "optimizer.pt").write_bytes(b"\x80\x04pickle")

        # The RECURSIVE, extension-only scan flags them — that is exactly what
        # made `soup draft distill` refuse its own trainer's output...
        with pytest.raises(ValueError, match="(?i)unsafe"):
            check_strict_safetensors(str(adapter), strict=True)
        # ...while the shallow weight-name scan (what the merge uses) accepts it.
        assert_safe_top_level_weights(str(adapter))

        # A top-level pickle ADAPTER — the file PEFT actually unpickles, i.e.
        # the real threat — is still refused.
        (adapter / "adapter_model.bin").write_bytes(b"\x80\x04pickle")
        with pytest.raises(ValueError, match="(?i)unsafe"):
            assert_safe_top_level_weights(str(adapter))

    def test_shallow_scan_ignores_trainer_bookkeeping_pickles(self, tmp_path):
        from soup_cli.utils.strict_safetensors import (
            assert_safe_top_level_weights,
            find_unsafe_weight_files_shallow,
        )

        adapter = tmp_path / "_adapter"
        (adapter / "checkpoint-60").mkdir(parents=True)
        (adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
        (adapter / "training_args.bin").write_bytes(b"\x80\x04pickle")
        (adapter / "checkpoint-60" / "optimizer.pt").write_bytes(b"\x80\x04pickle")

        assert find_unsafe_weight_files_shallow(str(adapter)) == ()
        assert_safe_top_level_weights(str(adapter))  # must not raise

    def test_shallow_scan_still_catches_a_pickle_model_file(self, tmp_path):
        """serve --auto-spec loads a registry draft dir: pytorch_model.bin is
        the file from_pretrained would unpickle."""
        from soup_cli.utils.strict_safetensors import assert_safe_top_level_weights

        model_dir = tmp_path / "draft"
        model_dir.mkdir()
        (model_dir / "config.json").write_text("{}", encoding="utf-8")
        (model_dir / "pytorch_model.bin").write_bytes(b"\x80\x04pickle")

        with pytest.raises(ValueError, match="(?i)unsafe"):
            assert_safe_top_level_weights(str(model_dir))

    def test_merge_loads_base_weights_from_the_base_model_not_out_dir(
        self, tmp_path, monkeypatch
    ):
        """The whole point of the CRITICAL fix: an adapter-only dir has no base
        weights, so the base must be loaded from a separate model id/path."""
        import sys
        import types

        from soup_cli.utils import adapter_fuse

        monkeypatch.chdir(tmp_path)
        adapter = tmp_path / "out" / "_adapter"
        adapter.mkdir(parents=True)
        loaded: dict = {}

        class _Merged:
            def save_pretrained(self, path):
                Path(path).joinpath("model.safetensors").write_bytes(b"dense")

        class _Base:
            @staticmethod
            def from_pretrained(model_id, **kw):
                loaded["base"] = model_id
                return _Base()

        class _Peft:
            @staticmethod
            def from_pretrained(base, adapter_dir, **kw):
                loaded["adapter"] = adapter_dir
                obj = _Peft()
                obj.merge_and_unload = lambda: _Merged()  # noqa: E731
                return obj

        class _Tok:
            @staticmethod
            def from_pretrained(model_id, **kw):
                return _Tok()

            def save_pretrained(self, path):
                Path(path).joinpath("tokenizer.json").write_text("{}", encoding="utf-8")

        fake_tf = types.ModuleType("transformers")
        fake_tf.AutoModelForCausalLM = _Base
        fake_tf.AutoTokenizer = _Tok
        fake_peft = types.ModuleType("peft")
        fake_peft.PeftModel = _Peft
        monkeypatch.setitem(sys.modules, "transformers", fake_tf)
        monkeypatch.setitem(sys.modules, "peft", fake_peft)

        adapter_fuse.merge_adapter_to_dense(
            base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
        )
        assert loaded["base"] == "org/tiny"
        assert loaded["adapter"] == str(adapter)
        # out/ is now the DENSE model, and the adapter subdir is gone.
        assert (tmp_path / "out" / "model.safetensors").exists()
        assert not (tmp_path / "out" / "_adapter").exists()
        assert list(tmp_path.glob(".fuse_*")) == []

    def test_containment_checked_before_any_model_load(self, tmp_path, monkeypatch):
        """The cheap containment check runs before the heavy imports/loads."""
        from soup_cli.utils import adapter_fuse

        seen: list[tuple[str, str]] = []

        def _fake_enforce(path: str, field: str) -> str:
            seen.append((path, field))
            raise ValueError("refused")

        monkeypatch.setattr(
            adapter_fuse, "enforce_under_cwd_and_no_symlink", _fake_enforce
        )
        with pytest.raises(ValueError, match="refused"):
            adapter_fuse.fuse_adapter_into(
                base_dir=str(tmp_path / "base"), adapter_dir=str(tmp_path / "ad")
            )
        assert seen and seen[0][0] == str(tmp_path / "base")

    def test_rechecks_immediately_before_the_destructive_swap(
        self, tmp_path, monkeypatch
    ):
        """TOCTOU: a mock that raised on EVERY call could not distinguish
        'checked once at entry' from 'rechecked right before rmtree'. This one
        succeeds first and fails on the SECOND call, so it can only pass if the
        code truly re-validates immediately before the swap."""
        import sys
        import types

        from soup_cli.utils import adapter_fuse

        monkeypatch.chdir(tmp_path)
        base = tmp_path / "base"
        base.mkdir()
        adapter = tmp_path / "out" / "_adapter"
        adapter.mkdir(parents=True)

        calls = {"n": 0}
        real = adapter_fuse.enforce_under_cwd_and_no_symlink

        def _flaky(path, field):
            calls["n"] += 1
            if calls["n"] >= 2:
                raise ValueError("swap-time refusal")
            return real(path, field)

        monkeypatch.setattr(adapter_fuse, "enforce_under_cwd_and_no_symlink", _flaky)

        class _Merged:
            def save_pretrained(self, path):
                Path(path).joinpath("model.safetensors").write_bytes(b"dense")

        class _Base:
            @staticmethod
            def from_pretrained(*a, **kw):
                return _Base()

        class _Peft:
            @staticmethod
            def from_pretrained(*a, **kw):
                obj = _Peft()
                obj.merge_and_unload = lambda: _Merged()  # noqa: E731
                return obj

        class _Tok:
            @staticmethod
            def from_pretrained(*a, **kw):
                return _Tok()

            def save_pretrained(self, path):
                pass

        fake_tf = types.ModuleType("transformers")
        fake_tf.AutoModelForCausalLM = _Base
        fake_tf.AutoTokenizer = _Tok
        fake_peft = types.ModuleType("peft")
        fake_peft.PeftModel = _Peft
        monkeypatch.setitem(sys.modules, "transformers", fake_tf)
        monkeypatch.setitem(sys.modules, "peft", fake_peft)

        with pytest.raises(ValueError, match="swap-time refusal"):
            adapter_fuse.merge_adapter_to_dense(
                base_model=str(base), adapter_dir=str(adapter), out_dir="out"
            )
        assert calls["n"] == 2, "must recheck immediately before the destructive swap"
        # The swap never completed: no dense model landed, and the staging dir
        # was cleaned up rather than orphaned.
        assert not (tmp_path / "out" / "model.safetensors").exists()
        assert list(tmp_path.glob(".fuse_*")) == []

    def test_merge_falls_back_to_base_tokenizer_when_adapter_lacks_one(
        self, tmp_path, monkeypatch
    ):
        import sys
        import types

        from soup_cli.utils import adapter_fuse

        monkeypatch.chdir(tmp_path)
        adapter = tmp_path / "out" / "_adapter"
        adapter.mkdir(parents=True)
        seen = {"tok_from": []}

        class _Merged:
            def save_pretrained(self, path):
                Path(path).joinpath("model.safetensors").write_bytes(b"dense")

        class _Base:
            @staticmethod
            def from_pretrained(*a, **kw):
                return _Base()

        class _Peft:
            @staticmethod
            def from_pretrained(*a, **kw):
                obj = _Peft()
                obj.merge_and_unload = lambda: _Merged()  # noqa: E731
                return obj

        class _Tok:
            @staticmethod
            def from_pretrained(model_id, **kw):
                seen["tok_from"].append(model_id)
                if model_id == str(adapter):
                    raise OSError("no tokenizer files here")
                return _Tok()

            def save_pretrained(self, path):
                pass

        fake_tf = types.ModuleType("transformers")
        fake_tf.AutoModelForCausalLM = _Base
        fake_tf.AutoTokenizer = _Tok
        fake_peft = types.ModuleType("peft")
        fake_peft.PeftModel = _Peft
        monkeypatch.setitem(sys.modules, "transformers", fake_tf)
        monkeypatch.setitem(sys.modules, "peft", fake_peft)

        adapter_fuse.merge_adapter_to_dense(
            base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
        )
        assert seen["tok_from"] == [str(adapter), "org/tiny"]


# ---------------------------------------------------------------------------
# Task 2 — pure acceptance kernel
# ---------------------------------------------------------------------------
class _FakeTok:
    """Duck-typed tokenizer: a fixed text -> ids table plus a vocab_size."""

    def __init__(self, vocab_size: int, table: dict[str, list[int]] | None = None):
        self.vocab_size = vocab_size
        self._table = table or {}

    def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
        if text in self._table:
            return list(self._table[text])
        # Deterministic default: codepoint sum, so two tokenizers with the same
        # table agree everywhere.
        return [sum(ord(ch) for ch in text) % max(self.vocab_size, 1)]


class TestComputeAcceptance:
    def test_all_match(self):
        from soup_cli.utils.draft import compute_acceptance

        assert compute_acceptance([1, 2, 3], [1, 2, 3]) == 1.0

    def test_none_match(self):
        from soup_cli.utils.draft import compute_acceptance

        assert compute_acceptance([9, 9, 9], [1, 2, 3]) == 0.0

    def test_partial(self):
        from soup_cli.utils.draft import compute_acceptance

        assert compute_acceptance([1, 2, 9], [1, 2, 3]) == pytest.approx(2 / 3)

    def test_empty_is_zero(self):
        from soup_cli.utils.draft import compute_acceptance

        assert compute_acceptance([], []) == 0.0

    def test_length_mismatch_raises(self):
        from soup_cli.utils.draft import compute_acceptance

        with pytest.raises(ValueError, match="same length"):
            compute_acceptance([1, 2], [1])


class TestCountAcceptedAndRate:
    def test_count_accepted(self):
        from soup_cli.utils.draft import count_accepted

        assert count_accepted([1, 2, 9], [1, 2, 3]) == 2
        assert count_accepted([], []) == 0

    def test_count_accepted_length_mismatch_raises(self):
        from soup_cli.utils.draft import count_accepted

        with pytest.raises(ValueError, match="same length"):
            count_accepted([1], [1, 2])

    def test_acceptance_rate(self):
        from soup_cli.utils.draft import acceptance_rate

        assert acceptance_rate(75, 100) == 0.75
        assert acceptance_rate(0, 0) == 0.0

    def test_acceptance_rate_rejects_impossible_counts(self):
        from soup_cli.utils.draft import acceptance_rate

        with pytest.raises(ValueError, match="exceeds"):
            acceptance_rate(5, 3)
        with pytest.raises(ValueError, match="non-negative"):
            acceptance_rate(-1, 3)

    def test_count_accepted_spans_uses_max_align_chars_cap(self, monkeypatch):
        """regression: count_accepted_spans respects the _MAX_ALIGN_CHARS cap."""
        import difflib

        from soup_cli.utils import uld
        from soup_cli.utils.draft import count_accepted_spans

        monkeypatch.setattr(uld, "_MAX_ALIGN_CHARS", 10)

        captured: list[tuple[str, str]] = []
        real_matcher = difflib.SequenceMatcher

        def _spy_matcher(isjunk, a, b, autojunk=False):
            captured.append((a, b))
            return real_matcher(isjunk, a, b, autojunk=autojunk)

        monkeypatch.setattr(difflib, "SequenceMatcher", _spy_matcher)

        # 16-character non-matching strings: d_text != t_text triggers SequenceMatcher.
        draft_pieces = ["abcdef", "ghijkl", "mnop"]
        target_pieces = ["_bcdef", "ghijkl", "mnop"]

        count_accepted_spans(draft_pieces, target_pieces)

        assert len(captured) == 1
        a_str, b_str = captured[0]
        assert a_str == "abcdefghij"
        assert b_str == "_bcdefghij"
        assert len(a_str) == 10
        assert len(b_str) == 10


class TestClassify:
    def test_boundary_exact(self):
        from soup_cli.utils.draft import classify_acceptance

        assert classify_acceptance(0.70) == "STRONG"
        assert classify_acceptance(0.6999) == "MODERATE"
        assert classify_acceptance(0.50) == "MODERATE"
        assert classify_acceptance(0.4999) == "WEAK"
        assert classify_acceptance(0.0) == "WEAK"
        assert classify_acceptance(1.0) == "STRONG"

    def test_rejects_bool(self):
        from soup_cli.utils.draft import classify_acceptance

        with pytest.raises(TypeError, match="bool"):
            classify_acceptance(True)

    def test_rejects_nonfinite(self):
        from soup_cli.utils.draft import classify_acceptance

        with pytest.raises(ValueError, match="finite"):
            classify_acceptance(float("nan"))

    def test_rejects_out_of_range(self):
        from soup_cli.utils.draft import classify_acceptance

        with pytest.raises(ValueError, match="between 0 and 1"):
            classify_acceptance(1.5)
        with pytest.raises(ValueError, match="between 0 and 1"):
            classify_acceptance(-0.1)


class TestSameTokenizer:
    def test_identical_is_true(self):
        from soup_cli.utils.draft import same_tokenizer

        tok = _FakeTok(32000)
        assert same_tokenizer(tok, tok) is True
        assert same_tokenizer(_FakeTok(32000), _FakeTok(32000)) is True

    def test_equal_vocab_size_but_different_ids_is_false(self):
        """The test that makes the guard meaningful.

        Two tokenizers can both report vocab_size 32000 and still disagree on
        every single token — a vocab_size check alone would wave that through
        and the draft's proposals would be pure noise.
        """
        from soup_cli.utils.draft import same_tokenizer

        probe = "Hello, world!"
        a = _FakeTok(32000, {probe: [1, 2, 3]})
        b = _FakeTok(32000, {probe: [7, 8, 9]})
        assert same_tokenizer(a, b) is False

    def test_different_vocab_size_is_false(self):
        from soup_cli.utils.draft import same_tokenizer

        assert same_tokenizer(_FakeTok(32000), _FakeTok(49152)) is False

    def test_missing_vocab_size_attribute_is_false(self):
        from soup_cli.utils.draft import same_tokenizer

        class _NoVocab:
            def encode(self, text, add_special_tokens=False):
                return [1, 2, 3]

        assert same_tokenizer(_NoVocab(), _FakeTok(32000)) is False
        assert same_tokenizer(_FakeTok(32000), _NoVocab()) is False

    def test_probe_corpus_is_non_trivial(self):
        """A single-ASCII-word probe would miss most real tokenizer splits."""
        from soup_cli.utils.draft import PROBE_CORPUS

        assert len(PROBE_CORPUS) >= 4
        joined = "".join(PROBE_CORPUS)
        assert any(ord(ch) > 127 for ch in joined), "probe must include non-ASCII"
        assert any(ch.isdigit() for ch in joined), "probe must include digits"

    def test_broken_tokenizer_encode_is_false_not_raise(self):
        from soup_cli.utils.draft import same_tokenizer

        class _Broken:
            vocab_size = 32000

            def encode(self, text, add_special_tokens=False):
                raise RuntimeError("boom")

        assert same_tokenizer(_FakeTok(32000), _Broken()) is False


class TestReport:
    def _report(self, **kw):
        from soup_cli.utils.draft import AcceptanceReport

        defaults = dict(
            target="t",
            draft="d",
            n_prompts=2,
            n_generated_tokens=100,
            acceptance_rate=0.75,
            verdict="STRONG",
            tok_s_plain=10.0,
            tok_s_assisted=12.0,
            speedup=1.2,
            num_assistant_tokens=5,
            soup_version="0.71.33",
        )
        defaults.update(kw)
        return AcceptanceReport(**defaults)

    def test_frozen(self):
        import dataclasses

        report = self._report()
        with pytest.raises(dataclasses.FrozenInstanceError):
            report.acceptance_rate = 0.1  # type: ignore[misc]

    def test_to_dict_round_trip(self):
        from soup_cli.utils.draft import draft_report_to_dict

        data = draft_report_to_dict(self._report())
        assert data["acceptance_rate"] == 0.75
        assert data["verdict"] == "STRONG"
        assert data["target"] == "t"
        assert data["speedup"] == 1.2

    def test_panel_names_the_verdict_and_rate(self):
        from io import StringIO

        from rich.console import Console

        from soup_cli.utils.draft import render_draft_panel

        buf = StringIO()
        Console(file=buf, width=100).print(render_draft_panel(self._report()))
        out = buf.getvalue()
        assert "STRONG" in out
        assert "75" in out  # the acceptance rate as a percentage

    def test_panel_renders_when_throughput_unmeasured(self):
        from io import StringIO

        from rich.console import Console

        from soup_cli.utils.draft import render_draft_panel

        buf = StringIO()
        report = self._report(tok_s_plain=None, tok_s_assisted=None, speedup=None)
        Console(file=buf, width=100).print(render_draft_panel(report))
        assert "STRONG" in buf.getvalue()


# ---------------------------------------------------------------------------
# Task 3 — local draft registry + spec_pairing lookup
# ---------------------------------------------------------------------------
@pytest.fixture()
def draft_registry(tmp_path, monkeypatch):
    """Point the draft registry at a temp file (never touch the real ~/.soup)."""
    path = tmp_path / "drafts.json"
    monkeypatch.setenv("SOUP_DRAFT_REGISTRY_PATH", str(path))
    return path


class TestDraftRegistry:
    def test_round_trip(self, draft_registry, tmp_path):
        from soup_cli.utils.draft import list_drafts, lookup_draft, register_draft

        draft = tmp_path / "mydraft"
        draft.mkdir()
        register_draft("HF/Target-7B", str(draft), 0.62)

        assert lookup_draft("HF/Target-7B") == os.path.realpath(str(draft))
        entries = list_drafts()
        assert len(entries) == 1
        assert entries[0]["acceptance_rate"] == 0.62
        assert entries[0]["target"] == "hf/target-7b"

    def test_lookup_is_case_insensitive(self, draft_registry, tmp_path):
        """Must match pick_draft_model's .lower() normalisation."""
        from soup_cli.utils.draft import lookup_draft, register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        register_draft("Qwen/Qwen2.5-72B", str(draft))
        assert lookup_draft("qwen/qwen2.5-72b") is not None
        assert lookup_draft("QWEN/QWEN2.5-72B") is not None

    def test_stale_dir_is_skipped(self, draft_registry, tmp_path):
        """A moved/deleted draft degrades to 'no draft', never a crash in serve."""
        import shutil

        from soup_cli.utils.draft import lookup_draft, register_draft

        draft = tmp_path / "gone"
        draft.mkdir()
        register_draft("hf/target", str(draft))
        shutil.rmtree(draft)
        assert lookup_draft("hf/target") is None

    def test_reregister_replaces(self, draft_registry, tmp_path):
        from soup_cli.utils.draft import list_drafts, lookup_draft, register_draft

        first = tmp_path / "one"
        first.mkdir()
        second = tmp_path / "two"
        second.mkdir()
        register_draft("hf/t", str(first))
        register_draft("hf/t", str(second))
        assert len(list_drafts()) == 1
        assert lookup_draft("hf/t") == os.path.realpath(str(second))

    def test_malformed_json_reads_as_empty(self, draft_registry):
        from soup_cli.utils.draft import list_drafts, lookup_draft

        draft_registry.write_text("{ not json", encoding="utf-8")
        assert list_drafts() == []
        assert lookup_draft("anything") is None

    def test_non_dict_payload_reads_as_empty(self, draft_registry):
        from soup_cli.utils.draft import list_drafts

        draft_registry.write_text('["nope"]', encoding="utf-8")
        assert list_drafts() == []

    def test_missing_file_reads_as_empty(self, draft_registry):
        from soup_cli.utils.draft import list_drafts, lookup_draft

        assert not draft_registry.exists()
        assert list_drafts() == []
        assert lookup_draft("x") is None

    def test_entry_cap_evicts_oldest_first(self, draft_registry, tmp_path):
        from soup_cli.utils.draft import (
            _MAX_REGISTRY_ENTRIES,
            list_drafts,
            register_draft,
        )

        draft = tmp_path / "d"
        draft.mkdir()
        for i in range(_MAX_REGISTRY_ENTRIES + 5):
            register_draft(f"hf/target-{i}", str(draft))
        targets = {entry["target"] for entry in list_drafts()}
        assert len(targets) == _MAX_REGISTRY_ENTRIES
        # oldest evicted, newest kept
        assert "hf/target-0" not in targets
        assert "hf/target-4" not in targets
        assert "hf/target-5" in targets
        assert f"hf/target-{_MAX_REGISTRY_ENTRIES + 4}" in targets

    @pytest.mark.requires_symlink
    def test_registry_symlink_is_not_followed_on_read(self, draft_registry, tmp_path):
        """O_NOFOLLOW: a symlink at the registry path must degrade to empty,
        not leak an arbitrary file's parsed content into serve --auto-spec."""
        from soup_cli.utils.draft import list_drafts

        secret = tmp_path / "secret.json"
        secret.write_text(
            '{"drafts": [{"target": "leaked", "draft": "/tmp"}]}', encoding="utf-8"
        )
        draft_registry.symlink_to(secret)
        assert list_drafts() == []

    def test_atomic_write_leaves_no_temp_file(self, draft_registry, tmp_path):
        from soup_cli.utils.draft import register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        register_draft("hf/t", str(draft))
        leftovers = [p.name for p in draft_registry.parent.glob(".soup.*")]
        assert leftovers == []

    def test_concurrent_registration_does_not_lose_an_entry(
        self, draft_registry, tmp_path
    ):
        """code-review MEDIUM: read-modify-write under a cross-process lock."""
        import threading

        from soup_cli.utils.draft import list_drafts, register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        errors: list[BaseException] = []

        def _register(idx: int) -> None:
            try:
                register_draft(f"hf/target-{idx}", str(draft))
            except BaseException as exc:  # noqa: BLE001
                errors.append(exc)

        threads = [threading.Thread(target=_register, args=(i,)) for i in range(12)]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()

        assert not errors, errors
        targets = {entry["target"] for entry in list_drafts()}
        assert targets == {f"hf/target-{i}" for i in range(12)}

    def test_rejects_empty_target_and_bad_rate(self, draft_registry, tmp_path):
        from soup_cli.utils.draft import register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        with pytest.raises(ValueError, match="non-empty"):
            register_draft("  ", str(draft))
        with pytest.raises(ValueError, match="between 0 and 1"):
            register_draft("hf/t", str(draft), 1.5)


class TestSpecPairingLocal:
    def test_local_registry_beats_static_map(self, draft_registry, tmp_path):
        """A draft you trained yourself wins over the built-in pairing."""
        from soup_cli.utils.draft import register_draft
        from soup_cli.utils.spec_pairing import pick_draft_model

        draft = tmp_path / "mydraft"
        draft.mkdir()
        register_draft("qwen/qwen2.5-72b", str(draft))  # also in the static map
        assert pick_draft_model("Qwen/Qwen2.5-72B") == os.path.realpath(str(draft))

    def test_falls_back_to_static_map(self, draft_registry):
        from soup_cli.utils.spec_pairing import pick_draft_model

        assert pick_draft_model("Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-0.5B"

    def test_unknown_target_still_none(self, draft_registry):
        from soup_cli.utils.spec_pairing import pick_draft_model

        assert pick_draft_model("nobody/nothing-1b") is None

    def test_url_target_still_rejected(self, draft_registry):
        from soup_cli.utils.spec_pairing import pick_draft_model

        assert pick_draft_model("https://evil.example/model") is None

    def test_registry_blowup_does_not_crash_serve(self, draft_registry, monkeypatch):
        """pick_draft_model is on serve's startup path — it must never raise."""
        from soup_cli.utils import spec_pairing

        def _boom(target):
            raise RuntimeError("registry on fire")

        monkeypatch.setattr(spec_pairing, "_lookup_local_draft", _boom)
        assert spec_pairing.pick_draft_model("Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-0.5B"


# ---------------------------------------------------------------------------
# Task 4 — torch-lazy measurement
# ---------------------------------------------------------------------------
class _TensorTok:
    """Minimal tokenizer over torch tensors."""

    pad_token_id = 0
    eos_token_id = 0

    def __init__(self, prompt_ids: list[int]):
        self._prompt_ids = prompt_ids

    def __call__(self, text, return_tensors=None):
        import torch

        ids = torch.tensor([self._prompt_ids])
        return {"input_ids": ids, "attention_mask": torch.ones_like(ids)}


class _CrossTokenizerMock:
    """Mock tokenizer for cross-tokenizer measurement tests."""

    pad_token_id = 0
    eos_token_id = 0

    def __init__(
        self,
        vocab_size: int,
        encode_map: dict[str, list[int]],
        decode_map: dict[int, str],
    ):
        self.vocab_size = vocab_size
        self._encode_map = encode_map
        self._decode_map = decode_map

    def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
        return list(self._encode_map.get(text, [0]))

    def decode(self, token_ids: Sequence[int], skip_special_tokens: bool = False) -> str:
        return "".join(self._decode_map.get(int(tid), "") for tid in token_ids)

    def __call__(self, text: str, return_tensors: str | None = None, **kwargs):
        import torch

        ids = torch.tensor([self.encode(text)])
        return {"input_ids": ids, "attention_mask": torch.ones_like(ids)}


class _FakeTarget:
    """Greedy-generates a continuation, applying repetition penalty when != 1.0."""

    def __init__(
        self,
        full_ids: list[int],
        n_prompt: int,
        default_repetition_penalty: float = 1.2,
    ):
        self._full_ids = list(full_ids)
        self._n_prompt = n_prompt
        self._default_repetition_penalty = default_repetition_penalty
        self.calls: list[dict] = []

    def parameters(self):
        import torch

        yield torch.zeros(1)

    def generate(self, **kwargs):
        import torch

        self.calls.append(kwargs)
        rep_penalty = kwargs.get("repetition_penalty", self._default_repetition_penalty)
        if rep_penalty is None:
            rep_penalty = self._default_repetition_penalty

        input_ids = kwargs.get("input_ids")
        if input_ids is not None:
            curr_ids = input_ids[0].cpu().tolist()
        else:
            curr_ids = list(self._full_ids[: self._n_prompt])

        target_gen = self._full_ids[self._n_prompt :]
        if not target_gen:
            return torch.tensor([curr_ids])

        generated = list(curr_ids)
        for expected_tok in target_gen:
            alt_tok = 99 if expected_tok != 99 else 98
            logits = {expected_tok: 10.0, alt_tok: 9.0}
            if rep_penalty != 1.0:
                seen = set(generated)
                for tok_id in list(logits.keys()):
                    if tok_id in seen:
                        logits[tok_id] = logits[tok_id] / rep_penalty
            best_tok = max(logits.keys(), key=lambda t: logits[t])
            generated.append(best_tok)

        return torch.tensor([generated])


class _FakeDraft:
    """Returns logits whose per-position argmax is a scripted sequence."""

    def __init__(self, argmax_per_position: list[int], vocab: int = 100):
        self._argmax = argmax_per_position
        self._vocab = vocab

    def parameters(self):
        import torch

        yield torch.zeros(1)

    def __call__(self, input_ids=None, **kwargs):
        import torch

        seq = int(input_ids.shape[1])
        logits = torch.zeros(1, seq, self._vocab)
        for pos in range(seq):
            logits[0, pos, self._argmax[pos]] = 10.0

        class _Out:
            pass

        out = _Out()
        out.logits = logits
        return out


class TestMeasureAcceptance:
    # prompt = [5, 6] (len 2); target generates [10, 11, 12] at positions 2,3,4.
    FULL_IDS = [5, 6, 10, 11, 12]
    N_PROMPT = 2
    # Draft argmax per position. Causal alignment: logits[p-1] predicts token p,
    # so the proposals for the generated positions 2,3,4 are ARGMAX[1:4].
    #   correct     -> ARGMAX[1:4] = [10, 99, 12]  vs [10, 11, 12] -> 2 matches
    #   off-by-one+ -> ARGMAX[2:5] = [99, 12, 99]  vs [10, 11, 12] -> 0 matches
    #   off-by-one- -> ARGMAX[0:3] = [99, 10, 99]  vs [10, 11, 12] -> 0 matches
    # The three alignments give DIFFERENT answers, so this test can actually
    # fail on an off-by-one (an unfalsifiable test is worse than none).
    ARGMAX = [99, 10, 99, 12, 99]

    def test_shift_convention_is_pinned(self):
        from soup_cli.utils.draft import measure_acceptance

        accepted, total = measure_acceptance(
            _FakeTarget(self.FULL_IDS, self.N_PROMPT),
            _FakeDraft(self.ARGMAX),
            _TensorTok([5, 6]),
            ["hello"],
            max_new_tokens=8,
        )
        assert (accepted, total) == (2, 3)

    def test_off_by_one_alternatives_would_score_differently(self):
        """Guards the guard: the fixture must discriminate, not just pass."""
        from soup_cli.utils.draft import compute_acceptance

        generated = [10, 11, 12]
        correct = self.ARGMAX[1:4]
        off_plus = self.ARGMAX[2:5]
        off_minus = self.ARGMAX[0:3]
        assert compute_acceptance(correct, generated) == pytest.approx(2 / 3)
        assert compute_acceptance(off_plus, generated) == 0.0
        assert compute_acceptance(off_minus, generated) == 0.0

    def test_multiple_prompts_accumulate(self):
        from soup_cli.utils.draft import measure_acceptance

        accepted, total = measure_acceptance(
            _FakeTarget(self.FULL_IDS, self.N_PROMPT),
            _FakeDraft(self.ARGMAX),
            _TensorTok([5, 6]),
            ["a", "b", "c"],
            max_new_tokens=8,
        )
        assert (accepted, total) == (6, 9)

    def test_empty_generation_is_skipped_not_counted(self):
        from soup_cli.utils.draft import measure_acceptance

        # Target returns the prompt unchanged -> nothing was generated.
        accepted, total = measure_acceptance(
            _FakeTarget([5, 6], 2),
            _FakeDraft([99, 99]),
            _TensorTok([5, 6]),
            ["a"],
            max_new_tokens=8,
        )
        assert (accepted, total) == (0, 0)

    def test_generation_is_greedy(self):
        """Sampling would make the acceptance rate non-deterministic."""
        from soup_cli.utils.draft import measure_acceptance

        target = _FakeTarget(self.FULL_IDS, self.N_PROMPT)
        measure_acceptance(
            target, _FakeDraft(self.ARGMAX), _TensorTok([5, 6]), ["a"],
            max_new_tokens=8,
        )
        assert target.calls[0]["do_sample"] is False
        assert target.calls[0]["repetition_penalty"] == 1.0
        assert target.calls[0]["max_new_tokens"] == 8

    def test_repetition_penalty_neutralized_for_self_acceptance(self):
        """When target and draft are identical, repetition_penalty is neutralized (Refs #345)."""
        from soup_cli.utils.draft import measure_acceptance

        # Target generates [10, 11, 12] from prompt [10, 6].
        # An identical draft produces predictions [10, 11, 12] for those positions.
        identical_draft_argmax = [99, 10, 11, 12, 99]
        target = _FakeTarget([10, 6, 10, 11, 12], 2)
        draft = _FakeDraft(identical_draft_argmax)

        accepted, total = measure_acceptance(
            target, draft, _TensorTok([10, 6]), ["hello"], max_new_tokens=8
        )
        assert target.calls[0]["repetition_penalty"] == 1.0
        assert total == 3
        assert accepted == 3

    def test_cross_tokenizer_unextendable_prompt_scores_zero_zero(self):
        """When draft tokenizer produces no continuation beyond draft_prompt_len,
        measure_acceptance skips counting and returns exactly (0, 0)."""
        from soup_cli.utils.draft import measure_acceptance

        target_tok = _CrossTokenizerMock(
            vocab_size=32000,
            encode_map={"hello": [5, 6], "hello world": [5, 6, 10, 11, 12]},
            decode_map={5: "hel", 6: "lo", 10: " ", 11: "wor", 12: "ld"},
        )
        # Draft tokenizer produces no new tokens beyond draft_prompt_len (both 3 tokens).
        draft_tok = _CrossTokenizerMock(
            vocab_size=49152,
            encode_map={"hello": [1, 2, 3], "hello world": [1, 2, 3]},
            decode_map={1: "hel", 2: "lo", 3: ""},
        )
        target = _FakeTarget([5, 6, 10, 11, 12], 2)
        draft = _FakeDraft([99, 99, 99, 99, 99])

        accepted, total = measure_acceptance(
            target,
            draft,
            target_tok,
            ["hello"],
            max_new_tokens=8,
            draft_tokenizer=draft_tok,
        )
        assert (accepted, total) == (0, 0)

    def test_cross_tokenizer_extendable_prompt_measures_acceptance(self):
        """When draft tokenizer produces tokens beyond draft_prompt_len,
        measure_acceptance computes span-aligned acceptance."""
        from soup_cli.utils.draft import measure_acceptance

        target_tok = _CrossTokenizerMock(
            vocab_size=32000,
            encode_map={"hello": [5, 6], "hello world": [5, 6, 10, 11, 12]},
            decode_map={5: "hel", 6: "lo", 10: " ", 11: "wor", 12: "ld"},
        )
        draft_tok = _CrossTokenizerMock(
            vocab_size=49152,
            encode_map={"hello": [1, 2], "hello world": [1, 2, 20, 21, 22]},
            decode_map={1: "hel", 2: "lo", 20: " ", 21: "wor", 22: "ld"},
        )
        target = _FakeTarget([5, 6, 10, 11, 12], 2)
        draft = _FakeDraft([99, 20, 21, 22, 99])

        accepted, total = measure_acceptance(
            target,
            draft,
            target_tok,
            ["hello"],
            max_new_tokens=8,
            draft_tokenizer=draft_tok,
        )
        assert (accepted, total) == (3, 3)



class TestMeasureThroughput:
    def test_discards_a_warmup_generate(self):
        from soup_cli.utils.draft import measure_throughput

        target = _FakeTarget([5, 6, 10, 11, 12], 2)
        rate = measure_throughput(
            target, _TensorTok([5, 6]), ["a", "b"], max_new_tokens=8
        )
        # 2 prompts + 1 warm-up = 3 generate calls; only 2 are timed.
        assert len(target.calls) == 3
        assert rate > 0

    def test_empty_prompts_is_zero(self):
        from soup_cli.utils.draft import measure_throughput

        target = _FakeTarget([5, 6], 2)
        assert measure_throughput(target, _TensorTok([5, 6]), []) == 0.0
        assert target.calls == []

    def test_assistant_model_is_forwarded(self):
        from soup_cli.utils.draft import measure_throughput

        target = _FakeTarget([5, 6, 10, 11, 12], 2)
        draft = _FakeDraft([0, 0, 0, 0, 0])
        measure_throughput(
            target,
            _TensorTok([5, 6]),
            ["a"],
            assistant_model=draft,
            num_assistant_tokens=7,
        )
        assert target.calls[0]["assistant_model"] is draft
        assert target.calls[0]["num_assistant_tokens"] == 7

    def test_plain_run_passes_no_assistant(self):
        from soup_cli.utils.draft import measure_throughput

        target = _FakeTarget([5, 6, 10, 11, 12], 2)
        measure_throughput(target, _TensorTok([5, 6]), ["a"])
        assert "assistant_model" not in target.calls[0]


class TestDraftNoTopLevelTorch:
    @pytest.mark.parametrize(
        "relpath",
        [("utils", "draft.py"), ("commands", "draft.py")],
    )
    def test_is_torch_free(self, relpath):
        import soup_cli

        path = Path(soup_cli.__file__).parent.joinpath(*relpath)
        tree = ast.parse(path.read_text(encoding="utf-8"))
        for node in tree.body:
            if isinstance(node, ast.Import):
                for alias in node.names:
                    assert not alias.name.startswith(
                        ("torch", "transformers", "peft")
                    ), f"top-level import of {alias.name}"
            elif isinstance(node, ast.ImportFrom) and node.module:
                assert not node.module.startswith(
                    ("torch", "transformers", "peft")
                ), f"top-level import from {node.module}"


# ---------------------------------------------------------------------------
# Task 5 — soup draft CLI
# ---------------------------------------------------------------------------
class _Cfg:
    """Stand-in for transformers AutoConfig output."""

    def __init__(self, vocab_size: int):
        self.vocab_size = vocab_size


@pytest.fixture()
def runner():
    from typer.testing import CliRunner

    return CliRunner()


@pytest.fixture()
def in_tmp_cwd(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    return tmp_path


def _write_jsonl(path: Path, rows: list[dict]) -> str:
    import json as _json

    path.write_text(
        "\n".join(_json.dumps(row) for row in rows) + "\n", encoding="utf-8"
    )
    return str(path)


class TestDraftCliPlumbing:
    def test_registered_on_main_app(self, runner):
        from soup_cli.cli import app

        result = runner.invoke(app, ["draft", "--help"])
        assert result.exit_code == 0, (result.output, repr(result.exception))
        for sub in ("distill", "measure", "list"):
            assert sub in result.output

    def test_subcommand_help(self, runner):
        from soup_cli.commands.draft import app

        for sub in ("distill", "measure", "list"):
            result = runner.invoke(app, [sub, "--help"])
            assert result.exit_code == 0, (sub, result.output)

    @pytest.mark.parametrize("sub", ["distill", "measure"])
    def test_trust_remote_code_flag_exists(self, runner, sub):
        """code-review MEDIUM: without it, any auto_map model is a dead end."""
        import re

        from soup_cli.commands.draft import app

        result = runner.invoke(app, [sub, "--help"])
        plain = re.sub(r"\x1b\[[0-9;]*m", "", result.output).replace("\n", " ")
        assert "--trust-remote-code" in re.sub(r"\s+", " ", plain)

    def test_trust_flag_is_threaded_into_resolution(self, monkeypatch):
        from soup_cli.commands import draft as draft_cmd

        seen: dict = {}

        def _fake_resolve(model_id, requested, console, requires_remote_code):
            seen["requested"] = requested
            return requested

        monkeypatch.setattr(
            "soup_cli.utils.trust_remote.resolve_trust_remote_code", _fake_resolve
        )
        monkeypatch.setattr(
            "soup_cli.utils.trust_remote.model_requires_trust_remote_code",
            lambda mid: False,
        )
        assert draft_cmd._resolve_trust("org/x", True) is True
        assert seen["requested"] is True


class TestDraftDistillCli:
    def _data(self, tmp_path, rows: int = 200):
        return _write_jsonl(
            tmp_path / "d.jsonl",
            [{"messages": [{"role": "user", "content": "hi"},
                           {"role": "assistant", "content": "yo"}]}] * rows,
        )

    def _patch_configs(self, monkeypatch, target_vocab, draft_vocab):
        from soup_cli.commands import draft as draft_cmd

        def _fake_vocab(model_id: str, trc: bool = False) -> int:
            return target_vocab if "target" in model_id else draft_vocab

        monkeypatch.setattr(draft_cmd, "_vocab_size_of", _fake_vocab)

        tok_target = _FakeTok(target_vocab)
        tok_draft = _FakeTok(draft_vocab)

        def _fake_tok(model_id: str, **kwargs):
            return tok_target if "target" in model_id else tok_draft

        monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _fake_tok)

    def test_plan_only_writes_nothing(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "task: distill" in result.output
        assert not (in_tmp_cwd / "draftout").exists()

    def test_vocab_mismatch_routes_to_cross_tokenizer_uld(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 151936)
        data = self._data(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 0
        assert "uld_strategy: wasserstein_aligned" in result.output
        assert "cross-tokenizer" in result.output.lower()

    def test_data_outside_cwd_rejected(self, runner, in_tmp_cwd, tmp_path_factory,
                                       monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        outside = tmp_path_factory.mktemp("outside")
        data = _write_jsonl(outside / "d.jsonl", [{"text": "x"}])
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "cwd" in result.output.lower() or "outside" in result.output.lower()

    def test_output_outside_cwd_rejected(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "../escape", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "cwd" in result.output.lower()

    def test_output_is_cwd_rejected(self, runner, in_tmp_cwd, monkeypatch):
        """security CRITICAL: -o . would rmtree the whole working dir on success."""
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", ".", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "current directory" in result.output.lower()

    def test_output_preexisting_nondraft_dir_rejected(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """security CRITICAL: overwriting an unrelated dir needs --force."""
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        victim = in_tmp_cwd / "important"
        victim.mkdir()
        (victim / "notes.txt").write_text("keep me", encoding="utf-8")
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "important", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "--force" in result.output

    def test_output_preexisting_draft_dir_allowed(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """Re-distilling into a prior draft (has config.json) is fine."""
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        prior = in_tmp_cwd / "draftout"
        prior.mkdir()
        (prior / "config.json").write_text("{}", encoding="utf-8")
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))

    def test_missing_data_file_rejected(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", "nope.jsonl", "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "unreadable" in result.output.lower()

    def test_force_overwrites_a_preexisting_nondraft_dir(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        victim = in_tmp_cwd / "important"
        victim.mkdir()
        (victim / "notes.txt").write_text("keep me", encoding="utf-8")
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "important", "--force", "--plan-only"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))

    def test_empty_preexisting_dir_allowed_without_force(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands.draft import app

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        (in_tmp_cwd / "draftout").mkdir()  # empty, no config.json
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))

    def test_config_unreadable_reports_friendly_error(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        def _boom(model_id, trc=False):
            raise OSError("model not found")

        monkeypatch.setattr(draft_cmd, "_vocab_size_of", _boom)
        data = self._data(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["distill", "--target", "org/nope", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--plan-only"],
        )
        assert result.exit_code == 1
        assert "could not read model config" in result.output.lower()

    def test_run_distill_surfaces_subprocess_failure(self, in_tmp_cwd, monkeypatch):
        import subprocess as _sp

        from soup_cli.commands import draft as draft_cmd

        class _Result:
            returncode = 1
            stdout = b"partial log\n"
            stderr = b"error: exploded\x1b[31m"

        monkeypatch.setattr(_sp, "run", lambda argv, **kw: _Result())
        self._data(in_tmp_cwd)
        with pytest.raises(RuntimeError, match="rc=1"):
            draft_cmd._run_distill(
                draft_base="org/tiny", target="org/target", data="d.jsonl",
                out_dir="draftout", steps=100, data_rows=200,
            )

    def test_run_distill_reports_rendered_config_validation_error(
        self, in_tmp_cwd, monkeypatch
    ):
        import typer

        from soup_cli.commands import draft as draft_cmd

        messages: list[str] = []
        monkeypatch.setattr(
            "soup_cli.config.loader.load_config_from_string",
            lambda _yaml: (_ for _ in ()).throw(ValueError("invalid rendered config")),
        )
        monkeypatch.setattr(draft_cmd.console, "print", lambda message: messages.append(message))

        with pytest.raises(typer.Exit) as exc_info:
            draft_cmd._run_distill(
                draft_base="org/tiny",
                target="org/target",
                data="d.jsonl",
                out_dir="draftout",
                steps=100,
                data_rows=200,
            )

        assert exc_info.value.exit_code == 1
        assert messages == ["[red]Invalid rendered distill config:[/] invalid rendered config"]

    def test_run_distill_sanitizes_rendered_config_validation_error(
        self, in_tmp_cwd, monkeypatch
    ):
        import typer

        from soup_cli.commands import draft as draft_cmd

        messages: list[str] = []
        monkeypatch.setattr(
            "soup_cli.config.loader.load_config_from_string",
            lambda _yaml: (_ for _ in ()).throw(
                ValueError("bad \x1b[2J[bold red]field[/]")
            ),
        )
        monkeypatch.setattr(draft_cmd.console, "print", lambda message: messages.append(message))

        with pytest.raises(typer.Exit) as exc_info:
            draft_cmd._run_distill(
                draft_base="org/tiny",
                target="org/target",
                data="d.jsonl",
                out_dir="draftout",
                steps=100,
                data_rows=200,
            )

        assert exc_info.value.exit_code == 1
        assert len(messages) == 1
        assert "\x1b" not in messages[0]
        assert "bad [2J\\[bold red]field\\[/]" in messages[0]

    def test_run_distill_timeout_raises_friendly_error(self, in_tmp_cwd, monkeypatch):
        import subprocess as _sp

        from soup_cli.commands import draft as draft_cmd

        def _boom(argv, **kw):
            raise _sp.TimeoutExpired(cmd=argv, timeout=kw.get("timeout", 1))

        monkeypatch.setattr(_sp, "run", _boom)
        self._data(in_tmp_cwd)
        with pytest.raises(RuntimeError, match="timeout"):
            draft_cmd._run_distill(
                draft_base="org/tiny", target="org/target", data="d.jsonl",
                out_dir="draftout", steps=100, data_rows=200,
            )

    def test_run_distill_cpu_device_hides_gpus(self, in_tmp_cwd, monkeypatch):
        import subprocess as _sp

        from soup_cli.commands import draft as draft_cmd

        captured: dict = {}

        class _Result:
            returncode = 0
            stdout = b""
            stderr = b""

        def _fake_run(argv, **kwargs):
            captured["env"] = kwargs.get("env")
            (Path("draftout") / draft_cmd._ADAPTER_SUBDIR).mkdir(
                parents=True, exist_ok=True
            )
            return _Result()

        monkeypatch.setattr(_sp, "run", _fake_run)
        monkeypatch.setattr(draft_cmd, "merge_adapter_to_dense", lambda **kw: None)
        self._data(in_tmp_cwd)
        draft_cmd._run_distill(
            draft_base="org/tiny", target="org/target", data="d.jsonl",
            out_dir="draftout", steps=100, data_rows=200, device="cpu",
        )
        assert captured["env"]["CUDA_VISIBLE_DEVICES"] == "-1"

    def test_generated_config_is_schema_valid(self, in_tmp_cwd):
        """The rendered YAML must actually load as a SoupConfig."""
        from soup_cli.commands.draft import _build_distill_config_yaml
        from soup_cli.config.loader import load_config_from_string

        data = self._data(in_tmp_cwd)
        yaml_text = _build_distill_config_yaml(
            draft_base="org/tiny",
            target="org/target",
            data=data,
            out_dir="draftout",
            steps=100,
            data_rows=200,
        )
        cfg = load_config_from_string(yaml_text)
        assert cfg.task == "distill"
        assert cfg.base == "org/tiny"
        assert cfg.training.teacher_model == "org/target"

    def test_trainer_output_goes_to_a_nested_adapter_dir(self, in_tmp_cwd):
        """code-review CRITICAL: the distill trainer only ever writes a LoRA
        adapter (never dense base weights), so it must NOT train straight into
        -o — the merge needs a separate adapter dir plus the base weights."""
        import yaml

        from soup_cli.commands.draft import _ADAPTER_SUBDIR, _build_distill_config_yaml

        yaml_text = _build_distill_config_yaml(
            draft_base="org/tiny",
            target="org/target",
            data="d.jsonl",
            out_dir=os.path.join("draftout", _ADAPTER_SUBDIR),
            steps=100,
            data_rows=200,
        )
        parsed = yaml.safe_load(yaml_text)
        assert parsed["output"].endswith(_ADAPTER_SUBDIR)
        assert parsed["base"] == "org/tiny"  # student = the draft base

    def test_run_distill_merges_base_plus_adapter_not_out_dir_into_itself(
        self, in_tmp_cwd, monkeypatch
    ):
        """The merge must load base weights from --draft-base and the adapter
        from the nested dir. Fusing out_dir into itself (the original bug) can
        never work: out_dir holds no base weights."""
        from soup_cli.commands import draft as draft_cmd

        calls: dict = {}

        class _Result:
            returncode = 0
            stdout = b""
            stderr = b""

        def _fake_run(argv, **kwargs):
            # Simulate the trainer writing ONLY an adapter into its output dir.
            adapter = Path("draftout") / draft_cmd._ADAPTER_SUBDIR
            adapter.mkdir(parents=True, exist_ok=True)
            (adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
            return _Result()

        def _fake_merge(*, base_model, adapter_dir, out_dir, trc=False):
            calls.update(
                base_model=base_model, adapter_dir=adapter_dir, out_dir=out_dir
            )

        import subprocess as _sp

        monkeypatch.setattr(_sp, "run", _fake_run)
        monkeypatch.setattr(draft_cmd, "merge_adapter_to_dense", _fake_merge)

        self._data(in_tmp_cwd)
        draft_cmd._run_distill(
            draft_base="org/tiny",
            target="org/target",
            data="d.jsonl",
            out_dir="draftout",
            steps=100,
            data_rows=200,
        )
        assert calls["base_model"] == "org/tiny", "base weights must come from --draft-base"
        assert calls["adapter_dir"].endswith(draft_cmd._ADAPTER_SUBDIR)
        assert calls["out_dir"] == "draftout"
        assert calls["adapter_dir"] != calls["out_dir"]

    def test_run_distill_fails_loudly_when_no_adapter_was_written(
        self, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd

        class _Result:
            returncode = 0
            stdout = b""
            stderr = b""

        import subprocess as _sp

        monkeypatch.setattr(_sp, "run", lambda argv, **kw: _Result())
        monkeypatch.setattr(
            draft_cmd, "merge_adapter_to_dense", lambda **kw: None
        )
        with pytest.raises(RuntimeError, match="no adapter"):
            draft_cmd._run_distill(
                draft_base="org/tiny",
                target="org/target",
                data="d.jsonl",
                out_dir="draftout",
                steps=100,
                data_rows=200,
            )

    def test_newline_in_model_id_cannot_inject_yaml_keys(self, in_tmp_cwd):
        """python-review CRITICAL: raw interpolation let a crafted --target
        smuggle sibling keys into the training: block. Every user string is
        json.dumps'd, so a newline stays inside the scalar."""
        import yaml

        from soup_cli.commands.draft import _build_distill_config_yaml

        hostile = "org/x\ntraining:\n  reward_fn: ../../evil.py"
        yaml_text = _build_distill_config_yaml(
            draft_base="org/tiny",
            target=hostile,
            data="d.jsonl",
            out_dir="draftout",
            steps=100,
            data_rows=200,
        )
        parsed = yaml.safe_load(yaml_text)
        # The payload landed as a VALUE, not as new keys.
        assert parsed["training"]["teacher_model"] == hostile
        assert "reward_fn" not in parsed["training"]
        assert parsed["task"] == "distill"

    def test_absurd_steps_over_tiny_data_is_refused(self, in_tmp_cwd):
        """500 steps over 4 rows = 125 epochs — refuse rather than emit it."""
        from soup_cli.commands.draft import _build_distill_config_yaml

        with pytest.raises(ValueError, match="epochs"):
            _build_distill_config_yaml(
                draft_base="org/tiny",
                target="org/target",
                data="d.jsonl",
                out_dir="draftout",
                steps=500,
                data_rows=4,
            )

    def test_happy_path_trains_fuses_and_registers(
        self, runner, in_tmp_cwd, monkeypatch, draft_registry
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app
        from soup_cli.utils.draft import lookup_draft

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)

        trained: dict = {}

        def _fake_train(**kwargs):
            trained.update(kwargs)
            # The real subprocess writes the adapter into out_dir; simulate the
            # fused dense draft landing in the output dir.
            Path(kwargs["out_dir"]).mkdir(parents=True, exist_ok=True)

        monkeypatch.setattr(draft_cmd, "_run_distill", _fake_train)

        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert trained["target"] == "org/target"
        registered = lookup_draft("org/target")
        assert registered == os.path.realpath(str(in_tmp_cwd / "draftout"))

    def test_no_register_flag_skips_registry(
        self, runner, in_tmp_cwd, monkeypatch, draft_registry
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app
        from soup_cli.utils.draft import lookup_draft

        self._patch_configs(monkeypatch, 49152, 49152)
        data = self._data(in_tmp_cwd)
        monkeypatch.setattr(
            draft_cmd,
            "_run_distill",
            lambda **kw: Path(kw["out_dir"]).mkdir(parents=True, exist_ok=True),
        )
        result = runner.invoke(
            app,
            ["distill", "--target", "org/target", "--draft-base", "org/tiny",
             "--data", data, "-o", "draftout", "--no-register"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert lookup_draft("org/target") is None


class TestDraftMeasureCli:
    def _prompts(self, tmp_path):
        return _write_jsonl(
            tmp_path / "p.jsonl", [{"prompt": "What is 2+2?"}, {"prompt": "Hi"}]
        )

    def _patch_load(self, monkeypatch, *, compatible=True):
        from soup_cli.commands import draft as draft_cmd

        tok_a = _FakeTok(49152)
        tok_b = _FakeTok(49152) if compatible else _FakeTok(151936)
        model_a = object()
        model_b = object()

        # measure now gates on config.vocab_size before loading (issue #344). Make
        # that gate pass (equal config vocab) so these tests keep exercising the
        # load / same_tokenizer / measurement paths; the tokenizer-vocab mismatch
        # `compatible=False` sets still trips same_tokenizer as before.
        monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)

        def _fake_load(model_id, **kwargs):
            tok = tok_a if "target" in model_id else tok_b
            model = model_a if "target" in model_id else model_b
            return model, tok, "cpu"

        monkeypatch.setattr(draft_cmd, "_load_pair_member", _fake_load)
        return (model_a, tok_a), (model_b, tok_b)

    def test_happy_path_writes_report_and_exits_zero(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        import json as _json

        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "-o", "report.json"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "STRONG" in _plain(result.output)
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["acceptance_rate"] == 0.75
        assert data["verdict"] == "STRONG"
        assert data["n_generated_tokens"] == 100

    def test_below_min_acceptance_exits_two(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (40, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "--min-acceptance", "0.6"],
        )
        assert result.exit_code == 2
        assert "60.0%" in _plain(result.output)
        assert "below" in _plain(result.output).lower()

    def test_mismatched_tokenizer_measures_cross_tokenizer(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        (_, _), (draft_model, draft_tok) = self._patch_load(
            monkeypatch, compatible=False
        )
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (60, 100))

        captured_kwargs: list[dict] = []

        def _spy_measure_throughput(*args, **kwargs):
            captured_kwargs.append(kwargs)
            return 15.0

        monkeypatch.setattr(draft_cmd, "measure_throughput", _spy_measure_throughput)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts],
        )
        assert result.exit_code == 0
        assert "Cross-tokenizer draft detected" in _plain(result.output)
        assert "60.0%" in _plain(result.output)

        assisted_calls = [kw for kw in captured_kwargs if "assistant_model" in kw]
        assert len(assisted_calls) == 1
        assisted = assisted_calls[0]
        assert assisted["assistant_model"] is draft_model
        assert assisted["assistant_tokenizer"] is draft_tok
        assert "max_new_tokens" in assisted
        assert "num_assistant_tokens" in assisted

    def test_mismatched_tokenizer_unsupported_uad_warns(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch, compatible=False)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (60, 100))

        def _boom(model, tok, prompts, *, assistant_model=None, **kw):
            if assistant_model is not None:
                raise RuntimeError("Universal Assisted Decoding requires transformers>=4.45.0")
            return 20.0

        monkeypatch.setattr(draft_cmd, "measure_throughput", _boom)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts],
        )
        assert result.exit_code == 0
        assert "Universal Assisted Decoding requires transformers>=4.45.0" in _plain(result.output)
        assert "could not be measured" in _plain(result.output)

    def test_prompts_outside_cwd_rejected(
        self, runner, in_tmp_cwd, tmp_path_factory, monkeypatch
    ):
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        outside = tmp_path_factory.mktemp("outside2")
        prompts = _write_jsonl(outside / "p.jsonl", [{"prompt": "hi"}])
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts],
        )
        assert result.exit_code == 1

    def test_empty_prompts_rejected(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        empty = in_tmp_cwd / "empty.jsonl"
        empty.write_text("", encoding="utf-8")
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", str(empty)],
        )
        assert result.exit_code == 1

    def test_bad_min_acceptance_rejected(self, runner, in_tmp_cwd, monkeypatch):
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "--min-acceptance", "1.5"],
        )
        assert result.exit_code != 0

    def test_model_load_failure_reports_friendly_error(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        def _boom(model_id, **kw):
            raise OSError("model not found")

        # Config gate passes (issue #344) so the flow reaches the load, which is
        # what this test exercises.
        monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
        monkeypatch.setattr(draft_cmd, "_load_pair_member", _boom)
        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/nope", "--draft", "org/tiny",
             "--prompts", prompts],
        )
        assert result.exit_code == 1
        assert "could not load the model pair" in result.output.lower()

    def test_report_output_outside_cwd_rejected(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)
        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "-o", "../escape.json"],
        )
        assert result.exit_code == 1
        assert "cwd" in result.output.lower()

    def test_zero_throughput_renders_as_na(self, runner, in_tmp_cwd, monkeypatch):
        import json as _json

        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 0.0)
        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "-o", "report.json"],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "n/a" in result.output.lower()
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["tok_s_plain"] is None
        assert data["tok_s_assisted"] is None
        assert data["speedup"] is None

    def test_zero_generated_tokens_is_not_a_crash(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """A target that emits EOS immediately must not divide by zero."""
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        self._patch_load(monkeypatch)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (0, 0))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 5.0)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts],
        )
        assert result.exit_code == 1
        assert "no tokens" in result.output.lower()


#: Qwen2.5's config.vocab_size with the padded embedding rows, and without them.
#: The tokenizers are identical; only these two numbers differ (issue #344).
_QWEN_PADDED_VOCAB = 152064
_QWEN_BASE_VOCAB = 151936
#: `LlavaConfig().get_text_config().vocab_size` on transformers 4.57.6 — the
#: composite shape whose top-level `vocab_size` is absent (#344 review).
_LLAVA_TEXT_VOCAB = 32000


class TestDraftMeasureVocabGate:
    """issue #344 — measure must refuse the SAME pairs distill refuses (on
    config.vocab_size, before loading), and must not discard a completed
    measurement when the assisted arm fails inside transformers."""

    def _prompts(self, tmp_path):
        return _write_jsonl(
            tmp_path / "p.jsonl", [{"prompt": "What is 2+2?"}, {"prompt": "Hi"}]
        )

    @pytest.mark.parametrize(
        ("target_vocab", "draft_vocab"),
        [
            (_QWEN_PADDED_VOCAB, _QWEN_BASE_VOCAB),
            # The mirror image. The gate is `!=`, not `>`: a draft with the LARGER
            # config vocab is refused too. Without this direction, mutating the
            # comparison to `>` survives (#344 review, finding 3).
            (_QWEN_BASE_VOCAB, _QWEN_PADDED_VOCAB),
        ],
    )
    def test_config_vocab_mismatch_refused_before_any_model_loads(
        self, runner, in_tmp_cwd, monkeypatch, target_vocab, draft_vocab
    ):
        # Qwen2.5 large<-small: identical tokenizers, but config.vocab_size differs
        # by padded embedding rows (152064 vs 151936). same_tokenizer() accepts it;
        # transformers' assisted generation does not. measure must gate on the
        # config vocab up front, exactly like distill.
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        monkeypatch.setattr(
            draft_cmd,
            "_vocab_size_of",
            lambda mid, trc=False: target_vocab if "target" in mid else draft_vocab,
        )
        loaded: list[str] = []

        def _fake_load(model_id, **kwargs):
            loaded.append(model_id)
            return object(), _FakeTok(151643), "cpu"

        # Everything past the gate is patched out, so the only thing that can make
        # this test pass is the gate itself refusing before the load.
        monkeypatch.setattr(draft_cmd, "_load_pair_member", _fake_load)
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target-qwen-32b", "--draft", "org/qwen-0_5b",
             "--prompts", prompts, "-o", "report.json"],
        )
        assert result.exit_code == 1, (result.output, repr(result.exception))
        assert "vocab" in result.output.lower()
        # Refused BEFORE the expensive load, and no report written.
        assert loaded == []
        assert not (in_tmp_cwd / "report.json").exists()

    def test_assisted_arm_failure_keeps_acceptance_and_plain_on_disk(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        import json as _json

        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
        monkeypatch.setattr(
            draft_cmd,
            "_load_pair_member",
            lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
        )
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))

        def _throughput(model, tok, prompts, *, assistant_model=None,
                        num_assistant_tokens=5, max_new_tokens=64):
            # The assisted arm is the one transformers refuses (issue #344); the
            # plain arm has already succeeded by the time it runs.
            if assistant_model is not None:
                raise ValueError(
                    "The main and assistant models have different tokenizers"
                )
            return 20.0

        monkeypatch.setattr(draft_cmd, "measure_throughput", _throughput)

        prompts = self._prompts(in_tmp_cwd)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "-o", "report.json"],
        )
        # A failed assisted arm is a loud warning, not a crash: the acceptance rate
        # and plain throughput already succeeded and must survive on disk.
        assert result.exit_code == 0, (result.output, repr(result.exception))
        report = in_tmp_cwd / "report.json"
        assert report.exists()
        data = _json.loads(report.read_text(encoding="utf-8"))
        assert data["acceptance_rate"] == 0.75
        assert data["tok_s_plain"] == 20.0
        assert data["tok_s_assisted"] is None
        assert data["speedup"] is None
        # #344 review: a crashed arm is recorded on disk (else it is byte-identical
        # to an untimed or interrupted one), and the loud warning is actually shown
        # (deleting the message fails this).
        assert data["assisted_status"] == "crash"
        assert "could not be measured" in _plain(result.output)

    def _run_measure(self, runner, in_tmp_cwd, monkeypatch, throughput):
        """Drive `measure` to the assisted arm with everything before it mocked;
        `throughput` decides the assisted arm's outcome."""
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
        monkeypatch.setattr(
            draft_cmd,
            "_load_pair_member",
            lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
        )
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
        monkeypatch.setattr(draft_cmd, "measure_throughput", throughput)
        prompts = self._prompts(in_tmp_cwd)
        return runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", prompts, "-o", "report.json"],
        )

    def test_assisted_arm_outcomes_are_distinguishable_on_disk(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """#344 review: crash / untimed / interrupt / complete wrote byte-identical
        JSON. Each now records a distinct ``assisted_status``."""
        import json as _json

        def _complete(model, tok, prompts, *, assistant_model=None, **kw):
            return 30.0 if assistant_model is not None else 20.0

        result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _complete)
        assert result.exit_code == 0, (result.output, repr(result.exception))
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["assisted_status"] == "complete"
        assert data["tok_s_assisted"] == 30.0
        assert data["speedup"] == 1.5

    def test_assisted_arm_untimed_is_recorded_and_silent(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """A non-positive assisted number is 'untimed', not 'crash', and prints no
        warning — the two must not collapse to the same on-disk state."""
        import json as _json

        def _untimed(model, tok, prompts, *, assistant_model=None, **kw):
            return 0.0 if assistant_model is not None else 20.0

        result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _untimed)
        assert result.exit_code == 0, (result.output, repr(result.exception))
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["assisted_status"] == "untimed"
        assert data["tok_s_assisted"] is None
        assert "could not be measured" not in _plain(result.output)

    def test_assisted_arm_interrupt_is_recorded_and_reraised(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """Ctrl-C during the arm is recorded as 'interrupted' and re-raised (so the
        exit is not 0). Widening the ``except`` to swallow BaseException — or
        dropping the KeyboardInterrupt handler — fails this."""
        import json as _json

        def _interrupt(model, tok, prompts, *, assistant_model=None, **kw):
            if assistant_model is not None:
                raise KeyboardInterrupt
            return 20.0

        result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _interrupt)
        # click converts the re-raised KeyboardInterrupt to exit 130 — the point
        # is that it is NOT swallowed into a 0 like crash/untimed are.
        assert result.exit_code == 130, (result.output, repr(result.exception))
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["assisted_status"] == "interrupted"
        assert data["tok_s_assisted"] is None

    def test_hard_death_in_the_assisted_arm_keeps_the_incremental_report(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """The incremental write earns its keep on a death NO handler catches.

        `SystemExit` derives from `BaseException`, so neither `except Exception`
        nor `except KeyboardInterrupt` catches it and nothing in `measure` runs
        after the arm dies here — the only reason acceptance + plain throughput
        are on disk is the write that already happened BEFORE the arm. A hard
        kill is the same shape with no Python frame at all. Delete just that
        write and every other test in this class stays green; this one goes red
        (#344 review, blocker 1). It is also the only assertion on `pending`,
        which is reachable on disk exactly in this case.
        """
        import json as _json

        def _hard_death(model, tok, prompts, *, assistant_model=None, **kw):
            if assistant_model is not None:
                raise SystemExit(137)
            return 20.0

        result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _hard_death)
        assert result.exit_code == 137, (result.output, repr(result.exception))
        report = in_tmp_cwd / "report.json"
        assert report.exists(), "acceptance + plain throughput were discarded"
        data = _json.loads(report.read_text(encoding="utf-8"))
        assert data["acceptance_rate"] == 0.75
        assert data["tok_s_plain"] == 20.0
        assert data["assisted_status"] == "pending"

    @pytest.mark.parametrize(
        ("outcome", "expected_exit"),
        [("crash", 0), ("interrupt", 130)],
    )
    def test_a_failed_status_write_does_not_mask_the_arm_outcome(
        self, runner, in_tmp_cwd, monkeypatch, outcome, expected_exit
    ):
        """#344 review nit: the status write inside the two handlers was unguarded,
        so an `OSError` there replaced the exception being handled — the crashed
        arm's warning became an unrelated traceback, and Ctrl-C lost its exit 130.
        Drop the guard in `_record_assisted_status` and both cases fail.
        """
        import json as _json

        from soup_cli.commands import draft as draft_cmd

        real_write = draft_cmd._write_draft_report
        writes = {"n": 0}

        def _fail_after_the_pre_arm_write(report, output):
            writes["n"] += 1
            if writes["n"] == 1:
                real_write(report, output)
                return
            raise OSError("no space left on device")

        monkeypatch.setattr(
            draft_cmd, "_write_draft_report", _fail_after_the_pre_arm_write
        )

        def _throughput(model, tok, prompts, *, assistant_model=None, **kw):
            if assistant_model is None:
                return 20.0
            if outcome == "interrupt":
                raise KeyboardInterrupt
            raise ValueError("boom")

        result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _throughput)
        assert result.exit_code == expected_exit, (
            result.output,
            repr(result.exception),
        )
        # The failed update is reported, not swallowed silently...
        assert "could not record the assisted-arm outcome" in _plain(result.output)
        # ...and the results the pre-arm write persisted are untouched on disk.
        data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
        assert data["acceptance_rate"] == 0.75
        assert data["tok_s_plain"] == 20.0

    def test_a_crashed_arm_without_an_output_path_still_warns(
        self, runner, in_tmp_cwd, monkeypatch
    ):
        """`measure` without `-o` has no report to update, so recording the outcome
        must be a no-op rather than a write against `None` — the arm still warns
        and the run still exits 0."""
        from soup_cli.commands import draft as draft_cmd
        from soup_cli.commands.draft import app

        monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
        monkeypatch.setattr(
            draft_cmd,
            "_load_pair_member",
            lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
        )
        monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))

        def _crash(model, tok, prompts, *, assistant_model=None, **kw):
            if assistant_model is not None:
                raise ValueError("boom")
            return 20.0

        monkeypatch.setattr(draft_cmd, "measure_throughput", _crash)
        result = runner.invoke(
            app,
            ["measure", "--target", "org/target", "--draft", "org/tiny",
             "--prompts", self._prompts(in_tmp_cwd)],
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "could not be measured" in _plain(result.output)
        assert not list(in_tmp_cwd.glob("*.json"))


class TestVocabSizeOfCompositeConfig:
    """#344 review, blocker 2 — `_vocab_size_of` reads a composite/multimodal
    config's `get_text_config()` when the top-level `vocab_size` is absent, so a
    VLM target is no longer refused with "could not read model config". It is the
    shared helper, so this covers `measure` and `distill` at once.

    Stubbed configs rather than a real `LlavaConfig` so changes in the supported
    Transformers range cannot satisfy the assertion without reaching the
    fallback. The shapes below are the ones originally measured on 4.57.6
    (`LlavaConfig().vocab_size` missing, `get_text_config().vocab_size == 32000`).
    """

    def _vocab_size_for(self, config):
        from unittest.mock import patch

        from soup_cli.commands.draft import _vocab_size_of

        with patch("transformers.AutoConfig.from_pretrained", return_value=config):
            return _vocab_size_of("org/model")

    @pytest.mark.parametrize(
        "text_vocab", [_LLAVA_TEXT_VOCAB, _QWEN_PADDED_VOCAB]
    )
    def test_composite_config_resolves_through_get_text_config(self, text_vocab):
        """Two sizes, because one would be satisfied by a fallback that returns a
        hardcoded 32000 rather than reading the sub-config."""
        import types

        config = types.SimpleNamespace(
            get_text_config=lambda: types.SimpleNamespace(vocab_size=text_vocab)
        )
        assert self._vocab_size_for(config) == text_vocab

    def test_top_level_vocab_size_still_wins(self):
        """The control. Without it a fallback that took PRIORITY would also pass
        the test above — here `get_text_config()` raises, so calling it at all is
        a failure, and the plain (Qwen-shaped) path must be untouched."""
        import types

        def _must_not_be_called():
            raise AssertionError("get_text_config() shadowed the top-level vocab_size")

        config = types.SimpleNamespace(
            vocab_size=_QWEN_BASE_VOCAB, get_text_config=_must_not_be_called
        )
        assert self._vocab_size_for(config) == _QWEN_BASE_VOCAB

    def test_neither_source_still_raises(self):
        """A config with no vocab size anywhere keeps the pre-existing error —
        the fallback must not turn an unreadable config into a silent default."""
        import types

        with pytest.raises(ValueError, match="vocab_size"):
            self._vocab_size_for(types.SimpleNamespace())


class TestDraftListCli:
    def test_lists_registered_drafts(self, runner, draft_registry, tmp_path):
        from soup_cli.commands.draft import app
        from soup_cli.utils.draft import register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        register_draft("org/target-7b", str(draft), 0.71)

        result = runner.invoke(app, ["list"])
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "org/target-7b" in result.output
        assert "71" in result.output

    def test_empty_registry_is_not_an_error(self, runner, draft_registry):
        from soup_cli.commands.draft import app

        result = runner.invoke(app, ["list"])
        assert result.exit_code == 0
        assert "no draft" in result.output.lower()

    def test_renders_dash_for_unmeasured_acceptance(
        self, runner, draft_registry, tmp_path
    ):
        from soup_cli.commands.draft import app
        from soup_cli.utils.draft import register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        register_draft("org/no-rate", str(draft))  # acceptance_rate=None
        result = runner.invoke(app, ["list"])
        assert result.exit_code == 0
        assert "org/no-rate" in result.output

    def test_escapes_markup_in_registered_target(
        self, runner, draft_registry, tmp_path
    ):
        from soup_cli.commands.draft import app
        from soup_cli.utils.draft import register_draft

        draft = tmp_path / "d"
        draft.mkdir()
        register_draft("[bold red]org/evil[/]", str(draft))
        result = runner.invoke(app, ["list"])
        assert result.exit_code == 0
        assert "org/evil" in result.output  # literal, not interpreted as markup


class TestMillionParamSizeGate:
    """Live-smoke regression: `soup draft distill` on SmolLM2-135M was REFUSED
    by the hardware-fit gate, which predicted 14 GB of weights — because
    model_size_from_name knew no "M" (millions) suffix and fell back to the 7B
    default. Every draft-sized model hit this. Same class as the v0.71.32
    whisper fix."""

    def test_million_suffix_is_parsed(self):
        from soup_cli.utils.gpu import model_size_from_name

        assert model_size_from_name(
            "HuggingFaceTB/SmolLM2-135M-Instruct"
        ) == pytest.approx(0.135)
        assert model_size_from_name(
            "HuggingFaceTB/SmolLM2-360M-Instruct"
        ) == pytest.approx(0.360)
        assert model_size_from_name("HuggingFaceTB/SmolVLM-256M") == pytest.approx(
            0.256
        )

    def test_billion_marker_still_wins_over_a_context_length_suffix(self):
        """`Qwen2.5-7B-Instruct-1M` is a 7B model with a 1M CONTEXT — the "1M"
        must not be read as 1M parameters."""
        from soup_cli.utils.gpu import model_size_from_name

        assert model_size_from_name("Qwen/Qwen2.5-7B-Instruct-1M") == 7

    def test_one_point_seven_b_is_not_seven_b(self):
        """"1.7b" contains "7b" — the marker list must match the longer one."""
        from soup_cli.utils.gpu import model_size_from_name

        assert model_size_from_name(
            "HuggingFaceTB/SmolLM2-1.7B-Instruct"
        ) == pytest.approx(1.7)

    def test_unknown_model_still_defaults_to_7b(self):
        from soup_cli.utils.gpu import model_size_from_name

        assert model_size_from_name("some-unknown-model") == 7.0


class TestPromptTexts:
    def test_extracts_messages_and_drops_unusable_rows(self):
        from soup_cli.commands.draft import _prompt_texts

        rows = [
            {"messages": [
                {"role": "system", "content": "sys"},
                {"role": "user", "content": "hi"},
            ]},
            {"prompt": "direct"},
            {"nothing": "useful"},  # dropped
            {"messages": [{"role": "assistant", "content": "no user turn"}]},  # dropped
        ]
        assert _prompt_texts(rows) == ["hi", "direct"]


class TestDistillStepBudget:
    """#364 — ``soup draft distill --steps N`` must deliver ~N optimiser steps.

    The epoch count that realises ``--steps`` used to divide the request by
    ``rows // batch_size``, ignoring that ``val_split`` removes rows from
    training and ``gradient_accumulation_steps`` micro-batches make one
    optimiser step. Both divide the budget, so ``--steps N`` delivered ~N/4.44.
    """

    @staticmethod
    def _epochs_from_yaml(yaml_text: str) -> int:
        import yaml  # noqa: PLC0415

        return int(yaml.safe_load(yaml_text)["training"]["epochs"])

    @staticmethod
    def _true_steps_per_epoch(rows: int, *, val_split: float, batch: int, accum: int) -> int:
        # The optimiser-step count one epoch actually delivers.
        train_rows = math.floor(rows * (1.0 - val_split))
        return max(1, train_rows // (batch * accum))

    def test_steps_are_delivered_not_quartered(self):
        """Asking for 100 steps must reach ~100, not ~22 (100 / 4.44)."""
        from soup_cli.commands import draft
        from soup_cli.config.schema import DataConfig, TrainingConfig

        rows, requested = 100, 100
        val_split = float(DataConfig.model_fields["val_split"].default)
        accum = int(TrainingConfig.model_fields["gradient_accumulation_steps"].default)
        yaml_text = draft._build_distill_config_yaml(
            draft_base="org/tiny", target="org/target", data="d.jsonl",
            out_dir="draft/_adapter", steps=requested, data_rows=rows,
        )
        epochs = self._epochs_from_yaml(yaml_text)
        per_epoch = self._true_steps_per_epoch(
            rows, val_split=val_split, batch=draft._DISTILL_BATCH_SIZE, accum=accum
        )
        delivered = epochs * per_epoch
        # Nearest reachable at this granularity: the request is met or overshot
        # by less than one epoch — never the old ~1/4.44 undershoot.
        assert delivered >= requested
        assert delivered - requested < per_epoch

    @pytest.mark.parametrize(
        "val_split,accum",
        [(0.0, 1), (0.1, 1), (0.0, 4), (0.1, 4), (0.2, 8)],
    )
    def test_delivered_steps_track_request_across_shapes(self, val_split, accum):
        """val_split and gradient_accumulation_steps each divide the budget —
        vary them independently (one arm at defaults is not discriminating)."""
        from soup_cli.commands import draft

        rows, requested = 500, 120
        per_epoch = draft._distill_steps_per_epoch(
            rows, val_split=val_split, batch_size=1, grad_accum=accum
        )
        epochs = draft._distill_epochs_for_steps(
            requested, rows, val_split=val_split, batch_size=1, grad_accum=accum
        )
        delivered = epochs * per_epoch
        assert delivered >= requested
        assert delivered - requested < per_epoch

    def test_emitted_config_pins_the_run_shape_and_loads(self):
        """The config must pin the shape the epoch math assumed, so a
        schema-default change cannot silently reintroduce the drift (#364)."""
        from soup_cli.commands import draft
        from soup_cli.config.loader import load_config_from_string

        yaml_text = draft._build_distill_config_yaml(
            draft_base="HuggingFaceTB/SmolLM2-135M", target="org/target",
            data="d.jsonl", out_dir="draft/_adapter", steps=50, data_rows=200,
        )
        cfg = load_config_from_string(yaml_text)
        assert cfg.data.val_split == draft._DISTILL_VAL_SPLIT
        assert cfg.training.gradient_accumulation_steps == draft._DISTILL_GRAD_ACCUM