Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""v0.71.33 — `soup draft`: train-your-own speculative-decoding draft.
Covers:
* ``utils/adapter_fuse.py`` — the shared LoRA -> dense fuse extracted from
``commands/shrink.py`` (v0.71.29) so ``shrink`` and ``draft`` share one path.
* ``utils/draft.py`` — pure acceptance kernel + tokenizer-equality guard +
local draft registry + torch-lazy measurement.
* ``utils/spec_pairing.py`` — local-registry lookup before the static map.
* ``commands/draft.py`` — ``soup draft distill / measure / list``.
"""
from __future__ import annotations
import ast
import math
import os
import re
from pathlib import Path
from typing import Sequence
import pytest
# Rich styles console output, so a phrase assertion must never read raw
# `result.output`: an escape sequence can land inside the phrase and a wrap can
# split it across a line. Strip ANSI, then collapse whitespace — the repo-wide
# rule enforced by `test_cli_help_assertions_are_ansi_safe.py`.
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def _plain(text: str) -> str:
return re.sub(r"\s+", " ", _ANSI_RE.sub("", text or ""))
# ---------------------------------------------------------------------------
# Task 1 — shared adapter fuse
# ---------------------------------------------------------------------------
class TestAdapterFuse:
def test_module_exports_fuse_and_release(self):
from soup_cli.utils.adapter_fuse import fuse_adapter_into, release_cuda
assert callable(fuse_adapter_into)
assert callable(release_cuda)
def test_shrink_reuses_the_shared_implementation(self):
"""shrink must not keep a second copy of the fuse (no behavioural drift)."""
from soup_cli.commands import shrink
from soup_cli.utils.adapter_fuse import fuse_adapter_into, release_cuda
assert shrink._fuse_adapter is fuse_adapter_into
assert shrink._release_cuda is release_cuda
def test_no_top_level_torch(self):
import soup_cli
path = Path(soup_cli.__file__).parent / "utils" / "adapter_fuse.py"
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
assert not alias.name.startswith(
("torch", "transformers", "peft")
), f"top-level import of {alias.name}"
elif isinstance(node, ast.ImportFrom) and node.module:
assert not node.module.startswith(
("torch", "transformers", "peft")
), f"top-level import from {node.module}"
def test_failed_save_does_not_orphan_the_staging_dir(self, tmp_path, monkeypatch):
"""python-review HIGH: a save that raises mid-write used to leave a
full model's worth of bytes in a .fuse_* dir next to the model."""
import sys
import types
from soup_cli.utils import adapter_fuse
base = tmp_path / "base"
base.mkdir()
(base / "config.json").write_text("{}", encoding="utf-8")
monkeypatch.chdir(tmp_path)
class _Merged:
def save_pretrained(self, path):
# The staging dir already exists at this point — this is exactly
# where a disk-full / interrupted write lands.
Path(path).joinpath("partial.bin").write_bytes(b"xxxx")
raise RuntimeError("disk full")
class _Base:
@staticmethod
def from_pretrained(*a, **kw):
return _Base()
class _Peft:
@staticmethod
def from_pretrained(*a, **kw):
obj = _Peft()
obj.merge_and_unload = lambda: _Merged() # noqa: E731
return obj
class _Tok:
@staticmethod
def from_pretrained(*a, **kw):
return _Tok()
def save_pretrained(self, path):
pass
fake_tf = types.ModuleType("transformers")
fake_tf.AutoModelForCausalLM = _Base
fake_tf.AutoTokenizer = _Tok
fake_peft = types.ModuleType("peft")
fake_peft.PeftModel = _Peft
monkeypatch.setitem(sys.modules, "transformers", fake_tf)
monkeypatch.setitem(sys.modules, "peft", fake_peft)
with pytest.raises(RuntimeError, match="disk full"):
adapter_fuse.fuse_adapter_into(
base_dir="base", adapter_dir=str(tmp_path / "ad")
)
assert list(tmp_path.glob(".fuse_*")) == [], "staging dir was orphaned"
assert base.exists(), "the base model must survive a failed fuse"
def test_fuse_adapter_into_is_the_in_place_special_case(self, monkeypatch):
"""shrink's in-place fuse == merge(base=out=base_dir)."""
from soup_cli.utils import adapter_fuse
seen: dict = {}
monkeypatch.setattr(
adapter_fuse,
"merge_adapter_to_dense",
lambda **kw: seen.update(kw),
)
adapter_fuse.fuse_adapter_into(base_dir="model", adapter_dir="ad", trc=True)
assert seen["base_model"] == "model"
assert seen["out_dir"] == "model"
assert seen["adapter_dir"] == "ad"
assert seen["trc"] is True
def test_merge_rejects_pickle_weights_in_adapter_dir(self, tmp_path, monkeypatch):
"""security CRITICAL: the adapter dir may be attacker-produced; refuse
pickle weights before PEFT torch.load's them."""
from soup_cli.utils import adapter_fuse
monkeypatch.chdir(tmp_path)
adapter = tmp_path / "out" / "_adapter"
adapter.mkdir(parents=True)
(adapter / "adapter_model.bin").write_bytes(b"\x80\x04pickle") # unsafe ext
with pytest.raises(ValueError, match="(?i)unsafe|pickle|safetensors"):
adapter_fuse.merge_adapter_to_dense(
base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
)
def test_trainer_checkpoint_pickles_are_not_mistaken_for_an_attack(
self, tmp_path, monkeypatch
):
"""Live-smoke regression: the HF Trainer writes checkpoint-N/optimizer.pt
(a pickle IT wrote). A recursive scan refused Soup's own distill output
and made `soup draft distill` impossible. Only TOP-LEVEL weights — the
ones PEFT actually loads — are the threat surface."""
from soup_cli.utils.strict_safetensors import (
assert_safe_top_level_weights,
check_strict_safetensors,
)
monkeypatch.chdir(tmp_path) # check_strict_safetensors is cwd-contained
adapter = tmp_path / "_adapter"
(adapter / "checkpoint-60").mkdir(parents=True)
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
# Both pickles below are written by the HF Trainer itself, and NEITHER is
# ever deserialized by from_pretrained. Both broke the live smoke.
(adapter / "training_args.bin").write_bytes(b"\x80\x04pickle")
(adapter / "checkpoint-60" / "optimizer.pt").write_bytes(b"\x80\x04pickle")
# The RECURSIVE, extension-only scan flags them — that is exactly what
# made `soup draft distill` refuse its own trainer's output...
with pytest.raises(ValueError, match="(?i)unsafe"):
check_strict_safetensors(str(adapter), strict=True)
# ...while the shallow weight-name scan (what the merge uses) accepts it.
assert_safe_top_level_weights(str(adapter))
# A top-level pickle ADAPTER — the file PEFT actually unpickles, i.e.
# the real threat — is still refused.
(adapter / "adapter_model.bin").write_bytes(b"\x80\x04pickle")
with pytest.raises(ValueError, match="(?i)unsafe"):
assert_safe_top_level_weights(str(adapter))
def test_shallow_scan_ignores_trainer_bookkeeping_pickles(self, tmp_path):
from soup_cli.utils.strict_safetensors import (
assert_safe_top_level_weights,
find_unsafe_weight_files_shallow,
)
adapter = tmp_path / "_adapter"
(adapter / "checkpoint-60").mkdir(parents=True)
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
(adapter / "training_args.bin").write_bytes(b"\x80\x04pickle")
(adapter / "checkpoint-60" / "optimizer.pt").write_bytes(b"\x80\x04pickle")
assert find_unsafe_weight_files_shallow(str(adapter)) == ()
assert_safe_top_level_weights(str(adapter)) # must not raise
def test_shallow_scan_still_catches_a_pickle_model_file(self, tmp_path):
"""serve --auto-spec loads a registry draft dir: pytorch_model.bin is
the file from_pretrained would unpickle."""
from soup_cli.utils.strict_safetensors import assert_safe_top_level_weights
model_dir = tmp_path / "draft"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "pytorch_model.bin").write_bytes(b"\x80\x04pickle")
with pytest.raises(ValueError, match="(?i)unsafe"):
assert_safe_top_level_weights(str(model_dir))
def test_merge_loads_base_weights_from_the_base_model_not_out_dir(
self, tmp_path, monkeypatch
):
"""The whole point of the CRITICAL fix: an adapter-only dir has no base
weights, so the base must be loaded from a separate model id/path."""
import sys
import types
from soup_cli.utils import adapter_fuse
monkeypatch.chdir(tmp_path)
adapter = tmp_path / "out" / "_adapter"
adapter.mkdir(parents=True)
loaded: dict = {}
class _Merged:
def save_pretrained(self, path):
Path(path).joinpath("model.safetensors").write_bytes(b"dense")
class _Base:
@staticmethod
def from_pretrained(model_id, **kw):
loaded["base"] = model_id
return _Base()
class _Peft:
@staticmethod
def from_pretrained(base, adapter_dir, **kw):
loaded["adapter"] = adapter_dir
obj = _Peft()
obj.merge_and_unload = lambda: _Merged() # noqa: E731
return obj
class _Tok:
@staticmethod
def from_pretrained(model_id, **kw):
return _Tok()
def save_pretrained(self, path):
Path(path).joinpath("tokenizer.json").write_text("{}", encoding="utf-8")
fake_tf = types.ModuleType("transformers")
fake_tf.AutoModelForCausalLM = _Base
fake_tf.AutoTokenizer = _Tok
fake_peft = types.ModuleType("peft")
fake_peft.PeftModel = _Peft
monkeypatch.setitem(sys.modules, "transformers", fake_tf)
monkeypatch.setitem(sys.modules, "peft", fake_peft)
adapter_fuse.merge_adapter_to_dense(
base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
)
assert loaded["base"] == "org/tiny"
assert loaded["adapter"] == str(adapter)
# out/ is now the DENSE model, and the adapter subdir is gone.
assert (tmp_path / "out" / "model.safetensors").exists()
assert not (tmp_path / "out" / "_adapter").exists()
assert list(tmp_path.glob(".fuse_*")) == []
def test_containment_checked_before_any_model_load(self, tmp_path, monkeypatch):
"""The cheap containment check runs before the heavy imports/loads."""
from soup_cli.utils import adapter_fuse
seen: list[tuple[str, str]] = []
def _fake_enforce(path: str, field: str) -> str:
seen.append((path, field))
raise ValueError("refused")
monkeypatch.setattr(
adapter_fuse, "enforce_under_cwd_and_no_symlink", _fake_enforce
)
with pytest.raises(ValueError, match="refused"):
adapter_fuse.fuse_adapter_into(
base_dir=str(tmp_path / "base"), adapter_dir=str(tmp_path / "ad")
)
assert seen and seen[0][0] == str(tmp_path / "base")
def test_rechecks_immediately_before_the_destructive_swap(
self, tmp_path, monkeypatch
):
"""TOCTOU: a mock that raised on EVERY call could not distinguish
'checked once at entry' from 'rechecked right before rmtree'. This one
succeeds first and fails on the SECOND call, so it can only pass if the
code truly re-validates immediately before the swap."""
import sys
import types
from soup_cli.utils import adapter_fuse
monkeypatch.chdir(tmp_path)
base = tmp_path / "base"
base.mkdir()
adapter = tmp_path / "out" / "_adapter"
adapter.mkdir(parents=True)
calls = {"n": 0}
real = adapter_fuse.enforce_under_cwd_and_no_symlink
def _flaky(path, field):
calls["n"] += 1
if calls["n"] >= 2:
raise ValueError("swap-time refusal")
return real(path, field)
monkeypatch.setattr(adapter_fuse, "enforce_under_cwd_and_no_symlink", _flaky)
class _Merged:
def save_pretrained(self, path):
Path(path).joinpath("model.safetensors").write_bytes(b"dense")
class _Base:
@staticmethod
def from_pretrained(*a, **kw):
return _Base()
class _Peft:
@staticmethod
def from_pretrained(*a, **kw):
obj = _Peft()
obj.merge_and_unload = lambda: _Merged() # noqa: E731
return obj
class _Tok:
@staticmethod
def from_pretrained(*a, **kw):
return _Tok()
def save_pretrained(self, path):
pass
fake_tf = types.ModuleType("transformers")
fake_tf.AutoModelForCausalLM = _Base
fake_tf.AutoTokenizer = _Tok
fake_peft = types.ModuleType("peft")
fake_peft.PeftModel = _Peft
monkeypatch.setitem(sys.modules, "transformers", fake_tf)
monkeypatch.setitem(sys.modules, "peft", fake_peft)
with pytest.raises(ValueError, match="swap-time refusal"):
adapter_fuse.merge_adapter_to_dense(
base_model=str(base), adapter_dir=str(adapter), out_dir="out"
)
assert calls["n"] == 2, "must recheck immediately before the destructive swap"
# The swap never completed: no dense model landed, and the staging dir
# was cleaned up rather than orphaned.
assert not (tmp_path / "out" / "model.safetensors").exists()
assert list(tmp_path.glob(".fuse_*")) == []
def test_merge_falls_back_to_base_tokenizer_when_adapter_lacks_one(
self, tmp_path, monkeypatch
):
import sys
import types
from soup_cli.utils import adapter_fuse
monkeypatch.chdir(tmp_path)
adapter = tmp_path / "out" / "_adapter"
adapter.mkdir(parents=True)
seen = {"tok_from": []}
class _Merged:
def save_pretrained(self, path):
Path(path).joinpath("model.safetensors").write_bytes(b"dense")
class _Base:
@staticmethod
def from_pretrained(*a, **kw):
return _Base()
class _Peft:
@staticmethod
def from_pretrained(*a, **kw):
obj = _Peft()
obj.merge_and_unload = lambda: _Merged() # noqa: E731
return obj
class _Tok:
@staticmethod
def from_pretrained(model_id, **kw):
seen["tok_from"].append(model_id)
if model_id == str(adapter):
raise OSError("no tokenizer files here")
return _Tok()
def save_pretrained(self, path):
pass
fake_tf = types.ModuleType("transformers")
fake_tf.AutoModelForCausalLM = _Base
fake_tf.AutoTokenizer = _Tok
fake_peft = types.ModuleType("peft")
fake_peft.PeftModel = _Peft
monkeypatch.setitem(sys.modules, "transformers", fake_tf)
monkeypatch.setitem(sys.modules, "peft", fake_peft)
adapter_fuse.merge_adapter_to_dense(
base_model="org/tiny", adapter_dir=str(adapter), out_dir="out"
)
assert seen["tok_from"] == [str(adapter), "org/tiny"]
# ---------------------------------------------------------------------------
# Task 2 — pure acceptance kernel
# ---------------------------------------------------------------------------
class _FakeTok:
"""Duck-typed tokenizer: a fixed text -> ids table plus a vocab_size."""
def __init__(self, vocab_size: int, table: dict[str, list[int]] | None = None):
self.vocab_size = vocab_size
self._table = table or {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
if text in self._table:
return list(self._table[text])
# Deterministic default: codepoint sum, so two tokenizers with the same
# table agree everywhere.
return [sum(ord(ch) for ch in text) % max(self.vocab_size, 1)]
class TestComputeAcceptance:
def test_all_match(self):
from soup_cli.utils.draft import compute_acceptance
assert compute_acceptance([1, 2, 3], [1, 2, 3]) == 1.0
def test_none_match(self):
from soup_cli.utils.draft import compute_acceptance
assert compute_acceptance([9, 9, 9], [1, 2, 3]) == 0.0
def test_partial(self):
from soup_cli.utils.draft import compute_acceptance
assert compute_acceptance([1, 2, 9], [1, 2, 3]) == pytest.approx(2 / 3)
def test_empty_is_zero(self):
from soup_cli.utils.draft import compute_acceptance
assert compute_acceptance([], []) == 0.0
def test_length_mismatch_raises(self):
from soup_cli.utils.draft import compute_acceptance
with pytest.raises(ValueError, match="same length"):
compute_acceptance([1, 2], [1])
class TestCountAcceptedAndRate:
def test_count_accepted(self):
from soup_cli.utils.draft import count_accepted
assert count_accepted([1, 2, 9], [1, 2, 3]) == 2
assert count_accepted([], []) == 0
def test_count_accepted_length_mismatch_raises(self):
from soup_cli.utils.draft import count_accepted
with pytest.raises(ValueError, match="same length"):
count_accepted([1], [1, 2])
def test_acceptance_rate(self):
from soup_cli.utils.draft import acceptance_rate
assert acceptance_rate(75, 100) == 0.75
assert acceptance_rate(0, 0) == 0.0
def test_acceptance_rate_rejects_impossible_counts(self):
from soup_cli.utils.draft import acceptance_rate
with pytest.raises(ValueError, match="exceeds"):
acceptance_rate(5, 3)
with pytest.raises(ValueError, match="non-negative"):
acceptance_rate(-1, 3)
def test_count_accepted_spans_uses_max_align_chars_cap(self, monkeypatch):
"""regression: count_accepted_spans respects the _MAX_ALIGN_CHARS cap."""
import difflib
from soup_cli.utils import uld
from soup_cli.utils.draft import count_accepted_spans
monkeypatch.setattr(uld, "_MAX_ALIGN_CHARS", 10)
captured: list[tuple[str, str]] = []
real_matcher = difflib.SequenceMatcher
def _spy_matcher(isjunk, a, b, autojunk=False):
captured.append((a, b))
return real_matcher(isjunk, a, b, autojunk=autojunk)
monkeypatch.setattr(difflib, "SequenceMatcher", _spy_matcher)
# 16-character non-matching strings: d_text != t_text triggers SequenceMatcher.
draft_pieces = ["abcdef", "ghijkl", "mnop"]
target_pieces = ["_bcdef", "ghijkl", "mnop"]
count_accepted_spans(draft_pieces, target_pieces)
assert len(captured) == 1
a_str, b_str = captured[0]
assert a_str == "abcdefghij"
assert b_str == "_bcdefghij"
assert len(a_str) == 10
assert len(b_str) == 10
class TestClassify:
def test_boundary_exact(self):
from soup_cli.utils.draft import classify_acceptance
assert classify_acceptance(0.70) == "STRONG"
assert classify_acceptance(0.6999) == "MODERATE"
assert classify_acceptance(0.50) == "MODERATE"
assert classify_acceptance(0.4999) == "WEAK"
assert classify_acceptance(0.0) == "WEAK"
assert classify_acceptance(1.0) == "STRONG"
def test_rejects_bool(self):
from soup_cli.utils.draft import classify_acceptance
with pytest.raises(TypeError, match="bool"):
classify_acceptance(True)
def test_rejects_nonfinite(self):
from soup_cli.utils.draft import classify_acceptance
with pytest.raises(ValueError, match="finite"):
classify_acceptance(float("nan"))
def test_rejects_out_of_range(self):
from soup_cli.utils.draft import classify_acceptance
with pytest.raises(ValueError, match="between 0 and 1"):
classify_acceptance(1.5)
with pytest.raises(ValueError, match="between 0 and 1"):
classify_acceptance(-0.1)
class TestSameTokenizer:
def test_identical_is_true(self):
from soup_cli.utils.draft import same_tokenizer
tok = _FakeTok(32000)
assert same_tokenizer(tok, tok) is True
assert same_tokenizer(_FakeTok(32000), _FakeTok(32000)) is True
def test_equal_vocab_size_but_different_ids_is_false(self):
"""The test that makes the guard meaningful.
Two tokenizers can both report vocab_size 32000 and still disagree on
every single token — a vocab_size check alone would wave that through
and the draft's proposals would be pure noise.
"""
from soup_cli.utils.draft import same_tokenizer
probe = "Hello, world!"
a = _FakeTok(32000, {probe: [1, 2, 3]})
b = _FakeTok(32000, {probe: [7, 8, 9]})
assert same_tokenizer(a, b) is False
def test_different_vocab_size_is_false(self):
from soup_cli.utils.draft import same_tokenizer
assert same_tokenizer(_FakeTok(32000), _FakeTok(49152)) is False
def test_missing_vocab_size_attribute_is_false(self):
from soup_cli.utils.draft import same_tokenizer
class _NoVocab:
def encode(self, text, add_special_tokens=False):
return [1, 2, 3]
assert same_tokenizer(_NoVocab(), _FakeTok(32000)) is False
assert same_tokenizer(_FakeTok(32000), _NoVocab()) is False
def test_probe_corpus_is_non_trivial(self):
"""A single-ASCII-word probe would miss most real tokenizer splits."""
from soup_cli.utils.draft import PROBE_CORPUS
assert len(PROBE_CORPUS) >= 4
joined = "".join(PROBE_CORPUS)
assert any(ord(ch) > 127 for ch in joined), "probe must include non-ASCII"
assert any(ch.isdigit() for ch in joined), "probe must include digits"
def test_broken_tokenizer_encode_is_false_not_raise(self):
from soup_cli.utils.draft import same_tokenizer
class _Broken:
vocab_size = 32000
def encode(self, text, add_special_tokens=False):
raise RuntimeError("boom")
assert same_tokenizer(_FakeTok(32000), _Broken()) is False
class TestReport:
def _report(self, **kw):
from soup_cli.utils.draft import AcceptanceReport
defaults = dict(
target="t",
draft="d",
n_prompts=2,
n_generated_tokens=100,
acceptance_rate=0.75,
verdict="STRONG",
tok_s_plain=10.0,
tok_s_assisted=12.0,
speedup=1.2,
num_assistant_tokens=5,
soup_version="0.71.33",
)
defaults.update(kw)
return AcceptanceReport(**defaults)
def test_frozen(self):
import dataclasses
report = self._report()
with pytest.raises(dataclasses.FrozenInstanceError):
report.acceptance_rate = 0.1 # type: ignore[misc]
def test_to_dict_round_trip(self):
from soup_cli.utils.draft import draft_report_to_dict
data = draft_report_to_dict(self._report())
assert data["acceptance_rate"] == 0.75
assert data["verdict"] == "STRONG"
assert data["target"] == "t"
assert data["speedup"] == 1.2
def test_panel_names_the_verdict_and_rate(self):
from io import StringIO
from rich.console import Console
from soup_cli.utils.draft import render_draft_panel
buf = StringIO()
Console(file=buf, width=100).print(render_draft_panel(self._report()))
out = buf.getvalue()
assert "STRONG" in out
assert "75" in out # the acceptance rate as a percentage
def test_panel_renders_when_throughput_unmeasured(self):
from io import StringIO
from rich.console import Console
from soup_cli.utils.draft import render_draft_panel
buf = StringIO()
report = self._report(tok_s_plain=None, tok_s_assisted=None, speedup=None)
Console(file=buf, width=100).print(render_draft_panel(report))
assert "STRONG" in buf.getvalue()
# ---------------------------------------------------------------------------
# Task 3 — local draft registry + spec_pairing lookup
# ---------------------------------------------------------------------------
@pytest.fixture()
def draft_registry(tmp_path, monkeypatch):
"""Point the draft registry at a temp file (never touch the real ~/.soup)."""
path = tmp_path / "drafts.json"
monkeypatch.setenv("SOUP_DRAFT_REGISTRY_PATH", str(path))
return path
class TestDraftRegistry:
def test_round_trip(self, draft_registry, tmp_path):
from soup_cli.utils.draft import list_drafts, lookup_draft, register_draft
draft = tmp_path / "mydraft"
draft.mkdir()
register_draft("HF/Target-7B", str(draft), 0.62)
assert lookup_draft("HF/Target-7B") == os.path.realpath(str(draft))
entries = list_drafts()
assert len(entries) == 1
assert entries[0]["acceptance_rate"] == 0.62
assert entries[0]["target"] == "hf/target-7b"
def test_lookup_is_case_insensitive(self, draft_registry, tmp_path):
"""Must match pick_draft_model's .lower() normalisation."""
from soup_cli.utils.draft import lookup_draft, register_draft
draft = tmp_path / "d"
draft.mkdir()
register_draft("Qwen/Qwen2.5-72B", str(draft))
assert lookup_draft("qwen/qwen2.5-72b") is not None
assert lookup_draft("QWEN/QWEN2.5-72B") is not None
def test_stale_dir_is_skipped(self, draft_registry, tmp_path):
"""A moved/deleted draft degrades to 'no draft', never a crash in serve."""
import shutil
from soup_cli.utils.draft import lookup_draft, register_draft
draft = tmp_path / "gone"
draft.mkdir()
register_draft("hf/target", str(draft))
shutil.rmtree(draft)
assert lookup_draft("hf/target") is None
def test_reregister_replaces(self, draft_registry, tmp_path):
from soup_cli.utils.draft import list_drafts, lookup_draft, register_draft
first = tmp_path / "one"
first.mkdir()
second = tmp_path / "two"
second.mkdir()
register_draft("hf/t", str(first))
register_draft("hf/t", str(second))
assert len(list_drafts()) == 1
assert lookup_draft("hf/t") == os.path.realpath(str(second))
def test_malformed_json_reads_as_empty(self, draft_registry):
from soup_cli.utils.draft import list_drafts, lookup_draft
draft_registry.write_text("{ not json", encoding="utf-8")
assert list_drafts() == []
assert lookup_draft("anything") is None
def test_non_dict_payload_reads_as_empty(self, draft_registry):
from soup_cli.utils.draft import list_drafts
draft_registry.write_text('["nope"]', encoding="utf-8")
assert list_drafts() == []
def test_missing_file_reads_as_empty(self, draft_registry):
from soup_cli.utils.draft import list_drafts, lookup_draft
assert not draft_registry.exists()
assert list_drafts() == []
assert lookup_draft("x") is None
def test_entry_cap_evicts_oldest_first(self, draft_registry, tmp_path):
from soup_cli.utils.draft import (
_MAX_REGISTRY_ENTRIES,
list_drafts,
register_draft,
)
draft = tmp_path / "d"
draft.mkdir()
for i in range(_MAX_REGISTRY_ENTRIES + 5):
register_draft(f"hf/target-{i}", str(draft))
targets = {entry["target"] for entry in list_drafts()}
assert len(targets) == _MAX_REGISTRY_ENTRIES
# oldest evicted, newest kept
assert "hf/target-0" not in targets
assert "hf/target-4" not in targets
assert "hf/target-5" in targets
assert f"hf/target-{_MAX_REGISTRY_ENTRIES + 4}" in targets
@pytest.mark.requires_symlink
def test_registry_symlink_is_not_followed_on_read(self, draft_registry, tmp_path):
"""O_NOFOLLOW: a symlink at the registry path must degrade to empty,
not leak an arbitrary file's parsed content into serve --auto-spec."""
from soup_cli.utils.draft import list_drafts
secret = tmp_path / "secret.json"
secret.write_text(
'{"drafts": [{"target": "leaked", "draft": "/tmp"}]}', encoding="utf-8"
)
draft_registry.symlink_to(secret)
assert list_drafts() == []
def test_atomic_write_leaves_no_temp_file(self, draft_registry, tmp_path):
from soup_cli.utils.draft import register_draft
draft = tmp_path / "d"
draft.mkdir()
register_draft("hf/t", str(draft))
leftovers = [p.name for p in draft_registry.parent.glob(".soup.*")]
assert leftovers == []
def test_concurrent_registration_does_not_lose_an_entry(
self, draft_registry, tmp_path
):
"""code-review MEDIUM: read-modify-write under a cross-process lock."""
import threading
from soup_cli.utils.draft import list_drafts, register_draft
draft = tmp_path / "d"
draft.mkdir()
errors: list[BaseException] = []
def _register(idx: int) -> None:
try:
register_draft(f"hf/target-{idx}", str(draft))
except BaseException as exc: # noqa: BLE001
errors.append(exc)
threads = [threading.Thread(target=_register, args=(i,)) for i in range(12)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert not errors, errors
targets = {entry["target"] for entry in list_drafts()}
assert targets == {f"hf/target-{i}" for i in range(12)}
def test_rejects_empty_target_and_bad_rate(self, draft_registry, tmp_path):
from soup_cli.utils.draft import register_draft
draft = tmp_path / "d"
draft.mkdir()
with pytest.raises(ValueError, match="non-empty"):
register_draft(" ", str(draft))
with pytest.raises(ValueError, match="between 0 and 1"):
register_draft("hf/t", str(draft), 1.5)
class TestSpecPairingLocal:
def test_local_registry_beats_static_map(self, draft_registry, tmp_path):
"""A draft you trained yourself wins over the built-in pairing."""
from soup_cli.utils.draft import register_draft
from soup_cli.utils.spec_pairing import pick_draft_model
draft = tmp_path / "mydraft"
draft.mkdir()
register_draft("qwen/qwen2.5-72b", str(draft)) # also in the static map
assert pick_draft_model("Qwen/Qwen2.5-72B") == os.path.realpath(str(draft))
def test_falls_back_to_static_map(self, draft_registry):
from soup_cli.utils.spec_pairing import pick_draft_model
assert pick_draft_model("Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-0.5B"
def test_unknown_target_still_none(self, draft_registry):
from soup_cli.utils.spec_pairing import pick_draft_model
assert pick_draft_model("nobody/nothing-1b") is None
def test_url_target_still_rejected(self, draft_registry):
from soup_cli.utils.spec_pairing import pick_draft_model
assert pick_draft_model("https://evil.example/model") is None
def test_registry_blowup_does_not_crash_serve(self, draft_registry, monkeypatch):
"""pick_draft_model is on serve's startup path — it must never raise."""
from soup_cli.utils import spec_pairing
def _boom(target):
raise RuntimeError("registry on fire")
monkeypatch.setattr(spec_pairing, "_lookup_local_draft", _boom)
assert spec_pairing.pick_draft_model("Qwen/Qwen2.5-72B") == "Qwen/Qwen2.5-0.5B"
# ---------------------------------------------------------------------------
# Task 4 — torch-lazy measurement
# ---------------------------------------------------------------------------
class _TensorTok:
"""Minimal tokenizer over torch tensors."""
pad_token_id = 0
eos_token_id = 0
def __init__(self, prompt_ids: list[int]):
self._prompt_ids = prompt_ids
def __call__(self, text, return_tensors=None):
import torch
ids = torch.tensor([self._prompt_ids])
return {"input_ids": ids, "attention_mask": torch.ones_like(ids)}
class _CrossTokenizerMock:
"""Mock tokenizer for cross-tokenizer measurement tests."""
pad_token_id = 0
eos_token_id = 0
def __init__(
self,
vocab_size: int,
encode_map: dict[str, list[int]],
decode_map: dict[int, str],
):
self.vocab_size = vocab_size
self._encode_map = encode_map
self._decode_map = decode_map
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
return list(self._encode_map.get(text, [0]))
def decode(self, token_ids: Sequence[int], skip_special_tokens: bool = False) -> str:
return "".join(self._decode_map.get(int(tid), "") for tid in token_ids)
def __call__(self, text: str, return_tensors: str | None = None, **kwargs):
import torch
ids = torch.tensor([self.encode(text)])
return {"input_ids": ids, "attention_mask": torch.ones_like(ids)}
class _FakeTarget:
"""Greedy-generates a continuation, applying repetition penalty when != 1.0."""
def __init__(
self,
full_ids: list[int],
n_prompt: int,
default_repetition_penalty: float = 1.2,
):
self._full_ids = list(full_ids)
self._n_prompt = n_prompt
self._default_repetition_penalty = default_repetition_penalty
self.calls: list[dict] = []
def parameters(self):
import torch
yield torch.zeros(1)
def generate(self, **kwargs):
import torch
self.calls.append(kwargs)
rep_penalty = kwargs.get("repetition_penalty", self._default_repetition_penalty)
if rep_penalty is None:
rep_penalty = self._default_repetition_penalty
input_ids = kwargs.get("input_ids")
if input_ids is not None:
curr_ids = input_ids[0].cpu().tolist()
else:
curr_ids = list(self._full_ids[: self._n_prompt])
target_gen = self._full_ids[self._n_prompt :]
if not target_gen:
return torch.tensor([curr_ids])
generated = list(curr_ids)
for expected_tok in target_gen:
alt_tok = 99 if expected_tok != 99 else 98
logits = {expected_tok: 10.0, alt_tok: 9.0}
if rep_penalty != 1.0:
seen = set(generated)
for tok_id in list(logits.keys()):
if tok_id in seen:
logits[tok_id] = logits[tok_id] / rep_penalty
best_tok = max(logits.keys(), key=lambda t: logits[t])
generated.append(best_tok)
return torch.tensor([generated])
class _FakeDraft:
"""Returns logits whose per-position argmax is a scripted sequence."""
def __init__(self, argmax_per_position: list[int], vocab: int = 100):
self._argmax = argmax_per_position
self._vocab = vocab
def parameters(self):
import torch
yield torch.zeros(1)
def __call__(self, input_ids=None, **kwargs):
import torch
seq = int(input_ids.shape[1])
logits = torch.zeros(1, seq, self._vocab)
for pos in range(seq):
logits[0, pos, self._argmax[pos]] = 10.0
class _Out:
pass
out = _Out()
out.logits = logits
return out
class TestMeasureAcceptance:
# prompt = [5, 6] (len 2); target generates [10, 11, 12] at positions 2,3,4.
FULL_IDS = [5, 6, 10, 11, 12]
N_PROMPT = 2
# Draft argmax per position. Causal alignment: logits[p-1] predicts token p,
# so the proposals for the generated positions 2,3,4 are ARGMAX[1:4].
# correct -> ARGMAX[1:4] = [10, 99, 12] vs [10, 11, 12] -> 2 matches
# off-by-one+ -> ARGMAX[2:5] = [99, 12, 99] vs [10, 11, 12] -> 0 matches
# off-by-one- -> ARGMAX[0:3] = [99, 10, 99] vs [10, 11, 12] -> 0 matches
# The three alignments give DIFFERENT answers, so this test can actually
# fail on an off-by-one (an unfalsifiable test is worse than none).
ARGMAX = [99, 10, 99, 12, 99]
def test_shift_convention_is_pinned(self):
from soup_cli.utils.draft import measure_acceptance
accepted, total = measure_acceptance(
_FakeTarget(self.FULL_IDS, self.N_PROMPT),
_FakeDraft(self.ARGMAX),
_TensorTok([5, 6]),
["hello"],
max_new_tokens=8,
)
assert (accepted, total) == (2, 3)
def test_off_by_one_alternatives_would_score_differently(self):
"""Guards the guard: the fixture must discriminate, not just pass."""
from soup_cli.utils.draft import compute_acceptance
generated = [10, 11, 12]
correct = self.ARGMAX[1:4]
off_plus = self.ARGMAX[2:5]
off_minus = self.ARGMAX[0:3]
assert compute_acceptance(correct, generated) == pytest.approx(2 / 3)
assert compute_acceptance(off_plus, generated) == 0.0
assert compute_acceptance(off_minus, generated) == 0.0
def test_multiple_prompts_accumulate(self):
from soup_cli.utils.draft import measure_acceptance
accepted, total = measure_acceptance(
_FakeTarget(self.FULL_IDS, self.N_PROMPT),
_FakeDraft(self.ARGMAX),
_TensorTok([5, 6]),
["a", "b", "c"],
max_new_tokens=8,
)
assert (accepted, total) == (6, 9)
def test_empty_generation_is_skipped_not_counted(self):
from soup_cli.utils.draft import measure_acceptance
# Target returns the prompt unchanged -> nothing was generated.
accepted, total = measure_acceptance(
_FakeTarget([5, 6], 2),
_FakeDraft([99, 99]),
_TensorTok([5, 6]),
["a"],
max_new_tokens=8,
)
assert (accepted, total) == (0, 0)
def test_generation_is_greedy(self):
"""Sampling would make the acceptance rate non-deterministic."""
from soup_cli.utils.draft import measure_acceptance
target = _FakeTarget(self.FULL_IDS, self.N_PROMPT)
measure_acceptance(
target, _FakeDraft(self.ARGMAX), _TensorTok([5, 6]), ["a"],
max_new_tokens=8,
)
assert target.calls[0]["do_sample"] is False
assert target.calls[0]["repetition_penalty"] == 1.0
assert target.calls[0]["max_new_tokens"] == 8
def test_repetition_penalty_neutralized_for_self_acceptance(self):
"""When target and draft are identical, repetition_penalty is neutralized (Refs #345)."""
from soup_cli.utils.draft import measure_acceptance
# Target generates [10, 11, 12] from prompt [10, 6].
# An identical draft produces predictions [10, 11, 12] for those positions.
identical_draft_argmax = [99, 10, 11, 12, 99]
target = _FakeTarget([10, 6, 10, 11, 12], 2)
draft = _FakeDraft(identical_draft_argmax)
accepted, total = measure_acceptance(
target, draft, _TensorTok([10, 6]), ["hello"], max_new_tokens=8
)
assert target.calls[0]["repetition_penalty"] == 1.0
assert total == 3
assert accepted == 3
def test_cross_tokenizer_unextendable_prompt_scores_zero_zero(self):
"""When draft tokenizer produces no continuation beyond draft_prompt_len,
measure_acceptance skips counting and returns exactly (0, 0)."""
from soup_cli.utils.draft import measure_acceptance
target_tok = _CrossTokenizerMock(
vocab_size=32000,
encode_map={"hello": [5, 6], "hello world": [5, 6, 10, 11, 12]},
decode_map={5: "hel", 6: "lo", 10: " ", 11: "wor", 12: "ld"},
)
# Draft tokenizer produces no new tokens beyond draft_prompt_len (both 3 tokens).
draft_tok = _CrossTokenizerMock(
vocab_size=49152,
encode_map={"hello": [1, 2, 3], "hello world": [1, 2, 3]},
decode_map={1: "hel", 2: "lo", 3: ""},
)
target = _FakeTarget([5, 6, 10, 11, 12], 2)
draft = _FakeDraft([99, 99, 99, 99, 99])
accepted, total = measure_acceptance(
target,
draft,
target_tok,
["hello"],
max_new_tokens=8,
draft_tokenizer=draft_tok,
)
assert (accepted, total) == (0, 0)
def test_cross_tokenizer_extendable_prompt_measures_acceptance(self):
"""When draft tokenizer produces tokens beyond draft_prompt_len,
measure_acceptance computes span-aligned acceptance."""
from soup_cli.utils.draft import measure_acceptance
target_tok = _CrossTokenizerMock(
vocab_size=32000,
encode_map={"hello": [5, 6], "hello world": [5, 6, 10, 11, 12]},
decode_map={5: "hel", 6: "lo", 10: " ", 11: "wor", 12: "ld"},
)
draft_tok = _CrossTokenizerMock(
vocab_size=49152,
encode_map={"hello": [1, 2], "hello world": [1, 2, 20, 21, 22]},
decode_map={1: "hel", 2: "lo", 20: " ", 21: "wor", 22: "ld"},
)
target = _FakeTarget([5, 6, 10, 11, 12], 2)
draft = _FakeDraft([99, 20, 21, 22, 99])
accepted, total = measure_acceptance(
target,
draft,
target_tok,
["hello"],
max_new_tokens=8,
draft_tokenizer=draft_tok,
)
assert (accepted, total) == (3, 3)
class TestMeasureThroughput:
def test_discards_a_warmup_generate(self):
from soup_cli.utils.draft import measure_throughput
target = _FakeTarget([5, 6, 10, 11, 12], 2)
rate = measure_throughput(
target, _TensorTok([5, 6]), ["a", "b"], max_new_tokens=8
)
# 2 prompts + 1 warm-up = 3 generate calls; only 2 are timed.
assert len(target.calls) == 3
assert rate > 0
def test_empty_prompts_is_zero(self):
from soup_cli.utils.draft import measure_throughput
target = _FakeTarget([5, 6], 2)
assert measure_throughput(target, _TensorTok([5, 6]), []) == 0.0
assert target.calls == []
def test_assistant_model_is_forwarded(self):
from soup_cli.utils.draft import measure_throughput
target = _FakeTarget([5, 6, 10, 11, 12], 2)
draft = _FakeDraft([0, 0, 0, 0, 0])
measure_throughput(
target,
_TensorTok([5, 6]),
["a"],
assistant_model=draft,
num_assistant_tokens=7,
)
assert target.calls[0]["assistant_model"] is draft
assert target.calls[0]["num_assistant_tokens"] == 7
def test_plain_run_passes_no_assistant(self):
from soup_cli.utils.draft import measure_throughput
target = _FakeTarget([5, 6, 10, 11, 12], 2)
measure_throughput(target, _TensorTok([5, 6]), ["a"])
assert "assistant_model" not in target.calls[0]
class TestDraftNoTopLevelTorch:
@pytest.mark.parametrize(
"relpath",
[("utils", "draft.py"), ("commands", "draft.py")],
)
def test_is_torch_free(self, relpath):
import soup_cli
path = Path(soup_cli.__file__).parent.joinpath(*relpath)
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
assert not alias.name.startswith(
("torch", "transformers", "peft")
), f"top-level import of {alias.name}"
elif isinstance(node, ast.ImportFrom) and node.module:
assert not node.module.startswith(
("torch", "transformers", "peft")
), f"top-level import from {node.module}"
# ---------------------------------------------------------------------------
# Task 5 — soup draft CLI
# ---------------------------------------------------------------------------
class _Cfg:
"""Stand-in for transformers AutoConfig output."""
def __init__(self, vocab_size: int):
self.vocab_size = vocab_size
@pytest.fixture()
def runner():
from typer.testing import CliRunner
return CliRunner()
@pytest.fixture()
def in_tmp_cwd(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
return tmp_path
def _write_jsonl(path: Path, rows: list[dict]) -> str:
import json as _json
path.write_text(
"\n".join(_json.dumps(row) for row in rows) + "\n", encoding="utf-8"
)
return str(path)
class TestDraftCliPlumbing:
def test_registered_on_main_app(self, runner):
from soup_cli.cli import app
result = runner.invoke(app, ["draft", "--help"])
assert result.exit_code == 0, (result.output, repr(result.exception))
for sub in ("distill", "measure", "list"):
assert sub in result.output
def test_subcommand_help(self, runner):
from soup_cli.commands.draft import app
for sub in ("distill", "measure", "list"):
result = runner.invoke(app, [sub, "--help"])
assert result.exit_code == 0, (sub, result.output)
@pytest.mark.parametrize("sub", ["distill", "measure"])
def test_trust_remote_code_flag_exists(self, runner, sub):
"""code-review MEDIUM: without it, any auto_map model is a dead end."""
import re
from soup_cli.commands.draft import app
result = runner.invoke(app, [sub, "--help"])
plain = re.sub(r"\x1b\[[0-9;]*m", "", result.output).replace("\n", " ")
assert "--trust-remote-code" in re.sub(r"\s+", " ", plain)
def test_trust_flag_is_threaded_into_resolution(self, monkeypatch):
from soup_cli.commands import draft as draft_cmd
seen: dict = {}
def _fake_resolve(model_id, requested, console, requires_remote_code):
seen["requested"] = requested
return requested
monkeypatch.setattr(
"soup_cli.utils.trust_remote.resolve_trust_remote_code", _fake_resolve
)
monkeypatch.setattr(
"soup_cli.utils.trust_remote.model_requires_trust_remote_code",
lambda mid: False,
)
assert draft_cmd._resolve_trust("org/x", True) is True
assert seen["requested"] is True
class TestDraftDistillCli:
def _data(self, tmp_path, rows: int = 200):
return _write_jsonl(
tmp_path / "d.jsonl",
[{"messages": [{"role": "user", "content": "hi"},
{"role": "assistant", "content": "yo"}]}] * rows,
)
def _patch_configs(self, monkeypatch, target_vocab, draft_vocab):
from soup_cli.commands import draft as draft_cmd
def _fake_vocab(model_id: str, trc: bool = False) -> int:
return target_vocab if "target" in model_id else draft_vocab
monkeypatch.setattr(draft_cmd, "_vocab_size_of", _fake_vocab)
tok_target = _FakeTok(target_vocab)
tok_draft = _FakeTok(draft_vocab)
def _fake_tok(model_id: str, **kwargs):
return tok_target if "target" in model_id else tok_draft
monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _fake_tok)
def test_plan_only_writes_nothing(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "task: distill" in result.output
assert not (in_tmp_cwd / "draftout").exists()
def test_vocab_mismatch_routes_to_cross_tokenizer_uld(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 151936)
data = self._data(in_tmp_cwd)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 0
assert "uld_strategy: wasserstein_aligned" in result.output
assert "cross-tokenizer" in result.output.lower()
def test_data_outside_cwd_rejected(self, runner, in_tmp_cwd, tmp_path_factory,
monkeypatch):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
outside = tmp_path_factory.mktemp("outside")
data = _write_jsonl(outside / "d.jsonl", [{"text": "x"}])
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 1
assert "cwd" in result.output.lower() or "outside" in result.output.lower()
def test_output_outside_cwd_rejected(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "../escape", "--plan-only"],
)
assert result.exit_code == 1
assert "cwd" in result.output.lower()
def test_output_is_cwd_rejected(self, runner, in_tmp_cwd, monkeypatch):
"""security CRITICAL: -o . would rmtree the whole working dir on success."""
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", ".", "--plan-only"],
)
assert result.exit_code == 1
assert "current directory" in result.output.lower()
def test_output_preexisting_nondraft_dir_rejected(
self, runner, in_tmp_cwd, monkeypatch
):
"""security CRITICAL: overwriting an unrelated dir needs --force."""
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
victim = in_tmp_cwd / "important"
victim.mkdir()
(victim / "notes.txt").write_text("keep me", encoding="utf-8")
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "important", "--plan-only"],
)
assert result.exit_code == 1
assert "--force" in result.output
def test_output_preexisting_draft_dir_allowed(
self, runner, in_tmp_cwd, monkeypatch
):
"""Re-distilling into a prior draft (has config.json) is fine."""
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
prior = in_tmp_cwd / "draftout"
prior.mkdir()
(prior / "config.json").write_text("{}", encoding="utf-8")
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
def test_missing_data_file_rejected(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", "nope.jsonl", "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 1
assert "unreadable" in result.output.lower()
def test_force_overwrites_a_preexisting_nondraft_dir(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
victim = in_tmp_cwd / "important"
victim.mkdir()
(victim / "notes.txt").write_text("keep me", encoding="utf-8")
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "important", "--force", "--plan-only"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
def test_empty_preexisting_dir_allowed_without_force(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands.draft import app
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
(in_tmp_cwd / "draftout").mkdir() # empty, no config.json
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
def test_config_unreadable_reports_friendly_error(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
def _boom(model_id, trc=False):
raise OSError("model not found")
monkeypatch.setattr(draft_cmd, "_vocab_size_of", _boom)
data = self._data(in_tmp_cwd)
result = runner.invoke(
app,
["distill", "--target", "org/nope", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--plan-only"],
)
assert result.exit_code == 1
assert "could not read model config" in result.output.lower()
def test_run_distill_surfaces_subprocess_failure(self, in_tmp_cwd, monkeypatch):
import subprocess as _sp
from soup_cli.commands import draft as draft_cmd
class _Result:
returncode = 1
stdout = b"partial log\n"
stderr = b"error: exploded\x1b[31m"
monkeypatch.setattr(_sp, "run", lambda argv, **kw: _Result())
self._data(in_tmp_cwd)
with pytest.raises(RuntimeError, match="rc=1"):
draft_cmd._run_distill(
draft_base="org/tiny", target="org/target", data="d.jsonl",
out_dir="draftout", steps=100, data_rows=200,
)
def test_run_distill_reports_rendered_config_validation_error(
self, in_tmp_cwd, monkeypatch
):
import typer
from soup_cli.commands import draft as draft_cmd
messages: list[str] = []
monkeypatch.setattr(
"soup_cli.config.loader.load_config_from_string",
lambda _yaml: (_ for _ in ()).throw(ValueError("invalid rendered config")),
)
monkeypatch.setattr(draft_cmd.console, "print", lambda message: messages.append(message))
with pytest.raises(typer.Exit) as exc_info:
draft_cmd._run_distill(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir="draftout",
steps=100,
data_rows=200,
)
assert exc_info.value.exit_code == 1
assert messages == ["[red]Invalid rendered distill config:[/] invalid rendered config"]
def test_run_distill_sanitizes_rendered_config_validation_error(
self, in_tmp_cwd, monkeypatch
):
import typer
from soup_cli.commands import draft as draft_cmd
messages: list[str] = []
monkeypatch.setattr(
"soup_cli.config.loader.load_config_from_string",
lambda _yaml: (_ for _ in ()).throw(
ValueError("bad \x1b[2J[bold red]field[/]")
),
)
monkeypatch.setattr(draft_cmd.console, "print", lambda message: messages.append(message))
with pytest.raises(typer.Exit) as exc_info:
draft_cmd._run_distill(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir="draftout",
steps=100,
data_rows=200,
)
assert exc_info.value.exit_code == 1
assert len(messages) == 1
assert "\x1b" not in messages[0]
assert "bad [2J\\[bold red]field\\[/]" in messages[0]
def test_run_distill_timeout_raises_friendly_error(self, in_tmp_cwd, monkeypatch):
import subprocess as _sp
from soup_cli.commands import draft as draft_cmd
def _boom(argv, **kw):
raise _sp.TimeoutExpired(cmd=argv, timeout=kw.get("timeout", 1))
monkeypatch.setattr(_sp, "run", _boom)
self._data(in_tmp_cwd)
with pytest.raises(RuntimeError, match="timeout"):
draft_cmd._run_distill(
draft_base="org/tiny", target="org/target", data="d.jsonl",
out_dir="draftout", steps=100, data_rows=200,
)
def test_run_distill_cpu_device_hides_gpus(self, in_tmp_cwd, monkeypatch):
import subprocess as _sp
from soup_cli.commands import draft as draft_cmd
captured: dict = {}
class _Result:
returncode = 0
stdout = b""
stderr = b""
def _fake_run(argv, **kwargs):
captured["env"] = kwargs.get("env")
(Path("draftout") / draft_cmd._ADAPTER_SUBDIR).mkdir(
parents=True, exist_ok=True
)
return _Result()
monkeypatch.setattr(_sp, "run", _fake_run)
monkeypatch.setattr(draft_cmd, "merge_adapter_to_dense", lambda **kw: None)
self._data(in_tmp_cwd)
draft_cmd._run_distill(
draft_base="org/tiny", target="org/target", data="d.jsonl",
out_dir="draftout", steps=100, data_rows=200, device="cpu",
)
assert captured["env"]["CUDA_VISIBLE_DEVICES"] == "-1"
def test_generated_config_is_schema_valid(self, in_tmp_cwd):
"""The rendered YAML must actually load as a SoupConfig."""
from soup_cli.commands.draft import _build_distill_config_yaml
from soup_cli.config.loader import load_config_from_string
data = self._data(in_tmp_cwd)
yaml_text = _build_distill_config_yaml(
draft_base="org/tiny",
target="org/target",
data=data,
out_dir="draftout",
steps=100,
data_rows=200,
)
cfg = load_config_from_string(yaml_text)
assert cfg.task == "distill"
assert cfg.base == "org/tiny"
assert cfg.training.teacher_model == "org/target"
def test_trainer_output_goes_to_a_nested_adapter_dir(self, in_tmp_cwd):
"""code-review CRITICAL: the distill trainer only ever writes a LoRA
adapter (never dense base weights), so it must NOT train straight into
-o — the merge needs a separate adapter dir plus the base weights."""
import yaml
from soup_cli.commands.draft import _ADAPTER_SUBDIR, _build_distill_config_yaml
yaml_text = _build_distill_config_yaml(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir=os.path.join("draftout", _ADAPTER_SUBDIR),
steps=100,
data_rows=200,
)
parsed = yaml.safe_load(yaml_text)
assert parsed["output"].endswith(_ADAPTER_SUBDIR)
assert parsed["base"] == "org/tiny" # student = the draft base
def test_run_distill_merges_base_plus_adapter_not_out_dir_into_itself(
self, in_tmp_cwd, monkeypatch
):
"""The merge must load base weights from --draft-base and the adapter
from the nested dir. Fusing out_dir into itself (the original bug) can
never work: out_dir holds no base weights."""
from soup_cli.commands import draft as draft_cmd
calls: dict = {}
class _Result:
returncode = 0
stdout = b""
stderr = b""
def _fake_run(argv, **kwargs):
# Simulate the trainer writing ONLY an adapter into its output dir.
adapter = Path("draftout") / draft_cmd._ADAPTER_SUBDIR
adapter.mkdir(parents=True, exist_ok=True)
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
return _Result()
def _fake_merge(*, base_model, adapter_dir, out_dir, trc=False):
calls.update(
base_model=base_model, adapter_dir=adapter_dir, out_dir=out_dir
)
import subprocess as _sp
monkeypatch.setattr(_sp, "run", _fake_run)
monkeypatch.setattr(draft_cmd, "merge_adapter_to_dense", _fake_merge)
self._data(in_tmp_cwd)
draft_cmd._run_distill(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir="draftout",
steps=100,
data_rows=200,
)
assert calls["base_model"] == "org/tiny", "base weights must come from --draft-base"
assert calls["adapter_dir"].endswith(draft_cmd._ADAPTER_SUBDIR)
assert calls["out_dir"] == "draftout"
assert calls["adapter_dir"] != calls["out_dir"]
def test_run_distill_fails_loudly_when_no_adapter_was_written(
self, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
class _Result:
returncode = 0
stdout = b""
stderr = b""
import subprocess as _sp
monkeypatch.setattr(_sp, "run", lambda argv, **kw: _Result())
monkeypatch.setattr(
draft_cmd, "merge_adapter_to_dense", lambda **kw: None
)
with pytest.raises(RuntimeError, match="no adapter"):
draft_cmd._run_distill(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir="draftout",
steps=100,
data_rows=200,
)
def test_newline_in_model_id_cannot_inject_yaml_keys(self, in_tmp_cwd):
"""python-review CRITICAL: raw interpolation let a crafted --target
smuggle sibling keys into the training: block. Every user string is
json.dumps'd, so a newline stays inside the scalar."""
import yaml
from soup_cli.commands.draft import _build_distill_config_yaml
hostile = "org/x\ntraining:\n reward_fn: ../../evil.py"
yaml_text = _build_distill_config_yaml(
draft_base="org/tiny",
target=hostile,
data="d.jsonl",
out_dir="draftout",
steps=100,
data_rows=200,
)
parsed = yaml.safe_load(yaml_text)
# The payload landed as a VALUE, not as new keys.
assert parsed["training"]["teacher_model"] == hostile
assert "reward_fn" not in parsed["training"]
assert parsed["task"] == "distill"
def test_absurd_steps_over_tiny_data_is_refused(self, in_tmp_cwd):
"""500 steps over 4 rows = 125 epochs — refuse rather than emit it."""
from soup_cli.commands.draft import _build_distill_config_yaml
with pytest.raises(ValueError, match="epochs"):
_build_distill_config_yaml(
draft_base="org/tiny",
target="org/target",
data="d.jsonl",
out_dir="draftout",
steps=500,
data_rows=4,
)
def test_happy_path_trains_fuses_and_registers(
self, runner, in_tmp_cwd, monkeypatch, draft_registry
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
from soup_cli.utils.draft import lookup_draft
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
trained: dict = {}
def _fake_train(**kwargs):
trained.update(kwargs)
# The real subprocess writes the adapter into out_dir; simulate the
# fused dense draft landing in the output dir.
Path(kwargs["out_dir"]).mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(draft_cmd, "_run_distill", _fake_train)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert trained["target"] == "org/target"
registered = lookup_draft("org/target")
assert registered == os.path.realpath(str(in_tmp_cwd / "draftout"))
def test_no_register_flag_skips_registry(
self, runner, in_tmp_cwd, monkeypatch, draft_registry
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
from soup_cli.utils.draft import lookup_draft
self._patch_configs(monkeypatch, 49152, 49152)
data = self._data(in_tmp_cwd)
monkeypatch.setattr(
draft_cmd,
"_run_distill",
lambda **kw: Path(kw["out_dir"]).mkdir(parents=True, exist_ok=True),
)
result = runner.invoke(
app,
["distill", "--target", "org/target", "--draft-base", "org/tiny",
"--data", data, "-o", "draftout", "--no-register"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert lookup_draft("org/target") is None
class TestDraftMeasureCli:
def _prompts(self, tmp_path):
return _write_jsonl(
tmp_path / "p.jsonl", [{"prompt": "What is 2+2?"}, {"prompt": "Hi"}]
)
def _patch_load(self, monkeypatch, *, compatible=True):
from soup_cli.commands import draft as draft_cmd
tok_a = _FakeTok(49152)
tok_b = _FakeTok(49152) if compatible else _FakeTok(151936)
model_a = object()
model_b = object()
# measure now gates on config.vocab_size before loading (issue #344). Make
# that gate pass (equal config vocab) so these tests keep exercising the
# load / same_tokenizer / measurement paths; the tokenizer-vocab mismatch
# `compatible=False` sets still trips same_tokenizer as before.
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
def _fake_load(model_id, **kwargs):
tok = tok_a if "target" in model_id else tok_b
model = model_a if "target" in model_id else model_b
return model, tok, "cpu"
monkeypatch.setattr(draft_cmd, "_load_pair_member", _fake_load)
return (model_a, tok_a), (model_b, tok_b)
def test_happy_path_writes_report_and_exits_zero(
self, runner, in_tmp_cwd, monkeypatch
):
import json as _json
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "-o", "report.json"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "STRONG" in _plain(result.output)
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["acceptance_rate"] == 0.75
assert data["verdict"] == "STRONG"
assert data["n_generated_tokens"] == 100
def test_below_min_acceptance_exits_two(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (40, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "--min-acceptance", "0.6"],
)
assert result.exit_code == 2
assert "60.0%" in _plain(result.output)
assert "below" in _plain(result.output).lower()
def test_mismatched_tokenizer_measures_cross_tokenizer(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
(_, _), (draft_model, draft_tok) = self._patch_load(
monkeypatch, compatible=False
)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (60, 100))
captured_kwargs: list[dict] = []
def _spy_measure_throughput(*args, **kwargs):
captured_kwargs.append(kwargs)
return 15.0
monkeypatch.setattr(draft_cmd, "measure_throughput", _spy_measure_throughput)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts],
)
assert result.exit_code == 0
assert "Cross-tokenizer draft detected" in _plain(result.output)
assert "60.0%" in _plain(result.output)
assisted_calls = [kw for kw in captured_kwargs if "assistant_model" in kw]
assert len(assisted_calls) == 1
assisted = assisted_calls[0]
assert assisted["assistant_model"] is draft_model
assert assisted["assistant_tokenizer"] is draft_tok
assert "max_new_tokens" in assisted
assert "num_assistant_tokens" in assisted
def test_mismatched_tokenizer_unsupported_uad_warns(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch, compatible=False)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (60, 100))
def _boom(model, tok, prompts, *, assistant_model=None, **kw):
if assistant_model is not None:
raise RuntimeError("Universal Assisted Decoding requires transformers>=4.45.0")
return 20.0
monkeypatch.setattr(draft_cmd, "measure_throughput", _boom)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts],
)
assert result.exit_code == 0
assert "Universal Assisted Decoding requires transformers>=4.45.0" in _plain(result.output)
assert "could not be measured" in _plain(result.output)
def test_prompts_outside_cwd_rejected(
self, runner, in_tmp_cwd, tmp_path_factory, monkeypatch
):
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
outside = tmp_path_factory.mktemp("outside2")
prompts = _write_jsonl(outside / "p.jsonl", [{"prompt": "hi"}])
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts],
)
assert result.exit_code == 1
def test_empty_prompts_rejected(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
empty = in_tmp_cwd / "empty.jsonl"
empty.write_text("", encoding="utf-8")
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", str(empty)],
)
assert result.exit_code == 1
def test_bad_min_acceptance_rejected(self, runner, in_tmp_cwd, monkeypatch):
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "--min-acceptance", "1.5"],
)
assert result.exit_code != 0
def test_model_load_failure_reports_friendly_error(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
def _boom(model_id, **kw):
raise OSError("model not found")
# Config gate passes (issue #344) so the flow reaches the load, which is
# what this test exercises.
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(draft_cmd, "_load_pair_member", _boom)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/nope", "--draft", "org/tiny",
"--prompts", prompts],
)
assert result.exit_code == 1
assert "could not load the model pair" in result.output.lower()
def test_report_output_outside_cwd_rejected(
self, runner, in_tmp_cwd, monkeypatch
):
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "-o", "../escape.json"],
)
assert result.exit_code == 1
assert "cwd" in result.output.lower()
def test_zero_throughput_renders_as_na(self, runner, in_tmp_cwd, monkeypatch):
import json as _json
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 0.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "-o", "report.json"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "n/a" in result.output.lower()
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["tok_s_plain"] is None
assert data["tok_s_assisted"] is None
assert data["speedup"] is None
def test_zero_generated_tokens_is_not_a_crash(
self, runner, in_tmp_cwd, monkeypatch
):
"""A target that emits EOS immediately must not divide by zero."""
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
self._patch_load(monkeypatch)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (0, 0))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 5.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts],
)
assert result.exit_code == 1
assert "no tokens" in result.output.lower()
#: Qwen2.5's config.vocab_size with the padded embedding rows, and without them.
#: The tokenizers are identical; only these two numbers differ (issue #344).
_QWEN_PADDED_VOCAB = 152064
_QWEN_BASE_VOCAB = 151936
#: `LlavaConfig().get_text_config().vocab_size` on transformers 4.57.6 — the
#: composite shape whose top-level `vocab_size` is absent (#344 review).
_LLAVA_TEXT_VOCAB = 32000
class TestDraftMeasureVocabGate:
"""issue #344 — measure must refuse the SAME pairs distill refuses (on
config.vocab_size, before loading), and must not discard a completed
measurement when the assisted arm fails inside transformers."""
def _prompts(self, tmp_path):
return _write_jsonl(
tmp_path / "p.jsonl", [{"prompt": "What is 2+2?"}, {"prompt": "Hi"}]
)
@pytest.mark.parametrize(
("target_vocab", "draft_vocab"),
[
(_QWEN_PADDED_VOCAB, _QWEN_BASE_VOCAB),
# The mirror image. The gate is `!=`, not `>`: a draft with the LARGER
# config vocab is refused too. Without this direction, mutating the
# comparison to `>` survives (#344 review, finding 3).
(_QWEN_BASE_VOCAB, _QWEN_PADDED_VOCAB),
],
)
def test_config_vocab_mismatch_refused_before_any_model_loads(
self, runner, in_tmp_cwd, monkeypatch, target_vocab, draft_vocab
):
# Qwen2.5 large<-small: identical tokenizers, but config.vocab_size differs
# by padded embedding rows (152064 vs 151936). same_tokenizer() accepts it;
# transformers' assisted generation does not. measure must gate on the
# config vocab up front, exactly like distill.
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
monkeypatch.setattr(
draft_cmd,
"_vocab_size_of",
lambda mid, trc=False: target_vocab if "target" in mid else draft_vocab,
)
loaded: list[str] = []
def _fake_load(model_id, **kwargs):
loaded.append(model_id)
return object(), _FakeTok(151643), "cpu"
# Everything past the gate is patched out, so the only thing that can make
# this test pass is the gate itself refusing before the load.
monkeypatch.setattr(draft_cmd, "_load_pair_member", _fake_load)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", lambda *a, **k: 20.0)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target-qwen-32b", "--draft", "org/qwen-0_5b",
"--prompts", prompts, "-o", "report.json"],
)
assert result.exit_code == 1, (result.output, repr(result.exception))
assert "vocab" in result.output.lower()
# Refused BEFORE the expensive load, and no report written.
assert loaded == []
assert not (in_tmp_cwd / "report.json").exists()
def test_assisted_arm_failure_keeps_acceptance_and_plain_on_disk(
self, runner, in_tmp_cwd, monkeypatch
):
import json as _json
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(
draft_cmd,
"_load_pair_member",
lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
def _throughput(model, tok, prompts, *, assistant_model=None,
num_assistant_tokens=5, max_new_tokens=64):
# The assisted arm is the one transformers refuses (issue #344); the
# plain arm has already succeeded by the time it runs.
if assistant_model is not None:
raise ValueError(
"The main and assistant models have different tokenizers"
)
return 20.0
monkeypatch.setattr(draft_cmd, "measure_throughput", _throughput)
prompts = self._prompts(in_tmp_cwd)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "-o", "report.json"],
)
# A failed assisted arm is a loud warning, not a crash: the acceptance rate
# and plain throughput already succeeded and must survive on disk.
assert result.exit_code == 0, (result.output, repr(result.exception))
report = in_tmp_cwd / "report.json"
assert report.exists()
data = _json.loads(report.read_text(encoding="utf-8"))
assert data["acceptance_rate"] == 0.75
assert data["tok_s_plain"] == 20.0
assert data["tok_s_assisted"] is None
assert data["speedup"] is None
# #344 review: a crashed arm is recorded on disk (else it is byte-identical
# to an untimed or interrupted one), and the loud warning is actually shown
# (deleting the message fails this).
assert data["assisted_status"] == "crash"
assert "could not be measured" in _plain(result.output)
def _run_measure(self, runner, in_tmp_cwd, monkeypatch, throughput):
"""Drive `measure` to the assisted arm with everything before it mocked;
`throughput` decides the assisted arm's outcome."""
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(
draft_cmd,
"_load_pair_member",
lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
monkeypatch.setattr(draft_cmd, "measure_throughput", throughput)
prompts = self._prompts(in_tmp_cwd)
return runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", prompts, "-o", "report.json"],
)
def test_assisted_arm_outcomes_are_distinguishable_on_disk(
self, runner, in_tmp_cwd, monkeypatch
):
"""#344 review: crash / untimed / interrupt / complete wrote byte-identical
JSON. Each now records a distinct ``assisted_status``."""
import json as _json
def _complete(model, tok, prompts, *, assistant_model=None, **kw):
return 30.0 if assistant_model is not None else 20.0
result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _complete)
assert result.exit_code == 0, (result.output, repr(result.exception))
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["assisted_status"] == "complete"
assert data["tok_s_assisted"] == 30.0
assert data["speedup"] == 1.5
def test_assisted_arm_untimed_is_recorded_and_silent(
self, runner, in_tmp_cwd, monkeypatch
):
"""A non-positive assisted number is 'untimed', not 'crash', and prints no
warning — the two must not collapse to the same on-disk state."""
import json as _json
def _untimed(model, tok, prompts, *, assistant_model=None, **kw):
return 0.0 if assistant_model is not None else 20.0
result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _untimed)
assert result.exit_code == 0, (result.output, repr(result.exception))
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["assisted_status"] == "untimed"
assert data["tok_s_assisted"] is None
assert "could not be measured" not in _plain(result.output)
def test_assisted_arm_interrupt_is_recorded_and_reraised(
self, runner, in_tmp_cwd, monkeypatch
):
"""Ctrl-C during the arm is recorded as 'interrupted' and re-raised (so the
exit is not 0). Widening the ``except`` to swallow BaseException — or
dropping the KeyboardInterrupt handler — fails this."""
import json as _json
def _interrupt(model, tok, prompts, *, assistant_model=None, **kw):
if assistant_model is not None:
raise KeyboardInterrupt
return 20.0
result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _interrupt)
# click converts the re-raised KeyboardInterrupt to exit 130 — the point
# is that it is NOT swallowed into a 0 like crash/untimed are.
assert result.exit_code == 130, (result.output, repr(result.exception))
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["assisted_status"] == "interrupted"
assert data["tok_s_assisted"] is None
def test_hard_death_in_the_assisted_arm_keeps_the_incremental_report(
self, runner, in_tmp_cwd, monkeypatch
):
"""The incremental write earns its keep on a death NO handler catches.
`SystemExit` derives from `BaseException`, so neither `except Exception`
nor `except KeyboardInterrupt` catches it and nothing in `measure` runs
after the arm dies here — the only reason acceptance + plain throughput
are on disk is the write that already happened BEFORE the arm. A hard
kill is the same shape with no Python frame at all. Delete just that
write and every other test in this class stays green; this one goes red
(#344 review, blocker 1). It is also the only assertion on `pending`,
which is reachable on disk exactly in this case.
"""
import json as _json
def _hard_death(model, tok, prompts, *, assistant_model=None, **kw):
if assistant_model is not None:
raise SystemExit(137)
return 20.0
result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _hard_death)
assert result.exit_code == 137, (result.output, repr(result.exception))
report = in_tmp_cwd / "report.json"
assert report.exists(), "acceptance + plain throughput were discarded"
data = _json.loads(report.read_text(encoding="utf-8"))
assert data["acceptance_rate"] == 0.75
assert data["tok_s_plain"] == 20.0
assert data["assisted_status"] == "pending"
@pytest.mark.parametrize(
("outcome", "expected_exit"),
[("crash", 0), ("interrupt", 130)],
)
def test_a_failed_status_write_does_not_mask_the_arm_outcome(
self, runner, in_tmp_cwd, monkeypatch, outcome, expected_exit
):
"""#344 review nit: the status write inside the two handlers was unguarded,
so an `OSError` there replaced the exception being handled — the crashed
arm's warning became an unrelated traceback, and Ctrl-C lost its exit 130.
Drop the guard in `_record_assisted_status` and both cases fail.
"""
import json as _json
from soup_cli.commands import draft as draft_cmd
real_write = draft_cmd._write_draft_report
writes = {"n": 0}
def _fail_after_the_pre_arm_write(report, output):
writes["n"] += 1
if writes["n"] == 1:
real_write(report, output)
return
raise OSError("no space left on device")
monkeypatch.setattr(
draft_cmd, "_write_draft_report", _fail_after_the_pre_arm_write
)
def _throughput(model, tok, prompts, *, assistant_model=None, **kw):
if assistant_model is None:
return 20.0
if outcome == "interrupt":
raise KeyboardInterrupt
raise ValueError("boom")
result = self._run_measure(runner, in_tmp_cwd, monkeypatch, _throughput)
assert result.exit_code == expected_exit, (
result.output,
repr(result.exception),
)
# The failed update is reported, not swallowed silently...
assert "could not record the assisted-arm outcome" in _plain(result.output)
# ...and the results the pre-arm write persisted are untouched on disk.
data = _json.loads((in_tmp_cwd / "report.json").read_text(encoding="utf-8"))
assert data["acceptance_rate"] == 0.75
assert data["tok_s_plain"] == 20.0
def test_a_crashed_arm_without_an_output_path_still_warns(
self, runner, in_tmp_cwd, monkeypatch
):
"""`measure` without `-o` has no report to update, so recording the outcome
must be a no-op rather than a write against `None` — the arm still warns
and the run still exits 0."""
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(
draft_cmd,
"_load_pair_member",
lambda model_id, **kw: (object(), _FakeTok(49152), "cpu"),
)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: (75, 100))
def _crash(model, tok, prompts, *, assistant_model=None, **kw):
if assistant_model is not None:
raise ValueError("boom")
return 20.0
monkeypatch.setattr(draft_cmd, "measure_throughput", _crash)
result = runner.invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", self._prompts(in_tmp_cwd)],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "could not be measured" in _plain(result.output)
assert not list(in_tmp_cwd.glob("*.json"))
class TestVocabSizeOfCompositeConfig:
"""#344 review, blocker 2 — `_vocab_size_of` reads a composite/multimodal
config's `get_text_config()` when the top-level `vocab_size` is absent, so a
VLM target is no longer refused with "could not read model config". It is the
shared helper, so this covers `measure` and `distill` at once.
Stubbed configs rather than a real `LlavaConfig` so changes in the supported
Transformers range cannot satisfy the assertion without reaching the
fallback. The shapes below are the ones originally measured on 4.57.6
(`LlavaConfig().vocab_size` missing, `get_text_config().vocab_size == 32000`).
"""
def _vocab_size_for(self, config):
from unittest.mock import patch
from soup_cli.commands.draft import _vocab_size_of
with patch("transformers.AutoConfig.from_pretrained", return_value=config):
return _vocab_size_of("org/model")
@pytest.mark.parametrize(
"text_vocab", [_LLAVA_TEXT_VOCAB, _QWEN_PADDED_VOCAB]
)
def test_composite_config_resolves_through_get_text_config(self, text_vocab):
"""Two sizes, because one would be satisfied by a fallback that returns a
hardcoded 32000 rather than reading the sub-config."""
import types
config = types.SimpleNamespace(
get_text_config=lambda: types.SimpleNamespace(vocab_size=text_vocab)
)
assert self._vocab_size_for(config) == text_vocab
def test_top_level_vocab_size_still_wins(self):
"""The control. Without it a fallback that took PRIORITY would also pass
the test above — here `get_text_config()` raises, so calling it at all is
a failure, and the plain (Qwen-shaped) path must be untouched."""
import types
def _must_not_be_called():
raise AssertionError("get_text_config() shadowed the top-level vocab_size")
config = types.SimpleNamespace(
vocab_size=_QWEN_BASE_VOCAB, get_text_config=_must_not_be_called
)
assert self._vocab_size_for(config) == _QWEN_BASE_VOCAB
def test_neither_source_still_raises(self):
"""A config with no vocab size anywhere keeps the pre-existing error —
the fallback must not turn an unreadable config into a silent default."""
import types
with pytest.raises(ValueError, match="vocab_size"):
self._vocab_size_for(types.SimpleNamespace())
class TestDraftListCli:
def test_lists_registered_drafts(self, runner, draft_registry, tmp_path):
from soup_cli.commands.draft import app
from soup_cli.utils.draft import register_draft
draft = tmp_path / "d"
draft.mkdir()
register_draft("org/target-7b", str(draft), 0.71)
result = runner.invoke(app, ["list"])
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "org/target-7b" in result.output
assert "71" in result.output
def test_empty_registry_is_not_an_error(self, runner, draft_registry):
from soup_cli.commands.draft import app
result = runner.invoke(app, ["list"])
assert result.exit_code == 0
assert "no draft" in result.output.lower()
def test_renders_dash_for_unmeasured_acceptance(
self, runner, draft_registry, tmp_path
):
from soup_cli.commands.draft import app
from soup_cli.utils.draft import register_draft
draft = tmp_path / "d"
draft.mkdir()
register_draft("org/no-rate", str(draft)) # acceptance_rate=None
result = runner.invoke(app, ["list"])
assert result.exit_code == 0
assert "org/no-rate" in result.output
def test_escapes_markup_in_registered_target(
self, runner, draft_registry, tmp_path
):
from soup_cli.commands.draft import app
from soup_cli.utils.draft import register_draft
draft = tmp_path / "d"
draft.mkdir()
register_draft("[bold red]org/evil[/]", str(draft))
result = runner.invoke(app, ["list"])
assert result.exit_code == 0
assert "org/evil" in result.output # literal, not interpreted as markup
class TestMillionParamSizeGate:
"""Live-smoke regression: `soup draft distill` on SmolLM2-135M was REFUSED
by the hardware-fit gate, which predicted 14 GB of weights — because
model_size_from_name knew no "M" (millions) suffix and fell back to the 7B
default. Every draft-sized model hit this. Same class as the v0.71.32
whisper fix."""
def test_million_suffix_is_parsed(self):
from soup_cli.utils.gpu import model_size_from_name
assert model_size_from_name(
"HuggingFaceTB/SmolLM2-135M-Instruct"
) == pytest.approx(0.135)
assert model_size_from_name(
"HuggingFaceTB/SmolLM2-360M-Instruct"
) == pytest.approx(0.360)
assert model_size_from_name("HuggingFaceTB/SmolVLM-256M") == pytest.approx(
0.256
)
def test_billion_marker_still_wins_over_a_context_length_suffix(self):
"""`Qwen2.5-7B-Instruct-1M` is a 7B model with a 1M CONTEXT — the "1M"
must not be read as 1M parameters."""
from soup_cli.utils.gpu import model_size_from_name
assert model_size_from_name("Qwen/Qwen2.5-7B-Instruct-1M") == 7
def test_one_point_seven_b_is_not_seven_b(self):
""""1.7b" contains "7b" — the marker list must match the longer one."""
from soup_cli.utils.gpu import model_size_from_name
assert model_size_from_name(
"HuggingFaceTB/SmolLM2-1.7B-Instruct"
) == pytest.approx(1.7)
def test_unknown_model_still_defaults_to_7b(self):
from soup_cli.utils.gpu import model_size_from_name
assert model_size_from_name("some-unknown-model") == 7.0
class TestPromptTexts:
def test_extracts_messages_and_drops_unusable_rows(self):
from soup_cli.commands.draft import _prompt_texts
rows = [
{"messages": [
{"role": "system", "content": "sys"},
{"role": "user", "content": "hi"},
]},
{"prompt": "direct"},
{"nothing": "useful"}, # dropped
{"messages": [{"role": "assistant", "content": "no user turn"}]}, # dropped
]
assert _prompt_texts(rows) == ["hi", "direct"]
class TestDistillStepBudget:
"""#364 — ``soup draft distill --steps N`` must deliver ~N optimiser steps.
The epoch count that realises ``--steps`` used to divide the request by
``rows // batch_size``, ignoring that ``val_split`` removes rows from
training and ``gradient_accumulation_steps`` micro-batches make one
optimiser step. Both divide the budget, so ``--steps N`` delivered ~N/4.44.
"""
@staticmethod
def _epochs_from_yaml(yaml_text: str) -> int:
import yaml # noqa: PLC0415
return int(yaml.safe_load(yaml_text)["training"]["epochs"])
@staticmethod
def _true_steps_per_epoch(rows: int, *, val_split: float, batch: int, accum: int) -> int:
# The optimiser-step count one epoch actually delivers.
train_rows = math.floor(rows * (1.0 - val_split))
return max(1, train_rows // (batch * accum))
def test_steps_are_delivered_not_quartered(self):
"""Asking for 100 steps must reach ~100, not ~22 (100 / 4.44)."""
from soup_cli.commands import draft
from soup_cli.config.schema import DataConfig, TrainingConfig
rows, requested = 100, 100
val_split = float(DataConfig.model_fields["val_split"].default)
accum = int(TrainingConfig.model_fields["gradient_accumulation_steps"].default)
yaml_text = draft._build_distill_config_yaml(
draft_base="org/tiny", target="org/target", data="d.jsonl",
out_dir="draft/_adapter", steps=requested, data_rows=rows,
)
epochs = self._epochs_from_yaml(yaml_text)
per_epoch = self._true_steps_per_epoch(
rows, val_split=val_split, batch=draft._DISTILL_BATCH_SIZE, accum=accum
)
delivered = epochs * per_epoch
# Nearest reachable at this granularity: the request is met or overshot
# by less than one epoch — never the old ~1/4.44 undershoot.
assert delivered >= requested
assert delivered - requested < per_epoch
@pytest.mark.parametrize(
"val_split,accum",
[(0.0, 1), (0.1, 1), (0.0, 4), (0.1, 4), (0.2, 8)],
)
def test_delivered_steps_track_request_across_shapes(self, val_split, accum):
"""val_split and gradient_accumulation_steps each divide the budget —
vary them independently (one arm at defaults is not discriminating)."""
from soup_cli.commands import draft
rows, requested = 500, 120
per_epoch = draft._distill_steps_per_epoch(
rows, val_split=val_split, batch_size=1, grad_accum=accum
)
epochs = draft._distill_epochs_for_steps(
requested, rows, val_split=val_split, batch_size=1, grad_accum=accum
)
delivered = epochs * per_epoch
assert delivered >= requested
assert delivered - requested < per_epoch
def test_emitted_config_pins_the_run_shape_and_loads(self):
"""The config must pin the shape the epoch math assumed, so a
schema-default change cannot silently reintroduce the drift (#364)."""
from soup_cli.commands import draft
from soup_cli.config.loader import load_config_from_string
yaml_text = draft._build_distill_config_yaml(
draft_base="HuggingFaceTB/SmolLM2-135M", target="org/target",
data="d.jsonl", out_dir="draft/_adapter", steps=50, data_rows=200,
)
cfg = load_config_from_string(yaml_text)
assert cfg.data.val_split == draft._DISTILL_VAL_SPLIT
assert cfg.training.gradient_accumulation_steps == draft._DISTILL_GRAD_ACCUM