Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#843: ``soup draft measure`` never said whether a pair pays, or at which k.
The acceptance band (STRONG >= 0.70) labelled Llama-3.1-8B <- Llama-3.2-1B STRONG at
0.813, while the measured assisted arm ran at 0.481x of plain. Whether a pair pays
depends on the draft/target latency ratio, which the command never measured.
The model here is the standard expected-tokens one, and it is a CEILING: i.i.d.
per-position acceptance, verification of k+1 tokens costed as one target step,
no framework overhead. The published figures below are the ones
``benchmarks/gate-h100-validation.md`` STEP 26 and #303's closing comment derived
from the same inputs, so pinning them pins the formula, not a fresh measurement.
"""
import json
import math
import re
import pytest
from rich.console import Console
# benchmarks/gate-h100-validation.md STEP 26: target and draft tok/s on one H100.
_TARGET_TOK_S = 39.28
_DRAFT_TOK_S = 65.90
# #303's closing comment: the unrounded measured acceptance. 0.813 (the rounded
# figure) gives 0.9556x at k=5, which rounds to 0.956 rather than the published 0.955.
_ALPHA = 0.81293
def _c():
from soup_cli.utils.draft import latency_ratio
return latency_ratio(_TARGET_TOK_S, _DRAFT_TOK_S)
class TestThePublishedFigures:
def test_latency_ratio(self):
assert _c() == pytest.approx(0.596, abs=5e-4)
def test_breakeven_at_the_shipped_k(self):
from soup_cli.utils.draft import breakeven_acceptance
assert breakeven_acceptance(5, _c()) == pytest.approx(0.832, abs=5e-4)
def test_a_perfect_draft_caps_at_1_507x_at_k5(self):
from soup_cli.utils.draft import modelled_speedup
assert modelled_speedup(1.0, 5, _c()) == pytest.approx(1.507, abs=5e-4)
def test_the_measured_alpha_is_below_breakeven_at_k5(self):
from soup_cli.utils.draft import modelled_speedup
assert modelled_speedup(_ALPHA, 5, _c()) == pytest.approx(0.955, abs=5e-4)
def test_k1_and_k2(self):
"""#843 derived these from the ROUNDED 0.813 (at 0.81293, k=2 gives 1.1285,
which rounds the other way). Each figure is pinned to its source's input."""
from soup_cli.utils.draft import modelled_speedup
assert modelled_speedup(0.813, 1, _c()) == pytest.approx(1.136, abs=5e-4)
assert modelled_speedup(0.813, 2, _c()) == pytest.approx(1.129, abs=5e-4)
def test_best_k_is_one_or_two(self):
from soup_cli.utils.draft import modelled_best_k
k, speedup = modelled_best_k(_ALPHA, _c())
assert k in {1, 2}
assert speedup == pytest.approx(1.136, abs=5e-4)
@pytest.mark.parametrize("k, expected", [(1, 0.596), (2, 0.701), (3, 0.763), (4, 0.803)])
def test_breakeven_rises_with_k(self, k, expected):
"""Derived in #843 from the same inputs; each is where S crosses 1."""
from soup_cli.utils.draft import breakeven_acceptance, modelled_speedup
a = breakeven_acceptance(k, _c())
assert a == pytest.approx(expected, abs=5e-4)
assert modelled_speedup(a, k, _c()) == pytest.approx(1.0, abs=1e-6)
class TestExpectedTokens:
def test_a_perfect_draft_yields_k_plus_one(self):
from soup_cli.utils.draft import expected_tokens_per_step
for k in (1, 5, 64):
assert expected_tokens_per_step(1.0, k) == k + 1
def test_a_useless_draft_yields_the_target_token_only(self):
from soup_cli.utils.draft import expected_tokens_per_step
assert expected_tokens_per_step(0.0, 5) == 1.0
def test_continuous_approaching_one(self):
"""The closed form divides by (1 - a); it must not jump at a -> 1."""
from soup_cli.utils.draft import expected_tokens_per_step
assert expected_tokens_per_step(1 - 1e-12, 5) == pytest.approx(6.0, abs=1e-6)
def test_matches_the_series(self):
from soup_cli.utils.draft import expected_tokens_per_step
a, k = 0.6, 7
assert expected_tokens_per_step(a, k) == pytest.approx(sum(a**i for i in range(k + 1)))
class TestNoRatePays:
@pytest.mark.parametrize("c", [1.0, 1.5])
def test_a_draft_no_faster_than_its_target_has_no_breakeven(self, c):
"""At c >= 1 even a = 1 gives (k+1)/(k*c+1) <= 1: the answer is "none",
not a number the operator might chase."""
from soup_cli.utils.draft import breakeven_acceptance
for k in (1, 5, 64):
assert breakeven_acceptance(k, c) is None
def test_just_faster_than_the_target_still_has_one(self):
from soup_cli.utils.draft import breakeven_acceptance
a = breakeven_acceptance(1, 0.99)
assert a is not None and 0.98 < a < 1.0
class TestLatencyRatio:
@pytest.mark.parametrize(
"plain, draft",
[(0.0, 10.0), (10.0, 0.0), (-1.0, 10.0), (float("nan"), 10.0),
(10.0, float("inf")), (None, 10.0), (10.0, None)],
)
def test_unusable_throughput_gives_no_ratio(self, plain, draft):
from soup_cli.utils.draft import latency_ratio
assert latency_ratio(plain, draft) is None
class TestInputValidation:
@pytest.mark.parametrize("a", [-0.1, 1.1, float("nan"), True, "0.5"])
def test_acceptance_must_be_a_probability(self, a):
from soup_cli.utils.draft import modelled_speedup
with pytest.raises((TypeError, ValueError)):
modelled_speedup(a, 5, 0.5)
@pytest.mark.parametrize("k", [0, 65, 2.0, True])
def test_k_is_the_cli_range(self, k):
from soup_cli.utils.draft import modelled_speedup
with pytest.raises((TypeError, ValueError)):
modelled_speedup(0.5, k, 0.5)
@pytest.mark.parametrize("c", [0.0, -0.5, float("inf"), float("nan"), False])
def test_latency_ratio_must_be_positive_and_finite(self, c):
from soup_cli.utils.draft import breakeven_acceptance
with pytest.raises((TypeError, ValueError)):
breakeven_acceptance(5, c)
class TestTheModelIsTorchFree:
def test_no_torch_import(self):
"""The four functions live in a module that imports torch lazily; calling
them must not pull it in."""
import subprocess
import sys
code = (
"import sys\n"
"from soup_cli.utils.draft import (breakeven_acceptance, latency_ratio,"
" modelled_best_k, modelled_speedup)\n"
"c = latency_ratio(39.28, 65.9)\n"
"breakeven_acceptance(5, c); modelled_best_k(0.8, c); modelled_speedup(0.8, 5, c)\n"
"print('torch' in sys.modules)\n"
)
out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert out.returncode == 0, out.stderr
assert out.stdout.strip() == "False"
class TestTheBandNoLongerClaimsToPay:
def test_docs_drop_the_paying_claim(self):
"""#843: 0.70 is an acceptance band, not where speculative decoding pays --
the one pair measured at scale was a 0.481x slowdown at 0.813."""
from pathlib import Path
docs = (
Path(__file__).resolve().parents[1] / "docs" / "serving-and-export.md"
).read_text(encoding="utf-8")
assert not re.search(r"starts?\s+paying", docs), "0.70 does not mark payback"
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
def _plain(text):
return " ".join(_ANSI.sub("", text).split())
class _FakeTok:
def __init__(self, vocab_size):
self.vocab_size = vocab_size
def encode(self, text, add_special_tokens=False):
return [ord(ch) % self.vocab_size for ch in text]
@pytest.fixture()
def in_tmp_cwd(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
return tmp_path
def _run(monkeypatch, tmp_path, throughput, extra=(), acceptance=(81, 100)):
from typer.testing import CliRunner
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
(tmp_path / "p.jsonl").write_text(
json.dumps({"prompt": "What is 2+2?"}) + "\n" + json.dumps({"prompt": "Hi"}) + "\n",
encoding="utf-8",
)
target_model, draft_model = object(), object()
# Pin the panel width. The module Console measures the terminal, and on a
# legacy Windows console Rich reports width - 1 (Console.size subtracts
# legacy_windows): an 80-column CI runner renders at 79, which wraps the
# 80-character "closest: k=1 -> 0.995x" row, fails the positive match, and
# makes the "not in" guard pass whatever is printed (#1042 review).
# file=None keeps writing to sys.stdout, which CliRunner captures.
monkeypatch.setattr(
draft_cmd,
"console",
Console(width=200, legacy_windows=False, force_terminal=False, color_system=None),
)
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(
draft_cmd,
"_load_pair_member",
lambda model_id, **kw: (
target_model if "target" in model_id else draft_model,
_FakeTok(49152),
"cpu",
),
)
monkeypatch.setattr(draft_cmd, "measure_acceptance", lambda *a, **k: acceptance)
def _throughput(model, tok, prompts, *, assistant_model=None, num_assistant_tokens=5,
**kw):
role = "draft" if model is draft_model else (
"assisted" if assistant_model is not None else "plain"
)
return throughput(role, num_assistant_tokens)
monkeypatch.setattr(draft_cmd, "measure_throughput", _throughput)
result = CliRunner().invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", "p.jsonl", "-o", "report.json", *extra],
)
report = tmp_path / "report.json"
data = json.loads(report.read_text(encoding="utf-8")) if report.exists() else None
return result, data
def _h100(role, k):
return {"plain": _TARGET_TOK_S, "draft": _DRAFT_TOK_S, "assisted": 18.90}[role]
class TestMeasureReportsTheModel:
def test_json_carries_the_modelled_fields(self, in_tmp_cwd, monkeypatch):
result, data = _run(monkeypatch, in_tmp_cwd, _h100)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["tok_s_draft"] == _DRAFT_TOK_S
assert data["draft_status"] == "complete"
assert data["latency_ratio"] == pytest.approx(0.596, abs=5e-4)
assert data["breakeven_acceptance"] == pytest.approx(0.832, abs=5e-4)
assert data["modelled_best_k"] in {1, 2}
assert data["modelled_speedup_best_k"] > 1.0
# Existing fields keep their names and meaning.
assert data["acceptance_rate"] == 0.81
assert data["tok_s_plain"] == _TARGET_TOK_S
assert data["tok_s_assisted"] == 18.90
assert data["num_assistant_tokens"] == 5
def test_breakeven_is_for_the_k_in_use(self, in_tmp_cwd, monkeypatch):
"""Break-even is reported at ``--num-assistant-tokens``, not at the default:
k=2 on the H100 inputs is 0.701, not k=5's 0.832."""
result, data = _run(
monkeypatch, in_tmp_cwd, _h100, extra=["--num-assistant-tokens", "2"]
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["num_assistant_tokens"] == 2
assert data["breakeven_acceptance"] == pytest.approx(0.701, abs=5e-4)
assert "70.1% acceptance at k=2" in _plain(result.output)
def test_panel_labels_the_assumption(self, in_tmp_cwd, monkeypatch):
result, _ = _run(monkeypatch, in_tmp_cwd, _h100)
out = _plain(result.output)
assert "83.2%" in out, out
assert "modelled" in out.lower()
assert "i.i.d." in out
assert "framework overhead" in out
def test_a_draft_no_faster_says_no_rate_pays(self, in_tmp_cwd, monkeypatch):
def _same_speed(role, k):
return 20.0
result, data = _run(monkeypatch, in_tmp_cwd, _same_speed)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["latency_ratio"] == 1.0
assert data["breakeven_acceptance"] is None
assert "no acceptance rate pays" in _plain(result.output).lower()
def test_a_best_k_that_is_still_a_slowdown_is_not_recommended(
self, in_tmp_cwd, monkeypatch
):
"""Seen live (gpt2 <- tiny-gpt2 on CPU, 0% acceptance): the panel printed
"k=1 -> 0.98x" as the best k. Below 1x no k pays, and the panel says so."""
result, data = _run(monkeypatch, in_tmp_cwd, _h100, acceptance=(0, 100))
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["modelled_speedup_best_k"] < 1.0
out = _plain(result.output).lower()
assert "no k pays at the measured acceptance" in out
def test_a_near_miss_is_not_rounded_up_to_one(self, in_tmp_cwd, monkeypatch):
"""c = 1.0 at 99% acceptance models k=1 at 0.995x. Two decimals printed
"no k pays ... 1.00x", which reads as a contradiction."""
result, data = _run(
monkeypatch, in_tmp_cwd, lambda role, k: 20.0, acceptance=(99, 100)
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["modelled_best_k"] == 1
assert data["modelled_speedup_best_k"] == pytest.approx(0.995)
out = _plain(result.output)
assert "no k pays" in out.lower()
assert "k=1 -> 0.995x" in out, out
# The measured "Speedup 1.00x" row is genuinely 1.00 here; only the
# modelled closest-k figure must not round up.
assert "k=1 -> 1.00x" not in out
def test_a_best_k_that_pays_is_recommended(self, in_tmp_cwd, monkeypatch):
result, data = _run(monkeypatch, in_tmp_cwd, _h100)
assert data["modelled_speedup_best_k"] > 1.0
out = _plain(result.output).lower()
assert "no k pays" not in out
assert f"k={data['modelled_best_k']} -> " in out
def test_a_crashed_draft_arm_keeps_everything_else(self, in_tmp_cwd, monkeypatch):
"""Best-effort, #344 pattern: acceptance, plain and assisted all survive,
and the report says the draft arm crashed rather than looking un-run."""
def _draft_crashes(role, k):
if role == "draft":
raise RuntimeError("draft OOM")
return _h100(role, k)
result, data = _run(monkeypatch, in_tmp_cwd, _draft_crashes)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["draft_status"] == "crash"
assert data["tok_s_draft"] is None
assert data["latency_ratio"] is None
assert data["breakeven_acceptance"] is None
assert data["acceptance_rate"] == 0.81
assert data["tok_s_assisted"] == 18.90
assert "draft-alone throughput could not be measured" in _plain(result.output)
def test_exit_two_below_min_acceptance_is_unchanged(self, in_tmp_cwd, monkeypatch):
result, _ = _run(monkeypatch, in_tmp_cwd, _h100, extra=["--min-acceptance", "0.9"])
assert result.exit_code == 2
class TestSweepK:
def test_measures_each_k_and_names_the_best(self, in_tmp_cwd, monkeypatch):
measured = {1: 44.0, 2: 41.0, 5: 18.9}
def _by_k(role, k):
return measured[k] if role == "assisted" else _h100(role, k)
calls = []
result, data = _run(
monkeypatch, in_tmp_cwd,
lambda role, k: (calls.append((role, k)), _by_k(role, k))[1],
extra=["--sweep-k", "1,2,5"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
swept = {row["k"]: row for row in data["k_sweep"]}
assert set(swept) == {1, 2, 5}
assert swept[1]["tok_s_assisted"] == 44.0
assert swept[1]["speedup"] == pytest.approx(44.0 / _TARGET_TOK_S)
assert data["measured_best_k"] == 1
assert {k for role, k in calls if role == "assisted"} >= {1, 2, 5}
out = _plain(result.output)
assert "measured best k" in out.lower()
def test_one_crashed_k_does_not_discard_the_others(self, in_tmp_cwd, monkeypatch):
def _k2_crashes(role, k):
if role == "assisted" and k == 2:
raise RuntimeError("boom")
return {1: 44.0, 5: 18.9}.get(k, 18.9) if role == "assisted" else _h100(role, k)
result, data = _run(monkeypatch, in_tmp_cwd, _k2_crashes, extra=["--sweep-k", "1,2,5"])
assert result.exit_code == 0, (result.output, repr(result.exception))
swept = {row["k"]: row for row in data["k_sweep"]}
assert swept[2]["status"] == "crash" and swept[2]["tok_s_assisted"] is None
assert swept[1]["status"] == "complete"
assert data["measured_best_k"] == 1
@pytest.mark.parametrize(
"value",
# "²" and "٣" pass str.isdigit(): the first used to raise from int() without
# naming the flag, the second was silently read as 3.
["0", "65", "1,x", "", "1,,2", "1,2,3,4,5,6,7,8,9", "2,2", "²", "1,٣"],
)
def test_bad_values_refused_before_any_model_loads(
self, in_tmp_cwd, monkeypatch, value
):
from typer.testing import CliRunner
from soup_cli.commands import draft as draft_cmd
from soup_cli.commands.draft import app
(in_tmp_cwd / "p.jsonl").write_text(json.dumps({"prompt": "Hi"}) + "\n")
loaded = []
monkeypatch.setattr(draft_cmd, "_vocab_size_of", lambda mid, trc=False: 49152)
monkeypatch.setattr(
draft_cmd, "_load_pair_member",
lambda model_id, **kw: loaded.append(model_id) or (object(), _FakeTok(49152), "cpu"),
)
result = CliRunner().invoke(
app,
["measure", "--target", "org/target", "--draft", "org/tiny",
"--prompts", "p.jsonl", "--sweep-k", value],
)
out = _plain(result.output).lower()
assert "no such option" not in out, "the option must exist and refuse the value"
assert result.exit_code == 1, (result.exit_code, out)
assert loaded == [], "refused before loading either model"
assert "--sweep-k" in out
@pytest.mark.parametrize("order", ["5,1", "1,5"])
def test_a_measured_tie_goes_to_the_smaller_k_whatever_the_order(
self, in_tmp_cwd, monkeypatch, order
):
"""Matches the modelled best k, which takes the smallest k on a tie."""
result, data = _run(
monkeypatch, in_tmp_cwd,
lambda role, k: 30.0 if role == "assisted" else _h100(role, k),
extra=["--sweep-k", order],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert data["measured_best_k"] == 1
def test_without_the_flag_no_sweep_runs(self, in_tmp_cwd, monkeypatch):
calls = []
result, data = _run(
monkeypatch, in_tmp_cwd, lambda role, k: (calls.append((role, k)), _h100(role, k))[1]
)
assert result.exit_code == 0
assert [c for c in calls if c[0] == "assisted"] == [("assisted", 5)]
assert data["k_sweep"] is None and data["measured_best_k"] is None
def test_isfinite_guard_used():
"""Sanity for the fixture numbers: all finite."""
assert all(math.isfinite(x) for x in (_TARGET_TOK_S, _DRAFT_TOK_S, _ALPHA))