Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#781 — inference prompted tuned models with special tokens training never saw.
``soup chat``, ``soup serve`` (transformers backend), ``soup infer``,
``soup diff``, the MoLE serve runtime, the ``live_eval`` generators behind the
eval gates, and ``soup data generate``'s local provider all rendered the chat
template to a string and then tokenized that string with the tokenizer's default
``add_special_tokens=True``. A template that renders ``{{ bos_token }}``
(Llama-3, Gemma, Mistral) on a tokenizer whose post-processor also prepends BOS
therefore sent ``[bos, bos, ...]`` — reproduced on the real tokenizers in the
issue. Soup's default SFT path (``train_on_responses_only``) builds its ids with
``add_special_tokens=False`` (``data/loss_mask.py``), so a model tuned there was
prompted with a sequence it never trained on.
The invariant pinned here: when a template rendered the prompt, the tokenizer
adds nothing. That is what training does and what HF's own
``apply_chat_template(tokenize=True)`` does. It is deliberately NOT "drop a
duplicate BOS": Soup's ``data.chat_template`` presets render no BOS at all, so
their adapters trained with none and must be prompted with none.
Every tokenizer below is a real ``transformers`` fast tokenizer built offline
from an in-memory vocab, so ``apply_chat_template`` is the genuine Jinja
renderer and the BOS/EOS come from a genuine ``tokenizers`` post-processor — no
network, and nothing that could agree with a wrong encoding.
"""
import ast
from functools import lru_cache
from pathlib import Path
import pytest
_SRC = Path(__file__).resolve().parent.parent / "src" / "soup_cli"
_SPECIALS = [
"<unk>", "<s>", "</s>",
"<|system|>", "<|user|>", "<|assistant|>", "<|end|>",
"<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>",
]
_WORDS = [
"You", "are", "terse", ".", "What", "is", "the", "capital", "of", "France", "?",
"Paris", "And", "Italy", "Rome", "System", "User", "Assistant", ":",
"system", "user", "assistant", "Generate", "1", "training", "examples", "now",
]
_BOS_ID = _SPECIALS.index("<s>")
_EOS_ID = _SPECIALS.index("</s>")
# Vendor shape (Llama-3 / Gemma / Mistral): the template renders BOS itself.
_BOS_TEMPLATE = (
"{{ bos_token }}"
"{% for m in messages %}<|{{ m['role'] }}|> {{ m['content'] }} <|end|> {% endfor %}"
"{% if add_generation_prompt %}<|assistant|>{% endif %}"
)
# Same rendering with {% generation %} markers, so training takes loss_mask's
# preferred assistant-mask path rather than the incremental fallback.
_GENERATION_TEMPLATE = (
"{{ bos_token }}"
"{% for m in messages %}"
"{% if m['role'] == 'assistant' %}"
"<|assistant|> {% generation %}{{ m['content'] }} <|end|>{% endgeneration %} "
"{% else %}<|{{ m['role'] }}|> {{ m['content'] }} <|end|> {% endif %}"
"{% endfor %}"
"{% if add_generation_prompt %}<|assistant|>{% endif %}"
)
_REJECTS_SYSTEM_TEMPLATE = (
"{% if messages[0]['role'] == 'system' %}"
"{{ raise_exception('System role not supported') }}"
"{% endif %}" + _BOS_TEMPLATE
)
_BROKEN_TEMPLATE = "{{ this_is_not_defined.boom() }}"
_SYSTEM = {"role": "system", "content": "You are terse."}
_USER = {"role": "user", "content": "What is the capital of France?"}
_ANSWER = {"role": "assistant", "content": "Paris."}
_USER_2 = {"role": "user", "content": "And Italy?"}
_ANSWER_2 = {"role": "assistant", "content": "Rome."}
_LEGACY = "System: You are terse.\nUser: What is the capital of France?\nAssistant:"
def _tokenizer(chat_template, *, post_processor="bos"):
"""A real fast tokenizer whose post-processor adds ``post_processor``.
``"bos"`` mirrors Llama-3 / Gemma / Mistral (``<s> $A``), ``"bos_eos"``
a tokenizer that also appends EOS, ``None`` one that adds nothing.
"""
transformers = pytest.importorskip("transformers")
tokenizers = pytest.importorskip("tokenizers")
from tokenizers import models, pre_tokenizers, processors
vocab = {token: index for index, token in enumerate(_SPECIALS + _WORDS)}
backend = tokenizers.Tokenizer(models.WordLevel(vocab=vocab, unk_token="<unk>"))
backend.pre_tokenizer = pre_tokenizers.Whitespace()
backend.add_special_tokens(_SPECIALS)
single = {"bos": "<s> $A", "bos_eos": "<s> $A </s>", None: None}[post_processor]
if single is not None:
backend.post_processor = processors.TemplateProcessing(
single=single,
special_tokens=[("<s>", _BOS_ID), ("</s>", _EOS_ID)],
)
tok = transformers.PreTrainedTokenizerFast(
tokenizer_object=backend, unk_token="<unk>", bos_token="<s>", eos_token="</s>"
)
tok.chat_template = chat_template
return tok
def _preset_tokenizer(name):
"""A BOS-adding tokenizer carrying one of Soup's own ``data.chat_template``
presets, applied through the same override ``soup train`` uses."""
from soup_cli.data.chat_templates import apply_chat_template_override
tok = _tokenizer(None)
apply_chat_template_override(tok, name)
return tok
def _hf_prompt_ids(tok, messages):
"""HF's own tokenize-in-one-step encoding — the reference."""
return list(
tok.apply_chat_template(
messages, tokenize=True, add_generation_prompt=True, return_dict=False
)
)
def _pre_fix_ids(tok, messages):
"""What every site sent before the fix: render to text, re-tokenize."""
text = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
return tok(text)["input_ids"]
class _RecordingModel:
"""Stands in for the causal LM. Records the ids a call site hands to
``generate`` and echoes one extra token so the decode paths still run.
Only the model is replaced; the tokenizer and the encoding are real."""
def __init__(self):
import torch
self.device = torch.device("cpu")
self.sent = None
def eval(self):
return self
def generate(self, input_ids=None, **kwargs):
import torch
self.sent = input_ids[0].tolist()
return torch.cat([input_ids, input_ids[:, -1:]], dim=1)
# ============================================================
# The shared encoder
# ============================================================
class TestEncodeChatPrompt:
def test_the_fixture_reproduces_the_doubled_bos(self):
"""CONTROL for this whole file: the pre-fix encoding on this fixture
really sends two BOS. Without it, a post-processor that silently added
nothing would let every "exactly one BOS" assertion pass vacuously."""
tok = _tokenizer(_BOS_TEMPLATE)
assert _pre_fix_ids(tok, [_USER])[:2] == [_BOS_ID, _BOS_ID]
def test_a_rendered_prompt_carries_exactly_one_bos(self):
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BOS_TEMPLATE)
ids = encode_chat_prompt([_USER], tok, fallback_on_error=False)["input_ids"]
assert ids[0] == _BOS_ID
assert ids.count(_BOS_ID) == 1
def test_ids_equal_hf_apply_chat_template_with_tokenize_true(self):
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BOS_TEMPLATE)
messages = [_SYSTEM, _USER]
ids = encode_chat_prompt(messages, tok, fallback_on_error=False)["input_ids"]
assert ids == _hf_prompt_ids(tok, messages)
def test_a_tokenizer_eos_is_not_appended_to_a_rendered_prompt(self):
"""A post-processor that appends EOS put an end-of-sequence token at the
END of the prompt, i.e. right where generation starts."""
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos_eos")
assert _pre_fix_ids(tok, [_USER])[-1] == _EOS_ID # the defect, on this fixture
ids = encode_chat_prompt([_USER], tok, fallback_on_error=False)["input_ids"]
assert _EOS_ID not in ids
assert ids == _hf_prompt_ids(tok, [_USER])
def test_control_a_tokenizer_that_adds_nothing_is_byte_identical(self):
"""CONTROL — with no post-processor there was never a defect, so the
fix must not move a single id."""
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BOS_TEMPLATE, post_processor=None)
ids = encode_chat_prompt([_SYSTEM, _USER], tok, fallback_on_error=False)["input_ids"]
assert ids == _pre_fix_ids(tok, [_SYSTEM, _USER])
def test_a_soup_preset_on_a_bos_adding_tokenizer_is_prompted_with_no_bos(self):
"""Soup's presets render no BOS, and training adds none, so an adapter
trained with ``data.chat_template: llama3`` never saw one. The pre-fix
encoding still sent the tokenizer's.
This is the case that separates the invariant from "drop BOS only when
the rendered text already starts with it": that rule would leave this
prompt with the BOS its training never had."""
from soup_cli.utils.vllm import encode_chat_prompt
tok = _preset_tokenizer("llama3")
text = tok.apply_chat_template([_USER], tokenize=False, add_generation_prompt=True)
assert not text.startswith("<s>")
assert _pre_fix_ids(tok, [_USER])[0] == _BOS_ID # the defect, on the preset
ids = encode_chat_prompt([_USER], tok, fallback_on_error=False)["input_ids"]
assert _BOS_ID not in ids
assert ids == _hf_prompt_ids(tok, [_USER])
def test_control_no_template_keeps_the_legacy_prompt_and_tokenizer_defaults(self):
"""CONTROL — a model that ships no template is served exactly as before:
the legacy role-prefixed text, with whatever the tokenizer adds."""
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(None)
ids = encode_chat_prompt([_SYSTEM, _USER], tok, fallback_on_error=False)["input_ids"]
assert ids == tok(_LEGACY)["input_ids"]
assert ids[0] == _BOS_ID
def test_a_template_that_fails_to_render_falls_back_with_tokenizer_defaults(self):
"""The flag must mean "the template rendered this text", not "the
tokenizer has a template". The fallback text never went through the
template, so the tokenizer's own BOS is the only one it gets."""
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BROKEN_TEMPLATE)
ids = encode_chat_prompt([_SYSTEM, _USER], tok, fallback_on_error=True)["input_ids"]
assert ids == tok(_LEGACY)["input_ids"]
assert ids[0] == _BOS_ID
def test_a_template_that_fails_to_render_raises_when_fallback_is_off(self):
jinja2 = pytest.importorskip("jinja2")
from soup_cli.utils.vllm import encode_chat_prompt
tok = _tokenizer(_BROKEN_TEMPLATE)
with pytest.raises(jinja2.exceptions.TemplateError):
encode_chat_prompt([_SYSTEM, _USER], tok, fallback_on_error=False)
@pytest.mark.parametrize(
("template", "expected_kwargs"),
[
(_BOS_TEMPLATE, {"add_special_tokens": False, "return_tensors": "pt"}),
# The legacy branch must be literally the call it always was: no
# add_special_tokens key at all, since tokenizer stand-ins elsewhere
# in the suite (and third-party tokenizer classes) take a fixed
# signature.
(None, {"return_tensors": "pt"}),
],
ids=["rendered", "legacy"],
)
def test_the_tokenizer_receives_exactly_these_keywords(self, template, expected_kwargs):
from soup_cli.utils.vllm import encode_chat_prompt
inner = _tokenizer(template)
seen = []
class _Spy:
chat_template = inner.chat_template
def apply_chat_template(self, *args, **kwargs):
return inner.apply_chat_template(*args, **kwargs)
def __call__(self, text, **kwargs):
seen.append(kwargs)
return inner(text, **kwargs)
encode_chat_prompt([_SYSTEM, _USER], _Spy(), fallback_on_error=False, return_tensors="pt")
assert seen == [expected_kwargs]
# ============================================================
# The contract the fix exists for: inference == training
# ============================================================
_CONVERSATIONS = {
"single_turn": ([_USER], _ANSWER),
"with_system": ([_SYSTEM, _USER], _ANSWER),
"multi_turn": ([_USER, _ANSWER, _USER_2], _ANSWER_2),
}
def _templated(kind):
if kind == "vendor_bos":
return _tokenizer(_BOS_TEMPLATE)
if kind == "generation_markers":
return _tokenizer(_GENERATION_TEMPLATE)
return _preset_tokenizer("llama3")
class TestTrainInferAgreement:
def test_the_fixtures_cover_both_training_mask_paths(self):
"""``loss_mask`` builds ids two ways; the agreement below means little
if both fixtures took the same one."""
from soup_cli.data.loss_mask import _apply_template_with_mask
conversation = [_USER, _ANSWER]
assert _apply_template_with_mask(_templated("generation_markers"), conversation)
assert _apply_template_with_mask(_templated("vendor_bos"), conversation) is None
@pytest.mark.parametrize("conversation", sorted(_CONVERSATIONS))
@pytest.mark.parametrize("kind", ["generation_markers", "soup_llama3_preset", "vendor_bos"])
def test_prompt_ids_are_the_prefix_soup_trained_on(self, kind, conversation):
from soup_cli.data.loss_mask import build_assistant_only_labels
from soup_cli.utils.vllm import encode_chat_prompt
tok = _templated(kind)
prompt, answer = _CONVERSATIONS[conversation]
trained = build_assistant_only_labels(prompt + [answer], tok, max_length=512)
ids = encode_chat_prompt(prompt, tok, fallback_on_error=False)["input_ids"]
assert trained["input_ids"][: len(ids)] == ids
assert len(trained["input_ids"]) > len(ids)
@pytest.mark.parametrize("kind", ["soup_llama3_preset", "vendor_bos"])
def test_control_the_pre_fix_encoding_was_not_that_prefix(self, kind):
"""CONTROL — the agreement above is only a finding if the old encoding
disagreed on the same fixture."""
from soup_cli.data.loss_mask import build_assistant_only_labels
tok = _templated(kind)
trained = build_assistant_only_labels([_USER, _ANSWER], tok, max_length=512)
old = _pre_fix_ids(tok, [_USER])
assert trained["input_ids"][: len(old)] != old
# ============================================================
# Every in-process generation path
# ============================================================
def _via_chat(tok, messages):
from soup_cli.commands.chat import _generate
model = _RecordingModel()
_generate(model, tok, list(messages), max_tokens=1, temperature=0.0)
return model.sent
def _via_serve(tok, messages):
from soup_cli.commands.serve import _generate_response
model = _RecordingModel()
_generate_response(model, tok, list(messages), max_tokens=1, temperature=0.0)
return model.sent
def _via_infer(tok, messages):
from soup_cli.commands.infer import _generate
model = _RecordingModel()
_generate(model, tok, list(messages), max_tokens=1, temperature=0.0)
return model.sent
def _via_diff(tok, messages):
from soup_cli.commands.diff import _generate
model = _RecordingModel()
_generate(model, tok, list(messages), max_tokens=1, temperature=0.0)
return model.sent
def _via_mole(tok, messages):
"""``LoadedMole.generate_text`` drives its own blended decode loop, so the
recorder replaces that loop; the encoding in front of it is the real one."""
from soup_cli.utils.mole_routing import LoadedMole
loaded = LoadedMole(object(), tok, None, ["task_0", "task_1"])
recorder = _RecordingModel()
loaded.generate = lambda input_ids, attention_mask, **kwargs: recorder.generate(input_ids)
loaded.generate_text(list(messages), max_tokens=1, temperature=0.0)
return recorder.sent
_SITES = {
"chat": _via_chat,
"diff": _via_diff,
"infer": _via_infer,
"mole": _via_mole,
"serve": _via_serve,
}
class TestEveryGenerationPathEncodesLikeTraining:
@pytest.mark.parametrize("site", sorted(_SITES))
def test_the_site_sends_hf_prompt_ids_with_one_bos(self, site):
tok = _tokenizer(_BOS_TEMPLATE)
messages = [_SYSTEM, _USER]
sent = _SITES[site](tok, messages)
assert sent == _hf_prompt_ids(tok, messages)
assert sent.count(_BOS_ID) == 1
@pytest.mark.parametrize("site", sorted(_SITES))
def test_control_without_a_template_the_site_sends_the_legacy_prompt_as_before(self, site):
tok = _tokenizer(None)
assert _SITES[site](tok, [_SYSTEM, _USER]) == tok(_LEGACY)["input_ids"]
@pytest.mark.parametrize("site", ["chat", "diff", "infer", "mole"])
def test_a_template_that_rejects_the_conversation_still_raises(self, site):
"""These sites always let a template error surface (e.g. ``soup chat
--system`` on a template with no system role). Routing them through the
shared encoder must not turn that into a quietly served legacy prompt
the model has never seen."""
jinja2 = pytest.importorskip("jinja2")
tok = _tokenizer(_REJECTS_SYSTEM_TEMPLATE)
with pytest.raises(jinja2.exceptions.TemplateError):
_SITES[site](tok, [_SYSTEM, _USER])
def test_serve_still_falls_back_and_counts_the_tokens_it_actually_sent(self):
from soup_cli.commands.serve import _generate_response
tok = _tokenizer(_REJECTS_SYSTEM_TEMPLATE)
model = _RecordingModel()
_, prompt_tokens, _ = _generate_response(
model, tok, [_SYSTEM, _USER], max_tokens=1, temperature=0.0
)
assert model.sent == tok(_LEGACY)["input_ids"]
assert prompt_tokens == len(model.sent)
def test_serve_usage_prompt_tokens_matches_the_rendered_prompt(self):
"""``usage.prompt_tokens`` is read off the encoded ids, so it drops by
the duplicated BOS — pinned so the reported number is the sent one."""
from soup_cli.commands.serve import _generate_response
tok = _tokenizer(_BOS_TEMPLATE)
model = _RecordingModel()
_, prompt_tokens, _ = _generate_response(
model, tok, [_SYSTEM, _USER], max_tokens=1, temperature=0.0
)
assert prompt_tokens == len(_hf_prompt_ids(tok, [_SYSTEM, _USER]))
class TestLiveEvalGenerators:
"""The closures behind training-time eval gates, ``soup ship``,
``soup diagnose`` and ``soup advise``."""
@staticmethod
def _sent(factory_name, tok, prompt):
from soup_cli.utils import live_eval
model = _RecordingModel()
factory = getattr(live_eval, factory_name)
generate = factory("unused", loaded=(model, tok, "cpu"))
if factory_name == "make_multi_generator":
generate(prompt, 1)
else:
generate(prompt)
return model.sent
@pytest.mark.parametrize("factory_name", ["make_generator", "make_multi_generator"])
def test_the_generator_sends_hf_prompt_ids_with_one_bos(self, factory_name):
tok = _tokenizer(_BOS_TEMPLATE)
sent = self._sent(factory_name, tok, _USER["content"])
assert sent == _hf_prompt_ids(tok, [_USER])
assert sent.count(_BOS_ID) == 1
@pytest.mark.parametrize("factory_name", ["make_generator", "make_multi_generator"])
@pytest.mark.parametrize("template", [None, _BROKEN_TEMPLATE], ids=["no-template", "broken"])
def test_control_an_unrendered_raw_prompt_is_tokenized_as_before(
self, factory_name, template
):
"""CONTROL — with no template, or one that fails to render, these
generators send the raw prompt; no template touched it, so the
tokenizer's own special tokens stay."""
tok = _tokenizer(template)
sent = self._sent(factory_name, tok, _USER["content"])
assert sent == tok(_USER["content"])["input_ids"]
class TestDataGenerateLocalProvider:
_PROMPT = "You are terse."
_REQUEST = "Generate 1 training examples now."
def _sent(self, monkeypatch, tok):
transformers = pytest.importorskip("transformers")
from soup_cli.commands import generate
model = _RecordingModel()
monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: tok)
monkeypatch.setattr(
transformers.AutoModelForCausalLM, "from_pretrained", lambda *a, **k: model
)
monkeypatch.setattr(
"soup_cli.utils.trust_remote.model_requires_trust_remote_code", lambda *a, **k: False
)
monkeypatch.setattr(
"soup_cli.utils.trust_remote.resolve_trust_remote_code", lambda *a, **k: False
)
generate._generate_local(
prompt="unused",
count=1,
fmt="alpaca",
model_name="local-model",
temperature=0.0,
seed_examples=[],
generation_prompt=self._PROMPT,
)
return model.sent
def test_the_local_provider_sends_hf_prompt_ids_with_one_bos(self, monkeypatch):
tok = _tokenizer(_BOS_TEMPLATE)
messages = [
{"role": "system", "content": self._PROMPT},
{"role": "user", "content": self._REQUEST},
]
sent = self._sent(monkeypatch, tok)
assert sent == _hf_prompt_ids(tok, messages)
assert sent.count(_BOS_ID) == 1
def test_control_without_a_template_its_own_prompt_is_tokenized_as_before(
self, monkeypatch
):
tok = _tokenizer(None)
sent = self._sent(monkeypatch, tok)
assert sent == tok(f"{self._PROMPT}\n\n{self._REQUEST}\n\n")["input_ids"]
# ============================================================
# Ratchet: no new scope may render a template and re-add special tokens
# ============================================================
# Functions whose return value is text a chat template rendered.
_RENDER_HELPERS = frozenset({
"_apply_prompt_template",
"_render_chat_prompt",
"_render_prompt_template",
"build_chat_prompt",
"render_raft_prompt",
})
_TOKENIZER_NAMES = frozenset({"_tokenizer", "processor", "teacher_tokenizer", "tok", "tokenizer"})
# Scopes that render and then let the tokenizer decide, each for a stated reason.
_ALLOWLIST = {
"trainer/sft.py::VisionLanguageDataCollator.__call__": (
"passes add_special_tokens through **processor_kwargs, decided per processor "
"by _processor_adds_leading_bos (#302)"
),
"commands/data.py::preprocess_dataset": (
"renders a template, then tokenizes with the tokenizer's default "
"add_special_tokens=True on purpose so the cache stays byte-identical to "
"main (truncation reservation and post-processor EOS preserved), and strips "
"only the one doubled leading BOS afterwards (#785/#788)"
),
"utils/live_eval.py::extract_layer_activations": (
"activation capture; changing the ids changes saved steering vectors and "
"probe calibration (#781, out of scope)"
),
"utils/steering.py::_capture_attn_heads": (
"activation capture, same reason as extract_layer_activations (#781)"
),
}
def _call_name(func):
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute):
return func.attr
return None
def _renders(call):
name = _call_name(call.func)
if name in _RENDER_HELPERS:
return True
return name == "apply_chat_template" and any(
keyword.arg == "tokenize"
and isinstance(keyword.value, ast.Constant)
and keyword.value.value is False
for keyword in call.keywords
)
def _tokenizes(call):
func = call.func
if _call_name(func) in _TOKENIZER_NAMES:
return True
return (
isinstance(func, ast.Attribute)
and func.attr == "encode"
and _call_name(func.value) in _TOKENIZER_NAMES
)
def _adds_special_tokens(call):
for keyword in call.keywords:
if keyword.arg == "add_special_tokens":
return isinstance(keyword.value, ast.Constant) and keyword.value.value is True
return True # the tokenizer default
_SCOPE_NODES = (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
def _scopes_of(source):
"""Per qualified function name (nested functions are their own scope): does it
render a template, and which tokenizer calls does it make?
Known blind spot, stated rather than hidden: a render in one function whose
text is tokenized in a DIFFERENT function is not seen. Every site this issue
found does both in one scope."""
scopes = {}
stack = [(ast.parse(source), "<module>")]
while stack:
node, name = stack.pop()
scope = scopes.setdefault(name, {"renders": False, "adds": [], "exact": []})
for child in ast.iter_child_nodes(node):
if isinstance(child, _SCOPE_NODES):
qualname = child.name if name == "<module>" else f"{name}.{child.name}"
stack.append((child, qualname))
continue
if isinstance(child, ast.Call):
if _renders(child):
scope["renders"] = True
if _tokenizes(child):
key = "adds" if _adds_special_tokens(child) else "exact"
scope[key].append(child.lineno)
stack.append((child, name))
return scopes
def _findings(scopes):
return sorted(name for name, scope in scopes.items() if scope["renders"] and scope["adds"])
@lru_cache(maxsize=1)
def _tree_scopes():
result = {}
for path in sorted(_SRC.rglob("*.py")):
module = path.relative_to(_SRC).as_posix()
for name, scope in _scopes_of(path.read_text(encoding="utf-8")).items():
result[f"{module}::{name}"] = scope
return result
class TestNoRenderThenReTokenize:
def test_no_scope_renders_a_template_and_lets_the_tokenizer_add_special_tokens(self):
found = set(_findings(_tree_scopes()))
assert found - set(_ALLOWLIST) == set()
def test_every_allowlist_entry_is_still_earned(self):
"""An entry whose scope was fixed, renamed or moved must leave the
allowlist, or it quietly becomes a hole."""
found = set(_findings(_tree_scopes()))
assert set(_ALLOWLIST) - found == set()
def test_the_scanner_sees_a_scope_that_already_does_it_right(self):
"""Not vacuous: ``_tokenize_pair`` renders and tokenizes in one scope,
so a scanner that could not see either would still report zero."""
scope = _tree_scopes()["utils/live_eval.py::_tokenize_pair"]
assert scope["renders"]
assert scope["exact"]
assert not scope["adds"]
@pytest.mark.parametrize(
"source",
[
# chat / infer / diff before the fix
"def _generate(model, tokenizer, messages):\n"
" text = tokenizer.apply_chat_template(\n"
" messages, tokenize=False, add_generation_prompt=True\n"
" )\n"
" return tokenizer(text, return_tensors='pt')\n",
# serve before the fix: the shared builder, then a bare call
"def _generate_response(model, tokenizer, messages):\n"
" text = build_chat_prompt(messages, tokenizer)\n"
" return tokenizer(text, return_tensors='pt')\n",
# live_eval before the fix: a nested closure
"def make_generator(tokenizer):\n"
" def _gen(prompt):\n"
" text = _apply_prompt_template(tokenizer, prompt)\n"
" return tokenizer(text, return_tensors='pt', truncation=True)\n"
" return _gen\n",
# the MoLE runtime shape: a method on self.tokenizer
"class Runtime:\n"
" def generate_text(self, messages):\n"
" text = self.tokenizer.apply_chat_template(messages, tokenize=False)\n"
" return self.tokenizer(text, return_tensors='pt')\n",
# an explicit True is the same defect spelled out
"def f(tokenizer, messages):\n"
" text = tokenizer.apply_chat_template(messages, tokenize=False)\n"
" return tokenizer(text, add_special_tokens=True)\n",
# .encode adds special tokens by default too
"def f(tok, messages):\n"
" text = tok.apply_chat_template(messages, tokenize=False)\n"
" return tok.encode(text)\n",
],
ids=["inline-render", "shared-builder", "nested-closure", "self-tokenizer",
"explicit-true", "encode"],
)
def test_the_scanner_flags_each_pre_fix_shape(self, source):
assert len(_findings(_scopes_of(source))) == 1
@pytest.mark.parametrize(
"source",
[
"def f(tokenizer, messages):\n"
" text = tokenizer.apply_chat_template(messages, tokenize=False)\n"
" return tokenizer(text, add_special_tokens=False)\n",
"def f(tokenizer, messages):\n"
" return encode_chat_prompt(messages, tokenizer, fallback_on_error=False)\n",
"def f(tokenizer, messages):\n"
" text, templated = _render_prompt_template(tokenizer, messages)\n"
" return encode_rendered_prompt(tokenizer, text, templated=templated)\n",
# tokenizing text no template rendered is not this defect
"def f(tokenizer, text):\n"
" return tokenizer(text, return_tensors='pt')\n",
# tokenize=True is HF's one-step encoding, not a render
"def f(tokenizer, messages, text):\n"
" ids = tokenizer.apply_chat_template(messages, tokenize=True)\n"
" return tokenizer(text)\n",
],
ids=["explicit-false", "shared-encoder", "rendered-encoder", "no-render",
"tokenize-true"],
)
def test_the_scanner_passes_the_fixed_shapes(self, source):
assert _findings(_scopes_of(source)) == []
def test_the_legacy_role_prefixed_prompt_exists_only_in_the_shared_fallback(self):
"""chat / infer / diff / mole each carried their own copy of the
role-prefixed fallback. One copy left means one place to be right."""
literal = 'f"User: {content}"'
copies = {
path.relative_to(_SRC).as_posix(): path.read_text(encoding="utf-8").count(literal)
for path in sorted(_SRC.rglob("*.py"))
}
assert {module: count for module, count in copies.items() if count} == {
"utils/vllm.py": 1
}