Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Issue #674: explicit mixed-precision QuEST engineering integration.
The quality result behind this route is evaluation-only. These tests protect
the engineering contract: exact route accounting, trainable fake quantisation,
metadata round-trips, resume fidelity and fail-closed configuration.
"""
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import SoupConfig
from tests.conftest import strip_ansi
_TEST_CALIBRATION_SHA256 = "ab" * 32
def _plain(text: str) -> str:
"""Return CLI output safe for assertions across Rich-capable terminals."""
return " ".join(strip_ansi(text).split())
def test_plain_joins_a_label_split_by_ansi():
decorated = "mixed W4/A4+A16 (QuEST \x1b[1mfake\x1b[0m quant)"
assert "mixed W4/A4+A16 (QuEST fake quant)" not in decorated
assert "mixed W4/A4+A16 (QuEST fake quant)" in _plain(decorated)
def _config(**training):
values = {
"quantization_aware": "quest",
"quantization": "none",
"batch_size": 2,
"lora": {"r": 0},
**training,
}
return SoupConfig(
base="ahxt/LiteLlama-460M-1T",
task="sft",
modality="text",
backend="transformers",
data={"train": "data.jsonl"},
training=values,
)
def _tiny_litellama(width: int = 128):
import torch
class Attention(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.q_proj = torch.nn.Linear(width, width, bias=False)
self.k_proj = torch.nn.Linear(width, width, bias=False)
self.v_proj = torch.nn.Linear(width, width, bias=False)
self.o_proj = torch.nn.Linear(width, width, bias=False)
class MLP(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.gate_proj = torch.nn.Linear(width, width, bias=False)
self.up_proj = torch.nn.Linear(width, width, bias=False)
self.down_proj = torch.nn.Linear(width, width, bias=False)
class Block(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.self_attn = Attention()
self.mlp = MLP()
class Model(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.model = torch.nn.Module()
self.model.layers = torch.nn.ModuleList(Block() for _ in range(24))
self.lm_head = torch.nn.Linear(width, 17, bias=False)
self.config = SimpleNamespace(
model_type="llama",
num_hidden_layers=24,
hidden_size=width,
intermediate_size=width,
)
return Model()
def _scales(model) -> dict[str, float]:
import torch
return {
name: 3.0
for name, module in model.named_modules()
if type(module) is torch.nn.Linear and name != "lm_head"
}
def _calibration_model():
import torch
class Model(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.linear = torch.nn.Linear(128, 128, bias=False)
def forward(self, input_ids, attention_mask):
del attention_mask
values = torch.nn.functional.one_hot(input_ids.remainder(128), num_classes=128).float()
return self.linear(values)
return Model()
def _calibration_rows(count: int = 32) -> list[dict[str, list[int]]]:
return [
{
"input_ids": [1, 2, 3],
"attention_mask": [1, 1, 1],
"labels": [-100, 2, 3],
}
for _ in range(count)
]
def test_schema_accepts_only_the_explicit_first_slice():
cfg = _config()
assert cfg.training.quantization_aware == "quest"
assert cfg.training.quantization == "none"
assert cfg.training.lora.r == 0
assert cfg.training.batch_size == 2
def test_config_loader_rejects_quest_auto_mixed_precision() -> None:
config = (
"base: ahxt/LiteLlama-460M-1T\n"
"task: sft\n"
"backend: transformers\n"
"modality: text\n"
"data:\n"
" train: data.jsonl\n"
"training:\n"
" quantization_aware: quest\n"
" quantization: none\n"
" batch_size: 2\n"
" lora:\n"
" r: 0\n"
)
with pytest.raises(ValueError) as error:
load_config_from_string(config + " auto_mixed_precision: true\n")
assert "training.quantization_aware" in str(error.value)
assert "training.auto_mixed_precision" in str(error.value)
assert load_config_from_string(config).training.quantization_aware == "quest"
allowed = load_config_from_string(config + " auto_mixed_precision: false\n")
assert allowed.training.quantization_aware == "quest"
def test_config_loader_keeps_non_quest_auto_mixed_precision() -> None:
cfg = load_config_from_string(
"base: org/model\n"
"data:\n"
" train: data.jsonl\n"
"training:\n"
" auto_mixed_precision: true\n"
)
assert cfg.training.auto_mixed_precision is True
@pytest.mark.parametrize(
"field,value,primary_error",
[
("task", "dpo", "task='sft'"),
("backend", "unsloth", "backend='transformers'"),
("modality", "vision", "modality='text'"),
],
)
def test_quest_route_error_precedes_auto_mixed_precision(
field: str, value: str, primary_error: str
) -> None:
payload = {
"base": "ahxt/LiteLlama-460M-1T",
"task": "sft",
"backend": "transformers",
"modality": "text",
"data": {"train": "data.jsonl"},
"training": {
"quantization_aware": "quest",
"auto_mixed_precision": True,
"quantization": "none",
"batch_size": 2,
"lora": {"r": 0},
},
}
payload[field] = value
with pytest.raises(ValueError, match=primary_error) as error:
load_config_from_string(yaml.safe_dump(payload))
assert "auto_mixed_precision" not in str(error.value)
@pytest.mark.parametrize(
"root,training,match",
[
({"task": "dpo"}, {}, "task='sft'"),
({"backend": "unsloth"}, {}, "backend='transformers'"),
({"modality": "vision"}, {}, "modality='text'"),
({}, {"quantization": "4bit"}, "quantization='none'"),
({}, {"lora": {"r": 8}}, "lora.r=0"),
({}, {"batch_size": "auto"}, "explicit training.batch_size"),
({}, {"auto_mixed_precision": True}, "auto_mixed_precision=false"),
({}, {"stream_layers": True}, "stream_layers=false"),
({}, {"nvfp4": True}, "nvfp4=false"),
({}, {"activation_offloading": "cpu"}, "activation_offloading"),
({}, {"activation_offloading": "disk"}, "activation_offloading"),
],
)
def test_schema_rejects_unwired_quest_combinations(root, training, match):
payload = {
"base": "ahxt/LiteLlama-460M-1T",
"task": "sft",
"modality": "text",
"backend": "transformers",
"data": {"train": "data.jsonl"},
"training": {
"quantization_aware": "quest",
"quantization": "none",
"batch_size": 2,
"lora": {"r": 0},
},
}
payload.update(root)
payload["training"].update(training)
with pytest.raises(ValueError, match=match):
SoupConfig(**payload)
@pytest.mark.parametrize(
"training",
[
{"freeze_layers": 12},
{"freeze_ratio": 0.5},
{"unfrozen_parameters": [".*q_proj.*"]},
{"lisa_enabled": True},
{"expand_layers": 1, "freeze_trainable_layers": 1},
],
)
def test_schema_rejects_partial_training_routes(training):
with pytest.raises(ValueError, match="unmodified full fine-tuning"):
_config(**training)
def test_schema_keeps_preload_cut_ce_available_for_quest():
cfg = _config(use_cut_ce=True)
assert cfg.training.use_cut_ce is True
def test_non_quest_defaults_are_unchanged():
cfg = SoupConfig(base="org/model", data={"train": "data.jsonl"})
assert cfg.training.quantization_aware is False
assert cfg.training.quantization == "4bit"
def test_cli_dry_run_reports_quest_without_qat_or_torchao(tmp_path, monkeypatch):
import builtins
from rich.console import Console
from typer.testing import CliRunner
import soup_cli.commands.train as train_mod
import soup_cli.utils.qat as qat
from soup_cli.cli import app
data_path = tmp_path / "train.jsonl"
data_path.write_text('{"text": "hello"}\n', encoding="utf-8")
config_path = tmp_path / "soup.yaml"
config_path.write_text(
"base: ahxt/LiteLlama-460M-1T\n"
"task: sft\n"
"backend: transformers\n"
"modality: text\n"
f"output: {tmp_path / 'out'}\n"
"data:\n"
f" train: {data_path}\n"
"training:\n"
" quantization_aware: quest\n"
" quantization: none\n"
" batch_size: 2\n"
" lora:\n"
" r: 0\n",
encoding="utf-8",
)
qat_calls = []
monkeypatch.setattr(train_mod, "detect_device", lambda backend=None: ("cpu", "CPU"))
monkeypatch.setattr(
train_mod,
"console",
Console(force_terminal=True, color_system="truecolor", width=200),
)
monkeypatch.setattr(
train_mod, "get_gpu_info", lambda backend=None: {"memory_total": "N/A"}
)
monkeypatch.setattr(
train_mod,
"load_dataset",
lambda *args, **kwargs: {"train": [{"text": "hello"}]},
)
monkeypatch.setattr(
qat,
"validate_qat_config",
lambda *args, **kwargs: qat_calls.append((args, kwargs)) or ["must not run"],
)
real_import = builtins.__import__
def reject_torchao(name, *args, **kwargs):
if name == "torchao" or name.startswith("torchao."):
raise AssertionError("QuEST dry-run must not require torchao")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", reject_torchao)
result = CliRunner().invoke(
app,
["train", "--config", str(config_path), "--dry-run", "--yes"],
)
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "\x1b[" in result.output
assert "mixed W4/A4+A16 (QuEST fake quant)" in _plain(result.output)
assert qat_calls == []
def test_install_accounts_for_every_weight_and_activation_route():
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
assert metadata["recipe"] == "w4a4-group128-block23-a16"
assert metadata["weight_routes"] == {"w4": 168}
assert metadata["activation_routes"] == {"a4": 161, "a16": 7}
assert metadata["pure_w4a4"] is False
assert metadata["training_quality_validated"] is False
assert metadata["packed_int4"] is False
assert metadata["transform"] == "full-width-normalized-hadamard"
assert metadata["weight_scale"] == 2.513930578568423
assert metadata["surrogate"] == "quest-trust-gradient"
assert metadata["calibration"]["rows_sha256"] == _TEST_CALIBRATION_SHA256
assert metadata["a16_modules"] == [
"model.layers.23.self_attn.q_proj",
"model.layers.23.self_attn.k_proj",
"model.layers.23.self_attn.v_proj",
"model.layers.23.self_attn.o_proj",
"model.layers.23.mlp.gate_proj",
"model.layers.23.mlp.up_proj",
"model.layers.23.mlp.down_proj",
]
def test_route_provenance_is_topology_only_and_bound_to_the_public_record():
from soup_cli.utils.quest import ROUTE_PROVENANCE
assert ROUTE_PROVENANCE == {
"schema_version": 1,
"scope": "topology_selection_only",
"measured_on": {
"mode": "evaluation_only",
"model": "ahxt/LiteLlama-460M-1T",
"examples": 704,
"targets": 25017,
"gap_nat": 0.0863441881,
"ci95": [0.0802412531, 0.0931898109],
"result_sha256": (
"94fee10542da29281f7753cbf221a3421ad5acf67f2b290e52d65acece359cdf"
),
},
"claims": {
"artifact_training_quality": False,
"cross_model_quality": False,
},
}
measured = ROUTE_PROVENANCE["measured_on"]
record = (
Path(__file__).parents[1] / "benchmarks" / "gate-674-quest-mixed-route.md"
).read_text(encoding="utf-8")
for binding in (
measured["result_sha256"],
"0.086344",
"[0.080241, 0.093190]",
"704 examples",
"25,017 targets",
):
assert str(binding) in record
def test_metadata_accepts_a_schema_valid_provenance_correction():
from soup_cli.utils.quest import install_mixed_quest, validate_metadata
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="other/model-with-the-same-topology",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
measured = metadata["route_provenance"]["measured_on"]
measured.update(
model="corrected/model-id",
examples=705,
targets=25018,
gap_nat=0.08,
ci95=[0.07, 0.09],
result_sha256="cd" * 32,
)
assert validate_metadata(metadata) is metadata
def test_provenance_correction_does_not_invalidate_resume_or_restore(tmp_path):
from soup_cli.utils.quest import (
install_mixed_quest,
restore_mixed_quest,
validate_resume_metadata,
write_metadata,
)
source = _tiny_litellama()
current = install_mixed_quest(
source,
activation_scales=_scales(source),
base_model="same/model",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
historical = json.loads(json.dumps(current))
historical["route_provenance"]["measured_on"].update(
model="corrected/model-id",
examples=705,
targets=25018,
gap_nat=0.08,
ci95=[0.07, 0.09],
result_sha256="cd" * 32,
)
write_metadata(tmp_path, historical)
validate_resume_metadata(tmp_path, current)
restored = _tiny_litellama()
assert restore_mixed_quest(restored, historical) is restored
assert int(restored.model.layers[0].self_attn.q_proj.quest_activation_bits) == 4
def test_v1_local_sidecar_restores_without_the_original_base_directory():
from soup_cli.utils.quest import install_mixed_quest, restore_mixed_quest
source = _tiny_litellama()
historical = install_mixed_quest(
source,
activation_scales=_scales(source),
base_model="C:/Users/old-account/models/base",
calibration_sha256=_TEST_CALIBRATION_SHA256,
format_version=1,
)
target = _tiny_litellama()
assert restore_mixed_quest(target, historical) is target
assert int(target.model.layers[0].self_attn.q_proj.quest_activation_bits) == 4
def test_quantizer_matches_the_retained_grid_and_trust_gradient():
import torch
from soup_cli.utils.quest import quantize_rotated
torch.manual_seed(674)
value = torch.randn(2, 3, 256, dtype=torch.float32, requires_grad=True)
scale = 3.0
actual = quantize_rotated(value, scale=scale, group=128)
grouped = value.detach().reshape(2, 3, 2, 128)
rms = grouped.square().mean(-1, keepdim=True).sqrt()
bound = rms * scale + 1e-8
spacing = 2 * bound / 15
expected = (grouped.clamp(-bound, bound) / spacing + 0.5).round() * spacing - spacing / 2
trusted = ((expected - grouped).abs() <= rms * (scale / 15)).float()
# The trust-gradient expression reconstructs the same grid value through
# one detached subtraction/addition, so the forward may differ by one FP32
# rounding ulp from the directly written reference expression.
torch.testing.assert_close(
actual.detach(),
expected.reshape_as(value),
rtol=0,
atol=torch.finfo(torch.float32).eps,
)
actual.sum().backward()
assert torch.equal(value.grad, trusted.reshape_as(value))
def test_all_weights_stay_w4_while_only_block23_activations_bypass_a4(monkeypatch):
import torch
import soup_cli.utils.quest as quest
model = _tiny_litellama()
quest.install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
calls: list[tuple[str, int]] = []
original = quest.quantize_rotated
def record(value, *, scale, group):
calls.append(("weight" if value.ndim == 2 else "activation", group))
return original(value, scale=scale, group=group)
monkeypatch.setattr(quest, "quantize_rotated", record)
value = torch.randn(2, 3, 128)
model.model.layers[22].self_attn.q_proj(value)
assert calls == [("activation", 128), ("weight", 128)]
calls.clear()
model.model.layers[23].self_attn.q_proj(value)
assert calls == [("weight", 128)]
def test_mixed_layer_backward_updates_the_original_master_weight():
import torch
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama()
parameter = model.model.layers[23].mlp.down_proj.weight
install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
value = torch.randn(2, 128, requires_grad=True)
model.model.layers[23].mlp.down_proj(value).square().mean().backward()
assert model.model.layers[23].mlp.down_proj.weight is parameter
assert parameter.grad is not None
assert torch.isfinite(parameter.grad).all()
assert torch.count_nonzero(parameter.grad)
def test_state_dict_roundtrip_preserves_the_exact_route_and_output():
import torch
from soup_cli.utils.quest import install_mixed_quest
torch.manual_seed(17)
source = _tiny_litellama()
scales = _scales(source)
install_mixed_quest(
source,
activation_scales=scales,
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
target = _tiny_litellama()
install_mixed_quest(
target,
activation_scales=scales,
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
target.load_state_dict(source.state_dict(), strict=True)
value = torch.randn(2, 128)
assert torch.equal(
source.model.layers[22].mlp.down_proj(value),
target.model.layers[22].mlp.down_proj(value),
)
assert torch.equal(
source.model.layers[23].mlp.down_proj(value),
target.model.layers[23].mlp.down_proj(value),
)
def test_route_buffers_do_not_pollute_the_transformers_checkpoint():
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama()
install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
assert not any("quest_" in name for name in model.state_dict())
def test_build_metadata_rejects_an_incomplete_activation_scale_table():
from soup_cli.utils.quest import build_metadata
scales = _scales(_tiny_litellama())
scales.pop("model.layers.23.mlp.down_proj")
with pytest.raises(ValueError, match="all 168"):
build_metadata(
activation_scales=scales,
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
def test_metadata_roundtrip_and_resume_fidelity(tmp_path: Path):
from soup_cli.utils.quest import (
install_mixed_quest,
load_metadata,
validate_resume_metadata,
write_metadata,
)
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
path = write_metadata(tmp_path, metadata)
assert path.name == "quest_mixed_precision.json"
loaded = load_metadata(tmp_path)
assert loaded == metadata
validate_resume_metadata(tmp_path, metadata)
def test_resume_rejects_missing_or_different_route(tmp_path: Path):
from soup_cli.utils.quest import (
install_mixed_quest,
validate_resume_metadata,
write_metadata,
)
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
with pytest.raises(ValueError, match="missing QuEST metadata"):
validate_resume_metadata(tmp_path, metadata)
changed = json.loads(json.dumps(metadata))
changed["activation_scales"]["model.layers.0.self_attn.q_proj"] = 4.0
write_metadata(tmp_path, changed)
with pytest.raises(ValueError, match="does not match"):
validate_resume_metadata(tmp_path, metadata)
@pytest.mark.parametrize(
"mutation,match",
[
(lambda value: value.update(format_version=True), "format_version"),
(lambda value: value.update(recipe="unknown"), "recipe"),
(lambda value: value.update(pure_w4a4=True), "pure_w4a4"),
(lambda value: value["a16_modules"].pop(), "seven"),
(lambda value: value["activation_routes"].update(a4=162), "route"),
(
lambda value: value["activation_scales"].update(
{"model.layers.0.self_attn.q_proj": float("nan")}
),
"scale",
),
(
lambda value: value["calibration"].update(rows_sha256="not-a-digest"),
"row binding",
),
(
lambda value: value["route_provenance"]["measured_on"]["ci95"].pop(),
"confidence interval",
),
(
lambda value: value["route_provenance"]["claims"].update(
artifact_training_quality=True
),
"must not claim artifact quality",
),
],
)
def test_metadata_validation_rejects_unknown_or_malformed_routes(tmp_path, mutation, match):
from soup_cli.utils.quest import install_mixed_quest, load_metadata, write_metadata
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
mutation(metadata)
write_metadata(tmp_path, metadata, validate=False)
with pytest.raises(ValueError, match=match):
load_metadata(tmp_path)
@pytest.mark.parametrize("count", [23, 25])
def test_install_refuses_any_topology_other_than_the_measured_24_blocks(count):
import copy
import torch
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama()
layers = list(model.model.layers)
if count == 23:
layers = layers[:23]
else:
layers.append(copy.deepcopy(layers[-1]))
model.model.layers = torch.nn.ModuleList(layers)
with pytest.raises(ValueError, match="24 blocks"):
install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="other/model",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
def test_install_refuses_partial_calibration_before_replacing_any_layer():
import torch
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama()
scales = _scales(model)
scales.pop("model.layers.23.mlp.down_proj")
with pytest.raises(ValueError, match="all 168"):
install_mixed_quest(
model,
activation_scales=scales,
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
assert all(
type(module) is torch.nn.Linear for name, module in model.named_modules() if name in scales
)
def test_install_refuses_non_fp32_masters_before_replacing_any_layer():
import torch
from soup_cli.utils.quest import install_mixed_quest
model = _tiny_litellama().to(dtype=torch.float64)
with pytest.raises(ValueError, match="FP32 master weights"):
install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
assert type(model.model.layers[0].self_attn.q_proj) is torch.nn.Linear
def test_calibration_selects_local_error_minimum_and_removes_hooks(monkeypatch):
import soup_cli.utils.quest as quest
model = _calibration_model()
model.train()
monkeypatch.setattr(quest, "EXPECTED_MODULES", ("linear",))
monkeypatch.setattr(
quest, "_validate_raw_topology", lambda candidate: {"linear": candidate.linear}
)
monkeypatch.setattr(quest, "rotate", lambda value: value)
def controlled_quantizer(value, *, scale, group):
del group
if value.ndim == 2:
return value
return value if scale == 3.0 else value * 0
monkeypatch.setattr(quest, "quantize_rotated", controlled_quantizer)
rows = _calibration_rows(33)
rows[-1] = {}
selected = quest.calibrate_activation_scales(model, rows)
assert selected == {"linear": 3.0}
assert model.training is True
assert not model.linear._forward_pre_hooks
def test_calibration_failure_removes_hooks_and_restores_mode(monkeypatch):
import soup_cli.utils.quest as quest
model = _calibration_model().eval()
monkeypatch.setattr(quest, "EXPECTED_MODULES", ("linear",))
monkeypatch.setattr(
quest, "_validate_raw_topology", lambda candidate: {"linear": candidate.linear}
)
rows = _calibration_rows()
rows[0].pop("labels")
with pytest.raises(ValueError, match="missing"):
quest.calibrate_activation_scales(model, rows)
assert model.training is False
assert not model.linear._forward_pre_hooks
def test_calibration_rejects_a_zero_energy_proxy(monkeypatch):
import soup_cli.utils.quest as quest
model = _calibration_model()
model.linear.weight.data.zero_()
monkeypatch.setattr(quest, "EXPECTED_MODULES", ("linear",))
monkeypatch.setattr(
quest, "_validate_raw_topology", lambda candidate: {"linear": candidate.linear}
)
with pytest.raises(ValueError, match="local error"):
quest.calibrate_activation_scales(model, _calibration_rows())
def test_calibration_refuses_an_unserialized_position_limit():
from soup_cli.utils.quest import calibrate_activation_scales
with pytest.raises(ValueError, match="calibration_position_limit at 16"):
calibrate_activation_scales(
_calibration_model(),
_calibration_rows(),
position_limit=8,
)
def test_calibration_row_binding_is_deterministic_and_sensitive():
from soup_cli.utils.quest import calibration_rows_sha256
first = _calibration_rows(33)
second = _calibration_rows(33)
second[-1]["input_ids"][0] = 99
assert calibration_rows_sha256(first) == calibration_rows_sha256(second)
second[0]["input_ids"][0] = 99
assert calibration_rows_sha256(first) != calibration_rows_sha256(second)
@pytest.mark.parametrize("mutation", ["precision", "missing"])
def test_runtime_route_assertion_kills_route_mutations(mutation):
import torch
from soup_cli.utils.quest import assert_route, install_mixed_quest
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
target = model.model.layers[0].self_attn.q_proj
if mutation == "precision":
target.quest_activation_bits.fill_(16)
else:
model.model.layers[0].self_attn.q_proj = torch.nn.Linear(128, 128)
with pytest.raises(ValueError, match="route"):
assert_route(model, metadata)
def test_loader_requires_metadata_and_reconstructs_the_route(tmp_path, monkeypatch):
from transformers import AutoModelForCausalLM
from soup_cli.utils.quest import (
install_mixed_quest,
load_mixed_quest_artifact,
write_metadata,
)
source = _tiny_litellama()
metadata = install_mixed_quest(
source,
activation_scales=_scales(source),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
write_metadata(tmp_path, metadata)
raw = _tiny_litellama()
raw.config.soup_quest = metadata
calls = []
def fake_load(path, **kwargs):
calls.append((path, kwargs))
return raw
monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", fake_load)
loaded = load_mixed_quest_artifact(tmp_path, local_files_only=True)
assert loaded is raw
assert calls == [(tmp_path, {"local_files_only": True})]
assert int(raw.model.layers[0].self_attn.q_proj.quest_activation_bits) == 4
assert int(raw.model.layers[23].mlp.down_proj.quest_activation_bits) == 16
def test_loader_refuses_config_sidecar_drift(tmp_path, monkeypatch):
from transformers import AutoModelForCausalLM
from soup_cli.utils.quest import (
install_mixed_quest,
load_mixed_quest_artifact,
write_metadata,
)
source = _tiny_litellama()
metadata = install_mixed_quest(
source,
activation_scales=_scales(source),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
write_metadata(tmp_path, metadata)
raw = _tiny_litellama()
raw.config.soup_quest = {**metadata, "recipe": "different"}
monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", lambda *args, **kwargs: raw)
with pytest.raises(ValueError, match="config declaration"):
load_mixed_quest_artifact(tmp_path)
def test_loader_refuses_a_missing_config_declaration(tmp_path, monkeypatch):
from transformers import AutoModelForCausalLM
from soup_cli.utils.quest import (
install_mixed_quest,
load_mixed_quest_artifact,
write_metadata,
)
source = _tiny_litellama()
metadata = install_mixed_quest(
source,
activation_scales=_scales(source),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
write_metadata(tmp_path, metadata)
raw = _tiny_litellama()
monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", lambda *args, **kwargs: raw)
with pytest.raises(ValueError, match="config declaration"):
load_mixed_quest_artifact(tmp_path)
def test_hardware_gate_requires_exactly_one_ampere_or_newer_gpu(monkeypatch):
import torch
from soup_cli.utils.quest import validate_cuda_hardware
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index: (8, 6))
monkeypatch.setattr(torch.cuda, "get_device_name", lambda index: "RTX 3090")
assert validate_cuda_hardware() == ("RTX 3090", (8, 6))
monkeypatch.setattr(torch.cuda, "device_count", lambda: 4)
with pytest.raises(RuntimeError, match="exactly one visible"):
validate_cuda_hardware()
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index: (7, 5))
with pytest.raises(RuntimeError, match="Ampere or newer"):
validate_cuda_hardware()
def test_hardware_gate_rejects_cpu(monkeypatch):
import torch
from soup_cli.utils.quest import validate_cuda_hardware
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
with pytest.raises(RuntimeError, match="no CUDA device"):
validate_cuda_hardware()
def test_setup_quest_rejects_every_distributed_route(monkeypatch):
from soup_cli.trainer import stream_setup
from soup_cli.trainer.sft import SFTTrainerWrapper
wrapper = object.__new__(SFTTrainerWrapper)
wrapper.deepspeed_config = None
wrapper.fsdp_config = None
monkeypatch.setattr(stream_setup, "_distributed_launch", lambda: True)
with pytest.raises(ValueError, match="single-GPU"):
wrapper._setup_quest(_calibration_rows())
def test_setup_quest_calibrates_then_installs_and_stores_metadata(monkeypatch):
import soup_cli.utils.quest as quest
from soup_cli.trainer import stream_setup
from soup_cli.trainer.sft import SFTTrainerWrapper
wrapper = object.__new__(SFTTrainerWrapper)
wrapper.deepspeed_config = None
wrapper.fsdp_config = None
wrapper.model = SimpleNamespace(config=SimpleNamespace())
wrapper.config = SimpleNamespace(base="ahxt/LiteLlama-460M-1T")
rows = _calibration_rows()
events = []
metadata = {"route": "sentinel"}
monkeypatch.setattr(stream_setup, "_distributed_launch", lambda: False)
monkeypatch.setattr(quest, "validate_cuda_hardware", lambda: ("RTX 3090", (8, 6)))
monkeypatch.setattr(
quest,
"calibrate_activation_scales",
lambda model, dataset: events.append(("calibrate", model, dataset)) or {"x": 3.0},
)
monkeypatch.setattr(
quest,
"calibration_rows_sha256",
lambda dataset: events.append(("fingerprint", dataset)) or _TEST_CALIBRATION_SHA256,
)
monkeypatch.setattr(
quest,
"install_mixed_quest",
lambda model, *, activation_scales, base_model, calibration_sha256: (
events.append(("install", model, activation_scales, base_model, calibration_sha256))
or metadata
),
)
wrapper._setup_quest(rows)
assert [event[0] for event in events] == [
"fingerprint",
"calibrate",
"install",
]
assert events[0][1] == rows
assert events[0][1] is events[1][2]
assert events[2][2:] == (
{"x": 3.0},
"ahxt/LiteLlama-460M-1T",
_TEST_CALIBRATION_SHA256,
)
assert wrapper._quest_metadata is metadata
assert wrapper.model.config.soup_quest is metadata
def test_transformers_setup_checks_hardware_before_loading(monkeypatch):
import soup_cli.utils.quest as quest
from soup_cli.trainer.sft import SFTTrainerWrapper
wrapper = object.__new__(SFTTrainerWrapper)
monkeypatch.setattr(
quest,
"validate_cuda_hardware",
lambda: (_ for _ in ()).throw(RuntimeError("hardware sentinel")),
)
with pytest.raises(RuntimeError, match="hardware sentinel"):
wrapper._setup_transformers(
SimpleNamespace(base="unused"),
SimpleNamespace(quantization_aware="quest"),
)
def test_quest_never_enters_normal_qat_or_v028_paths(monkeypatch):
import soup_cli.utils.qat as qat
import soup_cli.utils.v028_features as v028_features
from soup_cli.trainer.sft import SFTTrainerWrapper
def unexpected(*args, **kwargs):
raise AssertionError("QuEST must bypass the normal QAT and v0.28 paths")
monkeypatch.setattr(qat, "prepare_model_for_qat", unexpected)
monkeypatch.setattr(v028_features, "apply_v028_speed_memory", unexpected)
wrapper = object.__new__(SFTTrainerWrapper)
original_model = object()
wrapper.model = original_model
wrapper._apply_quantization_aware(SimpleNamespace(quantization_aware="quest"))
assert wrapper.model is original_model
def test_train_validates_resume_and_rewrites_final_metadata(monkeypatch, tmp_path):
import contextlib
import soup_cli.trainer.sft as sft
import soup_cli.utils.quest as quest
from soup_cli.trainer.sft import SFTTrainerWrapper
from soup_cli.utils import ebft_gdpo, peft_wiring
events = []
metadata = {"route": "sentinel"}
class FakeModel:
def named_parameters(self):
return []
class FakeTrainer:
def __init__(self):
self.model = FakeModel()
self.args = SimpleNamespace(fp16=False, bf16=False, should_save=True)
self.state = SimpleNamespace(log_history=[], global_step=9)
def train(self, *, resume_from_checkpoint):
events.append(("train", resume_from_checkpoint))
def save_model(self, output):
events.append(("save", output))
class FakeTokenizer:
def save_pretrained(self, output):
events.append(("tokenizer", output))
wrapper = object.__new__(SFTTrainerWrapper)
wrapper.config = _config()
wrapper._quest_metadata = metadata
wrapper._output_dir = str(tmp_path)
wrapper.trainer = FakeTrainer()
wrapper.tokenizer = FakeTokenizer()
wrapper._training_context = lambda context: contextlib.ExitStack()
wrapper._report_rewind = lambda: None
for name in (
"attach_loraplus_optimizer",
"attach_relora_callback",
"attach_lisa_callback",
"attach_curriculum_callback",
"attach_plugin_callback",
):
monkeypatch.setattr(peft_wiring, name, lambda *args, **kwargs: None)
monkeypatch.setattr(ebft_gdpo, "attach_ebft_compute_loss", lambda *args: None)
monkeypatch.setattr(sft, "align_trainable_dtype_for_fp16", lambda *args, **kwargs: None)
monkeypatch.setattr(
quest,
"validate_resume_metadata",
lambda checkpoint, value, **kwargs: events.append(("resume", checkpoint, value, kwargs)),
)
monkeypatch.setattr(
quest,
"write_metadata",
lambda output, value: events.append(("metadata", output, value)),
)
result = wrapper.train(resume_from_checkpoint="checkpoint-4")
assert events[:5] == [
(
"resume",
"checkpoint-4",
metadata,
# `legacy_base_identity` is the pre-resolved identity of `config.base`
# (#1199 fix 2). This wrapper is built with `object.__new__` and never
# resolved one, so it is None and the callee resolves as before.
{
"legacy_base_model": "ahxt/LiteLlama-460M-1T",
"legacy_base_identity": None,
},
),
("metadata", str(tmp_path), metadata),
("train", "checkpoint-4"),
("save", str(tmp_path)),
("metadata", str(tmp_path), metadata),
]
assert events[-1] == ("tokenizer", str(tmp_path))
assert result["quest_mixed_precision"] is metadata
assert result["total_steps"] == 9
events.clear()
monkeypatch.setattr(
quest,
"validate_resume_metadata",
lambda checkpoint, value, **kwargs: (
_ for _ in ()
).throw(ValueError("resume route mismatch")),
)
with pytest.raises(ValueError, match="resume route mismatch"):
wrapper.train(resume_from_checkpoint="checkpoint-5")
assert events == []
def test_callback_writes_the_same_metadata_into_periodic_checkpoint(tmp_path: Path):
from soup_cli.utils.quest import QuestMetadataCallback, load_metadata
model = _tiny_litellama()
from soup_cli.utils.quest import install_mixed_quest
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
checkpoint = tmp_path / "checkpoint-12"
checkpoint.mkdir()
callback = QuestMetadataCallback(str(tmp_path), metadata)
callback.on_save(
SimpleNamespace(output_dir=str(tmp_path), should_save=True),
SimpleNamespace(global_step=12),
None,
)
assert load_metadata(checkpoint) == metadata
def test_callback_does_not_write_on_non_saving_rank(tmp_path: Path):
from soup_cli.utils.quest import QuestMetadataCallback, install_mixed_quest
model = _tiny_litellama()
metadata = install_mixed_quest(
model,
activation_scales=_scales(model),
base_model="ahxt/LiteLlama-460M-1T",
calibration_sha256=_TEST_CALIBRATION_SHA256,
)
callback = QuestMetadataCallback(str(tmp_path), metadata)
callback.on_save(
SimpleNamespace(output_dir=str(tmp_path), should_save=False),
SimpleNamespace(global_step=12),
None,
)
assert not (tmp_path / "checkpoint-12" / "quest_mixed_precision.json").exists()
def test_heavy_dependencies_stay_lazy():
import ast
source = Path("src/soup_cli/utils/quest.py").read_text(encoding="utf-8")
tree = ast.parse(source)
for node in tree.body:
if isinstance(node, ast.Import):
assert all(
alias.name.split(".")[0] not in {"torch", "transformers"} for alias in node.names
)
elif isinstance(node, ast.ImportFrom):
assert (node.module or "").split(".")[0] not in {"torch", "transformers"}