Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Issue #1162: local QuEST bases use portable, content-bound identities."""
from __future__ import annotations
import json
import shutil
from pathlib import Path
from types import SimpleNamespace
import pytest
from soup_cli.utils.quest import (
EXPECTED_MODULES,
_selected_local_model_files,
build_metadata,
load_metadata,
resolve_base_model_identity,
validate_resume_metadata,
write_metadata,
)
def _base(directory: Path, *, sharded: bool = False) -> Path:
directory.mkdir()
(directory / "config.json").write_text('{"model_type":"llama"}', encoding="utf-8")
if sharded:
(directory / "model.safetensors.index.json").write_text(
json.dumps(
{
"metadata": {"total_size": 2},
"weight_map": {
"a": "model-00001-of-00002.safetensors",
"b": "model-00002-of-00002.safetensors",
},
}
),
encoding="utf-8",
)
(directory / "model-00001-of-00002.safetensors").write_bytes(b"first")
(directory / "model-00002-of-00002.safetensors").write_bytes(b"second")
else:
(directory / "model.safetensors").write_bytes(b"safe-weights")
return directory
def _metadata(base_model: str, *, format_version: int = 2) -> dict:
return build_metadata(
activation_scales={name: 3.0 for name in EXPECTED_MODULES},
base_model=base_model,
calibration_sha256="ab" * 32,
format_version=format_version,
)
def test_hub_id_stays_byte_identical():
assert resolve_base_model_identity("ahxt/LiteLlama-460M-1T") == "ahxt/LiteLlama-460M-1T"
def test_selected_safetensors_matches_real_transformers_loader(tmp_path):
from transformers import AutoModelForCausalLM, LlamaConfig, LlamaForCausalLM
base = tmp_path / "base"
model = LlamaForCausalLM(
LlamaConfig(
hidden_size=128,
intermediate_size=256,
# 24, not 1: #1199 refuses a local base whose config.json declares
# a layer count the measured route cannot cover, so a 1-layer toy
# is no longer a legal local QuEST base. These two tests pin which
# files are selected, for which the layer count is irrelevant.
num_hidden_layers=24,
num_attention_heads=4,
vocab_size=256,
)
)
model.save_pretrained(base, safe_serialization=True)
selected = resolve_base_model_identity(str(base))
(base / "pytorch_model.bin").write_bytes(b"invalid and deliberately unused")
assert isinstance(AutoModelForCausalLM.from_pretrained(base), LlamaForCausalLM)
assert resolve_base_model_identity(str(base)) == selected
def test_sharded_safetensors_matches_real_transformers_loader(tmp_path):
from transformers import AutoModelForCausalLM, LlamaConfig, LlamaForCausalLM
base = tmp_path / "sharded"
model = LlamaForCausalLM(
LlamaConfig(
hidden_size=128,
intermediate_size=256,
# 24, not 1: see the note in the safetensors test above.
num_hidden_layers=24,
num_attention_heads=4,
vocab_size=256,
)
)
model.save_pretrained(base, safe_serialization=True, max_shard_size="250KB")
assert (base / "model.safetensors.index.json").is_file()
first = resolve_base_model_identity(str(base))
assert isinstance(AutoModelForCausalLM.from_pretrained(base), LlamaForCausalLM)
(base / "unselected.safetensors").write_bytes(b"not in the index")
assert resolve_base_model_identity(str(base)) == first
def test_local_identity_survives_relocation_and_ignores_unread_files(tmp_path):
original = _base(tmp_path / "original")
first = resolve_base_model_identity(str(original))
assert first.startswith("local-sha256:")
relocated = tmp_path / "elsewhere" / "model"
relocated.parent.mkdir()
shutil.copytree(original, relocated)
(relocated / "README.md").write_text("not loaded", encoding="utf-8")
(relocated / "pytorch_model.bin").write_bytes(b"not selected when safetensors exists")
assert resolve_base_model_identity(str(relocated)) == first
checkpoint = tmp_path / "checkpoint"
write_metadata(checkpoint, _metadata(first))
validate_resume_metadata(checkpoint, _metadata(resolve_base_model_identity(str(relocated))))
assert json.dumps(str(original))[1:-1] not in (
checkpoint / "quest_mixed_precision.json"
).read_text(encoding="utf-8")
def test_setup_writes_only_the_local_digest_into_both_artifact_declarations(tmp_path, monkeypatch):
from transformers import LlamaConfig
import soup_cli.utils.quest as quest
from soup_cli.trainer import stream_setup
from soup_cli.trainer.sft import SFTTrainerWrapper
# Keep the fixture's source outside the artifact directory, as in a real run.
source = tmp_path / "private"
source.mkdir()
base = _base(source / "base")
wrapper = object.__new__(SFTTrainerWrapper)
wrapper.config = SimpleNamespace(base=str(base))
wrapper.model = SimpleNamespace(config=LlamaConfig())
wrapper.deepspeed_config = None
wrapper.fsdp_config = None
wrapper._quest_base_identity_before = resolve_base_model_identity(str(base))
monkeypatch.setattr(stream_setup, "_distributed_launch", lambda: False)
monkeypatch.setattr(quest, "validate_cuda_hardware", lambda: ("RTX 3090", (8, 6)))
monkeypatch.setattr(quest, "calibration_rows_sha256", lambda rows: "ab" * 32)
monkeypatch.setattr(
quest,
"calibrate_activation_scales",
lambda model, rows: {name: 3.0 for name in EXPECTED_MODULES},
)
monkeypatch.setattr(
quest,
"install_mixed_quest",
lambda model, *, activation_scales, base_model, calibration_sha256: build_metadata(
activation_scales=activation_scales,
base_model=base_model,
calibration_sha256=calibration_sha256,
),
)
wrapper._setup_quest([{"input_ids": [1]}] * 32)
assert wrapper._quest_metadata["base_model"].startswith("local-sha256:")
assert wrapper.model.config.soup_quest == wrapper._quest_metadata
escaped_base = json.dumps(str(base))[1:-1]
assert escaped_base not in json.dumps(wrapper.model.config.soup_quest)
artifact = tmp_path / "artifact"
wrapper.model.config.save_pretrained(artifact)
write_metadata(artifact, wrapper._quest_metadata)
assert escaped_base not in (artifact / "config.json").read_text(encoding="utf-8")
assert escaped_base not in (artifact / "quest_mixed_precision.json").read_text(encoding="utf-8")
(base / "model.safetensors").write_bytes(b"changed-after-model-load")
with pytest.raises(ValueError, match="changed while the model was loading"):
wrapper._setup_quest([{"input_ids": [1]}] * 32)
def test_content_change_at_the_same_path_is_refused(tmp_path):
base = _base(tmp_path / "base")
checkpoint = tmp_path / "checkpoint"
write_metadata(checkpoint, _metadata(resolve_base_model_identity(str(base))))
(base / "model.safetensors").write_bytes(b"changed-weights")
with pytest.raises(ValueError, match="does not match"):
validate_resume_metadata(checkpoint, _metadata(resolve_base_model_identity(str(base))))
def test_config_change_at_the_same_path_is_refused(tmp_path):
base = _base(tmp_path / "base")
first = resolve_base_model_identity(str(base))
(base / "config.json").write_text('{"model_type":"llama","hidden_size":256}')
assert resolve_base_model_identity(str(base)) != first
def test_generation_config_change_at_the_same_path_is_refused(tmp_path):
base = _base(tmp_path / "base")
(base / "generation_config.json").write_text('{"max_length":32}', encoding="utf-8")
first = resolve_base_model_identity(str(base))
(base / "generation_config.json").write_text('{"max_length":64}', encoding="utf-8")
assert resolve_base_model_identity(str(base)) != first
def test_tokenizer_change_at_the_same_path_is_refused(tmp_path):
base = _base(tmp_path / "base")
(base / "tokenizer.json").write_text('{"version":"1.0"}', encoding="utf-8")
first = resolve_base_model_identity(str(base))
(base / "tokenizer.json").write_text('{"version":"1.1"}', encoding="utf-8")
assert resolve_base_model_identity(str(base)) != first
def test_additional_chat_template_change_is_refused_after_relocation(tmp_path):
base = _base(tmp_path / "base")
templates = base / "additional_chat_templates"
templates.mkdir()
(templates / "tool_use.jinja").write_text("[A]{{ message }}", encoding="utf-8")
first = resolve_base_model_identity(str(base))
relocated = tmp_path / "relocated"
shutil.copytree(base, relocated)
assert resolve_base_model_identity(str(relocated)) == first
(relocated / "additional_chat_templates" / "tool_use.jinja").write_text(
"[B-CHANGED]{{ message }}", encoding="utf-8"
)
changed = resolve_base_model_identity(str(relocated))
assert changed != first
(relocated / "additional_chat_templates" / "analysis.jinja").write_text(
"{{ message }}", encoding="utf-8"
)
assert resolve_base_model_identity(str(relocated)) != changed
def test_renaming_an_additional_chat_template_changes_the_identity(tmp_path):
base = _base(tmp_path / "base")
templates = base / "additional_chat_templates"
templates.mkdir()
(templates / "tool_use.jinja").write_text("[A]{{ message }}", encoding="utf-8")
first = resolve_base_model_identity(str(base))
(templates / "tool_use.jinja").rename(templates / "rag.jinja")
assert resolve_base_model_identity(str(base)) != first
def test_additional_chat_templates_have_platform_independent_order(tmp_path):
base = _base(tmp_path / "base")
templates = base / "additional_chat_templates"
templates.mkdir()
for name in ("Tool.jinja", "analysis.jinja"):
(templates / name).write_text("{{ message }}", encoding="utf-8")
names = [
name
for name, _ in _selected_local_model_files(base)
if name.startswith("additional_chat_templates/")
]
assert names == sorted(names)
def test_additional_chat_template_identity_tracks_real_tokenizer_rendering(tmp_path):
from tokenizers import Tokenizer, models
from transformers import AutoTokenizer, PreTrainedTokenizerFast
base = _base(tmp_path / "base")
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]")),
unk_token="[UNK]",
)
tokenizer.chat_template = "[DEFAULT]{{ messages[0]['content'] }}"
tokenizer.save_pretrained(base)
templates = base / "additional_chat_templates"
templates.mkdir()
tool_use = templates / "tool_use.jinja"
tool_use.write_text("[A]{{ messages[0]['content'] }}", encoding="utf-8")
messages = [{"role": "user", "content": "hi"}]
tools = [{"type": "function", "function": {"name": "noop", "parameters": {}}}]
first = resolve_base_model_identity(str(base))
assert (
AutoTokenizer.from_pretrained(base).apply_chat_template(
messages, tools=tools, tokenize=False
)
== "[A]hi"
)
tool_use.write_text("[B-CHANGED]{{ messages[0]['content'] }}", encoding="utf-8")
assert (
AutoTokenizer.from_pretrained(base).apply_chat_template(
messages, tools=tools, tokenize=False
)
== "[B-CHANGED]hi"
)
assert resolve_base_model_identity(str(base)) != first
def test_custom_model_with_untracked_code_is_refused(tmp_path):
base = _base(tmp_path / "base")
(base / "config.json").write_text(
'{"model_type":"llama","auto_map":{"AutoModelForCausalLM":"custom.Model"}}',
encoding="utf-8",
)
with pytest.raises(ValueError, match="standard model config"):
resolve_base_model_identity(str(base))
def test_custom_tokenizer_with_untracked_code_is_refused(tmp_path):
base = _base(tmp_path / "base")
(base / "tokenizer_config.json").write_text(
'{"auto_map":{"AutoTokenizer":["custom.Tokenizer",null]}}', encoding="utf-8"
)
with pytest.raises(ValueError, match="standard tokenizer"):
resolve_base_model_identity(str(base))
def test_versioned_fast_tokenizer_files_are_refused(tmp_path):
base = _base(tmp_path / "base")
(base / "tokenizer_config.json").write_text(
'{"fast_tokenizer_files":["tokenizer.4.0.0.json"]}', encoding="utf-8"
)
(base / "tokenizer.4.0.0.json").write_text('{"version":"1.0"}', encoding="utf-8")
with pytest.raises(ValueError, match="fast_tokenizer_files"):
resolve_base_model_identity(str(base))
def test_sharded_identity_uses_index_and_only_referenced_shards(tmp_path):
base = _base(tmp_path / "base", sharded=True)
first = resolve_base_model_identity(str(base))
(base / "unused.safetensors").write_bytes(b"ignored")
assert resolve_base_model_identity(str(base)) == first
(base / "model-00002-of-00002.safetensors").write_bytes(b"changed")
assert resolve_base_model_identity(str(base)) != first
(base / "model.safetensors.index.json").write_text(
json.dumps({"metadata": {}, "weight_map": {"a": "../outside.safetensors"}}),
encoding="utf-8",
)
with pytest.raises(ValueError, match="shard"):
resolve_base_model_identity(str(base))
@pytest.mark.parametrize(
"shard_name",
["../outside.safetensors", "/outside.safetensors", "C:/outside.safetensors", "nested\\file"],
)
def test_shard_index_rejects_paths_outside_the_selected_base(tmp_path, shard_name):
base = _base(tmp_path / "base", sharded=True)
(base / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a": shard_name}}), encoding="utf-8"
)
with pytest.raises(ValueError, match="shard name"):
resolve_base_model_identity(str(base))
def test_shard_index_size_and_count_limits_are_enforced(tmp_path):
base = _base(tmp_path / "base", sharded=True)
index = base / "model.safetensors.index.json"
index.write_bytes(b" " * (4 * 1024 * 1024 + 1))
with pytest.raises(ValueError, match="exceeds 4 MiB"):
resolve_base_model_identity(str(base))
index.write_text(
json.dumps({"weight_map": {str(i): f"shard-{i}.safetensors" for i in range(1025)}}),
encoding="utf-8",
)
with pytest.raises(ValueError, match="too many shards"):
resolve_base_model_identity(str(base))
def test_changed_during_hash_is_refused(tmp_path, monkeypatch):
import soup_cli.utils.quest as quest
weights = _base(tmp_path / "base") / "model.safetensors"
original_fstat = quest.os.fstat
reads = 0
def changed_fstat(fd):
nonlocal reads
result = original_fstat(fd)
reads += 1
if reads == 2:
return SimpleNamespace(
st_dev=result.st_dev,
st_ino=result.st_ino,
st_size=result.st_size + 1,
st_mtime_ns=result.st_mtime_ns,
)
return result
monkeypatch.setattr(quest.os, "fstat", changed_fstat)
with pytest.raises(ValueError, match="changed during fingerprinting"):
quest._hash_local_file(weights)
def test_replaced_path_during_hash_is_refused(tmp_path, monkeypatch):
import soup_cli.utils.quest as quest
weights = _base(tmp_path / "base") / "model.safetensors"
original_stat = Path.stat
def replaced_stat(path, *args, **kwargs):
result = original_stat(path, *args, **kwargs)
if path != weights:
return result
return SimpleNamespace(
st_dev=result.st_dev,
st_ino=result.st_ino + 1,
st_size=result.st_size,
st_mtime_ns=result.st_mtime_ns,
)
monkeypatch.setattr(Path, "stat", replaced_stat)
with pytest.raises(ValueError, match="changed during fingerprinting"):
quest._hash_local_file(weights)
def test_base_move_during_hash_is_refused(tmp_path, monkeypatch):
import soup_cli.utils.quest as quest
base = _base(tmp_path / "base")
original_realpath = quest.os.path.realpath
reads = 0
def changed_realpath(path):
nonlocal reads
reads += 1
result = original_realpath(path)
return result + "-moved" if reads == 2 else result
monkeypatch.setattr(quest.os.path, "realpath", changed_realpath)
with pytest.raises(ValueError, match="moved during fingerprinting"):
resolve_base_model_identity(str(base))
def test_bin_fallback_when_no_safetensors_exist(tmp_path):
base = tmp_path / "base"
base.mkdir()
(base / "config.json").write_text('{"model_type":"llama"}', encoding="utf-8")
(base / "pytorch_model.bin").write_bytes(b"torch-weights")
first = resolve_base_model_identity(str(base))
(base / "pytorch_model.bin").write_bytes(b"different")
assert resolve_base_model_identity(str(base)) != first
def test_v1_sidecar_remains_readable_with_its_original_path_contract(tmp_path):
original = _base(tmp_path / "original")
historical = _metadata(str(original), format_version=1)
checkpoint = tmp_path / "checkpoint"
write_metadata(checkpoint, historical)
assert load_metadata(checkpoint) == historical
current = _metadata(resolve_base_model_identity(str(original)))
validate_resume_metadata(checkpoint, current, legacy_base_model=str(original))
with pytest.raises(ValueError, match="set base: to the exact path or Hub ID"):
validate_resume_metadata(checkpoint, current, legacy_base_model=str(tmp_path / "moved"))
with pytest.raises(ValueError, match="original base model reference"):
validate_resume_metadata(checkpoint, current)
def test_v1_and_v2_do_not_silently_compare_as_the_same_format(tmp_path):
base = _base(tmp_path / "base")
historical = _metadata(str(base), format_version=1)
checkpoint = tmp_path / "checkpoint"
write_metadata(checkpoint, historical)
current = _metadata(resolve_base_model_identity(str(base)))
current["activation_scales"][EXPECTED_MODULES[0]] = 4.0
with pytest.raises(ValueError, match="does not match"):
validate_resume_metadata(checkpoint, current, legacy_base_model=str(base))
def test_v1_reference_cannot_override_a_different_current_base(tmp_path):
original = _base(tmp_path / "original")
changed = _base(tmp_path / "changed")
(changed / "model.safetensors").write_bytes(b"different-weights")
checkpoint = tmp_path / "checkpoint"
write_metadata(checkpoint, _metadata(str(original), format_version=1))
current = _metadata(resolve_base_model_identity(str(changed)))
with pytest.raises(ValueError, match="does not match current identity"):
validate_resume_metadata(checkpoint, current, legacy_base_model=str(original))
@pytest.mark.parametrize("absolute", ["/home/alice/models/base", "C:\\Users\\alice\\models\\base"])
def test_v2_rejects_raw_absolute_paths(absolute):
with pytest.raises(ValueError, match="base_model"):
_metadata(absolute)
@pytest.mark.parametrize("digest", ["0" * 63, "0" * 63 + "g", "A" * 64])
def test_v2_rejects_malformed_local_digest(digest):
with pytest.raises(ValueError, match="local base_model digest"):
_metadata(f"local-sha256:{digest}")