Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for soup adapters check: pre-flight health audit (Issue #1721).
Acceptance coverage:
1. Healthy adapter: ||ΔW||_F > 0, live fraction 1.0, verdict `alive`, exit 0.
2. Synthetic adapter with all-zero lora_B: verdict `inactive: all lora_B layers are zero`, exit 2.
3. Synthetic adapter with .inner. wrapper keys: verdict `inactive: leaked .inner. keys`, exit 2.
4. Partially zero lora_B: live fraction < 1.0, verdict `inactive: lora_B layers are zero`, exit 2.
5. Non-finite weights (NaN/Inf): verdict `inactive: non-finite weights detected`, exit 2.
6. JSON mode: --json emits complete document matching terminal metrics.
7. Pathguard: outside-cwd and symlink rejections exit 1.
8. Mathematical identity: trace formula matches explicit (B @ A) Frobenius norm bit-for-bit.
9. rsLoRA scaling factor: alpha / sqrt(r).
10. Transposed Conv1D shapes: trace identity bit-exact.
11. Incomplete LoRA pairs: orphaned weights report inactive.
12. Standalone non-LoRA tensors: zero bias not flagged as zero lora_B.
13. Strict RFC 8259 JSON: null serialization under NaN weights.
14. Boolean rank guard: booleans in adapter_config.json safely ignored.
"""
from __future__ import annotations
import json
import math
import re
from pathlib import Path
import numpy as np
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app as soup_app
from soup_cli.utils import adapter_diff as adapter_diff_module
from soup_cli.utils.adapter_check import (
AdapterCheckReport,
check_adapter,
render_check_terminal,
)
from tests.conftest import strip_ansi
runner = CliRunner()
Q_LORA_A = "base_model.model.layers.0.self_attn.q_proj.lora_A.weight"
Q_LORA_B = "base_model.model.layers.0.self_attn.q_proj.lora_B.weight"
V_LORA_A = "base_model.model.layers.0.self_attn.v_proj.lora_A.weight"
V_LORA_B = "base_model.model.layers.0.self_attn.v_proj.lora_B.weight"
INNER_LORA_A = "base_model.model.layers.0.inner.self_attn.q_proj.lora_A.weight"
INNER_LORA_B = "base_model.model.layers.0.inner.self_attn.q_proj.lora_B.weight"
def _write_synthetic_adapter(
dir_path: Path,
weights: dict[str, np.ndarray],
r: int = 8,
alpha: float = 16.0,
extra_cfg: dict | None = None,
) -> None:
"""Write synthetic adapter directory with safetensors and adapter_config.json."""
pytest.importorskip("safetensors")
from safetensors.numpy import save_file
dir_path.mkdir(parents=True, exist_ok=True)
save_file(weights, str(dir_path / "adapter_model.safetensors"))
cfg = {
"peft_type": "LORA",
"r": r,
"lora_alpha": alpha,
"target_modules": ["q_proj", "v_proj"],
}
if extra_cfg:
cfg.update(extra_cfg)
(dir_path / "adapter_config.json").write_text(json.dumps(cfg), encoding="utf-8")
# ---------- Core Acceptance Tests ----------
def test_healthy_adapter_is_alive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""A healthy adapter with non-zero updates reports 'alive' and exits 0."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "healthy"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: 0.5 * np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("healthy")
assert report.verdict == "alive"
assert report.reason is None
assert report.verdict_line == "alive"
assert report.total_frobenius > 0.0
assert report.live_fraction == 1.0
assert report.live_layers == 1
assert report.total_layers == 1
assert len(report.all_zero_lora_b_layers) == 0
assert len(report.inner_keys) == 0
# CLI test
res = runner.invoke(soup_app, ["adapters", "check", "healthy"])
assert res.exit_code == 0
lines = res.stdout.strip().splitlines()
assert lines[-1] == "alive"
assert "Live fraction: 100.0%" in strip_ansi(res.stdout)
def test_all_zero_lora_b_reports_inactive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""An untrained adapter with all-zero lora_B reports inactive and exits 2."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "zero_b"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: np.zeros((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("zero_b")
assert report.verdict == "inactive"
assert report.reason == "all lora_B layers are zero (1/1)"
assert report.verdict_line == "inactive: all lora_B layers are zero (1/1)"
assert report.total_frobenius == 0.0
assert report.live_fraction == 0.0
assert report.live_layers == 0
assert len(report.all_zero_lora_b_layers) == 1
# CLI test
res = runner.invoke(soup_app, ["adapters", "check", "zero_b"])
assert res.exit_code == 2
lines = res.stdout.strip().splitlines()
assert lines[-1] == "inactive: all lora_B layers are zero (1/1)"
def test_inner_namespace_leak_reports_inactive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""An adapter with leaked .inner. wrapper keys reports inactive and exits 2."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "inner_leak"
weights = {
INNER_LORA_A: np.ones((8, 16), dtype=np.float32),
INNER_LORA_B: 0.5 * np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("inner_leak")
assert report.verdict == "inactive"
assert "leaked .inner. keys detected" in (report.reason or "")
assert len(report.inner_keys) == 2
assert INNER_LORA_A in report.inner_keys
# CLI test
res = runner.invoke(soup_app, ["adapters", "check", "inner_leak"])
assert res.exit_code == 2
lines = res.stdout.strip().splitlines()
assert lines[-1].startswith("inactive: leaked .inner. keys detected")
def test_partially_zero_lora_b(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""An adapter with some zero and some live lora_B layers reports inactive with layer ratio."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "partial"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: 0.5 * np.ones((16, 8), dtype=np.float32),
V_LORA_A: np.ones((8, 16), dtype=np.float32),
V_LORA_B: np.zeros((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("partial")
assert report.verdict == "inactive"
assert report.reason == "lora_B layers are zero (1/2)"
assert report.live_fraction == 0.5
assert report.live_layers == 1
assert report.total_layers == 2
assert len(report.all_zero_lora_b_layers) == 1
res = runner.invoke(soup_app, ["adapters", "check", "partial"])
assert res.exit_code == 2
lines = res.stdout.strip().splitlines()
assert lines[-1] == "inactive: lora_B layers are zero (1/2)"
def test_non_finite_weights_rejected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""NaN/Inf in weights causes inactive classification."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "nan_adapter"
nan_b = np.ones((16, 8), dtype=np.float32)
nan_b[0, 0] = np.nan
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: nan_b,
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("nan_adapter")
assert report.verdict == "inactive"
assert report.reason == "non-finite weights detected (NaN/Inf)"
assert math.isnan(report.total_frobenius)
res = runner.invoke(soup_app, ["adapters", "check", "nan_adapter"])
assert res.exit_code == 2
assert "inactive: non-finite weights detected" in res.stdout
def test_json_output_mode(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""--json emits valid parseable JSON document matching the terminal metrics."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "healthy"
weights = {
Q_LORA_A: np.ones((4, 8), dtype=np.float32),
Q_LORA_B: np.ones((8, 4), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=4, alpha=8.0)
res = runner.invoke(soup_app, ["adapters", "check", "healthy", "--json"])
assert res.exit_code == 0
doc = json.loads(res.stdout)
assert doc["verdict"] == "alive"
assert doc["verdict_line"] == "alive"
assert doc["live_fraction"] == 1.0
assert doc["live_layers"] == 1
assert doc["total_layers"] == 1
assert doc["total_frobenius"] > 0.0
assert "inner_keys" in doc and doc["inner_keys"] == []
assert "all_zero_lora_b_layers" in doc and doc["all_zero_lora_b_layers"] == []
assert "reason" in doc and doc["reason"] is None
assert len(doc["per_layer"]) == 1
# ---------- Security & Path Validation ----------
def test_outside_cwd_rejected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""Adapters outside cwd must be refused by the path guard."""
work_dir = tmp_path / "work"
work_dir.mkdir()
outside_dir = tmp_path / "outside_adapter"
outside_dir.mkdir()
monkeypatch.chdir(work_dir)
with pytest.raises(ValueError, match="must stay under cwd"):
check_adapter(str(outside_dir))
res = runner.invoke(soup_app, ["adapters", "check", str(outside_dir)])
assert res.exit_code == 1
assert "refused" in res.output.lower()
assert "not found" not in res.output.lower()
def test_missing_adapter_rejected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""Non-existent adapter directory exits 1."""
monkeypatch.chdir(tmp_path)
with pytest.raises(FileNotFoundError):
check_adapter("does_not_exist")
res = runner.invoke(soup_app, ["adapters", "check", "does_not_exist"])
assert res.exit_code == 1
assert "does_not_exist" in res.output or "not found" in res.output.lower()
# ---------- Multi-Layer RSS & Conv1D Transposed Contraction ----------
def test_two_layer_and_conv1d_transposed_adapter(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Assert total_frobenius == sqrt(sum of squares) and Conv1D transposed contraction."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "two_layer_conv1d"
# Layer 0: standard Linear (q_proj), r=8, out=16, in=32 -> B is (16, 8), A is (8, 32)
rng = np.random.default_rng(123)
q_b = rng.standard_normal((16, 8)).astype(np.float32)
q_a = rng.standard_normal((8, 32)).astype(np.float32)
# Layer 1: transposed Conv1D (v_proj), r=8, in=32, out=16 -> B is (8, 16), A is (32, 8)
v_b = rng.standard_normal((8, 16)).astype(np.float32)
v_a = rng.standard_normal((32, 8)).astype(np.float32)
weights = {
Q_LORA_A: q_a,
Q_LORA_B: q_b,
V_LORA_A: v_a,
V_LORA_B: v_b,
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("two_layer_conv1d")
assert report.verdict == "alive"
assert report.total_layers == 2
assert report.live_layers == 2
# Direct computation: scaling = 16.0 / 8 = 2.0
delta0 = 2.0 * np.matmul(q_b.astype(np.float64), q_a.astype(np.float64))
expected_fro0 = float(np.linalg.norm(delta0))
delta1 = 2.0 * np.matmul(v_a.astype(np.float64), v_b.astype(np.float64))
expected_fro1 = float(np.linalg.norm(delta1))
assert math.isclose(report.per_layer[0].frobenius, expected_fro0, rel_tol=1e-12)
assert math.isclose(report.per_layer[1].frobenius, expected_fro1, rel_tol=1e-12)
# Total frobenius must equal Root-Sum-of-Squares (RSS), not simple linear sum
expected_total = math.sqrt(expected_fro0**2 + expected_fro1**2)
assert math.isclose(report.total_frobenius, expected_total, rel_tol=1e-12)
# ---------- Hardening & Invariant Defenses ----------
def test_rslora_scaling_factor(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""When use_rslora is True, scaling factor must be alpha / sqrt(r)."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "rslora"
r = 16
alpha = 32.0
# Expected scaling: 32 / sqrt(16) = 32 / 4 = 8.0
weights = {
Q_LORA_A: np.ones((16, 8), dtype=np.float32),
Q_LORA_B: np.ones((8, 16), dtype=np.float32),
}
_write_synthetic_adapter(
adapter_dir, weights, r=r, alpha=alpha, extra_cfg={"use_rslora": True}
)
report = check_adapter("rslora")
assert report.verdict == "alive"
# Direct computation: 8.0 * ||B @ A||_F
b_mat = np.ones((8, 16), dtype=np.float64)
a_mat = np.ones((16, 8), dtype=np.float64)
expected_fro = 8.0 * float(np.sqrt(np.sum(np.matmul(b_mat, a_mat) ** 2)))
assert math.isclose(report.total_frobenius, expected_fro, rel_tol=1e-6)
def test_incomplete_lora_pair_reported_inactive(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""An adapter with orphaned LoRA weights (missing A or missing B) reports inactive."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "orphan"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
# Q_LORA_B is intentionally missing
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("orphan")
assert report.verdict == "inactive"
assert "incomplete LoRA pairs detected" in (report.reason or "")
assert len(report.orphaned_layers) == 1
res = runner.invoke(soup_app, ["adapters", "check", "orphan"])
assert res.exit_code == 2
assert "inactive: incomplete LoRA pairs detected" in res.stdout
def test_standalone_tensor_not_flagged_as_lora_b(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Zero-valued standalone non-LoRA tensors must not be counted as all-zero lora_B layers."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "with_bias"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: 0.5 * np.ones((16, 8), dtype=np.float32),
"base_model.model.custom_bias": np.zeros((16,), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("with_bias")
assert report.verdict == "alive"
assert len(report.all_zero_lora_b_layers) == 0
assert report.total_layers == 1
assert report.live_fraction == 1.0
assert len(report.standalone_tensors) == 1
def test_adapter_with_only_standalone_tensors_reports_no_lora(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""An adapter with only saved heads / biases and 0 LoRA pairs reports no lora weights."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "only_heads"
weights = {
"base_model.model.lm_head.weight": np.ones((32, 16), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("only_heads")
assert report.verdict == "inactive"
assert report.reason == "no lora weights found"
assert report.total_layers == 0
assert len(report.standalone_tensors) == 1
def test_json_mode_rfc8259_strict_under_nan(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Under NaN weights, render_check_json produces RFC 8259 compliant JSON with nulls."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "nan_adapter"
nan_b = np.ones((16, 8), dtype=np.float32)
nan_b[0, 0] = np.nan
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: nan_b,
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
res = runner.invoke(soup_app, ["adapters", "check", "nan_adapter", "--json"])
assert res.exit_code == 2
def reject_constant(c):
raise ValueError(f"Non-compliant JSON token: {c}")
doc = json.loads(res.stdout, parse_constant=reject_constant)
assert doc["verdict"] == "inactive"
assert doc["reason"] == "non-finite weights detected (NaN/Inf)"
assert doc["total_frobenius"] is None
assert "inner_keys" in doc
assert "all_zero_lora_b_layers" in doc
assert '"total_frobenius": null' in strip_ansi(res.stdout)
def test_boolean_rank_in_config_not_coerced(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Boolean 'r': true in adapter_config.json must not be coerced to integer 1."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "bool_cfg"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: 0.5 * np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(
adapter_dir, weights, r=8, alpha=16.0, extra_cfg={"r": True}
)
report = check_adapter("bool_cfg")
# Rank is invalid (bool), so fallback scaling 1.0 is used rather than 16/1 = 16.0
b_mat = 0.5 * np.ones((16, 8), dtype=np.float64)
a_mat = np.ones((8, 16), dtype=np.float64)
expected_fro = 1.0 * float(np.sqrt(np.sum(np.matmul(b_mat, a_mat) ** 2)))
assert math.isclose(report.total_frobenius, expected_fro, rel_tol=1e-6)
def test_lora_shape_mismatch_reported_inactive(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Mismatched A and B matrix shapes report inactive with shape mismatch reason."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "bad_shapes"
# Incompatible shapes: A is (7, 13), B is (11, 19)
weights = {
Q_LORA_A: np.ones((7, 13), dtype=np.float32),
Q_LORA_B: np.ones((11, 19), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
report = check_adapter("bad_shapes")
assert report.verdict == "inactive"
assert "shape mismatch in LoRA projections" in (report.reason or "")
assert len(report.shape_mismatches) == 1
assert math.isnan(report.total_frobenius)
res = runner.invoke(soup_app, ["adapters", "check", "bad_shapes"])
assert res.exit_code == 2
assert "inactive: shape mismatch in LoRA projections" in res.stdout
# ---------- Mutation Resistance Teeth ----------
def test_mutation_zero_b_detection_teeth():
"""Verify that if all-zero B check is removed, healthy and zero-B adapters would conflate."""
# Control: zero B must report is_b_zero True
zero_b = np.zeros((16, 8), dtype=np.float64)
assert bool(np.all(zero_b == 0.0)) is True
# Perturbed: even a 1e-7 perturbation must report is_b_zero False
live_b = zero_b.copy()
live_b[0, 0] = 1e-7
assert bool(np.all(live_b == 0.0)) is False
def test_corrupt_adapter_config_json_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""Corrupted adapter_config.json must raise ValueError and exit 1."""
pytest.importorskip("safetensors")
from safetensors.numpy import save_file
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "corrupt_cfg"
adapter_dir.mkdir(parents=True, exist_ok=True)
weights = {
Q_LORA_A: np.ones((16, 8), dtype=np.float32),
Q_LORA_B: np.ones((8, 16), dtype=np.float32),
}
save_file(weights, str(adapter_dir / "adapter_model.safetensors"))
(adapter_dir / "adapter_config.json").write_text("{\"r\": 8, invalid_json", encoding="utf-8")
with pytest.raises(ValueError, match="unreadable or invalid JSON"):
check_adapter("corrupt_cfg")
res = runner.invoke(soup_app, ["adapters", "check", "corrupt_cfg"])
assert res.exit_code == 1
def test_non_dict_adapter_config_json_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""Non-dict JSON in adapter_config.json must raise ValueError and exit 1."""
pytest.importorskip("safetensors")
from safetensors.numpy import save_file
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "nondict_cfg"
adapter_dir.mkdir(parents=True, exist_ok=True)
weights = {
Q_LORA_A: np.ones((16, 8), dtype=np.float32),
Q_LORA_B: np.ones((8, 16), dtype=np.float32),
}
save_file(weights, str(adapter_dir / "adapter_model.safetensors"))
(adapter_dir / "adapter_config.json").write_text("[1, 2, 3]", encoding="utf-8")
with pytest.raises(ValueError, match="expected JSON object"):
check_adapter("nondict_cfg")
res = runner.invoke(soup_app, ["adapters", "check", "nondict_cfg"])
assert res.exit_code == 1
def test_fp16_float64_accumulation_precision(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""FP16 inputs must accumulate in float64 without float32 precision loss (rel_tol=1e-12)."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "fp16_prec"
rng = np.random.default_rng(42)
b_fp16 = (rng.standard_normal((512, 16)) * 1000.0).astype(np.float16)
a_fp16 = (rng.standard_normal((16, 512)) * 1000.0).astype(np.float16)
weights = {
Q_LORA_A: a_fp16,
Q_LORA_B: b_fp16,
}
_write_synthetic_adapter(adapter_dir, weights, r=16, alpha=16.0)
report = check_adapter("fp16_prec")
assert report.verdict == "alive"
b64 = b_fp16.astype(np.float64)
a64 = a_fp16.astype(np.float64)
expected_fro = float(np.linalg.norm(b64 @ a64))
assert math.isclose(report.total_frobenius, expected_fro, rel_tol=1e-12)
@pytest.mark.requires_symlink
def test_symlinked_components_refused(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Symlinked adapter dir, adapter_model.safetensors, and adapter_config.json each exit 1."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
real_dir = tmp_path / "real_adapter"
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(real_dir, weights, r=8, alpha=16.0)
# 1. Symlinked directory
symlink_dir = tmp_path / "symlink_dir"
symlink_dir.symlink_to(real_dir, target_is_directory=True)
res1 = runner.invoke(soup_app, ["adapters", "check", "symlink_dir"])
assert res1.exit_code == 1
assert "must not be a symlink" in strip_ansi(res1.output)
# 2. Symlinked adapter_model.safetensors
adapter_symlink_weights = tmp_path / "symlink_weights"
adapter_symlink_weights.mkdir()
(adapter_symlink_weights / "adapter_config.json").write_text(
(real_dir / "adapter_config.json").read_text(encoding="utf-8"), encoding="utf-8"
)
(adapter_symlink_weights / "adapter_model.safetensors").symlink_to(
real_dir / "adapter_model.safetensors"
)
res2 = runner.invoke(soup_app, ["adapters", "check", "symlink_weights"])
assert res2.exit_code == 1
assert "must not be a symlink" in strip_ansi(res2.output)
# 3. Symlinked adapter_config.json
adapter_symlink_cfg = tmp_path / "symlink_cfg"
adapter_symlink_cfg.mkdir()
(adapter_symlink_cfg / "adapter_model.safetensors").write_bytes(
(real_dir / "adapter_model.safetensors").read_bytes()
)
(adapter_symlink_cfg / "adapter_config.json").symlink_to(
real_dir / "adapter_config.json"
)
res3 = runner.invoke(soup_app, ["adapters", "check", "symlink_cfg"])
assert res3.exit_code == 1
assert "must not be a symlink" in strip_ansi(res3.output)
def test_standard_lora_scaling_factor(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Standard LoRA scaling must compute scaling = alpha / r (e.g. 32 / 8 = 4.0)."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "standard_scaling"
r = 8
alpha = 32.0
weights = {
Q_LORA_A: np.ones((8, 16), dtype=np.float32),
Q_LORA_B: np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=r, alpha=alpha)
report = check_adapter("standard_scaling")
assert report.verdict == "alive"
b_mat = np.ones((16, 8), dtype=np.float64)
a_mat = np.ones((8, 16), dtype=np.float64)
expected_fro = 4.0 * float(np.linalg.norm(b_mat @ a_mat))
assert math.isclose(report.total_frobenius, expected_fro, rel_tol=1e-6)
def test_terminal_injection_control_bytes_stripped(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Control and ANSI escape sequences in tensor names must be stripped in terminal output."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "esc_adapter"
esc_key = "base_model.model.layers.0.inner.\x1b[2Jself_attn.q_proj.lora_A.weight"
weights = {
esc_key: np.ones((8, 16), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
res = runner.invoke(soup_app, ["adapters", "check", "esc_adapter"])
assert chr(27) not in res.stdout
assert "\x1b" not in res.stdout
def test_zero_lora_a_is_inactive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
"""When lora_A is all zeros, adapter is inactive with reason 'total ||ΔW||_F is 0.0'."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
weights = {
Q_LORA_A: np.zeros((8, 16), dtype=np.float32),
Q_LORA_B: np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(tmp_path / "za", weights, r=8, alpha=16.0)
report = check_adapter("za")
assert report.verdict == "inactive"
assert report.reason == "total ||ΔW||_F is 0.0"
assert report.verdict_line == "inactive: total ||ΔW||_F is 0.0"
assert report.total_frobenius == 0.0
assert report.live_fraction == 0.0
assert report.live_layers == 0
assert len(report.all_zero_lora_b_layers) == 0
res = runner.invoke(soup_app, ["adapters", "check", "za"])
assert res.exit_code == 2
out = strip_ansi(res.stdout)
assert out.strip().splitlines()[-1].startswith("inactive: total ||")
assert "inactive: total ||ΔW||_F is 0.0" in out
assert "Total ||ΔW||_F: 0.000000" in out
assert "Live fraction: 0.0% (0/1 layers)" in out
def test_terminal_sanitization_adapter_path():
"""Adapter path carrying escape bytes in terminal report must be stripped."""
report = AdapterCheckReport(
adapter="models/\x1b[2Jmalicious_run\x1b[0m",
verdict="alive",
reason=None,
total_frobenius=1.0,
live_fraction=1.0,
live_layers=1,
total_layers=1,
all_zero_lora_b_layers=(),
inner_keys=(),
per_layer=(),
)
rendered = render_check_terminal(report)
assert chr(27) not in rendered
assert "\x1b" not in rendered
assert "malicious_run" in rendered
def test_terminal_sanitization_shape_mismatches(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Shape-mismatched projection names with control bytes must be sanitized."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "mismatch_esc"
esc_shape_a = "base_model.model.layers.0.self_attn.\x1b[31mmismatch_layer\x1b[0m.lora_A.weight"
esc_shape_b = "base_model.model.layers.0.self_attn.\x1b[31mmismatch_layer\x1b[0m.lora_B.weight"
weights = {
esc_shape_a: np.ones((16, 8), dtype=np.float32),
esc_shape_b: np.ones((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=4, alpha=16.0)
res = runner.invoke(soup_app, ["adapters", "check", "mismatch_esc"])
assert res.exit_code == 2
assert chr(27) not in res.stdout
assert "\x1b" not in res.stdout
assert "Shape-mismatched LoRA pairs:" in res.stdout
assert "mismatch_layer" in res.stdout
def test_terminal_sanitization_zero_b_names(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Zero-B layer names carrying control bytes must be sanitized in terminal output."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
adapter_dir = tmp_path / "zero_b_esc"
esc_b_name = "base_model.model.layers.0.self_attn.\x1b[2Jzero_b_layer\x1b[0m.lora_B.weight"
esc_a_name = "base_model.model.layers.0.self_attn.\x1b[2Jzero_b_layer\x1b[0m.lora_A.weight"
weights = {
esc_a_name: np.ones((8, 16), dtype=np.float32),
esc_b_name: np.zeros((16, 8), dtype=np.float32),
}
_write_synthetic_adapter(adapter_dir, weights, r=8, alpha=16.0)
res = runner.invoke(soup_app, ["adapters", "check", "zero_b_esc"])
assert res.exit_code == 2
assert chr(27) not in res.stdout
assert "\x1b" not in res.stdout
assert "Zero lora_B projections:" in res.stdout
assert "zero_b_layer" in res.stdout
def test_terminal_sanitization_cli_errors(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""CLI errors and invalid JSON messages with control characters must be stripped on stderr."""
pytest.importorskip("safetensors")
from rich.console import Console
import soup_cli.commands.adapters as adapters_cmd
monkeypatch.setattr(
adapters_cmd, "_err_console", Console(stderr=True, force_terminal=False)
)
monkeypatch.chdir(tmp_path)
# 1. Non-existent path argument with escape bytes
res_path = runner.invoke(soup_app, ["adapters", "check", "nonexistent_\x1b[2J_dir"])
assert res_path.exit_code == 1
assert chr(27) not in res_path.output
assert "\x1b" not in res_path.output
# 2. Corrupt adapter_config.json with escape bytes
corrupt_dir = tmp_path / "corrupt_esc"
corrupt_dir.mkdir()
(corrupt_dir / "adapter_config.json").write_text(
"{\x1b[2J\"invalid_json\":", encoding="utf-8"
)
(corrupt_dir / "adapter_model.safetensors").write_bytes(b"empty")
res_cfg = runner.invoke(soup_app, ["adapters", "check", "corrupt_esc"])
assert res_cfg.exit_code == 1
assert chr(27) not in res_cfg.output
assert "\x1b" not in res_cfg.output
def test_terminal_sanitization_verdict_line():
"""Verdict line carrying control bytes in reason must be sanitized by verdict_line."""
report = AdapterCheckReport(
adapter="models/run",
verdict="inactive",
reason="injected\x1b[2Jreason\x1b[0m",
total_frobenius=0.0,
live_fraction=0.0,
live_layers=0,
total_layers=1,
all_zero_lora_b_layers=(),
inner_keys=(),
per_layer=(),
)
line = report.verdict_line
assert chr(27) not in line
assert "\x1b" not in line
assert line == "inactive: injected[2Jreason[0m"
def test_every_rendered_name_is_stripped(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""Every rendered name and CLI error path must have control bytes stripped."""
from rich.console import Console
import soup_cli.commands.adapters as adapters_cmd
monkeypatch.setattr(
adapters_cmd, "_err_console", Console(stderr=True, force_terminal=False)
)
evil = "x\x1b[2Jy\x9bz"
report = AdapterCheckReport(
adapter=evil,
verdict="inactive",
reason="r",
total_frobenius=0.0,
live_fraction=0.0,
live_layers=0,
total_layers=2,
all_zero_lora_b_layers=(evil,),
inner_keys=(evil,),
per_layer=(),
orphaned_layers=(evil,),
shape_mismatches=(evil,),
)
out = render_check_terminal(report)
assert chr(27) not in out and "\x9b" not in out
assert out.count("x[2Jyz") == 5
monkeypatch.chdir(tmp_path)
res = runner.invoke(soup_app, ["adapters", "check", "no\x1b[2Jdir"])
assert res.exit_code == 1 and chr(27) not in res.output
# ---------- bfloat16 adapters (#2058) ----------
def _bf16_bits(arr: np.ndarray) -> np.ndarray:
"""Round float32 to bfloat16 (round-to-nearest-even), as raw uint16 bits."""
bits = np.ascontiguousarray(arr, dtype=np.float32).view(np.uint32).astype(np.uint64)
bits = bits + 0x7FFF + ((bits >> 16) & 1)
return (bits >> 16).astype(np.uint16)
def _bf16_to_f32(bits: np.ndarray) -> np.ndarray:
return (bits.astype(np.uint32) << 16).view(np.float32)
def _write_raw_adapter(dir_path: Path, tensors: dict[str, tuple[str, tuple, bytes]]) -> None:
"""Write a safetensors adapter by hand: ``{name: (dtype tag, shape, payload bytes)}``."""
header: dict = {}
payload = b""
for name, (tag, shape, raw) in tensors.items():
header[name] = {
"dtype": tag,
"shape": list(shape),
"data_offsets": [len(payload), len(payload) + len(raw)],
}
payload += raw
hdr = json.dumps(header).encode("utf-8")
hdr += b" " * (-len(hdr) % 8)
dir_path.mkdir(parents=True, exist_ok=True)
(dir_path / "adapter_model.safetensors").write_bytes(
len(hdr).to_bytes(8, "little") + hdr + payload
)
cfg = {
"peft_type": "LORA", "r": 8, "lora_alpha": 16.0,
"target_modules": ["q_proj", "v_proj"],
}
(dir_path / "adapter_config.json").write_text(json.dumps(cfg), encoding="utf-8")
def _write_bf16_adapter(dir_path: Path, weights: dict[str, np.ndarray]) -> None:
"""Write a bfloat16 safetensors adapter by hand (header + uint16 payload), no torch."""
_write_raw_adapter(
dir_path,
{name: ("BF16", arr.shape, _bf16_bits(arr).tobytes()) for name, arr in weights.items()},
)
def _bf16_weights(b_value: float) -> tuple[dict, dict]:
rng = np.random.default_rng(0)
a = rng.standard_normal((8, 16)).astype(np.float32)
b = b_value * rng.standard_normal((16, 8)).astype(np.float32)
exact = {Q_LORA_A: _bf16_to_f32(_bf16_bits(a)), Q_LORA_B: _bf16_to_f32(_bf16_bits(b))}
return {Q_LORA_A: a, Q_LORA_B: b}, exact
def test_bf16_adapter_check_matches_float32(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
weights, exact = _bf16_weights(0.5)
_write_bf16_adapter(tmp_path / "bf16", weights)
_write_synthetic_adapter(tmp_path / "f32", exact)
got = check_adapter("bf16")
want = check_adapter("f32")
assert got.verdict == "alive"
assert got.total_frobenius == pytest.approx(want.total_frobenius, rel=1e-9)
assert got.live_fraction == want.live_fraction
res = runner.invoke(soup_app, ["adapters", "check", "bf16"])
assert res.exit_code == 0, (res.output, repr(res.exception))
assert res.stdout.strip().splitlines()[-1] == "alive"
def test_bf16_zero_lora_b_reports_inactive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
weights, _ = _bf16_weights(0.0)
_write_bf16_adapter(tmp_path / "bf16_zero", weights)
report = check_adapter("bf16_zero")
assert report.verdict == "inactive"
assert report.verdict_line == "inactive: all lora_B layers are zero (1/1)"
res = runner.invoke(soup_app, ["adapters", "check", "bf16_zero"])
assert res.exit_code == 2, (res.output, repr(res.exception))
def test_bf16_adapter_diff_matches_float32(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
pytest.importorskip("safetensors")
from soup_cli.utils.adapter_diff import compute_adapter_diff
monkeypatch.chdir(tmp_path)
w1, e1 = _bf16_weights(0.5)
w2, e2 = _bf16_weights(0.9)
_write_bf16_adapter(tmp_path / "bf_a", w1)
_write_bf16_adapter(tmp_path / "bf_b", w2)
_write_synthetic_adapter(tmp_path / "f_a", e1)
_write_synthetic_adapter(tmp_path / "f_b", e2)
got = compute_adapter_diff("bf_a", "bf_b")
want = compute_adapter_diff("f_a", "f_b")
assert got.shared_layers == want.shared_layers == 2
assert [d.frobenius for d in got.per_layer] == pytest.approx(
[d.frobenius for d in want.per_layer], rel=1e-9
)
assert got.effective_rank_a == pytest.approx(want.effective_rank_a, rel=1e-9)
@pytest.mark.parametrize("dtype", sorted(adapter_diff_module._NUMPY_DTYPES))
@pytest.mark.parametrize("shape", [(3, 5), (), (0, 4)], ids=["matrix", "scalar", "empty"])
def test_adapter_load_matches_safe_open_numpy(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, dtype: str, shape: tuple
):
"""Every dtype the reader maps comes back as ``safe_open(framework="numpy")`` gives it."""
pytest.importorskip("safetensors")
from safetensors import safe_open
from safetensors.numpy import save_file
from soup_cli.utils.adapter_diff import load_adapter_weights
monkeypatch.chdir(tmp_path)
rng = np.random.default_rng(1)
values = np.asarray(rng.standard_normal(shape) * 100)
if dtype == "bool":
array = np.asarray(values > 0)
elif np.issubdtype(np.dtype(dtype), np.unsignedinteger):
array = np.asarray(np.abs(values).astype(dtype))
else:
array = np.asarray(values.astype(dtype))
(tmp_path / "typed").mkdir()
target = tmp_path / "typed" / "adapter_model.safetensors"
save_file({Q_LORA_A: array}, str(target))
with safe_open(str(target), framework="numpy") as f:
want = f.get_tensor(Q_LORA_A)
got = load_adapter_weights("typed")[Q_LORA_A]
assert got.dtype == want.dtype, (got.dtype, want.dtype)
assert got.shape == want.shape, (got.shape, want.shape)
assert np.array_equal(got, want), (got, want)
def test_bf16_adapter_widens_exactly(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
pytest.importorskip("safetensors")
from soup_cli.utils.adapter_diff import load_adapter_weights
monkeypatch.chdir(tmp_path)
weights, exact = _bf16_weights(0.5)
_write_bf16_adapter(tmp_path / "bf16", weights)
loaded = load_adapter_weights("bf16")
for name, want in exact.items():
assert loaded[name].dtype == np.float32
assert loaded[name].shape == want.shape
assert np.array_equal(loaded[name], want)
def test_bf16_truncated_payload_is_a_named_refusal(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
weights, _ = _bf16_weights(0.5)
_write_bf16_adapter(tmp_path / "bf16", weights)
target = tmp_path / "bf16" / "adapter_model.safetensors"
target.write_bytes(target.read_bytes()[:-4])
with pytest.raises(ValueError, match="past the end of the file") as excinfo:
check_adapter("bf16")
assert str(excinfo.value).startswith(
"Corrupt or unreadable safetensors weights in adapter_model.safetensors: tensor "
), str(excinfo.value)
assert str(excinfo.value).count("adapter_model.safetensors") == 1, str(excinfo.value)
res = runner.invoke(soup_app, ["adapters", "check", "bf16"])
assert res.exit_code == 1, (res.output, repr(res.exception))
def test_bf16_adapter_over_the_tensor_cap_is_refused(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
pytest.importorskip("safetensors")
import soup_cli.utils.adapter_diff as adapter_diff
monkeypatch.chdir(tmp_path)
weights, _ = _bf16_weights(0.5)
_write_bf16_adapter(tmp_path / "bf16", weights)
monkeypatch.setattr(adapter_diff, "_MAX_LAYERS", 1)
with pytest.raises(ValueError, match="tensors"):
check_adapter("bf16")
with pytest.raises(ValueError, match="tensors"):
adapter_diff.load_adapter_weights("bf16")
def test_payload_shorter_than_its_header_is_a_named_refusal(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""The file shrinking after ``read_header`` checked it is refused, not reshaped."""
pytest.importorskip("safetensors")
import soup_cli.utils.adapter_diff as adapter_diff
from soup_cli.utils import safetensors_reader
monkeypatch.chdir(tmp_path)
weights, _ = _bf16_weights(0.5)
_write_bf16_adapter(tmp_path / "bf16", weights)
target = tmp_path / "bf16" / "adapter_model.safetensors"
real_read_header = safetensors_reader.read_header
def read_then_truncate(path: str):
entries = real_read_header(path)
target.write_bytes(target.read_bytes()[:-4])
return entries
monkeypatch.setattr(safetensors_reader, "read_header", read_then_truncate)
with pytest.raises(ValueError, match=f"tensor {Q_LORA_B!r} is truncated"):
adapter_diff.load_adapter_weights("bf16")
@pytest.mark.parametrize("tag", ["F8_E4M3", "F8_E5M2"])
def test_fp8_adapter_is_refused_with_tensor_and_dtype(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, tag: str
):
"""``read_header`` knows the FP8 spellings, the numpy reader does not (#2061)."""
pytest.importorskip("safetensors")
from soup_cli.utils import safetensors_reader
from soup_cli.utils.adapter_diff import load_adapter_weights
monkeypatch.chdir(tmp_path)
_write_raw_adapter(tmp_path / "fp8", {Q_LORA_A: (tag, (4, 2), bytes(8))})
dtype = safetensors_reader._DTYPES[tag]
want = f"tensor {Q_LORA_A!r} has unsupported dtype {dtype!r}"
with pytest.raises(ValueError, match=re.escape(want)):
check_adapter("fp8")
with pytest.raises(ValueError, match=re.escape(want)):
load_adapter_weights("fp8")
res = runner.invoke(soup_app, ["adapters", "check", "fp8"])
assert res.exit_code == 1, (res.output, repr(res.exception))
assert "KeyError" not in res.output, res.output
def test_unmapped_header_dtype_is_unsupported_not_corrupt(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""A dtype ``read_header`` refuses (U16) is reported as unsupported, file named once."""
pytest.importorskip("safetensors")
monkeypatch.chdir(tmp_path)
_write_raw_adapter(tmp_path / "u16", {Q_LORA_A: ("U16", (4, 2), bytes(16))})
with pytest.raises(ValueError) as excinfo:
check_adapter("u16")
message = str(excinfo.value)
assert message.startswith(
"Unsupported safetensors weights in adapter_model.safetensors: tensor "
), message
assert "Corrupt" not in message, message
assert message.count("adapter_model.safetensors") == 1, message