Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for Issue #813 — Gate commands exit-code unification.
Taxonomy contract:
EXIT_OK = 0 (PASS / OK / SHIP / valid data)
EXIT_GATE_FAILED = 2 (MAJOR / REGRESSION / DRIFT / DON'T SHIP / unusable data)
EXIT_USAGE_ERROR = 3 (bad flag, missing or unparseable input file, empty series)
EXIT_RUNTIME_ERROR = 1 (unexpected runtime failure)
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Tuple
import pytest
import yaml
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.utils.exit_codes import (
EXIT_GATE_FAILED,
EXIT_OK,
EXIT_USAGE_ERROR,
)
from soup_cli.utils.soup_lock import SoupLock, compute_lock_closure, write_lock
def _setup_eval_gate(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_tasks_file = tmp_path / "gate_tasks_pass.jsonl"
pass_tasks_file.write_text(
json.dumps({"prompt": "ping", "expected": ""}) + "\n",
encoding="utf-8",
)
fail_tasks_file = tmp_path / "gate_tasks_fail.jsonl"
fail_tasks_file.write_text(
json.dumps({"prompt": "ping", "expected": "definite_mismatch"}) + "\n",
encoding="utf-8",
)
pass_suite = tmp_path / "gate_pass.yaml"
pass_suite.write_text(
yaml.safe_dump({
"suite": "pass-gate",
"tasks": [
{
"type": "custom",
"name": "t_pass",
"scorer": "exact",
"threshold": 0.0,
"tasks": str(pass_tasks_file),
}
],
}),
encoding="utf-8",
)
fail_suite = tmp_path / "gate_fail.yaml"
fail_suite.write_text(
yaml.safe_dump({
"suite": "fail-gate",
"tasks": [
{
"type": "custom",
"name": "t_fail",
"scorer": "exact",
"threshold": 1.0,
"tasks": str(fail_tasks_file),
}
],
}),
encoding="utf-8",
)
pass_args = ["eval", "gate", "--suite", str(pass_suite)]
fail_args = ["eval", "gate", "--suite", str(fail_suite)]
missing_args = ["eval", "gate", "--suite", str(tmp_path / "missing_suite.yaml")]
return pass_args, fail_args, missing_args
def _setup_eval_against(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_args = [
"eval", "against", "run-pass-base", "--candidate", "run-pass-cand",
]
fail_args = [
"eval", "against", "run-fail-base", "--candidate", "run-fail-cand",
]
missing_args = [
"eval", "against", "run-base", "--candidate", "run-cand",
"--suite", str(tmp_path / "nonexistent_locked.json"),
]
return pass_args, fail_args, missing_args
def _setup_eval_checklist(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
spec_file = tmp_path / "checklist_spec.yaml"
spec_file.write_text(
yaml.safe_dump({
"tests": [
{"name": "t1", "kind": "mft", "prompts": ["hi"], "expected": ["hello"]}
]
}),
encoding="utf-8",
)
pass_ev = tmp_path / "checklist_pass.json"
pass_ev.write_text(json.dumps({"t1": ["hello"]}), encoding="utf-8")
fail_ev = tmp_path / "checklist_fail.json"
fail_ev.write_text(json.dumps({"t1": ["bad_response"]}), encoding="utf-8")
pass_args = ["eval", "checklist", str(spec_file), "--evidence", str(pass_ev)]
fail_args = ["eval", "checklist", str(spec_file), "--evidence", str(fail_ev)]
missing_args = ["eval", "checklist", str(tmp_path / "nonexistent_spec.yaml")]
return pass_args, fail_args, missing_args
def _setup_eval_behavior(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_ev = tmp_path / "behavior_pass.json"
pass_ev.write_text(
json.dumps({
"pre_responses": ["safe"] * 5,
"post_responses": ["safe"] * 5,
"oracle": ["safe"] * 5,
}),
encoding="utf-8",
)
fail_ev = tmp_path / "behavior_fail.json"
fail_ev.write_text(
json.dumps({
"pre_responses": ["safe"] * 5,
"post_responses": ["unsafe"] * 5,
"oracle": ["safe"] * 5,
}),
encoding="utf-8",
)
pass_args = [
"eval", "behavior", "run1", "--battery", "xstest", "--evidence", str(pass_ev),
]
fail_args = [
"eval", "behavior", "run1", "--battery", "xstest", "--evidence", str(fail_ev),
]
missing_args = [
"eval", "behavior", "run1", "--battery", "xstest",
"--evidence", str(tmp_path / "nonexistent_behavior.json"),
]
return pass_args, fail_args, missing_args
def _setup_eval_quant_check(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
before_model = tmp_path / "before.safetensors"
before_model.write_bytes(b"model_before")
after_model = tmp_path / "after.safetensors"
after_model.write_bytes(b"model_after")
tasks_file = tmp_path / "quant_tasks.jsonl"
tasks_file.write_text(
json.dumps({"prompt": "p", "expected": "hi"}) + "\n",
encoding="utf-8",
)
pass_args = [
"eval", "quant-check",
"--before", str(before_model),
"--after", str(after_model),
"--tasks", str(tasks_file),
]
fail_args = [
"eval", "quant-check",
"--before", str(before_model),
"--after", str(after_model),
"--tasks", str(tasks_file),
]
missing_args = [
"eval", "quant-check",
"--before", str(before_model),
"--after", str(after_model),
"--tasks", str(tmp_path / "nonexistent_tasks.jsonl"),
]
return pass_args, fail_args, missing_args
def _setup_lock_check(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
base_sha = "a" * 64
dataset_sha = "b" * 64
env_hash = "c" * 64
closure = compute_lock_closure(
base_model_sha=base_sha,
dataset_sha=dataset_sha,
env_hash=env_hash,
)
lock_file = tmp_path / "soup.lock"
lock = SoupLock(
soup_version="0.75.0",
base_model="test-model",
base_model_sha=base_sha,
dataset_sha=dataset_sha,
env_hash=env_hash,
closure_sha=closure,
created_at="2026-09-13T00:00:00Z",
)
write_lock(lock, str(lock_file))
pass_args = [
"lock", "check", str(lock_file),
"--base-model", "test-model",
"--base-sha", base_sha,
"--dataset-sha", dataset_sha,
"--env-hash", env_hash,
]
fail_args = [
"lock", "check", str(lock_file),
"--base-model", "test-model",
"--base-sha", "d" * 64, # causes drift
"--dataset-sha", dataset_sha,
"--env-hash", env_hash,
]
missing_args = [
"lock", "check", str(tmp_path / "nonexistent.lock"),
"--base-model", "test-model",
"--base-sha", base_sha,
"--dataset-sha", dataset_sha,
"--env-hash", env_hash,
]
return pass_args, fail_args, missing_args
def _setup_data_validate(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_file = tmp_path / "valid.jsonl"
pass_file.write_text(
json.dumps({"instruction": "hi", "output": "there"}) + "\n",
encoding="utf-8",
)
fail_file = tmp_path / "invalid.jsonl"
fail_file.write_text(
"".join(json.dumps({"instruction": None, "output": None}) + "\n" for _ in range(5)),
encoding="utf-8",
)
pass_args = ["data", "validate", str(pass_file), "--format", "alpaca"]
fail_args = ["data", "validate", str(fail_file), "--format", "alpaca"]
missing_args = ["data", "validate", str(tmp_path / "nonexistent.jsonl"), "--format", "alpaca"]
return pass_args, fail_args, missing_args
def _setup_data_lint(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_file = tmp_path / "dpo_pass.jsonl"
pass_file.write_text(
json.dumps({
"prompt": "hello",
"chosen": "good morning how are you",
"rejected": "good night see you tomorrow",
}) + "\n",
encoding="utf-8",
)
fail_file = tmp_path / "dpo_fail.jsonl"
fail_file.write_text(
json.dumps({
"prompt": "hello",
"chosen": "good morning how are you",
"rejected": "good morning how are you", # chosen == rejected triggers MAJOR
}) + "\n",
encoding="utf-8",
)
pass_args = ["data", "lint", str(pass_file), "--format", "dpo"]
fail_args = ["data", "lint", str(fail_file), "--format", "dpo"]
missing_args = ["data", "lint", str(tmp_path / "nonexistent.jsonl"), "--format", "dpo"]
return pass_args, fail_args, missing_args
def _setup_ship(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
pass_file = tmp_path / "ship_pass.json"
pass_file.write_text(
json.dumps({
"task": {"mode": "metric", "base": 0.4, "tuned": 0.8},
"benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
}),
encoding="utf-8",
)
fail_file = tmp_path / "ship_fail.json"
fail_file.write_text(
json.dumps({
"task": {"mode": "metric", "base": 0.8, "tuned": 0.2},
"benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
}),
encoding="utf-8",
)
pass_args = ["ship", "--evidence", str(pass_file)]
fail_args = ["ship", "--evidence", str(fail_file)]
missing_args = ["ship", "--evidence", str(tmp_path / "nonexistent.json")]
return pass_args, fail_args, missing_args
def _setup_expect(tmp_path: Path) -> Tuple[list[str], list[str], list[str]]:
suite_file = tmp_path / "expect_suite.yaml"
suite_file.write_text(
"expectations:\n - name: expect_no_pii\n",
encoding="utf-8",
)
pass_file = tmp_path / "expect_pass.jsonl"
pass_file.write_text('{"text": "clean output here"}\n', encoding="utf-8")
fail_file = tmp_path / "expect_fail.jsonl"
fail_file.write_text('{"text": "email me at [email protected]"}\n', encoding="utf-8")
pass_args = ["expect", str(pass_file), str(suite_file)]
fail_args = ["expect", str(fail_file), str(suite_file)]
missing_args = ["expect", str(tmp_path / "nonexistent.jsonl"), str(suite_file)]
return pass_args, fail_args, missing_args
SETUP_MAP = {
"eval gate": _setup_eval_gate,
"eval against": _setup_eval_against,
"eval checklist": _setup_eval_checklist,
"eval behavior": _setup_eval_behavior,
"eval quant-check": _setup_eval_quant_check,
"lock check": _setup_lock_check,
"data validate": _setup_data_validate,
"data lint": _setup_data_lint,
"ship": _setup_ship,
"expect": _setup_expect,
}
@pytest.mark.parametrize(
"command_name",
[
"eval gate",
"eval against",
"eval checklist",
"eval behavior",
"eval quant-check",
"lock check",
"data validate",
"data lint",
"ship",
"expect",
],
)
def test_gate_commands_follow_unified_exit_code_taxonomy(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
command_name: str,
) -> None:
"""Every gate command must exit:
0 = EXIT_OK on passing input
2 = EXIT_GATE_FAILED on failing input / regression / drift
3 = EXIT_USAGE_ERROR on missing input file
"""
monkeypatch.chdir(tmp_path)
# Command-specific mocks for isolated execution without GPU or network
if command_name == "eval against":
def fake_get_metric_series(self, run_id: str, metric: str):
if "pass" in run_id:
return [0.8, 0.8, 0.8] if "cand" in run_id else [0.4, 0.4, 0.4]
return [0.2, 0.2, 0.2] if "cand" in run_id else [0.8, 0.8, 0.8]
monkeypatch.setattr(
"soup_cli.experiment.tracker.ExperimentTracker.get_metric_series",
fake_get_metric_series,
)
if command_name == "eval quant-check":
class _MockState:
is_fail = False
state = _MockState()
def fake_make_model_gen(ref: str):
if state.is_fail:
return (lambda p: "bad") if "after" in str(ref) else (lambda p: "hi")
return lambda p: "hi"
monkeypatch.setattr(
"soup_cli.eval.quant_check.make_model_generator",
fake_make_model_gen,
)
setup_fn = SETUP_MAP[command_name]
pass_args, fail_args, missing_args = setup_fn(tmp_path)
runner = CliRunner()
# (a) Passing input -> 0 (EXIT_OK)
res_pass = runner.invoke(app, pass_args)
assert res_pass.exit_code == EXIT_OK, (
f"{command_name} pass: expected {EXIT_OK}, got {res_pass.exit_code}\n"
f"output: {res_pass.output}\nexc: {repr(res_pass.exception)}"
)
# (b) Failing input -> 2 (EXIT_GATE_FAILED)
if command_name == "eval quant-check":
state.is_fail = True
res_fail = runner.invoke(app, fail_args)
assert res_fail.exit_code == EXIT_GATE_FAILED, (
f"{command_name} fail: expected {EXIT_GATE_FAILED}, got {res_fail.exit_code}\n"
f"output: {res_fail.output}\nexc: {repr(res_fail.exception)}"
)
# (c) Missing input file -> 3 (EXIT_USAGE_ERROR)
res_missing = runner.invoke(app, missing_args)
assert res_missing.exit_code == EXIT_USAGE_ERROR, (
f"{command_name} missing file: expected {EXIT_USAGE_ERROR}, got {res_missing.exit_code}\n"
f"output: {res_missing.output}\nexc: {repr(res_missing.exception)}"
)
def test_eval_checklist_missing_evidence_exits_3(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from tests.conftest import strip_ansi
monkeypatch.chdir(tmp_path)
spec_file = tmp_path / "checklist_spec.yaml"
spec_file.write_text(
yaml.safe_dump({
"tests": [
{"name": "t1", "kind": "mft", "prompts": ["hi"], "expected": ["hello"]}
]
}),
encoding="utf-8",
)
runner = CliRunner()
res = runner.invoke(app, ["eval", "checklist", str(spec_file)])
assert res.exit_code == EXIT_USAGE_ERROR
clean_out = " ".join(strip_ansi(res.output).lower().split())
assert "--evidence is required" in clean_out
assert "checklist test name" in clean_out
def test_eval_behavior_missing_evidence_exits_3(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from tests.conftest import strip_ansi
monkeypatch.chdir(tmp_path)
runner = CliRunner()
res = runner.invoke(app, ["eval", "behavior", "run1", "--battery", "xstest"])
assert res.exit_code == EXIT_USAGE_ERROR
clean_out = strip_ansi(res.output).lower()
assert "--evidence is required" in clean_out
assert "pre_responses" in clean_out
def test_eval_quant_check_missing_model_exits_3(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path)
tasks_file = tmp_path / "tasks.jsonl"
tasks_file.write_text(
json.dumps({"prompt": "p", "expected": "hi"}) + "\n",
encoding="utf-8",
)
runner = CliRunner()
res = runner.invoke(
app,
[
"eval", "quant-check",
"--before", "nonexistent_before",
"--after", "nonexistent_after",
"--tasks", str(tasks_file),
],
)
assert res.exit_code == EXIT_USAGE_ERROR
def test_ci_workflow_exit_code_dispatch(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from soup_cli.utils.ci_workflow import render_soup_gate_workflow
monkeypatch.chdir(tmp_path)
data = tmp_path / "data.jsonl"
data.write_text('{"text": "hi"}\n', encoding="utf-8")
suite = tmp_path / "suite.yaml"
suite.write_text("expectations:\n - name: expect_no_pii\n", encoding="utf-8")
evidence = tmp_path / "ev.json"
evidence.write_text("{}", encoding="utf-8")
rendered = render_soup_gate_workflow(
data_path=str(data),
suite_path=str(suite),
evidence_path=str(evidence),
)
# Must be valid YAML
parsed = yaml.safe_load(rendered)
assert parsed["name"] == "Soup Fine-tuning Gate"
steps = parsed["jobs"]["soup-gate"]["steps"]
step_runs = [s.get("run", "") for s in steps if "run" in s]
# Validate data step has 0/2/3 rc dispatch
data_step = [r for r in step_runs if "soup data validate" in r][0]
assert 'set +e' in data_step and 'set -e' in data_step
assert '[ "$rc" -eq 2 ]' in data_step
assert 'exit 2' in data_step
assert '[ "$rc" -ne 0 ]' in data_step
assert 'exit "$rc"' in data_step
# Expect step has 0/2/3 rc dispatch
expect_step = [r for r in step_runs if "soup expect" in r][0]
assert 'set +e' in expect_step and 'set -e' in expect_step
assert '[ "$rc" -eq 2 ]' in expect_step
assert 'exit 2' in expect_step
assert '[ "$rc" -ne 0 ]' in expect_step
assert 'exit "$rc"' in expect_step
# Ship step has 0/2/3 rc dispatch
ship_step = [r for r in step_runs if "soup ship" in r][0]
assert 'set +e' in ship_step and 'set -e' in ship_step
assert '[ "$rc" -eq 2 ]' in ship_step
assert 'exit 2' in ship_step
assert '[ "$rc" -ne 0 ]' in ship_step
assert 'exit "$rc"' in ship_step
def test_eval_gate_hook_exit_code_dispatch(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from soup_cli.utils.eval_gate_hook import render_pre_push_hook
monkeypatch.chdir(tmp_path)
suite = tmp_path / "suite.json"
suite.write_text("{}", encoding="utf-8")
hook = render_pre_push_hook(
baseline_run_id="run-base-1",
suite_path=str(suite),
)
assert "set -euo pipefail" in hook
assert "set +e" in hook
assert 'soup eval against "$BASELINE_RUN_ID"' in hook
assert "rc=$?" in hook
assert "set -e" in hook
assert '[ "$rc" -eq 2 ]' in hook
assert "exit 2" in hook
assert '[ "$rc" -ne 0 ]' in hook
assert 'exit "$rc"' in hook
@pytest.mark.parametrize(
"cmd",
[
["eval", "gate", "--bogus"],
["eval", "quant-check", "--bogus"],
["eval", "against", "--bogus"],
["eval", "behavior", "--bogus"],
["eval", "checklist", "--bogus"],
["lock", "check", "--bogus"],
["expect", "--bogus"],
["data", "validate", "--bogus"],
["data", "lint", "--bogus"],
["ship", "--bogus"],
["recipes", "verify", "--bogus"],
],
)
def test_gate_command_invalid_flag_exits_3(cmd: list[str]) -> None:
"""Invalid CLI flags must trigger EXIT_USAGE_ERROR (3), not Click's default 2."""
runner = CliRunner()
result = runner.invoke(app, cmd)
assert result.exit_code == EXIT_USAGE_ERROR
@pytest.mark.parametrize(
"cmd",
[
["drift-alarm", "--bogus"],
["data", "forge", "--bogus"],
["recipes", "list", "--bogus"],
],
)
def test_non_gate_command_invalid_flag_exits_2(cmd: list[str]) -> None:
"""Non-gate commands preserve Click's standard exit 2 on invalid flags."""
runner = CliRunner()
result = runner.invoke(app, cmd)
assert result.exit_code == 2