Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for the eval platform: custom eval, judge, human eval, leaderboard, compare."""
import json
import re
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.eval.custom import (
MAX_EVAL_TASKS,
MAX_REGEX_INPUT_LEN,
MAX_REGEX_PATTERN_LEN,
EvalResult,
EvalResults,
EvalTask,
load_eval_tasks,
score_contains,
score_exact,
score_regex,
score_semantic,
score_task,
)
from soup_cli.eval.human import (
ELO_DEFAULT,
MAX_PROMPTS,
HumanEvalResults,
HumanJudgment,
_expected_score,
load_prompts,
load_results,
run_human_eval_session,
save_results,
)
from soup_cli.eval.judge import (
DEFAULT_RUBRIC,
JudgeEvaluator,
JudgeResults,
JudgeScore,
_build_judge_prompt,
_compute_weighted_score,
_parse_judge_response,
load_rubric,
validate_judge_api_base,
)
from soup_cli.eval.leaderboard import (
Leaderboard,
LeaderboardEntry,
build_leaderboard_from_tracker,
compare_runs,
export_leaderboard,
)
runner = CliRunner()
def _strip_ansi(text: str) -> str:
"""Remove ANSI escape codes from Rich-formatted output."""
return re.sub(r'\x1b\[[0-9;]*m', '', text)
# ═══════════════════════════════════════════════════════════
# Custom Eval — Scoring Functions
# ═══════════════════════════════════════════════════════════
class TestScoreExact:
def test_match(self):
assert score_exact("Hello", "hello") is True
def test_no_match(self):
assert score_exact("Hello", "World") is False
def test_whitespace(self):
assert score_exact(" hello ", "hello") is True
def test_empty(self):
assert score_exact("", "") is True
def test_case_insensitive(self):
assert score_exact("ANSWER", "answer") is True
class TestScoreContains:
def test_contains(self):
assert score_contains("The answer is 42.", "42") is True
def test_not_contains(self):
assert score_contains("The answer is 42.", "99") is False
def test_case_insensitive(self):
assert score_contains("Hello World", "hello") is True
def test_full_match(self):
assert score_contains("exact", "exact") is True
class TestScoreRegex:
def test_match(self):
assert score_regex("The answer is 42", r"\d+") is True
def test_no_match(self):
assert score_regex("no numbers here", r"\d+") is False
def test_invalid_regex(self):
assert score_regex("test", r"[invalid") is False
def test_case_insensitive(self):
assert score_regex("Hello World", r"hello") is True
class TestScoreSemantic:
def test_identical(self):
score = score_semantic("hello world", "hello world")
assert score == 1.0
def test_partial_overlap(self):
score = score_semantic("hello world foo", "hello world bar")
assert 0.0 < score < 1.0
def test_no_overlap(self):
score = score_semantic("foo bar", "baz qux")
assert score == 0.0
def test_empty_output(self):
score = score_semantic("", "hello")
assert score == 0.0
def test_empty_expected(self):
score = score_semantic("hello", "")
assert score == 0.0
# ═══════════════════════════════════════════════════════════
# Custom Eval — Task Loading
# ═══════════════════════════════════════════════════════════
class TestLoadEvalTasks:
def test_valid_jsonl(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text(
'{"prompt": "What is 2+2?", "expected": "4", "category": "math"}\n'
'{"prompt": "Capital of France?", "expected": "Paris"}\n',
encoding="utf-8",
)
tasks = load_eval_tasks(path)
assert len(tasks) == 2
assert tasks[0].prompt == "What is 2+2?"
assert tasks[0].expected == "4"
assert tasks[0].category == "math"
assert tasks[1].category == "default"
def test_file_not_found(self, tmp_path):
with pytest.raises(FileNotFoundError):
load_eval_tasks(tmp_path / "nonexistent.jsonl")
def test_wrong_extension(self, tmp_path):
path = tmp_path / "tasks.json"
path.write_text("{}", encoding="utf-8")
with pytest.raises(ValueError, match="Expected .jsonl"):
load_eval_tasks(path)
def test_invalid_json(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text("not json\n", encoding="utf-8")
with pytest.raises(ValueError, match="Invalid JSON"):
load_eval_tasks(path)
def test_missing_prompt(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text('{"expected": "4"}\n', encoding="utf-8")
with pytest.raises(ValueError, match="missing required field"):
load_eval_tasks(path)
def test_invalid_scoring(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text(
'{"prompt": "q", "expected": "a", "scoring": "invalid"}\n',
encoding="utf-8",
)
with pytest.raises(ValueError, match="invalid scoring"):
load_eval_tasks(path)
def test_empty_file(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text("", encoding="utf-8")
with pytest.raises(ValueError, match="No eval tasks"):
load_eval_tasks(path)
def test_max_tasks_exceeded(self, tmp_path):
path = tmp_path / "tasks.jsonl"
lines = '{"prompt": "q"}\n' * (MAX_EVAL_TASKS + 1)
path.write_text(lines, encoding="utf-8")
with pytest.raises(ValueError, match="exceeds maximum"):
load_eval_tasks(path)
def test_non_object_json(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text("[1, 2, 3]\n", encoding="utf-8")
with pytest.raises(ValueError, match="expected JSON object"):
load_eval_tasks(path)
def test_blank_lines_skipped(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text(
'{"prompt": "q1", "expected": "a1"}\n'
"\n"
'{"prompt": "q2", "expected": "a2"}\n',
encoding="utf-8",
)
tasks = load_eval_tasks(path)
assert len(tasks) == 2
def test_scoring_types(self, tmp_path):
path = tmp_path / "tasks.jsonl"
lines = []
for scoring in ["exact", "contains", "regex", "semantic"]:
lines.append(json.dumps({
"prompt": "q", "expected": "a", "scoring": scoring,
}))
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
tasks = load_eval_tasks(path)
assert len(tasks) == 4
assert tasks[0].scoring == "exact"
assert tasks[3].scoring == "semantic"
def test_metadata_preserved(self, tmp_path):
path = tmp_path / "tasks.jsonl"
path.write_text(
'{"prompt": "q", "expected": "a", "difficulty": "hard"}\n',
encoding="utf-8",
)
tasks = load_eval_tasks(path)
assert tasks[0].metadata == {"difficulty": "hard"}
# ═══════════════════════════════════════════════════════════
# Custom Eval — score_task
# ═══════════════════════════════════════════════════════════
class TestScoreTask:
def test_exact_match(self):
task = EvalTask(prompt="q", expected="answer", scoring="exact")
result = score_task(task, "answer")
assert result.matched is True
assert result.score == 1.0
def test_exact_no_match(self):
task = EvalTask(prompt="q", expected="answer", scoring="exact")
result = score_task(task, "wrong")
assert result.matched is False
assert result.score == 0.0
def test_contains_match(self):
task = EvalTask(prompt="q", expected="42", scoring="contains")
result = score_task(task, "The answer is 42")
assert result.matched is True
def test_regex_match(self):
task = EvalTask(prompt="q", expected=r"\d+", scoring="regex")
result = score_task(task, "42")
assert result.matched is True
def test_semantic_match(self):
task = EvalTask(
prompt="q", expected="hello world", scoring="semantic",
)
result = score_task(task, "hello world test")
assert result.matched is True
assert 0.0 < result.score <= 1.0
# ═══════════════════════════════════════════════════════════
# Custom Eval — EvalResults
# ═══════════════════════════════════════════════════════════
class TestEvalResults:
def test_compute_basic(self):
results = EvalResults(results=[
EvalResult(
task=EvalTask(prompt="q1", expected="a", category="math"),
output="a", score=1.0, matched=True,
),
EvalResult(
task=EvalTask(prompt="q2", expected="b", category="math"),
output="x", score=0.0, matched=False,
),
EvalResult(
task=EvalTask(prompt="q3", expected="c", category="code"),
output="c", score=1.0, matched=True,
),
])
results.compute()
assert results.total == 3
assert results.correct == 2
assert abs(results.accuracy - 2 / 3) < 1e-6
assert results.category_scores["math"]["total"] == 2
assert results.category_scores["math"]["correct"] == 1
assert results.category_scores["code"]["accuracy"] == 1.0
def test_empty_results(self):
results = EvalResults(results=[])
results.compute()
assert results.total == 0
assert results.accuracy == 0.0
# ═══════════════════════════════════════════════════════════
# Custom Eval — run_eval with mock generator
# ═══════════════════════════════════════════════════════════
class TestRunEval:
def test_run_eval_with_mock_generator(self):
from soup_cli.eval.custom import run_eval
tasks = [
EvalTask(prompt="What is 2+2?", expected="4", scoring="exact"),
EvalTask(prompt="Capital?", expected="Paris", scoring="contains"),
]
def mock_gen(prompt: str) -> str:
if "2+2" in prompt:
return "4"
return "The capital is Paris"
results = run_eval("dummy_path", tasks, generate_fn=mock_gen)
assert results.total == 2
assert results.correct == 2
assert results.accuracy == 1.0
def test_run_eval_empty_tasks(self):
from soup_cli.eval.custom import run_eval
results = run_eval("dummy", tasks=[], generate_fn=lambda p: "")
assert results.total == 0
assert results.accuracy == 0.0
def test_run_eval_partial_match(self):
from soup_cli.eval.custom import run_eval
tasks = [
EvalTask(prompt="q1", expected="yes", scoring="exact"),
EvalTask(prompt="q2", expected="no", scoring="exact"),
]
results = run_eval(
"dummy", tasks, generate_fn=lambda p: "yes",
)
assert results.total == 2
assert results.correct == 1
# ═══════════════════════════════════════════════════════════
# Human Eval — Edge cases
# ═══════════════════════════════════════════════════════════
class TestHumanEvalEdgeCases:
def test_invalid_winner_treated_as_tie(self):
"""Invalid winner value falls through to tie (else branch)."""
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="invalid",
))
results.compute_ratings()
# Both models should be near default (tie behavior)
assert abs(results.ratings["m1"].rating - ELO_DEFAULT) < 1e-6
assert results.ratings["m1"].ties == 1
def test_load_results_malformed_json(self, tmp_path):
"""Corrupted results file raises JSONDecodeError."""
path = tmp_path / "bad.json"
path.write_text("not valid json{{{", encoding="utf-8")
with pytest.raises(json.JSONDecodeError):
load_results(path)
# ═══════════════════════════════════════════════════════════
# Leaderboard — Edge cases
# ═══════════════════════════════════════════════════════════
class TestLeaderboardEdgeCases:
def test_export_unknown_format_defaults_to_json(self):
"""Unknown format falls through to JSON export."""
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.8),
])
lb.compute()
output = export_leaderboard(lb, fmt="xml")
# Should still be valid JSON
data = json.loads(output)
assert len(data) == 1
def test_ipv6_loopback_http_ok(self):
"""::1 is a valid loopback address for HTTP."""
validate_judge_api_base("http://[::1]:8000")
# ═══════════════════════════════════════════════════════════
# Judge — API Base Validation (SSRF)
# ═══════════════════════════════════════════════════════════
class TestValidateJudgeApiBase:
def test_none_is_ok(self):
validate_judge_api_base(None)
def test_https_ok(self):
validate_judge_api_base("https://api.openai.com")
def test_localhost_http_ok(self):
validate_judge_api_base("http://localhost:8000")
def test_127_http_ok(self):
validate_judge_api_base("http://127.0.0.1:8000")
def test_remote_http_blocked(self):
with pytest.raises(ValueError, match="HTTPS"):
validate_judge_api_base("http://evil.com")
def test_0000_http_blocked(self):
"""0.0.0.0 is a bind address, not a safe loopback for clients."""
with pytest.raises(ValueError, match="HTTPS"):
validate_judge_api_base("http://0.0.0.0:8000")
def test_ftp_blocked(self):
with pytest.raises(ValueError, match="Invalid scheme"):
validate_judge_api_base("ftp://example.com")
def test_file_blocked(self):
with pytest.raises(ValueError, match="Invalid scheme"):
validate_judge_api_base("file:///etc/passwd")
# ═══════════════════════════════════════════════════════════
# Judge — Rubric Loading
# ═══════════════════════════════════════════════════════════
class TestLoadRubric:
def test_valid_rubric(self, tmp_path):
rubric_path = tmp_path / "rubric.yaml"
rubric_path.write_text(
"criteria:\n"
" - name: quality\n"
" description: response quality\n"
" weight: 1.0\n"
"scale:\n"
" min: 1\n"
" max: 10\n",
encoding="utf-8",
)
rubric = load_rubric(rubric_path)
assert len(rubric["criteria"]) == 1
assert rubric["criteria"][0]["name"] == "quality"
assert rubric["scale"]["max"] == 10
def test_file_not_found(self, tmp_path):
with pytest.raises(FileNotFoundError):
load_rubric(tmp_path / "nonexistent.yaml")
def test_missing_criteria(self, tmp_path):
rubric_path = tmp_path / "rubric.yaml"
rubric_path.write_text("scale:\n min: 1\n", encoding="utf-8")
with pytest.raises(ValueError, match="criteria"):
load_rubric(rubric_path)
def test_empty_criteria(self, tmp_path):
rubric_path = tmp_path / "rubric.yaml"
rubric_path.write_text("criteria: []\n", encoding="utf-8")
with pytest.raises(ValueError, match="non-empty"):
load_rubric(rubric_path)
def test_invalid_criterion(self, tmp_path):
rubric_path = tmp_path / "rubric.yaml"
rubric_path.write_text(
"criteria:\n - name: test\n", encoding="utf-8",
)
with pytest.raises(ValueError, match="description"):
load_rubric(rubric_path)
def test_non_mapping(self, tmp_path):
rubric_path = tmp_path / "rubric.yaml"
rubric_path.write_text("- item1\n- item2\n", encoding="utf-8")
with pytest.raises(ValueError, match="mapping"):
load_rubric(rubric_path)
# ═══════════════════════════════════════════════════════════
# Judge — Prompt Building & Response Parsing
# ═══════════════════════════════════════════════════════════
class TestJudgePromptAndParsing:
def test_build_prompt(self):
prompt = _build_judge_prompt("question", "answer", DEFAULT_RUBRIC)
assert "question" in prompt
assert "answer" in prompt
assert "helpfulness" in prompt
assert "JSON" in prompt
def test_parse_valid_response(self):
response = json.dumps({
"scores": {"helpfulness": 4, "accuracy": 5, "safety": 3},
"reasoning": "Good response.",
})
scores, reasoning = _parse_judge_response(response, DEFAULT_RUBRIC)
assert scores["helpfulness"] == 4.0
assert scores["accuracy"] == 5.0
assert reasoning == "Good response."
def test_parse_clamps_scores(self):
response = json.dumps({
"scores": {"helpfulness": 10, "accuracy": -1, "safety": 3},
"reasoning": "Test",
})
scores, _ = _parse_judge_response(response, DEFAULT_RUBRIC)
assert scores["helpfulness"] == 5.0 # clamped to max
assert scores["accuracy"] == 1.0 # clamped to min
def test_parse_no_json(self):
with pytest.raises(ValueError, match="No JSON"):
_parse_judge_response("no json here", DEFAULT_RUBRIC)
def test_parse_invalid_json(self):
with pytest.raises(ValueError, match="No JSON"):
_parse_judge_response("not {valid json", DEFAULT_RUBRIC)
def test_compute_weighted_score(self):
scores = {"helpfulness": 4.0, "accuracy": 5.0, "safety": 3.0}
result = _compute_weighted_score(scores, DEFAULT_RUBRIC)
assert abs(result - 4.0) < 1e-6 # (4+5+3)/3 = 4.0
def test_compute_weighted_custom_weights(self):
rubric = {
"criteria": [
{"name": "a", "description": "x", "weight": 2.0},
{"name": "b", "description": "y", "weight": 1.0},
],
}
scores = {"a": 5.0, "b": 2.0}
result = _compute_weighted_score(scores, rubric)
expected = (5.0 * 2.0 + 2.0 * 1.0) / 3.0
assert abs(result - expected) < 1e-6
# ═══════════════════════════════════════════════════════════
# Judge — JudgeEvaluator
# ═══════════════════════════════════════════════════════════
class TestJudgeEvaluator:
def test_invalid_provider(self):
with pytest.raises(ValueError, match="Invalid provider"):
JudgeEvaluator(provider="bad")
def test_ssrf_blocked(self):
with pytest.raises(ValueError):
JudgeEvaluator(api_base="ftp://evil.com")
def test_valid_init(self):
evaluator = JudgeEvaluator(
provider="openai", model="gpt-4o-mini",
)
assert evaluator.provider == "openai"
assert evaluator.model == "gpt-4o-mini"
def test_evaluate_with_mock(self):
evaluator = JudgeEvaluator(provider="openai")
mock_response = json.dumps({
"scores": {"helpfulness": 4, "accuracy": 5, "safety": 4},
"reasoning": "Solid response.",
})
with patch.object(evaluator, "_call_llm", return_value=mock_response):
score = evaluator.evaluate("test prompt", "test response")
assert score.weighted_score > 0
assert score.reasoning == "Solid response."
def test_evaluate_batch_with_mock(self):
evaluator = JudgeEvaluator(provider="openai")
mock_response = json.dumps({
"scores": {"helpfulness": 3, "accuracy": 4, "safety": 5},
"reasoning": "Ok.",
})
items = [
{"prompt": "q1", "response": "a1", "category": "cat1"},
{"prompt": "q2", "response": "a2"},
]
with patch.object(evaluator, "_call_llm", return_value=mock_response):
results = evaluator.evaluate_batch(items)
assert len(results.scores) == 2
results.compute()
assert results.overall_score > 0
assert "cat1" in results.category_scores
# ═══════════════════════════════════════════════════════════
# Judge — JudgeResults
# ═══════════════════════════════════════════════════════════
class TestJudgeResults:
def test_compute_empty(self):
results = JudgeResults(scores=[])
results.compute()
assert results.overall_score == 0.0
def test_compute_with_scores(self):
scores = [
JudgeScore(
prompt="q1", response="a1",
scores={"h": 4, "a": 5}, weighted_score=4.5,
category="cat1",
),
JudgeScore(
prompt="q2", response="a2",
scores={"h": 3, "a": 4}, weighted_score=3.5,
category="cat2",
),
]
results = JudgeResults(scores=scores)
results.compute()
assert abs(results.overall_score - 4.0) < 1e-6
assert "cat1" in results.category_scores
assert "h" in results.criteria_averages
# ═══════════════════════════════════════════════════════════
# Human Eval — Elo Rating
# ═══════════════════════════════════════════════════════════
class TestEloRating:
def test_expected_score_equal(self):
assert abs(_expected_score(1500, 1500) - 0.5) < 1e-6
def test_expected_score_higher(self):
assert _expected_score(1700, 1500) > 0.5
def test_expected_score_lower(self):
assert _expected_score(1300, 1500) < 0.5
def test_elo_update_winner_a(self):
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="a",
))
results.compute_ratings()
assert results.ratings["m1"].rating > ELO_DEFAULT
assert results.ratings["m2"].rating < ELO_DEFAULT
assert results.ratings["m1"].wins == 1
assert results.ratings["m2"].losses == 1
def test_elo_update_winner_b(self):
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="b",
))
results.compute_ratings()
assert results.ratings["m1"].rating < ELO_DEFAULT
assert results.ratings["m2"].rating > ELO_DEFAULT
def test_elo_update_tie(self):
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="tie",
))
results.compute_ratings()
assert abs(results.ratings["m1"].rating - ELO_DEFAULT) < 1e-6
assert results.ratings["m1"].ties == 1
def test_multiple_judgments(self):
results = HumanEvalResults()
for _ in range(3):
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="a",
))
results.compute_ratings()
assert results.ratings["m1"].wins == 3
assert results.ratings["m1"].rating > ELO_DEFAULT + 30
def test_to_dict(self):
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="a",
))
results.compute_ratings()
data = results.to_dict()
assert len(data["judgments"]) == 1
assert "m1" in data["ratings"] or "m2" in data["ratings"]
# ═══════════════════════════════════════════════════════════
# Human Eval — Load/Save
# ═══════════════════════════════════════════════════════════
class TestHumanEvalIO:
def test_load_prompts(self, tmp_path):
path = tmp_path / "prompts.jsonl"
path.write_text(
'{"prompt": "Hello"}\n'
'{"prompt": "World", "category": "test"}\n',
encoding="utf-8",
)
prompts = load_prompts(path)
assert len(prompts) == 2
def test_load_prompts_not_found(self, tmp_path):
with pytest.raises(FileNotFoundError):
load_prompts(tmp_path / "nope.jsonl")
def test_load_prompts_invalid_json(self, tmp_path):
path = tmp_path / "prompts.jsonl"
path.write_text("not json\n", encoding="utf-8")
with pytest.raises(ValueError, match="Invalid JSON"):
load_prompts(path)
def test_load_prompts_missing_field(self, tmp_path):
path = tmp_path / "prompts.jsonl"
path.write_text('{"text": "hello"}\n', encoding="utf-8")
with pytest.raises(ValueError, match="missing required field"):
load_prompts(path)
def test_save_and_load_results(self, tmp_path):
results = HumanEvalResults()
results.judgments.append(HumanJudgment(
prompt="q", response_a="a", response_b="b",
model_a="m1", model_b="m2", winner="a",
))
results.compute_ratings()
path = tmp_path / "results.json"
save_results(results, path)
loaded = load_results(path)
assert len(loaded.judgments) == 1
assert loaded.judgments[0].winner == "a"
assert loaded.ratings["m1"].rating > ELO_DEFAULT
def test_load_results_not_found(self, tmp_path):
with pytest.raises(FileNotFoundError):
load_results(tmp_path / "nope.json")
class TestRunHumanEvalSession:
def test_with_precomputed_responses(self):
prompts = [{"prompt": "q1"}, {"prompt": "q2"}]
results = run_human_eval_session(
prompts=prompts,
model_a_name="m1",
model_b_name="m2",
responses_a=["resp_a1", "resp_a2"],
responses_b=["resp_b1", "resp_b2"],
)
assert len(results.judgments) == 2
assert results.judgments[0].response_a == "resp_a1"
# ═══════════════════════════════════════════════════════════
# Leaderboard
# ═══════════════════════════════════════════════════════════
class TestLeaderboard:
def test_compute(self):
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.8),
LeaderboardEntry(model_path="m1", benchmark="gsm8k", score=0.6),
LeaderboardEntry(model_path="m2", benchmark="mmlu", score=0.9),
])
lb.compute()
assert len(lb.models) == 2
assert lb.models["m1"]["mmlu"] == 0.8
assert lb.models["m2"]["mmlu"] == 0.9
def test_get_sorted_models_by_avg(self):
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.8),
LeaderboardEntry(model_path="m2", benchmark="mmlu", score=0.9),
])
lb.compute()
sorted_models = lb.get_sorted_models()
assert sorted_models[0][0] == "m2" # higher score first
def test_get_sorted_by_benchmark(self):
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.9),
LeaderboardEntry(model_path="m1", benchmark="gsm8k", score=0.3),
LeaderboardEntry(model_path="m2", benchmark="mmlu", score=0.7),
LeaderboardEntry(model_path="m2", benchmark="gsm8k", score=0.8),
])
lb.compute()
# Sort by gsm8k: m2 should be first
sorted_models = lb.get_sorted_models(sort_by="gsm8k")
assert sorted_models[0][0] == "m2"
def test_export_json(self):
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.8),
])
lb.compute()
output = export_leaderboard(lb, fmt="json")
data = json.loads(output)
assert len(data) == 1
assert data[0]["model"] == "m1"
def test_export_csv(self):
lb = Leaderboard(entries=[
LeaderboardEntry(model_path="m1", benchmark="mmlu", score=0.8),
LeaderboardEntry(model_path="m1", benchmark="gsm8k", score=0.6),
])
lb.compute()
output = export_leaderboard(lb, fmt="csv")
lines = output.strip().split("\n")
assert len(lines) == 2 # header + 1 model
assert "mmlu" in lines[0]
assert "gsm8k" in lines[0]
def test_empty_leaderboard(self):
lb = Leaderboard()
lb.compute()
assert lb.get_sorted_models() == []
# ═══════════════════════════════════════════════════════════
# Leaderboard — Compare Runs
# ═══════════════════════════════════════════════════════════
class TestCompareRuns:
def test_compare_basic(self):
tracker = MagicMock()
tracker.get_eval_results.side_effect = [
[{"benchmark": "mmlu", "score": 0.7}],
[{"benchmark": "mmlu", "score": 0.8}],
]
result = compare_runs(tracker, "run1", "run2")
assert len(result["comparisons"]) == 1
assert result["comparisons"][0]["delta"] == pytest.approx(0.1)
assert result["has_regressions"] is False
def test_compare_regression(self):
tracker = MagicMock()
tracker.get_eval_results.side_effect = [
[{"benchmark": "mmlu", "score": 0.9}],
[{"benchmark": "mmlu", "score": 0.7}],
]
result = compare_runs(tracker, "run1", "run2")
assert result["has_regressions"] is True
assert "mmlu" in result["regressions"]
def test_compare_different_benchmarks(self):
tracker = MagicMock()
tracker.get_eval_results.side_effect = [
[{"benchmark": "mmlu", "score": 0.7}],
[{"benchmark": "gsm8k", "score": 0.8}],
]
result = compare_runs(tracker, "run1", "run2")
assert len(result["comparisons"]) == 2
def test_build_leaderboard_from_tracker(self):
tracker = MagicMock()
tracker.get_eval_results.return_value = [
{"model_path": "m1", "benchmark": "mmlu", "score": 0.8,
"run_id": "r1", "created_at": "2026-01-01"},
]
lb = build_leaderboard_from_tracker(tracker)
assert len(lb.entries) == 1
assert lb.models["m1"]["mmlu"] == 0.8
# ═══════════════════════════════════════════════════════════
# Config — EvalConfig
# ═══════════════════════════════════════════════════════════
class TestEvalConfig:
def test_eval_config_default(self):
from soup_cli.config.schema import EvalConfig
config = EvalConfig()
assert config.auto_eval is False
assert config.benchmarks is None
assert config.custom_tasks is None
assert config.judge is None
def test_eval_config_with_values(self):
from soup_cli.config.schema import EvalConfig
config = EvalConfig(
auto_eval=True,
benchmarks=["mmlu", "gsm8k"],
custom_tasks="eval.jsonl",
judge={"model": "gpt-4o-mini", "provider": "openai"},
)
assert config.auto_eval is True
assert config.benchmarks == ["mmlu", "gsm8k"]
def test_soup_config_with_eval(self):
from soup_cli.config.schema import SoupConfig
config = SoupConfig(
base="test-model",
data={"train": "data.jsonl"},
eval={"auto_eval": True, "benchmarks": ["mmlu"]},
)
assert config.eval is not None
assert config.eval.auto_eval is True
assert config.eval.benchmarks == ["mmlu"]
def test_soup_config_without_eval(self):
from soup_cli.config.schema import SoupConfig
config = SoupConfig(
base="test-model",
data={"train": "data.jsonl"},
)
assert config.eval is None
# ═══════════════════════════════════════════════════════════
# Auto-eval after training
# ═══════════════════════════════════════════════════════════
def _tiny_llama_dir(root: Path) -> str:
import torch
from safetensors.torch import save_file
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast
torch.manual_seed(7)
config = LlamaConfig(
vocab_size=64, hidden_size=64, intermediate_size=128, num_hidden_layers=2,
num_attention_heads=4, num_key_value_heads=2, tie_word_embeddings=True,
max_position_embeddings=128,
)
model = LlamaForCausalLM(config).to(torch.float32).eval()
weights = root / "model"
weights.mkdir(parents=True, exist_ok=True)
state = {k: v.contiguous() for k, v in model.state_dict().items()}
state.pop("lm_head.weight", None)
save_file(state, str(weights / "model.safetensors"))
config.save_pretrained(str(weights))
words = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
for word in ("hello", "world", "hi", "good", "answer", "bad"):
words[word] = len(words)
tok = Tokenizer(models.WordLevel(vocab=words, unk_token="<unk>"))
tok.pre_tokenizer = pre_tokenizers.Whitespace()
PreTrainedTokenizerFast(
tokenizer_object=tok, unk_token="<unk>", bos_token="<s>", eos_token="</s>",
pad_token="<pad>",
).save_pretrained(str(weights))
return str(weights)
def _write_project(tmp_path: Path, auto_eval: bool, seed: int = 1) -> None:
weights = str(tmp_path / "model")
if not (tmp_path / "model" / "model.safetensors").exists():
weights = _tiny_llama_dir(tmp_path)
row = {"messages": [{"role": "user", "content": "hi"},
{"role": "assistant", "content": "good answer"}]}
(tmp_path / "chat.jsonl").write_text(
"\n".join(json.dumps(row) for _ in range(8)) + "\n", encoding="utf-8"
)
(tmp_path / "soup.yaml").write_text(
f"base: {json.dumps(weights)}\n"
"task: sft\n"
"data:\n train: chat.jsonl\n format: chatml\n chat_template: chatml\n"
" max_length: 64\n"
"training:\n epochs: 1\n batch_size: 2\n logging_steps: 1\n"
f" quantization: none\n seed: {seed}\n"
" lora:\n r: 4\n alpha: 8\n target_modules: [q_proj, v_proj]\n"
f"eval:\n auto_eval: {'true' if auto_eval else 'false'}\n benchmarks: [mmlu]\n"
"output: ./out\n",
encoding="utf-8",
)
def _record_benchmark(monkeypatch) -> list[dict]:
import soup_cli.commands.eval as eval_cmd
seen: list[dict] = []
def fake_benchmark(**kwargs):
model = Path(kwargs["model"])
adapter = model / "adapter_model.safetensors"
seen.append({
"model": str(model),
"adapter_bytes": adapter.read_bytes() if adapter.exists() else None,
})
monkeypatch.setattr(eval_cmd, "benchmark", fake_benchmark)
return seen
class TestAutoEvalAfterTraining:
def test_auto_eval_not_called_when_disabled(self):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
eval_config = EvalConfig(auto_eval=False)
with patch("soup_cli.commands.eval.benchmark") as mock_bench:
_run_auto_eval_after_training(
eval_config,
"/tmp/model",
"test_run",
)
mock_bench.assert_not_called()
def test_auto_eval_uses_saved_output_dir(self):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
eval_config = EvalConfig(
auto_eval=True,
benchmarks=["mmlu", "gsm8k"],
)
with patch("soup_cli.commands.eval.benchmark") as mock_bench:
_run_auto_eval_after_training(
eval_config,
"/tmp/saved-adapter",
"test_run",
)
mock_bench.assert_called_once()
assert mock_bench.call_args.kwargs["model"] == "/tmp/saved-adapter"
assert mock_bench.call_args.kwargs["benchmarks"] == "mmlu,gsm8k"
assert mock_bench.call_args.kwargs["run_id"] == "test_run"
def test_auto_eval_failure_does_not_fail_training(self):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
eval_config = EvalConfig(
auto_eval=True,
benchmarks=["mmlu"],
)
with patch(
"soup_cli.commands.eval.benchmark",
side_effect=RuntimeError("benchmark exploded"),
):
_run_auto_eval_after_training(
eval_config,
"/tmp/saved-adapter",
"test_run",
)
def test_auto_eval_runs_only_on_the_main_process(self, monkeypatch):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
monkeypatch.setenv("RANK", "1")
with patch("soup_cli.commands.eval.benchmark") as bench:
_run_auto_eval_after_training(
EvalConfig(auto_eval=True, benchmarks=["mmlu"]),
"out",
"run-1",
)
bench.assert_not_called()
def test_auto_eval_failure_prints_what_failed(self, capsys):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
with patch(
"soup_cli.commands.eval.benchmark",
side_effect=RuntimeError("benchmark exploded"),
):
_run_auto_eval_after_training(
EvalConfig(auto_eval=True, benchmarks=["mmlu"]),
"out",
"run-1",
)
out = " ".join(_strip_ansi(capsys.readouterr().out).split())
assert "Auto-eval benchmark failed" in out
assert "benchmark exploded" in out
def test_custom_eval_uses_the_saved_output_and_its_failure_does_not_raise(self):
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
with patch(
"soup_cli.commands.eval.custom",
side_effect=RuntimeError("custom exploded"),
) as custom:
_run_auto_eval_after_training(
EvalConfig(auto_eval=True, custom_tasks="tasks.jsonl"),
"/tmp/saved-adapter",
"run-1",
)
assert custom.call_args.kwargs["model"] == "/tmp/saved-adapter"
assert custom.call_args.kwargs["tasks"] == "tasks.jsonl"
def test_a_benchmark_that_exits_is_reported_as_skipped(self, capsys):
import typer
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
with patch("soup_cli.commands.eval.benchmark", side_effect=typer.Exit(1)):
_run_auto_eval_after_training(
EvalConfig(auto_eval=True, benchmarks=["mmlu"]), "out", "run-1"
)
out = " ".join(_strip_ansi(capsys.readouterr().out).split())
assert "Auto-eval benchmark skipped" in out
assert "Auto-eval benchmark failed" not in out
def test_a_custom_eval_that_exits_is_reported_as_skipped(self, capsys):
import typer
from soup_cli.commands.train import _run_auto_eval_after_training
from soup_cli.config.schema import EvalConfig
with patch("soup_cli.commands.eval.custom", side_effect=typer.Exit(1)):
_run_auto_eval_after_training(
EvalConfig(auto_eval=True, custom_tasks="tasks.jsonl"), "out", "run-1"
)
out = " ".join(_strip_ansi(capsys.readouterr().out).split())
assert "Auto-eval custom skipped" in out
assert "Auto-eval custom failed" not in out
def test_auto_eval_runs_on_the_adapter_this_run_saved(
self, tmp_path, monkeypatch
):
pytest.importorskip("trl")
monkeypatch.chdir(tmp_path)
seen = _record_benchmark(monkeypatch)
# The second run reuses ./out, where run 1's adapter sits; a different seed
# makes the two adapters differ byte for byte.
for seed in (1, 2):
_write_project(tmp_path, auto_eval=True, seed=seed)
result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--yes"])
assert result.exit_code == 0, (result.output, repr(result.exception))
saved = (tmp_path / "out" / "adapter_model.safetensors").read_bytes()
assert seen, "auto_eval: true ran no evaluation"
assert seen[-1]["adapter_bytes"] is not None, (
"the evaluation ran before the adapter was saved"
)
assert seen[-1]["adapter_bytes"] == saved, (
"the evaluation scored a different adapter than the one this run saved"
)
assert len(seen) == 2
def test_auto_eval_false_runs_nothing(
self, tmp_path, monkeypatch
):
pytest.importorskip("trl")
monkeypatch.chdir(tmp_path)
_write_project(tmp_path, auto_eval=False)
seen = _record_benchmark(monkeypatch)
result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--yes"])
assert result.exit_code == 0, (result.output, repr(result.exception))
assert seen == []
# ═══════════════════════════════════════════════════════════
# CLI — eval subcommands
# ═══════════════════════════════════════════════════════════
class TestEvalCLI:
def test_eval_help(self):
result = runner.invoke(app, ["eval", "--help"])
assert result.exit_code == 0
assert "benchmark" in result.output.lower()
assert "custom" in result.output.lower()
assert "judge" in result.output.lower()
assert "leaderboard" in result.output.lower()
def test_eval_benchmark_help(self):
result = runner.invoke(app, ["eval", "benchmark", "--help"])
assert result.exit_code == 0
assert "model" in result.output.lower()
assert "benchmarks" in result.output.lower()
def test_eval_custom_help(self):
result = runner.invoke(app, ["eval", "custom", "--help"])
assert result.exit_code == 0
assert "tasks" in result.output.lower()
def test_eval_judge_help(self):
result = runner.invoke(app, ["eval", "judge", "--help"])
assert result.exit_code == 0
assert "target" in result.output.lower()
assert "provider" in result.output.lower()
def test_eval_compare_help(self):
result = runner.invoke(app, ["eval", "compare", "--help"])
assert result.exit_code == 0
def test_eval_leaderboard_help(self):
result = runner.invoke(app, ["eval", "leaderboard", "--help"])
assert result.exit_code == 0
def test_eval_human_help(self):
result = runner.invoke(app, ["eval", "human", "--help"])
assert result.exit_code == 0
clean = _strip_ansi(result.output).lower()
assert "model-a" in clean
def test_eval_auto_help(self):
result = runner.invoke(app, ["eval", "auto", "--help"])
assert result.exit_code == 0
assert "config" in result.output.lower()
def test_eval_benchmark_missing_model(self):
result = runner.invoke(
app, ["eval", "benchmark", "--model", "nonexistent_path"],
)
assert result.exit_code == 1
assert "not found" in result.output.lower()
def test_eval_custom_missing_model(self, tmp_path):
tasks_path = tmp_path / "tasks.jsonl"
tasks_path.write_text(
'{"prompt": "q", "expected": "a"}\n', encoding="utf-8",
)
result = runner.invoke(
app, [
"eval", "custom",
"--tasks", str(tasks_path),
"--model", "nonexistent_path",
],
)
assert result.exit_code == 1
assert "not found" in result.output.lower()
def test_eval_judge_missing_target(self):
result = runner.invoke(
app, ["eval", "judge", "--target", "nonexistent.jsonl"],
)
assert result.exit_code == 1
assert "not found" in result.output.lower()
def test_eval_leaderboard_empty(self, tmp_path):
"""Leaderboard with no results should show a message."""
import os
db_path = tmp_path / "test.db"
with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}):
result = runner.invoke(app, ["eval", "leaderboard"])
assert result.exit_code == 1
assert "no eval results" in result.output.lower()
def test_eval_compare_run_not_found(self, tmp_path):
"""Compare with nonexistent run IDs."""
import os
db_path = tmp_path / "test.db"
with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}):
result = runner.invoke(
app, ["eval", "compare", "nonexistent1", "nonexistent2"],
)
assert result.exit_code == 1
assert "not found" in result.output.lower()
# ═══════════════════════════════════════════════════════════
# Security — SSRF, input validation
# ═══════════════════════════════════════════════════════════
class TestSecurity:
def test_custom_eval_schema_validation(self, tmp_path):
"""Ensure custom eval rejects invalid schemas."""
path = tmp_path / "bad.jsonl"
path.write_text("42\n", encoding="utf-8")
with pytest.raises(ValueError, match="expected JSON object"):
load_eval_tasks(path)
def test_custom_eval_task_cap(self, tmp_path):
"""Ensure eval tasks are capped at MAX_EVAL_TASKS."""
path = tmp_path / "tasks.jsonl"
lines = '{"prompt": "q"}\n' * (MAX_EVAL_TASKS + 1)
path.write_text(lines, encoding="utf-8")
with pytest.raises(ValueError, match="exceeds maximum"):
load_eval_tasks(path)
def test_judge_ssrf_remote_http(self):
"""Remote HTTP is blocked for judge API."""
with pytest.raises(ValueError, match="HTTPS"):
validate_judge_api_base("http://malicious-server.com/api")
def test_judge_ssrf_file_protocol(self):
"""file:// protocol blocked."""
with pytest.raises(ValueError):
validate_judge_api_base("file:///etc/passwd")
def test_leaderboard_read_only(self):
"""Leaderboard queries are read-only (no user input in SQL)."""
tracker = MagicMock()
tracker.get_eval_results.return_value = []
lb = build_leaderboard_from_tracker(tracker)
assert len(lb.entries) == 0
# Verify only get_eval_results was called (read-only)
tracker.get_eval_results.assert_called_once()
def test_regex_redos_pattern_length_guard(self):
"""Long regex patterns are rejected to prevent ReDoS."""
long_pattern = "a" * (MAX_REGEX_PATTERN_LEN + 1)
assert score_regex("test input", long_pattern) is False
def test_regex_redos_input_truncation(self):
"""Long inputs are truncated before regex matching."""
# Pattern that matches only at the end
long_input = "a" * (MAX_REGEX_INPUT_LEN + 100) + "FIND_ME"
# FIND_ME is beyond the truncation limit, so should not match
assert score_regex(long_input, "FIND_ME") is False
# But within the limit, it should match
short_input = "a" * 100 + "FIND_ME"
assert score_regex(short_input, "FIND_ME") is True
def test_judge_api_key_not_leaked_to_ollama(self):
"""OpenAI API key should NOT be auto-loaded for non-openai providers."""
import os
with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-secret-key"}):
evaluator = JudgeEvaluator(provider="ollama")
assert evaluator.api_key == ""
def test_judge_api_key_loaded_for_openai(self):
"""OpenAI API key SHOULD be auto-loaded for openai provider."""
import os
with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-secret-key"}):
evaluator = JudgeEvaluator(provider="openai")
assert evaluator.api_key == "sk-secret-key"
def test_judge_api_key_not_leaked_to_server(self):
"""OpenAI API key should NOT be auto-loaded for server provider."""
import os
with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-secret-key"}):
evaluator = JudgeEvaluator(
provider="server",
api_base="http://localhost:8000",
)
assert evaluator.api_key == ""
def test_human_prompts_cap(self, tmp_path):
"""Human eval prompts are capped at MAX_PROMPTS."""
path = tmp_path / "prompts.jsonl"
lines = '{"prompt": "q"}\n' * (MAX_PROMPTS + 1)
path.write_text(lines, encoding="utf-8")
with pytest.raises(ValueError, match="exceeds maximum"):
load_prompts(path)