Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Regression tests for issue #91: the live Aider Polyglot runner."""
from __future__ import annotations
import json
import re
import subprocess
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def _plain(text: str) -> str:
return " ".join(_ANSI_RE.sub("", text).split())
def _result(path: Path, *, passed: bool, **extra: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
payload = {"tests_outcomes": [passed], **extra}
path.write_text(json.dumps(payload), encoding="utf-8")
class TestAiderResultParser:
def test_aggregates_upstream_per_exercise_json(self, tmp_path):
from soup_cli.eval.aider_polyglot import parse_aider_results
_result(
tmp_path / "python/exercises/practice/clock/.aider.results.json",
passed=True,
num_error_outputs=2,
test_timeouts=0,
)
_result(
tmp_path / "rust/exercises/practice/clock/.aider.results.json",
passed=False,
num_error_outputs=1,
test_timeouts=1,
)
row = parse_aider_results(tmp_path, model="openai/example")
assert row["model"] == "openai/example"
assert row["task"] == "aider_polyglot"
assert row["score"] == 0.5
assert row["errors"] == 1
assert row["details"]["completed_tests"] == 2
assert row["details"]["error_outputs"] == 3
assert row["details"]["test_timeouts"] == 1
def test_rejects_missing_results(self, tmp_path):
from soup_cli.eval.aider_polyglot import AiderEvalError, parse_aider_results
with pytest.raises(AiderEvalError, match="No Aider result files"):
parse_aider_results(tmp_path, model="openai/example")
def test_rejects_malformed_json(self, tmp_path):
from soup_cli.eval.aider_polyglot import AiderEvalError, parse_aider_results
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
result.parent.mkdir(parents=True)
result.write_text("{broken", encoding="utf-8")
with pytest.raises(AiderEvalError, match="Malformed Aider result JSON"):
parse_aider_results(tmp_path, model="openai/example")
def test_rejects_oversized_json(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
result.parent.mkdir(parents=True)
result.write_text("{}", encoding="utf-8")
monkeypatch.setattr(aider_eval, "MAX_RESULT_FILE_BYTES", 1)
with pytest.raises(aider_eval.AiderEvalError, match="too large"):
aider_eval.parse_aider_results(tmp_path, model="openai/example")
@pytest.mark.parametrize(
"value",
[float("nan"), float("inf"), float("-inf")],
ids=["nan", "positive-infinity", "negative-infinity"],
)
def test_rejects_non_finite_counts(self, tmp_path, value):
from soup_cli.eval.aider_polyglot import AiderEvalError, parse_aider_results
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
_result(result, passed=True, num_error_outputs=value)
with pytest.raises(AiderEvalError, match="Invalid 'num_error_outputs' count"):
parse_aider_results(tmp_path, model="openai/example")
def test_rejects_too_many_result_files(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
_result(
tmp_path / "python/exercises/practice/clock/.aider.results.json",
passed=True,
)
_result(
tmp_path / "rust/exercises/practice/clock/.aider.results.json",
passed=True,
)
monkeypatch.setattr(aider_eval, "MAX_RESULT_FILES", 1)
with pytest.raises(aider_eval.AiderEvalError, match="Too many Aider result files"):
aider_eval.parse_aider_results(tmp_path, model="openai/example")
def test_rejects_result_set_over_total_size_cap(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
_result(result, passed=True)
monkeypatch.setattr(aider_eval, "MAX_RESULT_TOTAL_BYTES", 1)
with pytest.raises(aider_eval.AiderEvalError, match="result set is too large"):
aider_eval.parse_aider_results(tmp_path, model="openai/example")
def test_parse_loop_rejects_result_outside_root(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
outside = tmp_path.parent / "outside-aider-result.json"
outside.write_text('{"tests_outcomes": [true]}', encoding="utf-8")
monkeypatch.setattr(Path, "glob", lambda _path, _pattern: [outside])
with pytest.raises(aider_eval.AiderEvalError, match="escaped --output"):
aider_eval.parse_aider_results(tmp_path, model="openai/example")
@pytest.mark.requires_symlink
def test_rejects_symlinked_result(self, tmp_path):
from soup_cli.eval.aider_polyglot import AiderEvalError, parse_aider_results
outside = tmp_path.parent / "outside-aider-result.json"
outside.write_text('{"tests_outcomes": [true]}', encoding="utf-8")
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
result.parent.mkdir(parents=True)
result.symlink_to(outside)
with pytest.raises(AiderEvalError, match="Refusing symlinked"):
parse_aider_results(tmp_path, model="openai/example")
@pytest.mark.requires_symlink
def test_rejects_symlink_even_when_target_stays_inside_root(self, tmp_path):
from soup_cli.eval.aider_polyglot import AiderEvalError, parse_aider_results
target = tmp_path / "inside-aider-result.json"
target.write_text('{"tests_outcomes": [true]}', encoding="utf-8")
result = tmp_path / "python/exercises/practice/clock/.aider.results.json"
result.parent.mkdir(parents=True)
result.symlink_to(target)
with pytest.raises(AiderEvalError, match="Refusing symlinked"):
parse_aider_results(tmp_path, model="openai/example")
class TestAiderDockerRunner:
def test_image_cannot_be_parsed_as_a_docker_option(self, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.setattr(
aider_eval.shutil,
"which",
lambda _name: pytest.fail("invalid image must fail before Docker lookup"),
)
with pytest.raises(aider_eval.AiderEvalError, match="valid Docker image reference"):
aider_eval.preflight_docker("--privileged")
def test_missing_docker_has_friendly_error(self, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.setattr(aider_eval.shutil, "which", lambda _name: None)
with pytest.raises(aider_eval.AiderEvalError, match="Docker CLI was not found"):
aider_eval.preflight_docker("aider-benchmark")
def test_daemon_failure_has_friendly_error(self, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.setattr(aider_eval.shutil, "which", lambda _name: "/usr/bin/docker")
monkeypatch.setattr(
aider_eval.subprocess,
"run",
lambda *_a, **_kw: SimpleNamespace(returncode=1, stdout="", stderr="not running"),
)
with pytest.raises(aider_eval.AiderEvalError, match="daemon is unavailable"):
aider_eval.preflight_docker("aider-benchmark")
def test_missing_image_has_setup_hint(self, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
calls = iter(
[
SimpleNamespace(returncode=0, stdout="29.0", stderr=""),
SimpleNamespace(returncode=1, stdout="", stderr="No such image"),
]
)
monkeypatch.setattr(aider_eval.shutil, "which", lambda _name: "/usr/bin/docker")
monkeypatch.setattr(aider_eval.subprocess, "run", lambda *_a, **_kw: next(calls))
with pytest.raises(aider_eval.AiderEvalError, match="docker_build.sh"):
aider_eval.preflight_docker("aider-benchmark")
def test_preflight_timeout_is_friendly(self, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.setattr(aider_eval.shutil, "which", lambda _name: "/usr/bin/docker")
def timeout(*_args, **_kwargs):
raise subprocess.TimeoutExpired("docker info", 10)
monkeypatch.setattr(aider_eval.subprocess, "run", timeout)
with pytest.raises(aider_eval.AiderEvalError, match="timed out"):
aider_eval.preflight_docker("aider-benchmark")
def test_build_command_uses_fixed_argv_and_read_only_corpus(self, tmp_path, monkeypatch):
from soup_cli.eval.aider_polyglot import build_docker_command
corpus = tmp_path / "polyglot"
output = tmp_path / "results"
corpus.mkdir()
output.mkdir()
monkeypatch.setenv("OPENAI_API_KEY", "secret-never-in-argv")
command = build_docker_command(
docker="/usr/bin/docker",
image="aider-benchmark",
model="openai/example; touch /tmp/pwned",
exercises_dir=corpus,
output_dir=output,
threads=2,
num_tests=3,
)
assert isinstance(command, list)
assert command[-1] == "3"
assert "openai/example; touch /tmp/pwned" in command
assert not any("secret-never-in-argv" in arg for arg in command)
assert "--add-host=host.docker.internal:host-gateway" not in command
mount = command[command.index("--mount") + 1]
assert "readonly" in mount
assert "type=bind" in mount
def test_host_gateway_is_opt_in(self, tmp_path):
from soup_cli.eval.aider_polyglot import build_docker_command
command = build_docker_command(
docker="/usr/bin/docker",
image="registry.example:5000/aider-benchmark:test",
model="openai/example",
exercises_dir=tmp_path,
output_dir=tmp_path,
threads=1,
num_tests=1,
allow_host_services=True,
)
assert "--add-host=host.docker.internal:host-gateway" in command
def test_mount_rejects_comma_in_source_path(self):
import soup_cli.eval.aider_polyglot as aider_eval
with pytest.raises(aider_eval.AiderEvalError, match="must not contain commas"):
aider_eval._mount(Path("corpus,with-comma"), "/benchmarks", readonly=True)
class TestAiderCLI:
def test_aider_optional_extra_is_declared(self):
pyproject = Path(__file__).parents[1] / "pyproject.toml"
text = pyproject.read_text(encoding="utf-8")
assert 'aider = ["aider-chat>=' in text
def test_help_lists_required_flags(self):
result = runner.invoke(app, ["eval", "aider", "--help"])
output = _plain(result.output)
assert result.exit_code == 0
assert "--model" in output
assert "--output" in output
assert "--exercises-dir" in output
assert "--run-id" in output
assert "--allow-host-servic" in output
def test_output_escape_is_rejected_before_docker(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
corpus = tmp_path / "polyglot"
corpus.mkdir()
with patch("soup_cli.eval.aider_polyglot.preflight_docker") as preflight:
result = runner.invoke(
app,
[
"eval",
"aider",
"--model",
"openai/example",
"--output",
str(tmp_path.parent / "escape"),
"--exercises-dir",
str(corpus),
],
)
assert result.exit_code == 1
output = _plain(result.output)
assert "must stay under the current working" in output
assert "directory" in output
preflight.assert_not_called()
def test_exercises_outside_cwd_warns_and_stays_read_only(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
project = tmp_path / "project"
project.mkdir()
corpus = tmp_path / "polyglot"
corpus.mkdir()
monkeypatch.chdir(project)
monkeypatch.setattr(aider_eval, "preflight_docker", lambda _image: "/usr/bin/docker")
def fake_run(command, **_kwargs):
corpus_mount = command[command.index("--mount") + 1]
assert "readonly" in corpus_mount
return SimpleNamespace(returncode=17)
monkeypatch.setattr(aider_eval.subprocess, "run", fake_run)
result = runner.invoke(
app,
[
"eval",
"aider",
"--model",
"openai/example",
"--output",
"results",
"--exercises-dir",
str(corpus),
],
)
assert result.exit_code == 1
assert "resolves outside the current working directory" in _plain(result.output)
def test_success_writes_soup_json_and_tracks_run(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.chdir(tmp_path)
corpus = tmp_path / "polyglot"
corpus.mkdir()
output = tmp_path / "results"
db_path = tmp_path / "experiments.db"
monkeypatch.setenv("SOUP_DB_PATH", str(db_path))
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
tracker.start_run({}, "cpu", "test", {}, run_id="run_aider_test")
tracker.close()
monkeypatch.setattr(aider_eval, "preflight_docker", lambda _image: "/usr/bin/docker")
def fake_run(command, **kwargs):
assert kwargs["shell"] is False
assert kwargs["timeout"] == 600
_result(
output / "python/exercises/practice/clock/.aider.results.json",
passed=True,
)
return SimpleNamespace(returncode=0)
monkeypatch.setattr(aider_eval.subprocess, "run", fake_run)
result = runner.invoke(
app,
[
"eval",
"aider",
"--model",
"openai/example",
"--output",
"results",
"--exercises-dir",
"polyglot",
"--run-id",
"run_aider_test",
"--timeout",
"600",
"--num-tests",
"1",
],
)
assert result.exit_code == 0, result.output
payload = json.loads((output / "soup_result.json").read_text(encoding="utf-8"))
assert payload["task"] == "aider_polyglot"
assert payload["score"] == 1.0
tracker = ExperimentTracker()
saved = tracker.get_eval_results(run_id="run_aider_test")
assert len(saved) == 1
assert saved[0]["benchmark"] == "aider_polyglot"
tracker.close()
def test_subprocess_failure_does_not_emit_summary(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.chdir(tmp_path)
corpus = tmp_path / "polyglot"
corpus.mkdir()
monkeypatch.setattr(aider_eval, "preflight_docker", lambda _image: "/usr/bin/docker")
monkeypatch.setattr(
aider_eval.subprocess,
"run",
lambda *_a, **_kw: SimpleNamespace(returncode=17),
)
result = runner.invoke(
app,
[
"eval",
"aider",
"--model",
"openai/example",
"--output",
"results",
"--exercises-dir",
"polyglot",
],
)
assert result.exit_code == 1
assert "exited with status 17" in _plain(result.output)
assert not (tmp_path / "results/soup_result.json").exists()
def test_subprocess_timeout_is_friendly(self, tmp_path, monkeypatch):
import soup_cli.eval.aider_polyglot as aider_eval
monkeypatch.chdir(tmp_path)
corpus = tmp_path / "polyglot"
corpus.mkdir()
monkeypatch.setattr(aider_eval, "preflight_docker", lambda _image: "/usr/bin/docker")
def timeout(*_args, **_kwargs):
raise subprocess.TimeoutExpired("docker run", 60)
monkeypatch.setattr(aider_eval.subprocess, "run", timeout)
result = runner.invoke(
app,
[
"eval",
"aider",
"--model",
"openai/example",
"--output",
"results",
"--exercises-dir",
"polyglot",
"--timeout",
"60",
],
)
assert result.exit_code == 1
assert "timed out after 60 seconds" in _plain(result.output)