Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

6.8 KB
Raw Blame
"""The `test` cells of CI run the suite in parallel, one test file per worker.

Measured on 2026-10-07 with Python 3.12 (run 37577416935 of the pytest-xdist trial, #1380 and
#1672): ubuntu 2520 s -> 1133 s, Windows 4492 s -> 1390 s, macOS 2003 s -> 992 s, with the same
tests passing and coverage unchanged. These pins keep the three choices that made that run
clean:

* `--dist loadfile`, never `load`: the tests of one file stay in one process and in order;
* `-n logical` with the plugin's `psutil` extra installed, which is what counts the CPUs;
* nothing in `pyproject.toml` turns workers on, so a local `pytest tests/` stays serial.

pyproject is read with regular expressions, not ``tomllib``: the convention of
``tests/test_requires_python_bound.py``, set while the floor was 3.10, which has no ``tomllib``.
"""

from __future__ import annotations

import re
import shlex
from pathlib import Path
from typing import Any

import pytest
import yaml

ROOT = Path(__file__).resolve().parent.parent
CI = ROOT / ".github" / "workflows" / "ci.yml"
PYPROJECT = ROOT / "pyproject.toml"
CONTRIBUTING = ROOT / "CONTRIBUTING.md"

RUN_STEP = "Run unit tests with coverage"
# 2026-10-19: every cell but ubuntu / 3.11 (3.10 until #1805) runs this one, with
# `--no-cov`; 2026-10-21: so does every cell of a pull-request run, ubuntu / 3.11 included.
# Coverage is measured in one cell of a run that is not a pull request
# (tests/test_ci_full_matrix_on_approval.py::TestCoverageIsMeasuredInOneCell); the
# parallelism pins below hold for both steps, because most cells now run the second.
NO_COV_STEP = "Run unit tests without coverage"
RUN_STEPS = (RUN_STEP, NO_COV_STEP)
INSTALL_STEP = "Install dependencies"


def _jobs() -> dict[str, Any]:
    return yaml.safe_load(CI.read_text(encoding="utf-8"))["jobs"]


def _step(name: str) -> dict[str, Any]:
    steps = [step for step in _jobs()["test"]["steps"] if step.get("name") == name]
    assert len(steps) == 1, (name, steps)
    return steps[0]


def _pytest_args(step: str = RUN_STEP) -> list[str]:
    args = shlex.split(str(_step(step)["run"]))
    assert args[:2] == ["pytest", "tests/"], args
    return args


def _option(args: list[str], flag: str) -> list[str]:
    """Every value given to ``flag``, as ``flag value`` or ``flag=value``."""
    values = [args[index + 1] for index, arg in enumerate(args[:-1]) if arg == flag]
    values += [arg.split("=", 1)[1] for arg in args if arg.startswith(flag + "=")]
    return values


class TestTheTestJobRunsInParallel:
    @pytest.mark.parametrize("step", RUN_STEPS)
    def test_one_worker_per_logical_cpu(self, step: str) -> None:
        assert _option(_pytest_args(step), "-n") == ["logical"]

    @pytest.mark.parametrize("step", RUN_STEPS)
    def test_one_test_file_per_worker(self, step: str) -> None:
        """`load` hands the tests of one file to several workers: module-level state and
        test order inside a file are lost, and a timing test failed under it."""
        assert _option(_pytest_args(step), "--dist") == ["loadfile"]

    def test_the_plugin_is_installed_with_its_cpu_counter(self) -> None:
        """Without the `psutil` extra `-n logical` falls back to another count."""
        requirements = shlex.split(str(_step(INSTALL_STEP)["run"]))
        assert requirements[:2] == ["pip", "install"], requirements
        assert "pytest-xdist[psutil]" in requirements
        assert ".[dev]" in requirements

    def test_the_plugin_is_installed_before_the_tests_run(self) -> None:
        names = [step.get("name") for step in _jobs()["test"]["steps"]]
        for step in RUN_STEPS:
            assert names.index(INSTALL_STEP) < names.index(step)

    @pytest.mark.parametrize("step", RUN_STEPS)
    @pytest.mark.parametrize(
        "argument",
        [
            "--junitxml=report.xml",  # the nightly badge parses it, from an unmeasured cell
            "-v",
            "--tb=short",
        ],
    )
    def test_the_reports_the_later_steps_read_are_still_written(
        self, step: str, argument: str
    ) -> None:
        assert argument in _pytest_args(step)

    @pytest.mark.parametrize(
        "argument",
        [
            "--cov=soup_cli",
            "--cov-report=xml:coverage.xml",  # the Codecov upload sends it
            "--cov-report=term-missing:skip-covered",
        ],
    )
    def test_the_measuring_step_still_writes_the_coverage_reports(self, argument: str) -> None:
        assert argument in _pytest_args(RUN_STEP)

    def test_the_other_step_turns_coverage_off_and_writes_no_coverage_report(self) -> None:
        args = _pytest_args(NO_COV_STEP)
        assert "--no-cov" in args
        assert not [arg for arg in args if arg.startswith("--cov")], args

    def test_no_step_of_any_job_spreads_one_file_over_workers(self) -> None:
        text = CI.read_text(encoding="utf-8")
        runs = [
            str(step["run"])
            for job in _jobs().values()
            for step in job.get("steps", [])
            if "run" in step
        ]
        assert runs, text[:200]
        for run in runs:
            for mode in re.findall(r"--dist[ =](\S+)", run):
                assert mode == "loadfile", run


class TestALocalRunStaysSerial:
    def _addopts(self) -> list[str]:
        text = PYPROJECT.read_text(encoding="utf-8")
        table = re.search(
            r"^\[tool\.pytest\.ini_options\]\n(.*?)(?=^\[|\Z)", text, re.M | re.S
        )
        assert table is not None
        line = re.search(r'^addopts\s*=\s*"(.*)"\s*$', table.group(1), re.M)
        assert line is not None, table.group(1)
        return shlex.split(line.group(1))

    def _extra_requirements(self) -> list[str]:
        text = PYPROJECT.read_text(encoding="utf-8")
        table = re.search(
            r"^\[project\.optional-dependencies\]\n(.*?)(?=^\[|\Z)", text, re.M | re.S
        )
        assert table is not None
        requirements = re.findall(r'"([^"\n]*)"', table.group(1))
        assert any(requirement.startswith("pytest") for requirement in requirements)
        return requirements

    def test_the_default_options_start_no_workers(self) -> None:
        """Workers are a CI choice. In `addopts` they would also be forced on every local
        run and on every subset run."""
        args = self._addopts()
        assert _option(args, "-n") == []
        assert _option(args, "--numprocesses") == []
        assert _option(args, "--dist") == []

    def test_the_plugin_is_not_a_dev_dependency(self) -> None:
        for requirement in self._extra_requirements():
            assert "xdist" not in requirement.lower(), requirement


def test_contributing_says_what_a_parallel_run_asks_of_a_test() -> None:
    text = " ".join(CONTRIBUTING.read_text(encoding="utf-8").split())
    assert "CI runs the tests in parallel" in text
    assert "`-n logical --dist loadfile`" in text
    assert 'pip install "pytest-xdist[psutil]"' in text