Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
test: run the linear-growth timing tests on Linux only (#2148)
Alpamys Makazhan31b705796df4
diff --git a/tests/_growth_timing.py b/tests/_growth_timing.pyindex 9ebe2a97..f4712f1f 100644--- a/tests/_growth_timing.py+++ b/tests/_growth_timing.py@@ -28,15 +28,28 @@ growing past that length and pass whatever it did before it: give ``cap`` the le``TIMER_SLACK_SECONDS`` is added to the bound for the case where the call is so fast that thecap is reached before the reference is long. A quadratic scan fails by a wide margin at eithersize: 64x against a 32x limit.++Linux only+----------+Every test that calls ``assert_grows_linearly`` carries ``@linux_only`` (on the test, its class,+or the module's ``pytestmark``); ``tests/test_growth_timing.py`` fails a test that does not.+``re`` behaves the same on every OS, so what the tests pin (a scan is linear, not quadratic) is+decided on Linux, and every Linux CI cell runs them. A macOS or Windows runner adds only noise+to a wall-clock ratio: cold regex compilation and timer noise. To run them on any OS, for a+local check or a change to one of the scanners, set ``SOUP_RUN_GROWTH_TESTS=1``."""from __future__ import annotationsimport gc+import os+import sysimport timefrom collections.abc import Callablefrom typing import Any+import pytest+SCALE = 8LIMIT = 4.0 * SCALE # 4x slack over linear (8x), as in #1932; a quadratic scan measures 64xMIN_BASE_SECONDS = 0.005@@ -46,6 +59,26 @@ MIN_ROUNDS = 2 # a failing scan still gets this many rounds, however slowBUDGET_SECONDS = 2.0 # ...and no more once the rounds have spent this longTIMER_SLACK_SECONDS = 0.002+#: Set to ``1`` to run the growth tests on an OS that skips them (see "Linux only" above).+RUN_ENV_VAR = "SOUP_RUN_GROWTH_TESTS"+++def _runs_here() -> bool:+ """True on Linux, or anywhere when ``SOUP_RUN_GROWTH_TESTS=1``. Read when it is called."""+ return sys.platform.startswith("linux") or os.environ.get(RUN_ENV_VAR) == "1"+++#: THE gate for a test that calls ``assert_grows_linearly``. One definition; the test file+#: ``tests/test_growth_timing.py`` fails a growth test that does not carry it.+linux_only = pytest.mark.skipif(+ not _runs_here(),+ reason=(+ "linear-growth timing test, run on Linux only: re behaves the same on every OS, a "+ "macOS or Windows runner only adds cold-compile and timer noise to the ratio, and "+ "every Linux CI cell runs it. Set SOUP_RUN_GROWTH_TESTS=1 to run it on this OS."+ ),+)+def _seconds(call: Callable[[Any], object], payload: Any) -> float:start = time.perf_counter()diff --git a/tests/test_growth_timing.py b/tests/test_growth_timing.pynew file mode 100644index 00000000..707efa72--- /dev/null+++ b/tests/test_growth_timing.py@@ -0,0 +1,350 @@+"""``tests/_growth_timing.py``: the Linux-only gate on the linear-growth timing tests.++``assert_grows_linearly`` times a scan at ``n`` and at ``8n``. ``re`` behaves the same on every+OS, but a macOS or Windows runner adds cold regex compilation and timer noise that a Linux+runner does not, and every Linux CI cell runs these tests, so ``linux_only`` skips them+elsewhere. ``SOUP_RUN_GROWTH_TESTS=1`` runs them anywhere, for a local check.++Two things are pinned here:++* the gate itself (``TestTheGateRunsOnLinuxOrOnRequest``): a skipped test is the same colour as a+ passing one, so both edges of the condition are tested;+* that every test calling ``assert_grows_linearly`` carries the gate+ (``TestEveryGrowthTestCarriesTheGate``), so a sixth file cannot forget it. The scanner is+ shown able to fail on seeded sources, as in ``test_symlink_skips_are_capability_based.py``.+"""++from __future__ import annotations++import ast+import sys+import textwrap+from pathlib import Path++import pytest++from tests._growth_timing import RUN_ENV_VAR, _runs_here, linux_only++TESTS_DIR = Path(__file__).resolve().parent+HELPER = "assert_grows_linearly"+GATE = "linux_only"+++def _is_helper_module(name: str | None) -> bool:+ return bool(name) and name.split(".")[-1] == "_growth_timing"+++def _imported_names(tree: ast.Module) -> tuple[bool, set[str], set[str]]:+ """Whether the module imports ``_growth_timing``, and the local names of helper and gate."""+ imports = False+ helper_names = {HELPER}+ gate_names = {GATE}+ for node in ast.walk(tree):+ if isinstance(node, ast.ImportFrom):+ if _is_helper_module(node.module):+ imports = True+ for alias in node.names:+ if alias.name == HELPER and alias.asname:+ helper_names.add(alias.asname)+ if alias.name == GATE and alias.asname:+ gate_names.add(alias.asname)+ elif any(_is_helper_module(alias.name) for alias in node.names):+ imports = True+ elif isinstance(node, ast.Import):+ imports = imports or any(_is_helper_module(alias.name) for alias in node.names)+ return imports, helper_names, gate_names+++def _names_in(node: ast.AST, names: set[str]) -> bool:+ """Whether ``node`` is, or holds, a reference to one of ``names`` (``gate`` or ``mod.gate``)."""+ return any(+ (isinstance(part, ast.Name) and part.id in names)+ or (isinstance(part, ast.Attribute) and part.attr in names)+ for part in ast.walk(node)+ )+++def _carries(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, names: set[str]) -> bool:+ return any(_names_in(decorator, names) for decorator in node.decorator_list)+++def scan_source(source: str, filename: str = "<string>") -> tuple[bool, int, list[str]]:+ """Return ``(imports the helper, calls found, one problem string per ungated call)``."""+ tree = ast.parse(source, filename=filename)+ imports, helper_names, gate_names = _imported_names(tree)+ if not imports:+ return False, 0, []++ module_gated = any(+ isinstance(node, ast.Assign)+ and any(isinstance(t, ast.Name) and t.id == "pytestmark" for t in node.targets)+ and _names_in(node.value, gate_names)+ for node in tree.body+ )+ problems: list[str] = []+ calls = 0++ def visit(node: ast.AST, scope: list[ast.AST]) -> None:+ nonlocal calls+ if (+ isinstance(node, ast.Call)+ and (+ (isinstance(node.func, ast.Name) and node.func.id in helper_names)+ or (isinstance(node.func, ast.Attribute) and node.func.attr == HELPER)+ )+ ):+ calls += 1+ where = f"{filename}:{node.lineno}"+ functions = [s for s in scope if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef))]+ if module_gated:+ pass+ elif not functions:+ problems.append(+ f"{where}: {HELPER} runs at import time, where no mark can skip it; "+ f"move it into a test or set pytestmark = {GATE}"+ )+ else:+ entry = functions[0] # the outermost function: the test, or the helper it hides in+ if not entry.name.startswith("test"):+ problems.append(+ f"{where}: {HELPER} is called from {entry.name}(), not from a test, so "+ f"the mark on its callers cannot be checked here; set module-level "+ f"pytestmark = {GATE}"+ )+ elif not any(+ _carries(s, gate_names)+ for s in scope+ if s is entry or isinstance(s, ast.ClassDef)+ ):+ problems.append(+ f"{where}: {entry.name} calls {HELPER} but neither it, its class nor the "+ f"module carries @{GATE} (tests/_growth_timing.py)"+ )+ child_scope = scope + [node] if isinstance(+ node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)+ ) else scope+ for child in ast.iter_child_nodes(node):+ visit(child, child_scope)++ visit(tree, [])+ return True, calls, problems+++class TestTheGateRunsOnLinuxOrOnRequest:+ """A skipped test and a passing test are the same colour: pin both edges."""++ @pytest.mark.parametrize("platform", ["linux", "linux2"])+ def test_it_runs_on_linux(self, monkeypatch, platform):+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.delenv(RUN_ENV_VAR, raising=False)+ assert _runs_here() is True++ @pytest.mark.parametrize("platform", ["darwin", "win32", "cygwin", "freebsd14"])+ def test_it_does_not_run_elsewhere_by_default(self, monkeypatch, platform):+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.delenv(RUN_ENV_VAR, raising=False)+ assert _runs_here() is False++ @pytest.mark.parametrize("platform", ["darwin", "win32"])+ def test_the_environment_variable_runs_it_anywhere(self, monkeypatch, platform):+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.setenv(RUN_ENV_VAR, "1")+ assert _runs_here() is True++ @pytest.mark.parametrize("value", ["", "0", "no", "true"])+ def test_only_one_turns_the_escape_hatch_on(self, monkeypatch, value):+ """A near miss ("true", "0") must not silently run a 30-second timing test."""+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setenv(RUN_ENV_VAR, value)+ assert _runs_here() is False++ def test_the_variable_is_named_in_the_reason(self):+ reason = linux_only.kwargs["reason"]+ assert RUN_ENV_VAR == "SOUP_RUN_GROWTH_TESTS"+ assert f"{RUN_ENV_VAR}=1" in reason+ assert "Linux" in reason and "OS" in reason++ def test_the_mark_is_a_skipif_on_the_gate(self):+ assert linux_only.name == "skipif"+ assert linux_only.args == (not _runs_here(),)+++class TestEveryGrowthTestCarriesTheGate:+ def test_every_test_that_times_growth_is_gated(self):+ importers: dict[str, int] = {}+ problems: list[str] = []+ for path in sorted(TESTS_DIR.rglob("*.py")):+ if path.name == "_growth_timing.py":+ continue+ text = path.read_text(encoding="utf-8")+ if "_growth_timing" not in text:+ continue+ try:+ imports, calls, found = scan_source(text, path.name)+ except SyntaxError:+ continue+ if imports:+ importers[path.name] = calls+ problems += found+ assert not problems, "\n".join(problems)++ # The scan must be looking at something: the five files of #2069, each with a call.+ known = {+ "test_issue1226_clause_scanner_is_linear.py",+ "test_issue1329_email_pattern_is_linear.py",+ "test_issue1346_rewrites_are_linear.py",+ "test_issue1347_boxed_scan.py",+ "test_issue1528_contains_is_linear.py",+ }+ assert known <= set(importers), sorted(known - set(importers))+ assert all(importers[name] for name in known), importers++ def test_a_sixth_file_without_the_gate_is_caught(self):+ source = textwrap.dedent(+ """+ from tests._growth_timing import assert_grows_linearly++ def test_new_scan_is_linear():+ assert_grows_linearly(len, lambda k: "a" * k, 10)+ """+ )+ imports, calls, problems = scan_source(source, "test_new.py")+ assert (imports, calls) == (True, 1)+ assert len(problems) == 1 and "test_new_scan_is_linear" in problems[0]+ assert "@linux_only" in problems[0]++ @pytest.mark.parametrize(+ "source",+ [+ # on the test+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ @linux_only+ def test_a():+ assert_grows_linearly(len, str, 1)+ """,+ # under a parametrize, imported under another name+ """+ import pytest+ from tests._growth_timing import assert_grows_linearly as grows+ from tests._growth_timing import linux_only as only_linux++ @pytest.mark.parametrize("n", [1, 2])+ @only_linux+ def test_a(n):+ grows(len, str, n)+ """,+ # on the class+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ @linux_only+ class TestA:+ def test_a(self):+ assert_grows_linearly(len, str, 1)+ """,+ # module-wide, as a list+ """+ import pytest+ from tests._growth_timing import assert_grows_linearly, linux_only++ pytestmark = [pytest.mark.unit, linux_only]++ def _check():+ assert_grows_linearly(len, str, 1)++ def test_a():+ _check()+ """,+ # through the module+ """+ from tests import _growth_timing++ @_growth_timing.linux_only+ def test_a():+ _growth_timing.assert_grows_linearly(len, str, 1)+ """,+ ],+ )+ def test_every_way_of_carrying_the_gate_is_accepted(self, source):+ imports, calls, problems = scan_source(textwrap.dedent(source))+ assert imports and calls == 1 and problems == []++ @pytest.mark.parametrize(+ "source, expected",+ [+ # the mark is on a neighbour, not on the test that times+ (+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ @linux_only+ def test_a():+ pass++ def test_b():+ assert_grows_linearly(len, str, 1)+ """,+ "test_b",+ ),+ # the class holds a gated test and an ungated one+ (+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ class TestA:+ @linux_only+ def test_a(self):+ pass++ def test_b(self):+ assert_grows_linearly(len, str, 1)+ """,+ "test_b",+ ),+ # a decorator of the same name that is not the shared one+ (+ """+ import pytest+ from tests._growth_timing import assert_grows_linearly++ @pytest.mark.skipif(False, reason="x")+ def test_a():+ assert_grows_linearly(len, str, 1)+ """,+ "test_a",+ ),+ # a helper hides the call and the module is not gated+ (+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ def _check():+ assert_grows_linearly(len, str, 1)++ @linux_only+ def test_a():+ _check()+ """,+ "_check()",+ ),+ # a call that runs at import time+ (+ """+ from tests._growth_timing import assert_grows_linearly, linux_only++ assert_grows_linearly(len, str, 1)+ """,+ "import time",+ ),+ ],+ )+ def test_each_way_of_missing_the_gate_is_caught(self, source, expected):+ _imports, calls, problems = scan_source(textwrap.dedent(source), "test_x.py")+ assert calls == 1 and len(problems) == 1, problems+ assert expected in problems[0]++ def test_a_file_that_does_not_import_the_helper_is_not_scanned(self):+ source = "def test_a():\n assert_grows_linearly(len, str, 1)\n"+ assert scan_source(source) == (False, 0, [])diff --git a/tests/test_issue1226_clause_scanner_is_linear.py b/tests/test_issue1226_clause_scanner_is_linear.pyindex b1103223..1e83bd57 100644--- a/tests/test_issue1226_clause_scanner_is_linear.py+++ b/tests/test_issue1226_clause_scanner_is_linear.py@@ -25,7 +25,7 @@ import pytestfrom soup_cli.trainer.rewards import accuracy_reward, math_verify_rewardfrom soup_cli.utils.final_answer import parse_completion-from tests._growth_timing import assert_grows_linearly+from tests._growth_timing import assert_grows_linearly, linux_onlydef _msg(text: str) -> list[list[dict]]:@@ -61,6 +61,7 @@ WHITESPACE_RUNS = {_SCALE_N = 100 # the starting size; the helper doubles it until the run is long enough to time+@linux_onlyclass TestTheClauseScannerIsLinear:@pytest.mark.parametrize("build", list(WHITESPACE_RUNS.values()), ids=list(WHITESPACE_RUNS))def test_time_grows_linearly_with_a_whitespace_run(self, build):diff --git a/tests/test_issue1329_email_pattern_is_linear.py b/tests/test_issue1329_email_pattern_is_linear.pyindex 7bc168e3..eb6a1592 100644--- a/tests/test_issue1329_email_pattern_is_linear.py+++ b/tests/test_issue1329_email_pattern_is_linear.py@@ -12,7 +12,7 @@ from typer.testing import CliRunnerfrom soup_cli.cli import appfrom soup_cli.utils import data_score-from tests._growth_timing import assert_grows_linearly+from tests._growth_timing import assert_grows_linearly, linux_only_BASELINE_EMAIL_PATTERN = re.compile(r"\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b")_PATHOLOGICAL_RUNS = ("1-", "1.", "a.", "a-", "a+")@@ -34,6 +34,7 @@ def _row(run: str, length: int) -> str:return (run * ((length + len(run) - 1) // len(run)))[:length]+@linux_onlydef test_email_pattern_growth_is_linear() -> None:"""The pattern alone: an 8x longer row must not take 32x the time (linear 8x, quadratic 64x).@@ -48,6 +49,7 @@ def test_email_pattern_growth_is_linear() -> None:)+@linux_onlydef test_email_detection_growth_is_subquadratic(monkeypatch: pytest.MonkeyPatch) -> None:"""``detect_pii`` end to end (all four patterns), within the rows it scans in full."""monkeypatch.setattr(data_score, "_presidio_pii", lambda _text: None)diff --git a/tests/test_issue1346_rewrites_are_linear.py b/tests/test_issue1346_rewrites_are_linear.pyindex 140d93ed..866b4070 100644--- a/tests/test_issue1346_rewrites_are_linear.py+++ b/tests/test_issue1346_rewrites_are_linear.py@@ -11,7 +11,7 @@ from __future__ import annotationsimport pytestfrom soup_cli.trainer.rewards import accuracy_reward, math_verify_reward-from tests._growth_timing import assert_grows_linearly+from tests._growth_timing import assert_grows_linearly, linux_only_SCALE_N = 100 # the starting size; the helper doubles it until the run is long enough to time@@ -41,6 +41,7 @@ def _score(text: str) -> None:math_verify_reward(_msg(text), answer=["42"])+@linux_only@pytest.mark.parametrize("build", list(REWRITE_INPUTS.values()), ids=list(REWRITE_INPUTS))def test_each_rewrite_grows_linearly(build):assert_grows_linearly(_score, build, _SCALE_N)diff --git a/tests/test_issue1347_boxed_scan.py b/tests/test_issue1347_boxed_scan.pyindex 671256b7..49cad3d5 100644--- a/tests/test_issue1347_boxed_scan.py+++ b/tests/test_issue1347_boxed_scan.py@@ -15,7 +15,7 @@ letter. Neither owns a regex any more.from __future__ import annotationsfrom soup_cli.utils.final_answer import BoxedAnswer, iter_boxed_answers, parse_completion-from tests._growth_timing import assert_grows_linearly+from tests._growth_timing import assert_grows_linearly, linux_only# The size the growth checks start from. ``tests/_growth_timing.py`` doubles it until one scan# lasts long enough to time, then compares it with a scan of 8x the input. It replaced a@@ -75,6 +75,7 @@ class TestIssue1347TheScanReadsEveryClosedBox:assert parsed is not None and parsed.text == box.answer+@linux_onlyclass TestIssue1347TheScanGrowsLinearly:def test_unclosed_boxes_cost_linear_time(self):"""A run of ``\\boxed{`` that never closes is the scan's worst case: every brace is@@ -91,6 +92,7 @@ class TestIssue1347TheScanGrowsLinearly:class TestIssue1347NestedBoxes:+ @linux_onlydef test_nested_closed_boxes_cost_linear_time(self):"""Every box closes and holds the next, so the boxes' contents overlap. Skipping theboxes that hold a box is what keeps this linear: normalising them all would be quadraticdiff --git a/tests/test_issue1528_contains_is_linear.py b/tests/test_issue1528_contains_is_linear.pyindex 70501072..873e334a 100644--- a/tests/test_issue1528_contains_is_linear.py+++ b/tests/test_issue1528_contains_is_linear.py@@ -15,7 +15,7 @@ import reimport pytestfrom soup_cli.eval.custom import score_contains-from tests._growth_timing import assert_grows_linearly+from tests._growth_timing import assert_grows_linearly, linux_only_SCALE_N = 250 # the starting size; the helper doubles it until the run is long enough to time@@ -27,6 +27,7 @@ SHAPES = {}+@linux_only@pytest.mark.parametrize("build", list(SHAPES.values()), ids=list(SHAPES))def test_contains_grows_linearly(build):assert_grows_linearly(lambda pair: score_contains(*pair), build, _SCALE_N)