Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

test: run the linear-growth timing tests on Linux only (#2148)

Alpamys Makazhan31b705796df4
diff --git a/tests/_growth_timing.py b/tests/_growth_timing.py
index 9ebe2a97..f4712f1f 100644
--- a/tests/_growth_timing.py
+++ b/tests/_growth_timing.py
@@ -28,15 +28,28 @@ growing past that length and pass whatever it did before it: give ``cap`` the le
``TIMER_SLACK_SECONDS`` is added to the bound for the case where the call is so fast that the
cap is reached before the reference is long. A quadratic scan fails by a wide margin at either
size: 64x against a 32x limit.
+
+Linux only
+----------
+Every test that calls ``assert_grows_linearly`` carries ``@linux_only`` (on the test, its class,
+or the module's ``pytestmark``); ``tests/test_growth_timing.py`` fails a test that does not.
+``re`` behaves the same on every OS, so what the tests pin (a scan is linear, not quadratic) is
+decided on Linux, and every Linux CI cell runs them. A macOS or Windows runner adds only noise
+to a wall-clock ratio: cold regex compilation and timer noise. To run them on any OS, for a
+local check or a change to one of the scanners, set ``SOUP_RUN_GROWTH_TESTS=1``.
"""
from __future__ import annotations
import gc
+import os
+import sys
import time
from collections.abc import Callable
from typing import Any
+import pytest
+
SCALE = 8
LIMIT = 4.0 * SCALE # 4x slack over linear (8x), as in #1932; a quadratic scan measures 64x
MIN_BASE_SECONDS = 0.005
@@ -46,6 +59,26 @@ MIN_ROUNDS = 2 # a failing scan still gets this many rounds, however slow
BUDGET_SECONDS = 2.0 # ...and no more once the rounds have spent this long
TIMER_SLACK_SECONDS = 0.002
+#: Set to ``1`` to run the growth tests on an OS that skips them (see "Linux only" above).
+RUN_ENV_VAR = "SOUP_RUN_GROWTH_TESTS"
+
+
+def _runs_here() -> bool:
+ """True on Linux, or anywhere when ``SOUP_RUN_GROWTH_TESTS=1``. Read when it is called."""
+ return sys.platform.startswith("linux") or os.environ.get(RUN_ENV_VAR) == "1"
+
+
+#: THE gate for a test that calls ``assert_grows_linearly``. One definition; the test file
+#: ``tests/test_growth_timing.py`` fails a growth test that does not carry it.
+linux_only = pytest.mark.skipif(
+ not _runs_here(),
+ reason=(
+ "linear-growth timing test, run on Linux only: re behaves the same on every OS, a "
+ "macOS or Windows runner only adds cold-compile and timer noise to the ratio, and "
+ "every Linux CI cell runs it. Set SOUP_RUN_GROWTH_TESTS=1 to run it on this OS."
+ ),
+)
+
def _seconds(call: Callable[[Any], object], payload: Any) -> float:
start = time.perf_counter()
diff --git a/tests/test_growth_timing.py b/tests/test_growth_timing.py
new file mode 100644
index 00000000..707efa72
--- /dev/null
+++ b/tests/test_growth_timing.py
@@ -0,0 +1,350 @@
+"""``tests/_growth_timing.py``: the Linux-only gate on the linear-growth timing tests.
+
+``assert_grows_linearly`` times a scan at ``n`` and at ``8n``. ``re`` behaves the same on every
+OS, but a macOS or Windows runner adds cold regex compilation and timer noise that a Linux
+runner does not, and every Linux CI cell runs these tests, so ``linux_only`` skips them
+elsewhere. ``SOUP_RUN_GROWTH_TESTS=1`` runs them anywhere, for a local check.
+
+Two things are pinned here:
+
+* the gate itself (``TestTheGateRunsOnLinuxOrOnRequest``): a skipped test is the same colour as a
+ passing one, so both edges of the condition are tested;
+* that every test calling ``assert_grows_linearly`` carries the gate
+ (``TestEveryGrowthTestCarriesTheGate``), so a sixth file cannot forget it. The scanner is
+ shown able to fail on seeded sources, as in ``test_symlink_skips_are_capability_based.py``.
+"""
+
+from __future__ import annotations
+
+import ast
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from tests._growth_timing import RUN_ENV_VAR, _runs_here, linux_only
+
+TESTS_DIR = Path(__file__).resolve().parent
+HELPER = "assert_grows_linearly"
+GATE = "linux_only"
+
+
+def _is_helper_module(name: str | None) -> bool:
+ return bool(name) and name.split(".")[-1] == "_growth_timing"
+
+
+def _imported_names(tree: ast.Module) -> tuple[bool, set[str], set[str]]:
+ """Whether the module imports ``_growth_timing``, and the local names of helper and gate."""
+ imports = False
+ helper_names = {HELPER}
+ gate_names = {GATE}
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ImportFrom):
+ if _is_helper_module(node.module):
+ imports = True
+ for alias in node.names:
+ if alias.name == HELPER and alias.asname:
+ helper_names.add(alias.asname)
+ if alias.name == GATE and alias.asname:
+ gate_names.add(alias.asname)
+ elif any(_is_helper_module(alias.name) for alias in node.names):
+ imports = True
+ elif isinstance(node, ast.Import):
+ imports = imports or any(_is_helper_module(alias.name) for alias in node.names)
+ return imports, helper_names, gate_names
+
+
+def _names_in(node: ast.AST, names: set[str]) -> bool:
+ """Whether ``node`` is, or holds, a reference to one of ``names`` (``gate`` or ``mod.gate``)."""
+ return any(
+ (isinstance(part, ast.Name) and part.id in names)
+ or (isinstance(part, ast.Attribute) and part.attr in names)
+ for part in ast.walk(node)
+ )
+
+
+def _carries(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, names: set[str]) -> bool:
+ return any(_names_in(decorator, names) for decorator in node.decorator_list)
+
+
+def scan_source(source: str, filename: str = "<string>") -> tuple[bool, int, list[str]]:
+ """Return ``(imports the helper, calls found, one problem string per ungated call)``."""
+ tree = ast.parse(source, filename=filename)
+ imports, helper_names, gate_names = _imported_names(tree)
+ if not imports:
+ return False, 0, []
+
+ module_gated = any(
+ isinstance(node, ast.Assign)
+ and any(isinstance(t, ast.Name) and t.id == "pytestmark" for t in node.targets)
+ and _names_in(node.value, gate_names)
+ for node in tree.body
+ )
+ problems: list[str] = []
+ calls = 0
+
+ def visit(node: ast.AST, scope: list[ast.AST]) -> None:
+ nonlocal calls
+ if (
+ isinstance(node, ast.Call)
+ and (
+ (isinstance(node.func, ast.Name) and node.func.id in helper_names)
+ or (isinstance(node.func, ast.Attribute) and node.func.attr == HELPER)
+ )
+ ):
+ calls += 1
+ where = f"{filename}:{node.lineno}"
+ functions = [s for s in scope if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef))]
+ if module_gated:
+ pass
+ elif not functions:
+ problems.append(
+ f"{where}: {HELPER} runs at import time, where no mark can skip it; "
+ f"move it into a test or set pytestmark = {GATE}"
+ )
+ else:
+ entry = functions[0] # the outermost function: the test, or the helper it hides in
+ if not entry.name.startswith("test"):
+ problems.append(
+ f"{where}: {HELPER} is called from {entry.name}(), not from a test, so "
+ f"the mark on its callers cannot be checked here; set module-level "
+ f"pytestmark = {GATE}"
+ )
+ elif not any(
+ _carries(s, gate_names)
+ for s in scope
+ if s is entry or isinstance(s, ast.ClassDef)
+ ):
+ problems.append(
+ f"{where}: {entry.name} calls {HELPER} but neither it, its class nor the "
+ f"module carries @{GATE} (tests/_growth_timing.py)"
+ )
+ child_scope = scope + [node] if isinstance(
+ node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+ ) else scope
+ for child in ast.iter_child_nodes(node):
+ visit(child, child_scope)
+
+ visit(tree, [])
+ return True, calls, problems
+
+
+class TestTheGateRunsOnLinuxOrOnRequest:
+ """A skipped test and a passing test are the same colour: pin both edges."""
+
+ @pytest.mark.parametrize("platform", ["linux", "linux2"])
+ def test_it_runs_on_linux(self, monkeypatch, platform):
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.delenv(RUN_ENV_VAR, raising=False)
+ assert _runs_here() is True
+
+ @pytest.mark.parametrize("platform", ["darwin", "win32", "cygwin", "freebsd14"])
+ def test_it_does_not_run_elsewhere_by_default(self, monkeypatch, platform):
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.delenv(RUN_ENV_VAR, raising=False)
+ assert _runs_here() is False
+
+ @pytest.mark.parametrize("platform", ["darwin", "win32"])
+ def test_the_environment_variable_runs_it_anywhere(self, monkeypatch, platform):
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setenv(RUN_ENV_VAR, "1")
+ assert _runs_here() is True
+
+ @pytest.mark.parametrize("value", ["", "0", "no", "true"])
+ def test_only_one_turns_the_escape_hatch_on(self, monkeypatch, value):
+ """A near miss ("true", "0") must not silently run a 30-second timing test."""
+ monkeypatch.setattr(sys, "platform", "win32")
+ monkeypatch.setenv(RUN_ENV_VAR, value)
+ assert _runs_here() is False
+
+ def test_the_variable_is_named_in_the_reason(self):
+ reason = linux_only.kwargs["reason"]
+ assert RUN_ENV_VAR == "SOUP_RUN_GROWTH_TESTS"
+ assert f"{RUN_ENV_VAR}=1" in reason
+ assert "Linux" in reason and "OS" in reason
+
+ def test_the_mark_is_a_skipif_on_the_gate(self):
+ assert linux_only.name == "skipif"
+ assert linux_only.args == (not _runs_here(),)
+
+
+class TestEveryGrowthTestCarriesTheGate:
+ def test_every_test_that_times_growth_is_gated(self):
+ importers: dict[str, int] = {}
+ problems: list[str] = []
+ for path in sorted(TESTS_DIR.rglob("*.py")):
+ if path.name == "_growth_timing.py":
+ continue
+ text = path.read_text(encoding="utf-8")
+ if "_growth_timing" not in text:
+ continue
+ try:
+ imports, calls, found = scan_source(text, path.name)
+ except SyntaxError:
+ continue
+ if imports:
+ importers[path.name] = calls
+ problems += found
+ assert not problems, "\n".join(problems)
+
+ # The scan must be looking at something: the five files of #2069, each with a call.
+ known = {
+ "test_issue1226_clause_scanner_is_linear.py",
+ "test_issue1329_email_pattern_is_linear.py",
+ "test_issue1346_rewrites_are_linear.py",
+ "test_issue1347_boxed_scan.py",
+ "test_issue1528_contains_is_linear.py",
+ }
+ assert known <= set(importers), sorted(known - set(importers))
+ assert all(importers[name] for name in known), importers
+
+ def test_a_sixth_file_without_the_gate_is_caught(self):
+ source = textwrap.dedent(
+ """
+ from tests._growth_timing import assert_grows_linearly
+
+ def test_new_scan_is_linear():
+ assert_grows_linearly(len, lambda k: "a" * k, 10)
+ """
+ )
+ imports, calls, problems = scan_source(source, "test_new.py")
+ assert (imports, calls) == (True, 1)
+ assert len(problems) == 1 and "test_new_scan_is_linear" in problems[0]
+ assert "@linux_only" in problems[0]
+
+ @pytest.mark.parametrize(
+ "source",
+ [
+ # on the test
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ @linux_only
+ def test_a():
+ assert_grows_linearly(len, str, 1)
+ """,
+ # under a parametrize, imported under another name
+ """
+ import pytest
+ from tests._growth_timing import assert_grows_linearly as grows
+ from tests._growth_timing import linux_only as only_linux
+
+ @pytest.mark.parametrize("n", [1, 2])
+ @only_linux
+ def test_a(n):
+ grows(len, str, n)
+ """,
+ # on the class
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ @linux_only
+ class TestA:
+ def test_a(self):
+ assert_grows_linearly(len, str, 1)
+ """,
+ # module-wide, as a list
+ """
+ import pytest
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ pytestmark = [pytest.mark.unit, linux_only]
+
+ def _check():
+ assert_grows_linearly(len, str, 1)
+
+ def test_a():
+ _check()
+ """,
+ # through the module
+ """
+ from tests import _growth_timing
+
+ @_growth_timing.linux_only
+ def test_a():
+ _growth_timing.assert_grows_linearly(len, str, 1)
+ """,
+ ],
+ )
+ def test_every_way_of_carrying_the_gate_is_accepted(self, source):
+ imports, calls, problems = scan_source(textwrap.dedent(source))
+ assert imports and calls == 1 and problems == []
+
+ @pytest.mark.parametrize(
+ "source, expected",
+ [
+ # the mark is on a neighbour, not on the test that times
+ (
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ @linux_only
+ def test_a():
+ pass
+
+ def test_b():
+ assert_grows_linearly(len, str, 1)
+ """,
+ "test_b",
+ ),
+ # the class holds a gated test and an ungated one
+ (
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ class TestA:
+ @linux_only
+ def test_a(self):
+ pass
+
+ def test_b(self):
+ assert_grows_linearly(len, str, 1)
+ """,
+ "test_b",
+ ),
+ # a decorator of the same name that is not the shared one
+ (
+ """
+ import pytest
+ from tests._growth_timing import assert_grows_linearly
+
+ @pytest.mark.skipif(False, reason="x")
+ def test_a():
+ assert_grows_linearly(len, str, 1)
+ """,
+ "test_a",
+ ),
+ # a helper hides the call and the module is not gated
+ (
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ def _check():
+ assert_grows_linearly(len, str, 1)
+
+ @linux_only
+ def test_a():
+ _check()
+ """,
+ "_check()",
+ ),
+ # a call that runs at import time
+ (
+ """
+ from tests._growth_timing import assert_grows_linearly, linux_only
+
+ assert_grows_linearly(len, str, 1)
+ """,
+ "import time",
+ ),
+ ],
+ )
+ def test_each_way_of_missing_the_gate_is_caught(self, source, expected):
+ _imports, calls, problems = scan_source(textwrap.dedent(source), "test_x.py")
+ assert calls == 1 and len(problems) == 1, problems
+ assert expected in problems[0]
+
+ def test_a_file_that_does_not_import_the_helper_is_not_scanned(self):
+ source = "def test_a():\n assert_grows_linearly(len, str, 1)\n"
+ assert scan_source(source) == (False, 0, [])
diff --git a/tests/test_issue1226_clause_scanner_is_linear.py b/tests/test_issue1226_clause_scanner_is_linear.py
index b1103223..1e83bd57 100644
--- a/tests/test_issue1226_clause_scanner_is_linear.py
+++ b/tests/test_issue1226_clause_scanner_is_linear.py
@@ -25,7 +25,7 @@ import pytest
from soup_cli.trainer.rewards import accuracy_reward, math_verify_reward
from soup_cli.utils.final_answer import parse_completion
-from tests._growth_timing import assert_grows_linearly
+from tests._growth_timing import assert_grows_linearly, linux_only
def _msg(text: str) -> list[list[dict]]:
@@ -61,6 +61,7 @@ WHITESPACE_RUNS = {
_SCALE_N = 100 # the starting size; the helper doubles it until the run is long enough to time
+@linux_only
class TestTheClauseScannerIsLinear:
@pytest.mark.parametrize("build", list(WHITESPACE_RUNS.values()), ids=list(WHITESPACE_RUNS))
def test_time_grows_linearly_with_a_whitespace_run(self, build):
diff --git a/tests/test_issue1329_email_pattern_is_linear.py b/tests/test_issue1329_email_pattern_is_linear.py
index 7bc168e3..eb6a1592 100644
--- a/tests/test_issue1329_email_pattern_is_linear.py
+++ b/tests/test_issue1329_email_pattern_is_linear.py
@@ -12,7 +12,7 @@ from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.utils import data_score
-from tests._growth_timing import assert_grows_linearly
+from tests._growth_timing import assert_grows_linearly, linux_only
_BASELINE_EMAIL_PATTERN = re.compile(r"\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b")
_PATHOLOGICAL_RUNS = ("1-", "1.", "a.", "a-", "a+")
@@ -34,6 +34,7 @@ def _row(run: str, length: int) -> str:
return (run * ((length + len(run) - 1) // len(run)))[:length]
+@linux_only
def test_email_pattern_growth_is_linear() -> None:
"""The pattern alone: an 8x longer row must not take 32x the time (linear 8x, quadratic 64x).
@@ -48,6 +49,7 @@ def test_email_pattern_growth_is_linear() -> None:
)
+@linux_only
def test_email_detection_growth_is_subquadratic(monkeypatch: pytest.MonkeyPatch) -> None:
"""``detect_pii`` end to end (all four patterns), within the rows it scans in full."""
monkeypatch.setattr(data_score, "_presidio_pii", lambda _text: None)
diff --git a/tests/test_issue1346_rewrites_are_linear.py b/tests/test_issue1346_rewrites_are_linear.py
index 140d93ed..866b4070 100644
--- a/tests/test_issue1346_rewrites_are_linear.py
+++ b/tests/test_issue1346_rewrites_are_linear.py
@@ -11,7 +11,7 @@ from __future__ import annotations
import pytest
from soup_cli.trainer.rewards import accuracy_reward, math_verify_reward
-from tests._growth_timing import assert_grows_linearly
+from tests._growth_timing import assert_grows_linearly, linux_only
_SCALE_N = 100 # the starting size; the helper doubles it until the run is long enough to time
@@ -41,6 +41,7 @@ def _score(text: str) -> None:
math_verify_reward(_msg(text), answer=["42"])
+@linux_only
@pytest.mark.parametrize("build", list(REWRITE_INPUTS.values()), ids=list(REWRITE_INPUTS))
def test_each_rewrite_grows_linearly(build):
assert_grows_linearly(_score, build, _SCALE_N)
diff --git a/tests/test_issue1347_boxed_scan.py b/tests/test_issue1347_boxed_scan.py
index 671256b7..49cad3d5 100644
--- a/tests/test_issue1347_boxed_scan.py
+++ b/tests/test_issue1347_boxed_scan.py
@@ -15,7 +15,7 @@ letter. Neither owns a regex any more.
from __future__ import annotations
from soup_cli.utils.final_answer import BoxedAnswer, iter_boxed_answers, parse_completion
-from tests._growth_timing import assert_grows_linearly
+from tests._growth_timing import assert_grows_linearly, linux_only
# The size the growth checks start from. ``tests/_growth_timing.py`` doubles it until one scan
# lasts long enough to time, then compares it with a scan of 8x the input. It replaced a
@@ -75,6 +75,7 @@ class TestIssue1347TheScanReadsEveryClosedBox:
assert parsed is not None and parsed.text == box.answer
+@linux_only
class TestIssue1347TheScanGrowsLinearly:
def test_unclosed_boxes_cost_linear_time(self):
"""A run of ``\\boxed{`` that never closes is the scan's worst case: every brace is
@@ -91,6 +92,7 @@ class TestIssue1347TheScanGrowsLinearly:
class TestIssue1347NestedBoxes:
+ @linux_only
def test_nested_closed_boxes_cost_linear_time(self):
"""Every box closes and holds the next, so the boxes' contents overlap. Skipping the
boxes that hold a box is what keeps this linear: normalising them all would be quadratic
diff --git a/tests/test_issue1528_contains_is_linear.py b/tests/test_issue1528_contains_is_linear.py
index 70501072..873e334a 100644
--- a/tests/test_issue1528_contains_is_linear.py
+++ b/tests/test_issue1528_contains_is_linear.py
@@ -15,7 +15,7 @@ import re
import pytest
from soup_cli.eval.custom import score_contains
-from tests._growth_timing import assert_grows_linearly
+from tests._growth_timing import assert_grows_linearly, linux_only
_SCALE_N = 250 # the starting size; the helper doubles it until the run is long enough to time
@@ -27,6 +27,7 @@ SHAPES = {
}
+@linux_only
@pytest.mark.parametrize("build", list(SHAPES.values()), ids=list(SHAPES))
def test_contains_grows_linearly(build):
assert_grows_linearly(lambda pair: score_contains(*pair), build, _SCALE_N)