Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tests: fail grouped tests by name when --dist loadgroup stops grouping (pytest 9.2) (#15638)

Daniel Hane54ae4fdafa2
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
index 5d817b327a1..101ba3294cb 100644
--- a/.github/workflows/version-compat-ci.yml
+++ b/.github/workflows/version-compat-ci.yml
@@ -59,7 +59,7 @@ jobs:
# packaging: import_fixes.py, loaded by path, imports it at module top.
run: |
python -m pip install --upgrade pip
- pip install 'pytest>=8' 'pytest-xdist>=3' packaging
+ pip install 'pytest>=8,<9.2' 'pytest-xdist>=3' packaging
- name: Resolve the transformers matrix once
# Resolve the matrix once so xdist workers collect identical parameters.
env:
@@ -107,7 +107,7 @@ jobs:
- name: Install pytest only
run: |
python -m pip install --upgrade pip
- pip install 'pytest>=8' packaging pyyaml
+ pip install 'pytest>=8,<9.2' packaging pyyaml
- name: Assert every site declares the same window
# Keep on one line: PowerShell on windows-latest turns a `\` continuation into a literal path.
run: |
@@ -198,7 +198,7 @@ jobs:
# torchcodec is required on transformers 5.x: audio_utils reads its metadata unconditionally.
# ABI-coupled to torch, so the pair is chosen per lane. huggingface_hub unpinned: 4.x wants <1.0, 5.x >=1.5.0.
pip install ${{ matrix.pkg_pins }} \
- sentencepiece protobuf safetensors numpy 'pytest>=8' \
+ sentencepiece protobuf safetensors numpy 'pytest>=8,<9.2' \
tqdm packaging psutil Pillow requests
pip install --no-deps -e "$RUNNER_TEMP/unsloth-zoo"
pip install --no-deps -e ./unsloth
@@ -307,7 +307,7 @@ jobs:
${{ matrix.torch_pins }}
# TRL last so it pulls the transformers/peft it requires.
pip install ${{ matrix.pkg_pins }} \
- sentencepiece protobuf safetensors numpy 'pytest>=8' \
+ sentencepiece protobuf safetensors numpy 'pytest>=8,<9.2' \
tqdm packaging psutil Pillow requests
pip install ${{ matrix.trl_pin }}
pip install --no-deps -e "$RUNNER_TEMP/unsloth-zoo"
@@ -390,7 +390,7 @@ jobs:
with:
python-version: '3.12'
- name: Install pytest
- run: pip install 'pytest>=8'
+ run: pip install 'pytest>=8,<9.2'
- name: Run all version-compat suites in one process (no cache)
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 692d8212c51..22fddb398fd 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -103,6 +103,13 @@ def _no_vae_structural_verdict_cache(monkeypatch):
monkeypatch.setenv("UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE", "0")
[email protected](trylast = True)
+def pytest_collection_modifyitems(config, items):
+ """After xdist tags grouped tests; see tests/_shared/xdist_group_check.py."""
+ from xdist_group_check import check
+ check(config, items)
+
+
@pytest.fixture(autouse = True)
def _no_deleted_torch_config_entries():
"""A test that leaves a dynamo / inductor config entry deleted errors at its own teardown; see
diff --git a/tests/_shared/xdist_group_check.py b/tests/_shared/xdist_group_check.py
new file mode 100644
index 00000000000..96f90437121
--- /dev/null
+++ b/tests/_shared/xdist_group_check.py
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Fails grouped tests by name when ``--dist loadgroup`` stops grouping, instead of letting them flake.
+
+pytest-xdist puts an ``xdist_group`` test on its group's worker by rewriting ``item._nodeid`` to ``<id>@<group>``.
+pytest 9.2 builds ``Item.nodeid`` from a structured id instead, so that write is lost (pytest-xdist 3.8.0): every
+grouped test is scheduled like an ungrouped one, and pairs that must share a worker, such as the
+``*_reset_between_tests`` guards, fail whenever they land apart."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def ungrouped(config: Any, items: list) -> list[str]:
+ """Node ids carrying an ``xdist_group`` mark that loadgroup did not tag; empty outside an xdist loadgroup worker."""
+ if not getattr(config.option, "loadgroup", False) or not hasattr(config, "workerinput"):
+ return []
+ return [
+ item.nodeid
+ for item in items
+ if item.get_closest_marker("xdist_group") and "@" not in item.nodeid
+ ]
+
+
+def check(config: Any, items: list) -> None:
+ """Each grouped test that lost its tag fails with the reason; a collection error here is swallowed by xdist."""
+ lost = set(ungrouped(config, items))
+ if not lost:
+ return
+ import pytest
+
+ message = (
+ "--dist loadgroup did not tag this xdist_group test, so its group can run on different workers. This "
+ "pytest / pytest-xdist pair drops the group suffix; keep pytest below the release that breaks it (CI pins "
+ "pytest<9.2)."
+ )
+
+ def _fail(*args, **kwargs):
+ pytest.fail(message, pytrace = False)
+
+ for item in items:
+ if item.nodeid in lost:
+ item.runtest = _fail
diff --git a/tests/conftest.py b/tests/conftest.py
index c95c9b8a9fe..f46e8bde06f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -86,6 +86,13 @@ def _no_deleted_torch_config_entries():
check_after_test()
[email protected](trylast = True)
+def pytest_collection_modifyitems(config, items):
+ """After xdist tags grouped tests; see tests/_shared/xdist_group_check.py."""
+ from xdist_group_check import check
+ check(config, items)
+
+
def _has_real_accelerator() -> bool:
"""Calls the probe before any spoof; the name is pinned by a test that reads this file's AST."""
from real_accelerator import has_real_accelerator
diff --git a/tests/test_xdist_group_check.py b/tests/test_xdist_group_check.py
new file mode 100644
index 00000000000..0f0f1df2fdb
--- /dev/null
+++ b/tests/test_xdist_group_check.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""tests/_shared/xdist_group_check.py: on a loadgroup worker, a grouped test that lost its group tag fails by name."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+import xdist_group_check
+
+REPO = Path(__file__).resolve().parents[1]
+
+
+class _Item:
+ def __init__(self, nodeid, grouped):
+ self.nodeid = nodeid
+ self._grouped = grouped
+
+ def get_closest_marker(self, name):
+ return object() if name == "xdist_group" and self._grouped else None
+
+
+def _config(loadgroup = True, worker = True):
+ config = SimpleNamespace(option = SimpleNamespace(loadgroup = loadgroup))
+ if worker:
+ config.workerinput = {}
+ return config
+
+
+def test_a_grouped_test_without_its_group_tag_fails_with_the_reason():
+ lost, plain = _Item("t.py::test_a", True), _Item("t.py::test_b", False)
+ assert xdist_group_check.ungrouped(_config(), [lost, plain]) == ["t.py::test_a"]
+ xdist_group_check.check(_config(), [lost, plain])
+ with pytest.raises(pytest.fail.Exception, match = "did not tag this xdist_group test"):
+ lost.runtest()
+ assert "runtest" not in vars(plain)
+
+
+def test_tagged_and_ungrouped_tests_are_left_alone():
+ items = [_Item("t.py::test_a@g", True), _Item("t.py::test_b", False)]
+ xdist_group_check.check(_config(), items)
+ assert all("runtest" not in vars(item) for item in items)
+
+
[email protected]("config", [_config(loadgroup = False), _config(worker = False)])
+def test_only_a_loadgroup_worker_is_checked(config):
+ item = _Item("t.py::test_a", True)
+ xdist_group_check.check(config, [item])
+ assert "runtest" not in vars(item)
+
+
[email protected]("conftest", ["tests/conftest.py", "studio/backend/tests/conftest.py"])
+def test_both_suites_run_the_check_after_xdist_tags_groups(conftest):
+ tree = ast.parse((REPO / conftest).read_text(encoding = "utf-8"))
+ hook = next(
+ (
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "pytest_collection_modifyitems"
+ ),
+ None,
+ )
+ assert hook is not None, f"{conftest} lost the loadgroup check"
+ assert "trylast" in ast.unparse(hook.decorator_list[0])
+ assert "check" in [ast.unparse(n.func) for n in ast.walk(hook) if isinstance(n, ast.Call)]
+
+
+ "workflow, pin, unpinned",
+ [
+ (".github/workflows/studio-backend-ci.yml", "'pytest<9.2'", " pytest pytest-"),
+ (".github/workflows/version-compat-ci.yml", "'pytest>=8,<9.2'", "'pytest>=8'"),
+ ],
+)
+def test_ci_keeps_pytest_below_the_release_that_drops_xdist_groups(workflow, pin, unpinned):
+ text = (REPO / workflow).read_text(encoding = "utf-8")
+ assert pin in text
+ assert unpinned not in text, f"{workflow} installs an unpinned pytest"